xref: /freebsd/crypto/openssl/crypto/rsa/rsa_sign.c (revision 9e5787d2284e187abb5b654d924394a65772e004)
1 /*
2  * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
3  *
4  * Licensed under the OpenSSL license (the "License").  You may not use
5  * this file except in compliance with the License.  You can obtain a copy
6  * in the file LICENSE in the source distribution or at
7  * https://www.openssl.org/source/license.html
8  */
9 
10 #include <stdio.h>
11 #include "internal/cryptlib.h"
12 #include <openssl/bn.h>
13 #include <openssl/rsa.h>
14 #include <openssl/objects.h>
15 #include <openssl/x509.h>
16 #include "crypto/x509.h"
17 #include "rsa_local.h"
18 
19 /* Size of an SSL signature: MD5+SHA1 */
20 #define SSL_SIG_LENGTH  36
21 
22 /*
23  * encode_pkcs1 encodes a DigestInfo prefix of hash |type| and digest |m|, as
24  * described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
25  * encodes the DigestInfo (T and tLen) but does not add the padding.
26  *
27  * On success, it returns one and sets |*out| to a newly allocated buffer
28  * containing the result and |*out_len| to its length. The caller must free
29  * |*out| with |OPENSSL_free|. Otherwise, it returns zero.
30  */
31 static int encode_pkcs1(unsigned char **out, int *out_len, int type,
32                         const unsigned char *m, unsigned int m_len)
33 {
34     X509_SIG sig;
35     X509_ALGOR algor;
36     ASN1_TYPE parameter;
37     ASN1_OCTET_STRING digest;
38     uint8_t *der = NULL;
39     int len;
40 
41     sig.algor = &algor;
42     sig.algor->algorithm = OBJ_nid2obj(type);
43     if (sig.algor->algorithm == NULL) {
44         RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
45         return 0;
46     }
47     if (OBJ_length(sig.algor->algorithm) == 0) {
48         RSAerr(RSA_F_ENCODE_PKCS1,
49                RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
50         return 0;
51     }
52     parameter.type = V_ASN1_NULL;
53     parameter.value.ptr = NULL;
54     sig.algor->parameter = &parameter;
55 
56     sig.digest = &digest;
57     sig.digest->data = (unsigned char *)m;
58     sig.digest->length = m_len;
59 
60     len = i2d_X509_SIG(&sig, &der);
61     if (len < 0)
62         return 0;
63 
64     *out = der;
65     *out_len = len;
66     return 1;
67 }
68 
69 int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
70              unsigned char *sigret, unsigned int *siglen, RSA *rsa)
71 {
72     int encrypt_len, encoded_len = 0, ret = 0;
73     unsigned char *tmps = NULL;
74     const unsigned char *encoded = NULL;
75 
76     if (rsa->meth->rsa_sign) {
77         return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
78     }
79 
80     /* Compute the encoded digest. */
81     if (type == NID_md5_sha1) {
82         /*
83          * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
84          * earlier. It has no DigestInfo wrapper but otherwise is
85          * RSASSA-PKCS1-v1_5.
86          */
87         if (m_len != SSL_SIG_LENGTH) {
88             RSAerr(RSA_F_RSA_SIGN, RSA_R_INVALID_MESSAGE_LENGTH);
89             return 0;
90         }
91         encoded_len = SSL_SIG_LENGTH;
92         encoded = m;
93     } else {
94         if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
95             goto err;
96         encoded = tmps;
97     }
98 
99     if (encoded_len > RSA_size(rsa) - RSA_PKCS1_PADDING_SIZE) {
100         RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
101         goto err;
102     }
103     encrypt_len = RSA_private_encrypt(encoded_len, encoded, sigret, rsa,
104                                       RSA_PKCS1_PADDING);
105     if (encrypt_len <= 0)
106         goto err;
107 
108     *siglen = encrypt_len;
109     ret = 1;
110 
111 err:
112     OPENSSL_clear_free(tmps, (size_t)encoded_len);
113     return ret;
114 }
115 
116 /*
117  * int_rsa_verify verifies an RSA signature in |sigbuf| using |rsa|. It may be
118  * called in two modes. If |rm| is NULL, it verifies the signature for digest
119  * |m|. Otherwise, it recovers the digest from the signature, writing the digest
120  * to |rm| and the length to |*prm_len|. |type| is the NID of the digest
121  * algorithm to use. It returns one on successful verification and zero
122  * otherwise.
123  */
124 int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
125                    unsigned char *rm, size_t *prm_len,
126                    const unsigned char *sigbuf, size_t siglen, RSA *rsa)
127 {
128     int decrypt_len, ret = 0, encoded_len = 0;
129     unsigned char *decrypt_buf = NULL, *encoded = NULL;
130 
131     if (siglen != (size_t)RSA_size(rsa)) {
132         RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
133         return 0;
134     }
135 
136     /* Recover the encoded digest. */
137     decrypt_buf = OPENSSL_malloc(siglen);
138     if (decrypt_buf == NULL) {
139         RSAerr(RSA_F_INT_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
140         goto err;
141     }
142 
143     decrypt_len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
144                                      RSA_PKCS1_PADDING);
145     if (decrypt_len <= 0)
146         goto err;
147 
148     if (type == NID_md5_sha1) {
149         /*
150          * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
151          * earlier. It has no DigestInfo wrapper but otherwise is
152          * RSASSA-PKCS1-v1_5.
153          */
154         if (decrypt_len != SSL_SIG_LENGTH) {
155             RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
156             goto err;
157         }
158 
159         if (rm != NULL) {
160             memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
161             *prm_len = SSL_SIG_LENGTH;
162         } else {
163             if (m_len != SSL_SIG_LENGTH) {
164                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
165                 goto err;
166             }
167 
168             if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
169                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
170                 goto err;
171             }
172         }
173     } else if (type == NID_mdc2 && decrypt_len == 2 + 16
174                && decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) {
175         /*
176          * Oddball MDC2 case: signature can be OCTET STRING. check for correct
177          * tag and length octets.
178          */
179         if (rm != NULL) {
180             memcpy(rm, decrypt_buf + 2, 16);
181             *prm_len = 16;
182         } else {
183             if (m_len != 16) {
184                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
185                 goto err;
186             }
187 
188             if (memcmp(m, decrypt_buf + 2, 16) != 0) {
189                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
190                 goto err;
191             }
192         }
193     } else {
194         /*
195          * If recovering the digest, extract a digest-sized output from the end
196          * of |decrypt_buf| for |encode_pkcs1|, then compare the decryption
197          * output as in a standard verification.
198          */
199         if (rm != NULL) {
200             const EVP_MD *md = EVP_get_digestbynid(type);
201             if (md == NULL) {
202                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
203                 goto err;
204             }
205 
206             m_len = EVP_MD_size(md);
207             if (m_len > (size_t)decrypt_len) {
208                 RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
209                 goto err;
210             }
211             m = decrypt_buf + decrypt_len - m_len;
212         }
213 
214         /* Construct the encoded digest and ensure it matches. */
215         if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
216             goto err;
217 
218         if (encoded_len != decrypt_len
219             || memcmp(encoded, decrypt_buf, encoded_len) != 0) {
220             RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
221             goto err;
222         }
223 
224         /* Output the recovered digest. */
225         if (rm != NULL) {
226             memcpy(rm, m, m_len);
227             *prm_len = m_len;
228         }
229     }
230 
231     ret = 1;
232 
233 err:
234     OPENSSL_clear_free(encoded, (size_t)encoded_len);
235     OPENSSL_clear_free(decrypt_buf, siglen);
236     return ret;
237 }
238 
239 int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
240                const unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
241 {
242 
243     if (rsa->meth->rsa_verify) {
244         return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);
245     }
246 
247     return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
248 }
249