xref: /freebsd/crypto/openssh/ssh-rsa.c (revision f374ba41f55c1a127303d92d830dd58eef2f5243)
1*f374ba41SEd Maste /* $OpenBSD: ssh-rsa.c,v 1.78 2022/10/28 02:47:04 djm Exp $ */
21e8db6e2SBrian Feldman /*
3d95e11bfSDag-Erling Smørgrav  * Copyright (c) 2000, 2003 Markus Friedl <markus@openbsd.org>
41e8db6e2SBrian Feldman  *
5d95e11bfSDag-Erling Smørgrav  * Permission to use, copy, modify, and distribute this software for any
6d95e11bfSDag-Erling Smørgrav  * purpose with or without fee is hereby granted, provided that the above
7d95e11bfSDag-Erling Smørgrav  * copyright notice and this permission notice appear in all copies.
81e8db6e2SBrian Feldman  *
9d95e11bfSDag-Erling Smørgrav  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10d95e11bfSDag-Erling Smørgrav  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11d95e11bfSDag-Erling Smørgrav  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12d95e11bfSDag-Erling Smørgrav  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13d95e11bfSDag-Erling Smørgrav  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14d95e11bfSDag-Erling Smørgrav  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15d95e11bfSDag-Erling Smørgrav  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
161e8db6e2SBrian Feldman  */
17761efaa7SDag-Erling Smørgrav 
181e8db6e2SBrian Feldman #include "includes.h"
19761efaa7SDag-Erling Smørgrav 
20bc5531deSDag-Erling Smørgrav #ifdef WITH_OPENSSL
21bc5531deSDag-Erling Smørgrav 
22761efaa7SDag-Erling Smørgrav #include <sys/types.h>
231e8db6e2SBrian Feldman 
241e8db6e2SBrian Feldman #include <openssl/evp.h>
251e8db6e2SBrian Feldman #include <openssl/err.h>
261e8db6e2SBrian Feldman 
27761efaa7SDag-Erling Smørgrav #include <stdarg.h>
28761efaa7SDag-Erling Smørgrav #include <string.h>
29761efaa7SDag-Erling Smørgrav 
30a0ee8cc6SDag-Erling Smørgrav #include "sshbuf.h"
311e8db6e2SBrian Feldman #include "compat.h"
32a0ee8cc6SDag-Erling Smørgrav #include "ssherr.h"
33a0ee8cc6SDag-Erling Smørgrav #define SSHKEY_INTERNAL
34a0ee8cc6SDag-Erling Smørgrav #include "sshkey.h"
35f7167e0eSDag-Erling Smørgrav #include "digest.h"
3647dd1d1bSDag-Erling Smørgrav #include "log.h"
371e8db6e2SBrian Feldman 
382a01feabSEd Maste #include "openbsd-compat/openssl-compat.h"
392a01feabSEd Maste 
40a0ee8cc6SDag-Erling Smørgrav static int openssh_RSA_verify(int, u_char *, size_t, u_char *, size_t, RSA *);
414b17dab0SDag-Erling Smørgrav 
42*f374ba41SEd Maste static u_int
43*f374ba41SEd Maste ssh_rsa_size(const struct sshkey *key)
44*f374ba41SEd Maste {
45*f374ba41SEd Maste 	const BIGNUM *rsa_n;
46*f374ba41SEd Maste 
47*f374ba41SEd Maste 	if (key->rsa == NULL)
48*f374ba41SEd Maste 		return 0;
49*f374ba41SEd Maste 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
50*f374ba41SEd Maste 	return BN_num_bits(rsa_n);
51*f374ba41SEd Maste }
52*f374ba41SEd Maste 
53*f374ba41SEd Maste static int
54*f374ba41SEd Maste ssh_rsa_alloc(struct sshkey *k)
55*f374ba41SEd Maste {
56*f374ba41SEd Maste 	if ((k->rsa = RSA_new()) == NULL)
57*f374ba41SEd Maste 		return SSH_ERR_ALLOC_FAIL;
58*f374ba41SEd Maste 	return 0;
59*f374ba41SEd Maste }
60*f374ba41SEd Maste 
61*f374ba41SEd Maste static void
62*f374ba41SEd Maste ssh_rsa_cleanup(struct sshkey *k)
63*f374ba41SEd Maste {
64*f374ba41SEd Maste 	RSA_free(k->rsa);
65*f374ba41SEd Maste 	k->rsa = NULL;
66*f374ba41SEd Maste }
67*f374ba41SEd Maste 
68*f374ba41SEd Maste static int
69*f374ba41SEd Maste ssh_rsa_equal(const struct sshkey *a, const struct sshkey *b)
70*f374ba41SEd Maste {
71*f374ba41SEd Maste 	const BIGNUM *rsa_e_a, *rsa_n_a;
72*f374ba41SEd Maste 	const BIGNUM *rsa_e_b, *rsa_n_b;
73*f374ba41SEd Maste 
74*f374ba41SEd Maste 	if (a->rsa == NULL || b->rsa == NULL)
75*f374ba41SEd Maste 		return 0;
76*f374ba41SEd Maste 	RSA_get0_key(a->rsa, &rsa_n_a, &rsa_e_a, NULL);
77*f374ba41SEd Maste 	RSA_get0_key(b->rsa, &rsa_n_b, &rsa_e_b, NULL);
78*f374ba41SEd Maste 	if (rsa_e_a == NULL || rsa_e_b == NULL)
79*f374ba41SEd Maste 		return 0;
80*f374ba41SEd Maste 	if (rsa_n_a == NULL || rsa_n_b == NULL)
81*f374ba41SEd Maste 		return 0;
82*f374ba41SEd Maste 	if (BN_cmp(rsa_e_a, rsa_e_b) != 0)
83*f374ba41SEd Maste 		return 0;
84*f374ba41SEd Maste 	if (BN_cmp(rsa_n_a, rsa_n_b) != 0)
85*f374ba41SEd Maste 		return 0;
86*f374ba41SEd Maste 	return 1;
87*f374ba41SEd Maste }
88*f374ba41SEd Maste 
89*f374ba41SEd Maste static int
90*f374ba41SEd Maste ssh_rsa_serialize_public(const struct sshkey *key, struct sshbuf *b,
91*f374ba41SEd Maste     enum sshkey_serialize_rep opts)
92*f374ba41SEd Maste {
93*f374ba41SEd Maste 	int r;
94*f374ba41SEd Maste 	const BIGNUM *rsa_n, *rsa_e;
95*f374ba41SEd Maste 
96*f374ba41SEd Maste 	if (key->rsa == NULL)
97*f374ba41SEd Maste 		return SSH_ERR_INVALID_ARGUMENT;
98*f374ba41SEd Maste 	RSA_get0_key(key->rsa, &rsa_n, &rsa_e, NULL);
99*f374ba41SEd Maste 	if ((r = sshbuf_put_bignum2(b, rsa_e)) != 0 ||
100*f374ba41SEd Maste 	    (r = sshbuf_put_bignum2(b, rsa_n)) != 0)
101*f374ba41SEd Maste 		return r;
102*f374ba41SEd Maste 
103*f374ba41SEd Maste 	return 0;
104*f374ba41SEd Maste }
105*f374ba41SEd Maste 
106*f374ba41SEd Maste static int
107*f374ba41SEd Maste ssh_rsa_serialize_private(const struct sshkey *key, struct sshbuf *b,
108*f374ba41SEd Maste     enum sshkey_serialize_rep opts)
109*f374ba41SEd Maste {
110*f374ba41SEd Maste 	int r;
111*f374ba41SEd Maste 	const BIGNUM *rsa_n, *rsa_e, *rsa_d, *rsa_iqmp, *rsa_p, *rsa_q;
112*f374ba41SEd Maste 
113*f374ba41SEd Maste 	RSA_get0_key(key->rsa, &rsa_n, &rsa_e, &rsa_d);
114*f374ba41SEd Maste 	RSA_get0_factors(key->rsa, &rsa_p, &rsa_q);
115*f374ba41SEd Maste 	RSA_get0_crt_params(key->rsa, NULL, NULL, &rsa_iqmp);
116*f374ba41SEd Maste 
117*f374ba41SEd Maste 	if (!sshkey_is_cert(key)) {
118*f374ba41SEd Maste 		/* Note: can't reuse ssh_rsa_serialize_public: e, n vs. n, e */
119*f374ba41SEd Maste 		if ((r = sshbuf_put_bignum2(b, rsa_n)) != 0 ||
120*f374ba41SEd Maste 		    (r = sshbuf_put_bignum2(b, rsa_e)) != 0)
121*f374ba41SEd Maste 			return r;
122*f374ba41SEd Maste 	}
123*f374ba41SEd Maste 	if ((r = sshbuf_put_bignum2(b, rsa_d)) != 0 ||
124*f374ba41SEd Maste 	    (r = sshbuf_put_bignum2(b, rsa_iqmp)) != 0 ||
125*f374ba41SEd Maste 	    (r = sshbuf_put_bignum2(b, rsa_p)) != 0 ||
126*f374ba41SEd Maste 	    (r = sshbuf_put_bignum2(b, rsa_q)) != 0)
127*f374ba41SEd Maste 		return r;
128*f374ba41SEd Maste 
129*f374ba41SEd Maste 	return 0;
130*f374ba41SEd Maste }
131*f374ba41SEd Maste 
132*f374ba41SEd Maste static int
133*f374ba41SEd Maste ssh_rsa_generate(struct sshkey *k, int bits)
134*f374ba41SEd Maste {
135*f374ba41SEd Maste 	RSA *private = NULL;
136*f374ba41SEd Maste 	BIGNUM *f4 = NULL;
137*f374ba41SEd Maste 	int ret = SSH_ERR_INTERNAL_ERROR;
138*f374ba41SEd Maste 
139*f374ba41SEd Maste 	if (bits < SSH_RSA_MINIMUM_MODULUS_SIZE ||
140*f374ba41SEd Maste 	    bits > SSHBUF_MAX_BIGNUM * 8)
141*f374ba41SEd Maste 		return SSH_ERR_KEY_LENGTH;
142*f374ba41SEd Maste 	if ((private = RSA_new()) == NULL || (f4 = BN_new()) == NULL) {
143*f374ba41SEd Maste 		ret = SSH_ERR_ALLOC_FAIL;
144*f374ba41SEd Maste 		goto out;
145*f374ba41SEd Maste 	}
146*f374ba41SEd Maste 	if (!BN_set_word(f4, RSA_F4) ||
147*f374ba41SEd Maste 	    !RSA_generate_key_ex(private, bits, f4, NULL)) {
148*f374ba41SEd Maste 		ret = SSH_ERR_LIBCRYPTO_ERROR;
149*f374ba41SEd Maste 		goto out;
150*f374ba41SEd Maste 	}
151*f374ba41SEd Maste 	k->rsa = private;
152*f374ba41SEd Maste 	private = NULL;
153*f374ba41SEd Maste 	ret = 0;
154*f374ba41SEd Maste  out:
155*f374ba41SEd Maste 	RSA_free(private);
156*f374ba41SEd Maste 	BN_free(f4);
157*f374ba41SEd Maste 	return ret;
158*f374ba41SEd Maste }
159*f374ba41SEd Maste 
160*f374ba41SEd Maste static int
161*f374ba41SEd Maste ssh_rsa_copy_public(const struct sshkey *from, struct sshkey *to)
162*f374ba41SEd Maste {
163*f374ba41SEd Maste 	const BIGNUM *rsa_n, *rsa_e;
164*f374ba41SEd Maste 	BIGNUM *rsa_n_dup = NULL, *rsa_e_dup = NULL;
165*f374ba41SEd Maste 	int r = SSH_ERR_INTERNAL_ERROR;
166*f374ba41SEd Maste 
167*f374ba41SEd Maste 	RSA_get0_key(from->rsa, &rsa_n, &rsa_e, NULL);
168*f374ba41SEd Maste 	if ((rsa_n_dup = BN_dup(rsa_n)) == NULL ||
169*f374ba41SEd Maste 	    (rsa_e_dup = BN_dup(rsa_e)) == NULL) {
170*f374ba41SEd Maste 		r = SSH_ERR_ALLOC_FAIL;
171*f374ba41SEd Maste 		goto out;
172*f374ba41SEd Maste 	}
173*f374ba41SEd Maste 	if (!RSA_set0_key(to->rsa, rsa_n_dup, rsa_e_dup, NULL)) {
174*f374ba41SEd Maste 		r = SSH_ERR_LIBCRYPTO_ERROR;
175*f374ba41SEd Maste 		goto out;
176*f374ba41SEd Maste 	}
177*f374ba41SEd Maste 	rsa_n_dup = rsa_e_dup = NULL; /* transferred */
178*f374ba41SEd Maste 	/* success */
179*f374ba41SEd Maste 	r = 0;
180*f374ba41SEd Maste  out:
181*f374ba41SEd Maste 	BN_clear_free(rsa_n_dup);
182*f374ba41SEd Maste 	BN_clear_free(rsa_e_dup);
183*f374ba41SEd Maste 	return r;
184*f374ba41SEd Maste }
185*f374ba41SEd Maste 
186*f374ba41SEd Maste static int
187*f374ba41SEd Maste ssh_rsa_deserialize_public(const char *ktype, struct sshbuf *b,
188*f374ba41SEd Maste     struct sshkey *key)
189*f374ba41SEd Maste {
190*f374ba41SEd Maste 	int ret = SSH_ERR_INTERNAL_ERROR;
191*f374ba41SEd Maste 	BIGNUM *rsa_n = NULL, *rsa_e = NULL;
192*f374ba41SEd Maste 
193*f374ba41SEd Maste 	if (sshbuf_get_bignum2(b, &rsa_e) != 0 ||
194*f374ba41SEd Maste 	    sshbuf_get_bignum2(b, &rsa_n) != 0) {
195*f374ba41SEd Maste 		ret = SSH_ERR_INVALID_FORMAT;
196*f374ba41SEd Maste 		goto out;
197*f374ba41SEd Maste 	}
198*f374ba41SEd Maste 	if (!RSA_set0_key(key->rsa, rsa_n, rsa_e, NULL)) {
199*f374ba41SEd Maste 		ret = SSH_ERR_LIBCRYPTO_ERROR;
200*f374ba41SEd Maste 		goto out;
201*f374ba41SEd Maste 	}
202*f374ba41SEd Maste 	rsa_n = rsa_e = NULL; /* transferred */
203*f374ba41SEd Maste 	if ((ret = sshkey_check_rsa_length(key, 0)) != 0)
204*f374ba41SEd Maste 		goto out;
205*f374ba41SEd Maste #ifdef DEBUG_PK
206*f374ba41SEd Maste 	RSA_print_fp(stderr, key->rsa, 8);
207*f374ba41SEd Maste #endif
208*f374ba41SEd Maste 	/* success */
209*f374ba41SEd Maste 	ret = 0;
210*f374ba41SEd Maste  out:
211*f374ba41SEd Maste 	BN_clear_free(rsa_n);
212*f374ba41SEd Maste 	BN_clear_free(rsa_e);
213*f374ba41SEd Maste 	return ret;
214*f374ba41SEd Maste }
215*f374ba41SEd Maste 
216*f374ba41SEd Maste static int
217*f374ba41SEd Maste ssh_rsa_deserialize_private(const char *ktype, struct sshbuf *b,
218*f374ba41SEd Maste     struct sshkey *key)
219*f374ba41SEd Maste {
220*f374ba41SEd Maste 	int r;
221*f374ba41SEd Maste 	BIGNUM *rsa_n = NULL, *rsa_e = NULL, *rsa_d = NULL;
222*f374ba41SEd Maste 	BIGNUM *rsa_iqmp = NULL, *rsa_p = NULL, *rsa_q = NULL;
223*f374ba41SEd Maste 
224*f374ba41SEd Maste 	/* Note: can't reuse ssh_rsa_deserialize_public: e, n vs. n, e */
225*f374ba41SEd Maste 	if (!sshkey_is_cert(key)) {
226*f374ba41SEd Maste 		if ((r = sshbuf_get_bignum2(b, &rsa_n)) != 0 ||
227*f374ba41SEd Maste 		    (r = sshbuf_get_bignum2(b, &rsa_e)) != 0)
228*f374ba41SEd Maste 			goto out;
229*f374ba41SEd Maste 		if (!RSA_set0_key(key->rsa, rsa_n, rsa_e, NULL)) {
230*f374ba41SEd Maste 			r = SSH_ERR_LIBCRYPTO_ERROR;
231*f374ba41SEd Maste 			goto out;
232*f374ba41SEd Maste 		}
233*f374ba41SEd Maste 		rsa_n = rsa_e = NULL; /* transferred */
234*f374ba41SEd Maste 	}
235*f374ba41SEd Maste 	if ((r = sshbuf_get_bignum2(b, &rsa_d)) != 0 ||
236*f374ba41SEd Maste 	    (r = sshbuf_get_bignum2(b, &rsa_iqmp)) != 0 ||
237*f374ba41SEd Maste 	    (r = sshbuf_get_bignum2(b, &rsa_p)) != 0 ||
238*f374ba41SEd Maste 	    (r = sshbuf_get_bignum2(b, &rsa_q)) != 0)
239*f374ba41SEd Maste 		goto out;
240*f374ba41SEd Maste 	if (!RSA_set0_key(key->rsa, NULL, NULL, rsa_d)) {
241*f374ba41SEd Maste 		r = SSH_ERR_LIBCRYPTO_ERROR;
242*f374ba41SEd Maste 		goto out;
243*f374ba41SEd Maste 	}
244*f374ba41SEd Maste 	rsa_d = NULL; /* transferred */
245*f374ba41SEd Maste 	if (!RSA_set0_factors(key->rsa, rsa_p, rsa_q)) {
246*f374ba41SEd Maste 		r = SSH_ERR_LIBCRYPTO_ERROR;
247*f374ba41SEd Maste 		goto out;
248*f374ba41SEd Maste 	}
249*f374ba41SEd Maste 	rsa_p = rsa_q = NULL; /* transferred */
250*f374ba41SEd Maste 	if ((r = sshkey_check_rsa_length(key, 0)) != 0)
251*f374ba41SEd Maste 		goto out;
252*f374ba41SEd Maste 	if ((r = ssh_rsa_complete_crt_parameters(key, rsa_iqmp)) != 0)
253*f374ba41SEd Maste 		goto out;
254*f374ba41SEd Maste 	if (RSA_blinding_on(key->rsa, NULL) != 1) {
255*f374ba41SEd Maste 		r = SSH_ERR_LIBCRYPTO_ERROR;
256*f374ba41SEd Maste 		goto out;
257*f374ba41SEd Maste 	}
258*f374ba41SEd Maste 	/* success */
259*f374ba41SEd Maste 	r = 0;
260*f374ba41SEd Maste  out:
261*f374ba41SEd Maste 	BN_clear_free(rsa_n);
262*f374ba41SEd Maste 	BN_clear_free(rsa_e);
263*f374ba41SEd Maste 	BN_clear_free(rsa_d);
264*f374ba41SEd Maste 	BN_clear_free(rsa_p);
265*f374ba41SEd Maste 	BN_clear_free(rsa_q);
266*f374ba41SEd Maste 	BN_clear_free(rsa_iqmp);
267*f374ba41SEd Maste 	return r;
268*f374ba41SEd Maste }
269*f374ba41SEd Maste 
270acc1a9efSDag-Erling Smørgrav static const char *
271acc1a9efSDag-Erling Smørgrav rsa_hash_alg_ident(int hash_alg)
272acc1a9efSDag-Erling Smørgrav {
273acc1a9efSDag-Erling Smørgrav 	switch (hash_alg) {
274acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA1:
275acc1a9efSDag-Erling Smørgrav 		return "ssh-rsa";
276acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA256:
277acc1a9efSDag-Erling Smørgrav 		return "rsa-sha2-256";
278acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA512:
279acc1a9efSDag-Erling Smørgrav 		return "rsa-sha2-512";
280acc1a9efSDag-Erling Smørgrav 	}
281acc1a9efSDag-Erling Smørgrav 	return NULL;
282acc1a9efSDag-Erling Smørgrav }
283acc1a9efSDag-Erling Smørgrav 
284190cef3dSDag-Erling Smørgrav /*
285190cef3dSDag-Erling Smørgrav  * Returns the hash algorithm ID for a given algorithm identifier as used
286190cef3dSDag-Erling Smørgrav  * inside the signature blob,
287190cef3dSDag-Erling Smørgrav  */
288acc1a9efSDag-Erling Smørgrav static int
289190cef3dSDag-Erling Smørgrav rsa_hash_id_from_ident(const char *ident)
290acc1a9efSDag-Erling Smørgrav {
291190cef3dSDag-Erling Smørgrav 	if (strcmp(ident, "ssh-rsa") == 0)
292acc1a9efSDag-Erling Smørgrav 		return SSH_DIGEST_SHA1;
293acc1a9efSDag-Erling Smørgrav 	if (strcmp(ident, "rsa-sha2-256") == 0)
294acc1a9efSDag-Erling Smørgrav 		return SSH_DIGEST_SHA256;
295acc1a9efSDag-Erling Smørgrav 	if (strcmp(ident, "rsa-sha2-512") == 0)
296acc1a9efSDag-Erling Smørgrav 		return SSH_DIGEST_SHA512;
297acc1a9efSDag-Erling Smørgrav 	return -1;
298acc1a9efSDag-Erling Smørgrav }
299acc1a9efSDag-Erling Smørgrav 
300190cef3dSDag-Erling Smørgrav /*
301190cef3dSDag-Erling Smørgrav  * Return the hash algorithm ID for the specified key name. This includes
302190cef3dSDag-Erling Smørgrav  * all the cases of rsa_hash_id_from_ident() but also the certificate key
303190cef3dSDag-Erling Smørgrav  * types.
304190cef3dSDag-Erling Smørgrav  */
305190cef3dSDag-Erling Smørgrav static int
306190cef3dSDag-Erling Smørgrav rsa_hash_id_from_keyname(const char *alg)
307190cef3dSDag-Erling Smørgrav {
308190cef3dSDag-Erling Smørgrav 	int r;
309190cef3dSDag-Erling Smørgrav 
310190cef3dSDag-Erling Smørgrav 	if ((r = rsa_hash_id_from_ident(alg)) != -1)
311190cef3dSDag-Erling Smørgrav 		return r;
312190cef3dSDag-Erling Smørgrav 	if (strcmp(alg, "ssh-rsa-cert-v01@openssh.com") == 0)
313190cef3dSDag-Erling Smørgrav 		return SSH_DIGEST_SHA1;
314190cef3dSDag-Erling Smørgrav 	if (strcmp(alg, "rsa-sha2-256-cert-v01@openssh.com") == 0)
315190cef3dSDag-Erling Smørgrav 		return SSH_DIGEST_SHA256;
316190cef3dSDag-Erling Smørgrav 	if (strcmp(alg, "rsa-sha2-512-cert-v01@openssh.com") == 0)
317190cef3dSDag-Erling Smørgrav 		return SSH_DIGEST_SHA512;
318190cef3dSDag-Erling Smørgrav 	return -1;
319190cef3dSDag-Erling Smørgrav }
320190cef3dSDag-Erling Smørgrav 
321acc1a9efSDag-Erling Smørgrav static int
322acc1a9efSDag-Erling Smørgrav rsa_hash_alg_nid(int type)
323acc1a9efSDag-Erling Smørgrav {
324acc1a9efSDag-Erling Smørgrav 	switch (type) {
325acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA1:
326acc1a9efSDag-Erling Smørgrav 		return NID_sha1;
327acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA256:
328acc1a9efSDag-Erling Smørgrav 		return NID_sha256;
329acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA512:
330acc1a9efSDag-Erling Smørgrav 		return NID_sha512;
331acc1a9efSDag-Erling Smørgrav 	default:
332acc1a9efSDag-Erling Smørgrav 		return -1;
333acc1a9efSDag-Erling Smørgrav 	}
334acc1a9efSDag-Erling Smørgrav }
335acc1a9efSDag-Erling Smørgrav 
3364f52dfbbSDag-Erling Smørgrav int
3372a01feabSEd Maste ssh_rsa_complete_crt_parameters(struct sshkey *key, const BIGNUM *iqmp)
3384f52dfbbSDag-Erling Smørgrav {
3392a01feabSEd Maste 	const BIGNUM *rsa_p, *rsa_q, *rsa_d;
3402a01feabSEd Maste 	BIGNUM *aux = NULL, *d_consttime = NULL;
3412a01feabSEd Maste 	BIGNUM *rsa_dmq1 = NULL, *rsa_dmp1 = NULL, *rsa_iqmp = NULL;
3424f52dfbbSDag-Erling Smørgrav 	BN_CTX *ctx = NULL;
3434f52dfbbSDag-Erling Smørgrav 	int r;
3444f52dfbbSDag-Erling Smørgrav 
3454f52dfbbSDag-Erling Smørgrav 	if (key == NULL || key->rsa == NULL ||
3464f52dfbbSDag-Erling Smørgrav 	    sshkey_type_plain(key->type) != KEY_RSA)
3474f52dfbbSDag-Erling Smørgrav 		return SSH_ERR_INVALID_ARGUMENT;
3484f52dfbbSDag-Erling Smørgrav 
3492a01feabSEd Maste 	RSA_get0_key(key->rsa, NULL, NULL, &rsa_d);
3502a01feabSEd Maste 	RSA_get0_factors(key->rsa, &rsa_p, &rsa_q);
3512a01feabSEd Maste 
3524f52dfbbSDag-Erling Smørgrav 	if ((ctx = BN_CTX_new()) == NULL)
3534f52dfbbSDag-Erling Smørgrav 		return SSH_ERR_ALLOC_FAIL;
3542a01feabSEd Maste 	if ((aux = BN_new()) == NULL ||
3552a01feabSEd Maste 	    (rsa_dmq1 = BN_new()) == NULL ||
3562a01feabSEd Maste 	    (rsa_dmp1 = BN_new()) == NULL)
3572a01feabSEd Maste 		return SSH_ERR_ALLOC_FAIL;
3582a01feabSEd Maste 	if ((d_consttime = BN_dup(rsa_d)) == NULL ||
3592a01feabSEd Maste 	    (rsa_iqmp = BN_dup(iqmp)) == NULL) {
3604f52dfbbSDag-Erling Smørgrav 		r = SSH_ERR_ALLOC_FAIL;
3614f52dfbbSDag-Erling Smørgrav 		goto out;
3624f52dfbbSDag-Erling Smørgrav 	}
36347dd1d1bSDag-Erling Smørgrav 	BN_set_flags(aux, BN_FLG_CONSTTIME);
3642a01feabSEd Maste 	BN_set_flags(d_consttime, BN_FLG_CONSTTIME);
3654f52dfbbSDag-Erling Smørgrav 
3662a01feabSEd Maste 	if ((BN_sub(aux, rsa_q, BN_value_one()) == 0) ||
3672a01feabSEd Maste 	    (BN_mod(rsa_dmq1, d_consttime, aux, ctx) == 0) ||
3682a01feabSEd Maste 	    (BN_sub(aux, rsa_p, BN_value_one()) == 0) ||
3692a01feabSEd Maste 	    (BN_mod(rsa_dmp1, d_consttime, aux, ctx) == 0)) {
3704f52dfbbSDag-Erling Smørgrav 		r = SSH_ERR_LIBCRYPTO_ERROR;
3714f52dfbbSDag-Erling Smørgrav 		goto out;
3724f52dfbbSDag-Erling Smørgrav 	}
3732a01feabSEd Maste 	if (!RSA_set0_crt_params(key->rsa, rsa_dmp1, rsa_dmq1, rsa_iqmp)) {
3742a01feabSEd Maste 		r = SSH_ERR_LIBCRYPTO_ERROR;
3752a01feabSEd Maste 		goto out;
3762a01feabSEd Maste 	}
3772a01feabSEd Maste 	rsa_dmp1 = rsa_dmq1 = rsa_iqmp = NULL; /* transferred */
3782a01feabSEd Maste 	/* success */
3794f52dfbbSDag-Erling Smørgrav 	r = 0;
3804f52dfbbSDag-Erling Smørgrav  out:
3814f52dfbbSDag-Erling Smørgrav 	BN_clear_free(aux);
3822a01feabSEd Maste 	BN_clear_free(d_consttime);
3832a01feabSEd Maste 	BN_clear_free(rsa_dmp1);
3842a01feabSEd Maste 	BN_clear_free(rsa_dmq1);
3852a01feabSEd Maste 	BN_clear_free(rsa_iqmp);
3864f52dfbbSDag-Erling Smørgrav 	BN_CTX_free(ctx);
3874f52dfbbSDag-Erling Smørgrav 	return r;
3884f52dfbbSDag-Erling Smørgrav }
3894f52dfbbSDag-Erling Smørgrav 
3901e8db6e2SBrian Feldman /* RSASSA-PKCS1-v1_5 (PKCS #1 v2.0 signature) with SHA1 */
391*f374ba41SEd Maste static int
392*f374ba41SEd Maste ssh_rsa_sign(struct sshkey *key,
393*f374ba41SEd Maste     u_char **sigp, size_t *lenp,
394*f374ba41SEd Maste     const u_char *data, size_t datalen,
395*f374ba41SEd Maste     const char *alg, const char *sk_provider, const char *sk_pin, u_int compat)
3961e8db6e2SBrian Feldman {
3972a01feabSEd Maste 	const BIGNUM *rsa_n;
398a0ee8cc6SDag-Erling Smørgrav 	u_char digest[SSH_DIGEST_MAX_LENGTH], *sig = NULL;
39947dd1d1bSDag-Erling Smørgrav 	size_t slen = 0;
400*f374ba41SEd Maste 	u_int hlen, len;
401acc1a9efSDag-Erling Smørgrav 	int nid, hash_alg, ret = SSH_ERR_INTERNAL_ERROR;
402a0ee8cc6SDag-Erling Smørgrav 	struct sshbuf *b = NULL;
4031e8db6e2SBrian Feldman 
404a0ee8cc6SDag-Erling Smørgrav 	if (lenp != NULL)
405a0ee8cc6SDag-Erling Smørgrav 		*lenp = 0;
406a0ee8cc6SDag-Erling Smørgrav 	if (sigp != NULL)
407a0ee8cc6SDag-Erling Smørgrav 		*sigp = NULL;
408a0ee8cc6SDag-Erling Smørgrav 
409*f374ba41SEd Maste 	if (alg == NULL || strlen(alg) == 0)
410acc1a9efSDag-Erling Smørgrav 		hash_alg = SSH_DIGEST_SHA1;
411acc1a9efSDag-Erling Smørgrav 	else
412*f374ba41SEd Maste 		hash_alg = rsa_hash_id_from_keyname(alg);
413acc1a9efSDag-Erling Smørgrav 	if (key == NULL || key->rsa == NULL || hash_alg == -1 ||
4144f52dfbbSDag-Erling Smørgrav 	    sshkey_type_plain(key->type) != KEY_RSA)
415a0ee8cc6SDag-Erling Smørgrav 		return SSH_ERR_INVALID_ARGUMENT;
4162a01feabSEd Maste 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
4172a01feabSEd Maste 	if (BN_num_bits(rsa_n) < SSH_RSA_MINIMUM_MODULUS_SIZE)
4184f52dfbbSDag-Erling Smørgrav 		return SSH_ERR_KEY_LENGTH;
419a0ee8cc6SDag-Erling Smørgrav 	slen = RSA_size(key->rsa);
420a0ee8cc6SDag-Erling Smørgrav 	if (slen <= 0 || slen > SSHBUF_MAX_BIGNUM)
421a0ee8cc6SDag-Erling Smørgrav 		return SSH_ERR_INVALID_ARGUMENT;
422f7167e0eSDag-Erling Smørgrav 
423f7167e0eSDag-Erling Smørgrav 	/* hash the data */
424acc1a9efSDag-Erling Smørgrav 	nid = rsa_hash_alg_nid(hash_alg);
425*f374ba41SEd Maste 	if ((hlen = ssh_digest_bytes(hash_alg)) == 0)
426a0ee8cc6SDag-Erling Smørgrav 		return SSH_ERR_INTERNAL_ERROR;
427a0ee8cc6SDag-Erling Smørgrav 	if ((ret = ssh_digest_memory(hash_alg, data, datalen,
428a0ee8cc6SDag-Erling Smørgrav 	    digest, sizeof(digest))) != 0)
429a0ee8cc6SDag-Erling Smørgrav 		goto out;
430a0ee8cc6SDag-Erling Smørgrav 
431a0ee8cc6SDag-Erling Smørgrav 	if ((sig = malloc(slen)) == NULL) {
432a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_ALLOC_FAIL;
433a0ee8cc6SDag-Erling Smørgrav 		goto out;
434f7167e0eSDag-Erling Smørgrav 	}
4351e8db6e2SBrian Feldman 
436*f374ba41SEd Maste 	if (RSA_sign(nid, digest, hlen, sig, &len, key->rsa) != 1) {
437a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_LIBCRYPTO_ERROR;
438a0ee8cc6SDag-Erling Smørgrav 		goto out;
4391e8db6e2SBrian Feldman 	}
4401e8db6e2SBrian Feldman 	if (len < slen) {
441a0ee8cc6SDag-Erling Smørgrav 		size_t diff = slen - len;
4421e8db6e2SBrian Feldman 		memmove(sig + diff, sig, len);
443b83788ffSDag-Erling Smørgrav 		explicit_bzero(sig, diff);
4441e8db6e2SBrian Feldman 	} else if (len > slen) {
445a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INTERNAL_ERROR;
446a0ee8cc6SDag-Erling Smørgrav 		goto out;
4471e8db6e2SBrian Feldman 	}
4481e8db6e2SBrian Feldman 	/* encode signature */
449a0ee8cc6SDag-Erling Smørgrav 	if ((b = sshbuf_new()) == NULL) {
450a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_ALLOC_FAIL;
451a0ee8cc6SDag-Erling Smørgrav 		goto out;
452a0ee8cc6SDag-Erling Smørgrav 	}
453acc1a9efSDag-Erling Smørgrav 	if ((ret = sshbuf_put_cstring(b, rsa_hash_alg_ident(hash_alg))) != 0 ||
454a0ee8cc6SDag-Erling Smørgrav 	    (ret = sshbuf_put_string(b, sig, slen)) != 0)
455a0ee8cc6SDag-Erling Smørgrav 		goto out;
456a0ee8cc6SDag-Erling Smørgrav 	len = sshbuf_len(b);
457a0ee8cc6SDag-Erling Smørgrav 	if (sigp != NULL) {
458a0ee8cc6SDag-Erling Smørgrav 		if ((*sigp = malloc(len)) == NULL) {
459a0ee8cc6SDag-Erling Smørgrav 			ret = SSH_ERR_ALLOC_FAIL;
460a0ee8cc6SDag-Erling Smørgrav 			goto out;
461a0ee8cc6SDag-Erling Smørgrav 		}
462a0ee8cc6SDag-Erling Smørgrav 		memcpy(*sigp, sshbuf_ptr(b), len);
463a0ee8cc6SDag-Erling Smørgrav 	}
4644b17dab0SDag-Erling Smørgrav 	if (lenp != NULL)
4654b17dab0SDag-Erling Smørgrav 		*lenp = len;
466a0ee8cc6SDag-Erling Smørgrav 	ret = 0;
467a0ee8cc6SDag-Erling Smørgrav  out:
468a0ee8cc6SDag-Erling Smørgrav 	explicit_bzero(digest, sizeof(digest));
46947dd1d1bSDag-Erling Smørgrav 	freezero(sig, slen);
470a0ee8cc6SDag-Erling Smørgrav 	sshbuf_free(b);
471557f75e5SDag-Erling Smørgrav 	return ret;
4721e8db6e2SBrian Feldman }
4731e8db6e2SBrian Feldman 
474*f374ba41SEd Maste static int
475a0ee8cc6SDag-Erling Smørgrav ssh_rsa_verify(const struct sshkey *key,
476*f374ba41SEd Maste     const u_char *sig, size_t siglen,
477*f374ba41SEd Maste     const u_char *data, size_t dlen, const char *alg, u_int compat,
478*f374ba41SEd Maste     struct sshkey_sig_details **detailsp)
4791e8db6e2SBrian Feldman {
4802a01feabSEd Maste 	const BIGNUM *rsa_n;
48147dd1d1bSDag-Erling Smørgrav 	char *sigtype = NULL;
482190cef3dSDag-Erling Smørgrav 	int hash_alg, want_alg, ret = SSH_ERR_INTERNAL_ERROR;
483*f374ba41SEd Maste 	size_t len = 0, diff, modlen, hlen;
484a0ee8cc6SDag-Erling Smørgrav 	struct sshbuf *b = NULL;
485a0ee8cc6SDag-Erling Smørgrav 	u_char digest[SSH_DIGEST_MAX_LENGTH], *osigblob, *sigblob = NULL;
4861e8db6e2SBrian Feldman 
487a0ee8cc6SDag-Erling Smørgrav 	if (key == NULL || key->rsa == NULL ||
488a0ee8cc6SDag-Erling Smørgrav 	    sshkey_type_plain(key->type) != KEY_RSA ||
489076ad2f8SDag-Erling Smørgrav 	    sig == NULL || siglen == 0)
490a0ee8cc6SDag-Erling Smørgrav 		return SSH_ERR_INVALID_ARGUMENT;
4912a01feabSEd Maste 	RSA_get0_key(key->rsa, &rsa_n, NULL, NULL);
4922a01feabSEd Maste 	if (BN_num_bits(rsa_n) < SSH_RSA_MINIMUM_MODULUS_SIZE)
4934f52dfbbSDag-Erling Smørgrav 		return SSH_ERR_KEY_LENGTH;
494f7167e0eSDag-Erling Smørgrav 
495acc1a9efSDag-Erling Smørgrav 	if ((b = sshbuf_from(sig, siglen)) == NULL)
496a0ee8cc6SDag-Erling Smørgrav 		return SSH_ERR_ALLOC_FAIL;
49747dd1d1bSDag-Erling Smørgrav 	if (sshbuf_get_cstring(b, &sigtype, NULL) != 0) {
498a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INVALID_FORMAT;
499a0ee8cc6SDag-Erling Smørgrav 		goto out;
5001e8db6e2SBrian Feldman 	}
501190cef3dSDag-Erling Smørgrav 	if ((hash_alg = rsa_hash_id_from_ident(sigtype)) == -1) {
502190cef3dSDag-Erling Smørgrav 		ret = SSH_ERR_KEY_TYPE_MISMATCH;
503190cef3dSDag-Erling Smørgrav 		goto out;
504190cef3dSDag-Erling Smørgrav 	}
505190cef3dSDag-Erling Smørgrav 	/*
506190cef3dSDag-Erling Smørgrav 	 * Allow ssh-rsa-cert-v01 certs to generate SHA2 signatures for
507190cef3dSDag-Erling Smørgrav 	 * legacy reasons, but otherwise the signature type should match.
508190cef3dSDag-Erling Smørgrav 	 */
509190cef3dSDag-Erling Smørgrav 	if (alg != NULL && strcmp(alg, "ssh-rsa-cert-v01@openssh.com") != 0) {
510190cef3dSDag-Erling Smørgrav 		if ((want_alg = rsa_hash_id_from_keyname(alg)) == -1) {
511190cef3dSDag-Erling Smørgrav 			ret = SSH_ERR_INVALID_ARGUMENT;
512190cef3dSDag-Erling Smørgrav 			goto out;
513190cef3dSDag-Erling Smørgrav 		}
514190cef3dSDag-Erling Smørgrav 		if (hash_alg != want_alg) {
51547dd1d1bSDag-Erling Smørgrav 			ret = SSH_ERR_SIGNATURE_INVALID;
51647dd1d1bSDag-Erling Smørgrav 			goto out;
51747dd1d1bSDag-Erling Smørgrav 		}
5181e8db6e2SBrian Feldman 	}
519a0ee8cc6SDag-Erling Smørgrav 	if (sshbuf_get_string(b, &sigblob, &len) != 0) {
520a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INVALID_FORMAT;
521a0ee8cc6SDag-Erling Smørgrav 		goto out;
522a0ee8cc6SDag-Erling Smørgrav 	}
523a0ee8cc6SDag-Erling Smørgrav 	if (sshbuf_len(b) != 0) {
524a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_UNEXPECTED_TRAILING_DATA;
525a0ee8cc6SDag-Erling Smørgrav 		goto out;
5261e8db6e2SBrian Feldman 	}
527545d5ecaSDag-Erling Smørgrav 	/* RSA_verify expects a signature of RSA_size */
528545d5ecaSDag-Erling Smørgrav 	modlen = RSA_size(key->rsa);
529545d5ecaSDag-Erling Smørgrav 	if (len > modlen) {
530a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_KEY_BITS_MISMATCH;
531a0ee8cc6SDag-Erling Smørgrav 		goto out;
532545d5ecaSDag-Erling Smørgrav 	} else if (len < modlen) {
533a0ee8cc6SDag-Erling Smørgrav 		diff = modlen - len;
534a0ee8cc6SDag-Erling Smørgrav 		osigblob = sigblob;
535a0ee8cc6SDag-Erling Smørgrav 		if ((sigblob = realloc(sigblob, modlen)) == NULL) {
536a0ee8cc6SDag-Erling Smørgrav 			sigblob = osigblob; /* put it back for clear/free */
537a0ee8cc6SDag-Erling Smørgrav 			ret = SSH_ERR_ALLOC_FAIL;
538a0ee8cc6SDag-Erling Smørgrav 			goto out;
539a0ee8cc6SDag-Erling Smørgrav 		}
540545d5ecaSDag-Erling Smørgrav 		memmove(sigblob + diff, sigblob, len);
541b83788ffSDag-Erling Smørgrav 		explicit_bzero(sigblob, diff);
542545d5ecaSDag-Erling Smørgrav 		len = modlen;
543545d5ecaSDag-Erling Smørgrav 	}
544*f374ba41SEd Maste 	if ((hlen = ssh_digest_bytes(hash_alg)) == 0) {
545a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INTERNAL_ERROR;
546a0ee8cc6SDag-Erling Smørgrav 		goto out;
5471e8db6e2SBrian Feldman 	}
548*f374ba41SEd Maste 	if ((ret = ssh_digest_memory(hash_alg, data, dlen,
549a0ee8cc6SDag-Erling Smørgrav 	    digest, sizeof(digest))) != 0)
550a0ee8cc6SDag-Erling Smørgrav 		goto out;
5511e8db6e2SBrian Feldman 
552*f374ba41SEd Maste 	ret = openssh_RSA_verify(hash_alg, digest, hlen, sigblob, len,
553f7167e0eSDag-Erling Smørgrav 	    key->rsa);
554a0ee8cc6SDag-Erling Smørgrav  out:
55547dd1d1bSDag-Erling Smørgrav 	freezero(sigblob, len);
55647dd1d1bSDag-Erling Smørgrav 	free(sigtype);
557a0ee8cc6SDag-Erling Smørgrav 	sshbuf_free(b);
558a0ee8cc6SDag-Erling Smørgrav 	explicit_bzero(digest, sizeof(digest));
5591e8db6e2SBrian Feldman 	return ret;
5601e8db6e2SBrian Feldman }
5614b17dab0SDag-Erling Smørgrav 
5624b17dab0SDag-Erling Smørgrav /*
5634b17dab0SDag-Erling Smørgrav  * See:
5644b17dab0SDag-Erling Smørgrav  * http://www.rsasecurity.com/rsalabs/pkcs/pkcs-1/
5654b17dab0SDag-Erling Smørgrav  * ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-1/pkcs-1v2-1.asn
5664b17dab0SDag-Erling Smørgrav  */
567acc1a9efSDag-Erling Smørgrav 
5684b17dab0SDag-Erling Smørgrav /*
5694b17dab0SDag-Erling Smørgrav  * id-sha1 OBJECT IDENTIFIER ::= { iso(1) identified-organization(3)
5704b17dab0SDag-Erling Smørgrav  *	oiw(14) secsig(3) algorithms(2) 26 }
5714b17dab0SDag-Erling Smørgrav  */
5724b17dab0SDag-Erling Smørgrav static const u_char id_sha1[] = {
5734b17dab0SDag-Erling Smørgrav 	0x30, 0x21, /* type Sequence, length 0x21 (33) */
5744b17dab0SDag-Erling Smørgrav 	0x30, 0x09, /* type Sequence, length 0x09 */
5754b17dab0SDag-Erling Smørgrav 	0x06, 0x05, /* type OID, length 0x05 */
5764b17dab0SDag-Erling Smørgrav 	0x2b, 0x0e, 0x03, 0x02, 0x1a, /* id-sha1 OID */
5774b17dab0SDag-Erling Smørgrav 	0x05, 0x00, /* NULL */
5784b17dab0SDag-Erling Smørgrav 	0x04, 0x14  /* Octet string, length 0x14 (20), followed by sha1 hash */
5794b17dab0SDag-Erling Smørgrav };
5804b17dab0SDag-Erling Smørgrav 
581acc1a9efSDag-Erling Smørgrav /*
582acc1a9efSDag-Erling Smørgrav  * See http://csrc.nist.gov/groups/ST/crypto_apps_infra/csor/algorithms.html
583acc1a9efSDag-Erling Smørgrav  * id-sha256 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840)
584acc1a9efSDag-Erling Smørgrav  *      organization(1) gov(101) csor(3) nistAlgorithm(4) hashAlgs(2)
585acc1a9efSDag-Erling Smørgrav  *      id-sha256(1) }
586acc1a9efSDag-Erling Smørgrav  */
587acc1a9efSDag-Erling Smørgrav static const u_char id_sha256[] = {
588acc1a9efSDag-Erling Smørgrav 	0x30, 0x31, /* type Sequence, length 0x31 (49) */
589acc1a9efSDag-Erling Smørgrav 	0x30, 0x0d, /* type Sequence, length 0x0d (13) */
590acc1a9efSDag-Erling Smørgrav 	0x06, 0x09, /* type OID, length 0x09 */
591acc1a9efSDag-Erling Smørgrav 	0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, /* id-sha256 */
592acc1a9efSDag-Erling Smørgrav 	0x05, 0x00, /* NULL */
593acc1a9efSDag-Erling Smørgrav 	0x04, 0x20  /* Octet string, length 0x20 (32), followed by sha256 hash */
594acc1a9efSDag-Erling Smørgrav };
595acc1a9efSDag-Erling Smørgrav 
596acc1a9efSDag-Erling Smørgrav /*
597acc1a9efSDag-Erling Smørgrav  * See http://csrc.nist.gov/groups/ST/crypto_apps_infra/csor/algorithms.html
598acc1a9efSDag-Erling Smørgrav  * id-sha512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840)
599acc1a9efSDag-Erling Smørgrav  *      organization(1) gov(101) csor(3) nistAlgorithm(4) hashAlgs(2)
600acc1a9efSDag-Erling Smørgrav  *      id-sha256(3) }
601acc1a9efSDag-Erling Smørgrav  */
602acc1a9efSDag-Erling Smørgrav static const u_char id_sha512[] = {
603acc1a9efSDag-Erling Smørgrav 	0x30, 0x51, /* type Sequence, length 0x51 (81) */
604acc1a9efSDag-Erling Smørgrav 	0x30, 0x0d, /* type Sequence, length 0x0d (13) */
605acc1a9efSDag-Erling Smørgrav 	0x06, 0x09, /* type OID, length 0x09 */
606acc1a9efSDag-Erling Smørgrav 	0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, /* id-sha512 */
607acc1a9efSDag-Erling Smørgrav 	0x05, 0x00, /* NULL */
608acc1a9efSDag-Erling Smørgrav 	0x04, 0x40  /* Octet string, length 0x40 (64), followed by sha512 hash */
609acc1a9efSDag-Erling Smørgrav };
610acc1a9efSDag-Erling Smørgrav 
611acc1a9efSDag-Erling Smørgrav static int
612acc1a9efSDag-Erling Smørgrav rsa_hash_alg_oid(int hash_alg, const u_char **oidp, size_t *oidlenp)
613acc1a9efSDag-Erling Smørgrav {
614acc1a9efSDag-Erling Smørgrav 	switch (hash_alg) {
615acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA1:
616acc1a9efSDag-Erling Smørgrav 		*oidp = id_sha1;
617acc1a9efSDag-Erling Smørgrav 		*oidlenp = sizeof(id_sha1);
618acc1a9efSDag-Erling Smørgrav 		break;
619acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA256:
620acc1a9efSDag-Erling Smørgrav 		*oidp = id_sha256;
621acc1a9efSDag-Erling Smørgrav 		*oidlenp = sizeof(id_sha256);
622acc1a9efSDag-Erling Smørgrav 		break;
623acc1a9efSDag-Erling Smørgrav 	case SSH_DIGEST_SHA512:
624acc1a9efSDag-Erling Smørgrav 		*oidp = id_sha512;
625acc1a9efSDag-Erling Smørgrav 		*oidlenp = sizeof(id_sha512);
626acc1a9efSDag-Erling Smørgrav 		break;
627acc1a9efSDag-Erling Smørgrav 	default:
628acc1a9efSDag-Erling Smørgrav 		return SSH_ERR_INVALID_ARGUMENT;
629acc1a9efSDag-Erling Smørgrav 	}
630acc1a9efSDag-Erling Smørgrav 	return 0;
631acc1a9efSDag-Erling Smørgrav }
632acc1a9efSDag-Erling Smørgrav 
6334b17dab0SDag-Erling Smørgrav static int
634a0ee8cc6SDag-Erling Smørgrav openssh_RSA_verify(int hash_alg, u_char *hash, size_t hashlen,
635a0ee8cc6SDag-Erling Smørgrav     u_char *sigbuf, size_t siglen, RSA *rsa)
6364b17dab0SDag-Erling Smørgrav {
637acc1a9efSDag-Erling Smørgrav 	size_t rsasize = 0, oidlen = 0, hlen = 0;
638acc1a9efSDag-Erling Smørgrav 	int ret, len, oidmatch, hashmatch;
6394b17dab0SDag-Erling Smørgrav 	const u_char *oid = NULL;
6404b17dab0SDag-Erling Smørgrav 	u_char *decrypted = NULL;
6414b17dab0SDag-Erling Smørgrav 
642acc1a9efSDag-Erling Smørgrav 	if ((ret = rsa_hash_alg_oid(hash_alg, &oid, &oidlen)) != 0)
643acc1a9efSDag-Erling Smørgrav 		return ret;
644a0ee8cc6SDag-Erling Smørgrav 	ret = SSH_ERR_INTERNAL_ERROR;
645acc1a9efSDag-Erling Smørgrav 	hlen = ssh_digest_bytes(hash_alg);
6464b17dab0SDag-Erling Smørgrav 	if (hashlen != hlen) {
647a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INVALID_ARGUMENT;
6484b17dab0SDag-Erling Smørgrav 		goto done;
6494b17dab0SDag-Erling Smørgrav 	}
6504b17dab0SDag-Erling Smørgrav 	rsasize = RSA_size(rsa);
651a0ee8cc6SDag-Erling Smørgrav 	if (rsasize <= 0 || rsasize > SSHBUF_MAX_BIGNUM ||
652a0ee8cc6SDag-Erling Smørgrav 	    siglen == 0 || siglen > rsasize) {
653a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INVALID_ARGUMENT;
6544b17dab0SDag-Erling Smørgrav 		goto done;
6554b17dab0SDag-Erling Smørgrav 	}
656a0ee8cc6SDag-Erling Smørgrav 	if ((decrypted = malloc(rsasize)) == NULL) {
657a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_ALLOC_FAIL;
658a0ee8cc6SDag-Erling Smørgrav 		goto done;
659a0ee8cc6SDag-Erling Smørgrav 	}
6604b17dab0SDag-Erling Smørgrav 	if ((len = RSA_public_decrypt(siglen, sigbuf, decrypted, rsa,
6614b17dab0SDag-Erling Smørgrav 	    RSA_PKCS1_PADDING)) < 0) {
662a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_LIBCRYPTO_ERROR;
6634b17dab0SDag-Erling Smørgrav 		goto done;
6644b17dab0SDag-Erling Smørgrav 	}
665a0ee8cc6SDag-Erling Smørgrav 	if (len < 0 || (size_t)len != hlen + oidlen) {
666a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_INVALID_FORMAT;
6674b17dab0SDag-Erling Smørgrav 		goto done;
6684b17dab0SDag-Erling Smørgrav 	}
669e2f6069cSDag-Erling Smørgrav 	oidmatch = timingsafe_bcmp(decrypted, oid, oidlen) == 0;
670e2f6069cSDag-Erling Smørgrav 	hashmatch = timingsafe_bcmp(decrypted + oidlen, hash, hlen) == 0;
671a0ee8cc6SDag-Erling Smørgrav 	if (!oidmatch || !hashmatch) {
672a0ee8cc6SDag-Erling Smørgrav 		ret = SSH_ERR_SIGNATURE_INVALID;
6734b17dab0SDag-Erling Smørgrav 		goto done;
6744b17dab0SDag-Erling Smørgrav 	}
675a0ee8cc6SDag-Erling Smørgrav 	ret = 0;
6764b17dab0SDag-Erling Smørgrav done:
67747dd1d1bSDag-Erling Smørgrav 	freezero(decrypted, rsasize);
6784b17dab0SDag-Erling Smørgrav 	return ret;
6794b17dab0SDag-Erling Smørgrav }
680*f374ba41SEd Maste 
681*f374ba41SEd Maste static const struct sshkey_impl_funcs sshkey_rsa_funcs = {
682*f374ba41SEd Maste 	/* .size = */		ssh_rsa_size,
683*f374ba41SEd Maste 	/* .alloc = */		ssh_rsa_alloc,
684*f374ba41SEd Maste 	/* .cleanup = */	ssh_rsa_cleanup,
685*f374ba41SEd Maste 	/* .equal = */		ssh_rsa_equal,
686*f374ba41SEd Maste 	/* .ssh_serialize_public = */ ssh_rsa_serialize_public,
687*f374ba41SEd Maste 	/* .ssh_deserialize_public = */ ssh_rsa_deserialize_public,
688*f374ba41SEd Maste 	/* .ssh_serialize_private = */ ssh_rsa_serialize_private,
689*f374ba41SEd Maste 	/* .ssh_deserialize_private = */ ssh_rsa_deserialize_private,
690*f374ba41SEd Maste 	/* .generate = */	ssh_rsa_generate,
691*f374ba41SEd Maste 	/* .copy_public = */	ssh_rsa_copy_public,
692*f374ba41SEd Maste 	/* .sign = */		ssh_rsa_sign,
693*f374ba41SEd Maste 	/* .verify = */		ssh_rsa_verify,
694*f374ba41SEd Maste };
695*f374ba41SEd Maste 
696*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_impl = {
697*f374ba41SEd Maste 	/* .name = */		"ssh-rsa",
698*f374ba41SEd Maste 	/* .shortname = */	"RSA",
699*f374ba41SEd Maste 	/* .sigalg = */		NULL,
700*f374ba41SEd Maste 	/* .type = */		KEY_RSA,
701*f374ba41SEd Maste 	/* .nid = */		0,
702*f374ba41SEd Maste 	/* .cert = */		0,
703*f374ba41SEd Maste 	/* .sigonly = */	0,
704*f374ba41SEd Maste 	/* .keybits = */	0,
705*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
706*f374ba41SEd Maste };
707*f374ba41SEd Maste 
708*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_cert_impl = {
709*f374ba41SEd Maste 	/* .name = */		"ssh-rsa-cert-v01@openssh.com",
710*f374ba41SEd Maste 	/* .shortname = */	"RSA-CERT",
711*f374ba41SEd Maste 	/* .sigalg = */		NULL,
712*f374ba41SEd Maste 	/* .type = */		KEY_RSA_CERT,
713*f374ba41SEd Maste 	/* .nid = */		0,
714*f374ba41SEd Maste 	/* .cert = */		1,
715*f374ba41SEd Maste 	/* .sigonly = */	0,
716*f374ba41SEd Maste 	/* .keybits = */	0,
717*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
718*f374ba41SEd Maste };
719*f374ba41SEd Maste 
720*f374ba41SEd Maste /* SHA2 signature algorithms */
721*f374ba41SEd Maste 
722*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_sha256_impl = {
723*f374ba41SEd Maste 	/* .name = */		"rsa-sha2-256",
724*f374ba41SEd Maste 	/* .shortname = */	"RSA",
725*f374ba41SEd Maste 	/* .sigalg = */		NULL,
726*f374ba41SEd Maste 	/* .type = */		KEY_RSA,
727*f374ba41SEd Maste 	/* .nid = */		0,
728*f374ba41SEd Maste 	/* .cert = */		0,
729*f374ba41SEd Maste 	/* .sigonly = */	1,
730*f374ba41SEd Maste 	/* .keybits = */	0,
731*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
732*f374ba41SEd Maste };
733*f374ba41SEd Maste 
734*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_sha512_impl = {
735*f374ba41SEd Maste 	/* .name = */		"rsa-sha2-512",
736*f374ba41SEd Maste 	/* .shortname = */	"RSA",
737*f374ba41SEd Maste 	/* .sigalg = */		NULL,
738*f374ba41SEd Maste 	/* .type = */		KEY_RSA,
739*f374ba41SEd Maste 	/* .nid = */		0,
740*f374ba41SEd Maste 	/* .cert = */		0,
741*f374ba41SEd Maste 	/* .sigonly = */	1,
742*f374ba41SEd Maste 	/* .keybits = */	0,
743*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
744*f374ba41SEd Maste };
745*f374ba41SEd Maste 
746*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_sha256_cert_impl = {
747*f374ba41SEd Maste 	/* .name = */		"rsa-sha2-256-cert-v01@openssh.com",
748*f374ba41SEd Maste 	/* .shortname = */	"RSA-CERT",
749*f374ba41SEd Maste 	/* .sigalg = */		"rsa-sha2-256",
750*f374ba41SEd Maste 	/* .type = */		KEY_RSA_CERT,
751*f374ba41SEd Maste 	/* .nid = */		0,
752*f374ba41SEd Maste 	/* .cert = */		1,
753*f374ba41SEd Maste 	/* .sigonly = */	1,
754*f374ba41SEd Maste 	/* .keybits = */	0,
755*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
756*f374ba41SEd Maste };
757*f374ba41SEd Maste 
758*f374ba41SEd Maste const struct sshkey_impl sshkey_rsa_sha512_cert_impl = {
759*f374ba41SEd Maste 	/* .name = */		"rsa-sha2-512-cert-v01@openssh.com",
760*f374ba41SEd Maste 	/* .shortname = */	"RSA-CERT",
761*f374ba41SEd Maste 	/* .sigalg = */		"rsa-sha2-512",
762*f374ba41SEd Maste 	/* .type = */		KEY_RSA_CERT,
763*f374ba41SEd Maste 	/* .nid = */		0,
764*f374ba41SEd Maste 	/* .cert = */		1,
765*f374ba41SEd Maste 	/* .sigonly = */	1,
766*f374ba41SEd Maste 	/* .keybits = */	0,
767*f374ba41SEd Maste 	/* .funcs = */		&sshkey_rsa_funcs,
768*f374ba41SEd Maste };
769bc5531deSDag-Erling Smørgrav #endif /* WITH_OPENSSL */
770