xref: /titanic_44/usr/src/cmd/ssh/libssh/common/ssh-rsa.c (revision 2df1fe9ca32bb227b9158c67f5c00b54c20b10fd)
1  /*
2   * Copyright (c) 2000 Markus Friedl.  All rights reserved.
3   *
4   * Redistribution and use in source and binary forms, with or without
5   * modification, are permitted provided that the following conditions
6   * are met:
7   * 1. Redistributions of source code must retain the above copyright
8   *    notice, this list of conditions and the following disclaimer.
9   * 2. Redistributions in binary form must reproduce the above copyright
10   *    notice, this list of conditions and the following disclaimer in the
11   *    documentation and/or other materials provided with the distribution.
12   *
13   * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14   * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16   * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19   * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20   * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22   * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23   */
24  
25  #include "includes.h"
26  RCSID("$OpenBSD: ssh-rsa.c,v 1.26 2002/08/27 17:13:56 stevesk Exp $");
27  
28  #pragma ident	"%Z%%M%	%I%	%E% SMI"
29  
30  #include <openssl/evp.h>
31  #include <openssl/err.h>
32  
33  #include "xmalloc.h"
34  #include "log.h"
35  #include "buffer.h"
36  #include "bufaux.h"
37  #include "key.h"
38  #include "ssh-rsa.h"
39  #include "compat.h"
40  #include "ssh.h"
41  
42  static int openssh_RSA_verify(int, u_char *, u_int, u_char *, u_int , RSA *);
43  
44  /* RSASSA-PKCS1-v1_5 (PKCS #1 v2.0 signature) with SHA1 */
45  int
46  ssh_rsa_sign(Key *key, u_char **sigp, u_int *lenp,
47      u_char *data, u_int datalen)
48  {
49  	const EVP_MD *evp_md;
50  	EVP_MD_CTX md;
51  	u_char digest[EVP_MAX_MD_SIZE], *sig;
52  	u_int slen, dlen, len;
53  	int ok, nid;
54  	Buffer b;
55  
56  	if (key == NULL || key->type != KEY_RSA || key->rsa == NULL) {
57  		error("ssh_rsa_sign: no RSA key");
58  		return -1;
59  	}
60  	nid = (datafellows & SSH_BUG_RSASIGMD5) ? NID_md5 : NID_sha1;
61  	if ((evp_md = EVP_get_digestbynid(nid)) == NULL) {
62  		error("ssh_rsa_sign: EVP_get_digestbynid %d failed", nid);
63  		return -1;
64  	}
65  	EVP_DigestInit(&md, evp_md);
66  	EVP_DigestUpdate(&md, data, datalen);
67  	EVP_DigestFinal(&md, digest, &dlen);
68  
69  	slen = RSA_size(key->rsa);
70  	sig = xmalloc(slen);
71  
72  	ok = RSA_sign(nid, digest, dlen, sig, &len, key->rsa);
73  	memset(digest, 'd', sizeof(digest));
74  
75  	if (ok != 1) {
76  		int ecode = ERR_get_error();
77  		error("ssh_rsa_sign: RSA_sign failed: %s",
78  		    ERR_error_string(ecode, NULL));
79  		xfree(sig);
80  		return -1;
81  	}
82  	if (len < slen) {
83  		u_int diff = slen - len;
84  		debug("slen %u > len %u", slen, len);
85  		memmove(sig + diff, sig, len);
86  		memset(sig, 0, diff);
87  	} else if (len > slen) {
88  		error("ssh_rsa_sign: slen %u slen2 %u", slen, len);
89  		xfree(sig);
90  		return -1;
91  	}
92  	/* encode signature */
93  	buffer_init(&b);
94  	buffer_put_cstring(&b, "ssh-rsa");
95  	buffer_put_string(&b, sig, slen);
96  	len = buffer_len(&b);
97  	if (lenp != NULL)
98  		*lenp = len;
99  	if (sigp != NULL) {
100  		*sigp = xmalloc(len);
101  		memcpy(*sigp, buffer_ptr(&b), len);
102  	}
103  	buffer_free(&b);
104  	memset(sig, 's', slen);
105  	xfree(sig);
106  
107  	return 0;
108  }
109  
110  int
111  ssh_rsa_verify(Key *key, u_char *signature, u_int signaturelen,
112      u_char *data, u_int datalen)
113  {
114  	Buffer b;
115  	const EVP_MD *evp_md;
116  	EVP_MD_CTX md;
117  	char *ktype;
118  	u_char digest[EVP_MAX_MD_SIZE], *sigblob;
119  	u_int len, dlen, modlen;
120  	int rlen, ret, nid;
121  
122  	if (key == NULL || key->type != KEY_RSA || key->rsa == NULL) {
123  		error("ssh_rsa_verify: no RSA key");
124  		return -1;
125  	}
126  	if (BN_num_bits(key->rsa->n) < SSH_RSA_MINIMUM_MODULUS_SIZE) {
127  		error("ssh_rsa_verify: RSA modulus too small: %d < minimum %d bits",
128  		    BN_num_bits(key->rsa->n), SSH_RSA_MINIMUM_MODULUS_SIZE);
129  		return -1;
130  	}
131  	buffer_init(&b);
132  	buffer_append(&b, signature, signaturelen);
133  	ktype = buffer_get_string(&b, NULL);
134  	if (strcmp("ssh-rsa", ktype) != 0) {
135  		error("ssh_rsa_verify: cannot handle type %s", ktype);
136  		buffer_free(&b);
137  		xfree(ktype);
138  		return -1;
139  	}
140  	xfree(ktype);
141  	sigblob = buffer_get_string(&b, &len);
142  	rlen = buffer_len(&b);
143  	buffer_free(&b);
144  	if (rlen != 0) {
145  		error("ssh_rsa_verify: remaining bytes in signature %d", rlen);
146  		xfree(sigblob);
147  		return -1;
148  	}
149  	/* RSA_verify expects a signature of RSA_size */
150  	modlen = RSA_size(key->rsa);
151  	if (len > modlen) {
152  		error("ssh_rsa_verify: len %u > modlen %u", len, modlen);
153  		xfree(sigblob);
154  		return -1;
155  	} else if (len < modlen) {
156  		u_int diff = modlen - len;
157  		debug("ssh_rsa_verify: add padding: modlen %u > len %u",
158  		    modlen, len);
159  		sigblob = xrealloc(sigblob, modlen);
160  		memmove(sigblob + diff, sigblob, len);
161  		memset(sigblob, 0, diff);
162  		len = modlen;
163  	}
164  	nid = (datafellows & SSH_BUG_RSASIGMD5) ? NID_md5 : NID_sha1;
165  	if ((evp_md = EVP_get_digestbynid(nid)) == NULL) {
166  		error("ssh_rsa_verify: EVP_get_digestbynid %d failed", nid);
167  		xfree(sigblob);
168  		return -1;
169  	}
170  	EVP_DigestInit(&md, evp_md);
171  	EVP_DigestUpdate(&md, data, datalen);
172  	EVP_DigestFinal(&md, digest, &dlen);
173  
174  	ret = openssh_RSA_verify(nid, digest, dlen, sigblob, len, key->rsa);
175  	memset(digest, 'd', sizeof(digest));
176  	memset(sigblob, 's', len);
177  	xfree(sigblob);
178  	debug("ssh_rsa_verify: signature %scorrect", (ret==0) ? "in" : "");
179  	return ret;
180  }
181  
182  /*
183   * See:
184   * http://www.rsasecurity.com/rsalabs/pkcs/pkcs-1/
185   * ftp://ftp.rsasecurity.com/pub/pkcs/pkcs-1/pkcs-1v2-1.asn
186   */
187  /*
188   * id-sha1 OBJECT IDENTIFIER ::= { iso(1) identified-organization(3)
189   *	oiw(14) secsig(3) algorithms(2) 26 }
190   */
191  static const u_char id_sha1[] = {
192  	0x30, 0x21, /* type Sequence, length 0x21 (33) */
193  	0x30, 0x09, /* type Sequence, length 0x09 */
194  	0x06, 0x05, /* type OID, length 0x05 */
195  	0x2b, 0x0e, 0x03, 0x02, 0x1a, /* id-sha1 OID */
196  	0x05, 0x00, /* NULL */
197  	0x04, 0x14  /* Octet string, length 0x14 (20), followed by sha1 hash */
198  };
199  /*
200   * id-md5 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840)
201   *	rsadsi(113549) digestAlgorithm(2) 5 }
202   */
203  static const u_char id_md5[] = {
204  	0x30, 0x20, /* type Sequence, length 0x20 (32) */
205  	0x30, 0x0c, /* type Sequence, length 0x09 */
206  	0x06, 0x08, /* type OID, length 0x05 */
207  	0x2a, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x02, 0x05, /* id-md5 */
208  	0x05, 0x00, /* NULL */
209  	0x04, 0x10  /* Octet string, length 0x10 (16), followed by md5 hash */
210  };
211  
212  static int
213  openssh_RSA_verify(int type, u_char *hash, u_int hashlen,
214      u_char *sigbuf, u_int siglen, RSA *rsa)
215  {
216  	u_int ret, rsasize, oidlen = 0, hlen = 0;
217  	int len;
218  	const u_char *oid = NULL;
219  	u_char *decrypted = NULL;
220  
221  	ret = 0;
222  	switch (type) {
223  	case NID_sha1:
224  		oid = id_sha1;
225  		oidlen = sizeof(id_sha1);
226  		hlen = 20;
227  		break;
228  	case NID_md5:
229  		oid = id_md5;
230  		oidlen = sizeof(id_md5);
231  		hlen = 16;
232  		break;
233  	default:
234  		goto done;
235  		break;
236  	}
237  	if (hashlen != hlen) {
238  		error("bad hashlen");
239  		goto done;
240  	}
241  	rsasize = RSA_size(rsa);
242  	if (siglen == 0 || siglen > rsasize) {
243  		error("bad siglen");
244  		goto done;
245  	}
246  	decrypted = xmalloc(rsasize);
247  	if ((len = RSA_public_decrypt(siglen, sigbuf, decrypted, rsa,
248  	    RSA_PKCS1_PADDING)) < 0) {
249  		error("RSA_public_decrypt failed: %s",
250  		    ERR_error_string(ERR_get_error(), NULL));
251  		goto done;
252  	}
253  	if (len != hlen + oidlen) {
254  		error("bad decrypted len: %d != %d + %d", len, hlen, oidlen);
255  		goto done;
256  	}
257  	if (memcmp(decrypted, oid, oidlen) != 0) {
258  		error("oid mismatch");
259  		goto done;
260  	}
261  	if (memcmp(decrypted + oidlen, hash, hlen) != 0) {
262  		error("hash mismatch");
263  		goto done;
264  	}
265  	ret = 1;
266  done:
267  	if (decrypted)
268  		xfree(decrypted);
269  	return ret;
270  }
271