xref: /freebsd/contrib/wpa/src/tls/pkcs1.c (revision 40a8ac8f62b535d30349faf28cf47106b7041b83)
1 /*
2  * PKCS #1 (RSA Encryption)
3  * Copyright (c) 2006-2009, Jouni Malinen <j@w1.fi>
4  *
5  * This software may be distributed under the terms of the BSD license.
6  * See README for more details.
7  */
8 
9 #include "includes.h"
10 
11 #include "common.h"
12 #include "rsa.h"
13 #include "pkcs1.h"
14 
15 
16 static int pkcs1_generate_encryption_block(u8 block_type, size_t modlen,
17 					   const u8 *in, size_t inlen,
18 					   u8 *out, size_t *outlen)
19 {
20 	size_t ps_len;
21 	u8 *pos;
22 
23 	/*
24 	 * PKCS #1 v1.5, 8.1:
25 	 *
26 	 * EB = 00 || BT || PS || 00 || D
27 	 * BT = 00 or 01 for private-key operation; 02 for public-key operation
28 	 * PS = k-3-||D||; at least eight octets
29 	 * (BT=0: PS=0x00, BT=1: PS=0xff, BT=2: PS=pseudorandom non-zero)
30 	 * k = length of modulus in octets (modlen)
31 	 */
32 
33 	if (modlen < 12 || modlen > *outlen || inlen > modlen - 11) {
34 		wpa_printf(MSG_DEBUG, "PKCS #1: %s - Invalid buffer "
35 			   "lengths (modlen=%lu outlen=%lu inlen=%lu)",
36 			   __func__, (unsigned long) modlen,
37 			   (unsigned long) *outlen,
38 			   (unsigned long) inlen);
39 		return -1;
40 	}
41 
42 	pos = out;
43 	*pos++ = 0x00;
44 	*pos++ = block_type; /* BT */
45 	ps_len = modlen - inlen - 3;
46 	switch (block_type) {
47 	case 0:
48 		os_memset(pos, 0x00, ps_len);
49 		pos += ps_len;
50 		break;
51 	case 1:
52 		os_memset(pos, 0xff, ps_len);
53 		pos += ps_len;
54 		break;
55 	case 2:
56 		if (os_get_random(pos, ps_len) < 0) {
57 			wpa_printf(MSG_DEBUG, "PKCS #1: %s - Failed to get "
58 				   "random data for PS", __func__);
59 			return -1;
60 		}
61 		while (ps_len--) {
62 			if (*pos == 0x00)
63 				*pos = 0x01;
64 			pos++;
65 		}
66 		break;
67 	default:
68 		wpa_printf(MSG_DEBUG, "PKCS #1: %s - Unsupported block type "
69 			   "%d", __func__, block_type);
70 		return -1;
71 	}
72 	*pos++ = 0x00;
73 	os_memcpy(pos, in, inlen); /* D */
74 
75 	return 0;
76 }
77 
78 
79 int pkcs1_encrypt(int block_type, struct crypto_rsa_key *key,
80 		  int use_private, const u8 *in, size_t inlen,
81 		  u8 *out, size_t *outlen)
82 {
83 	size_t modlen;
84 
85 	modlen = crypto_rsa_get_modulus_len(key);
86 
87 	if (pkcs1_generate_encryption_block(block_type, modlen, in, inlen,
88 					    out, outlen) < 0)
89 		return -1;
90 
91 	return crypto_rsa_exptmod(out, modlen, out, outlen, key, use_private);
92 }
93 
94 
95 int pkcs1_v15_private_key_decrypt(struct crypto_rsa_key *key,
96 				  const u8 *in, size_t inlen,
97 				  u8 *out, size_t *outlen)
98 {
99 	int res;
100 	u8 *pos, *end;
101 
102 	res = crypto_rsa_exptmod(in, inlen, out, outlen, key, 1);
103 	if (res)
104 		return res;
105 
106 	if (*outlen < 2 || out[0] != 0 || out[1] != 2)
107 		return -1;
108 
109 	/* Skip PS (pseudorandom non-zero octets) */
110 	pos = out + 2;
111 	end = out + *outlen;
112 	while (*pos && pos < end)
113 		pos++;
114 	if (pos == end)
115 		return -1;
116 	pos++;
117 
118 	*outlen -= pos - out;
119 
120 	/* Strip PKCS #1 header */
121 	os_memmove(out, pos, *outlen);
122 
123 	return 0;
124 }
125 
126 
127 int pkcs1_decrypt_public_key(struct crypto_rsa_key *key,
128 			     const u8 *crypt, size_t crypt_len,
129 			     u8 *plain, size_t *plain_len)
130 {
131 	size_t len;
132 	u8 *pos;
133 
134 	len = *plain_len;
135 	if (crypto_rsa_exptmod(crypt, crypt_len, plain, &len, key, 0) < 0)
136 		return -1;
137 
138 	/*
139 	 * PKCS #1 v1.5, 8.1:
140 	 *
141 	 * EB = 00 || BT || PS || 00 || D
142 	 * BT = 00 or 01
143 	 * PS = k-3-||D|| times (00 if BT=00) or (FF if BT=01)
144 	 * k = length of modulus in octets
145 	 */
146 
147 	if (len < 3 + 8 + 16 /* min hash len */ ||
148 	    plain[0] != 0x00 || (plain[1] != 0x00 && plain[1] != 0x01)) {
149 		wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature EB "
150 			   "structure");
151 		return -1;
152 	}
153 
154 	pos = plain + 3;
155 	if (plain[1] == 0x00) {
156 		/* BT = 00 */
157 		if (plain[2] != 0x00) {
158 			wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature "
159 				   "PS (BT=00)");
160 			return -1;
161 		}
162 		while (pos + 1 < plain + len && *pos == 0x00 && pos[1] == 0x00)
163 			pos++;
164 	} else {
165 		/* BT = 01 */
166 		if (plain[2] != 0xff) {
167 			wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature "
168 				   "PS (BT=01)");
169 			return -1;
170 		}
171 		while (pos < plain + len && *pos == 0xff)
172 			pos++;
173 	}
174 
175 	if (pos - plain - 2 < 8) {
176 		/* PKCS #1 v1.5, 8.1: At least eight octets long PS */
177 		wpa_printf(MSG_INFO, "LibTomCrypt: Too short signature "
178 			   "padding");
179 		return -1;
180 	}
181 
182 	if (pos + 16 /* min hash len */ >= plain + len || *pos != 0x00) {
183 		wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature EB "
184 			   "structure (2)");
185 		return -1;
186 	}
187 	pos++;
188 	len -= pos - plain;
189 
190 	/* Strip PKCS #1 header */
191 	os_memmove(plain, pos, len);
192 	*plain_len = len;
193 
194 	return 0;
195 }
196