xref: /freebsd/contrib/bearssl/src/rsa/rsa_oaep_unpad.c (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*
2  * Copyright (c) 2018 Thomas Pornin <pornin@bolet.org>
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining
5  * a copy of this software and associated documentation files (the
6  * "Software"), to deal in the Software without restriction, including
7  * without limitation the rights to use, copy, modify, merge, publish,
8  * distribute, sublicense, and/or sell copies of the Software, and to
9  * permit persons to whom the Software is furnished to do so, subject to
10  * the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be
13  * included in all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 
25 #include "inner.h"
26 
27 /*
28  * Hash some data and XOR the result into the provided buffer. This is put
29  * as a separate function so that stack allocation of the hash function
30  * context is done only for the duration of the hash.
31  */
32 static void
33 xor_hash_data(const br_hash_class *dig, void *dst, const void *src, size_t len)
34 {
35 	br_hash_compat_context hc;
36 	unsigned char tmp[64];
37 	unsigned char *buf;
38 	size_t u, hlen;
39 
40 	hc.vtable = dig;
41 	dig->init(&hc.vtable);
42 	dig->update(&hc.vtable, src, len);
43 	dig->out(&hc.vtable, tmp);
44 	buf = dst;
45 	hlen = br_digest_size(dig);
46 	for (u = 0; u < hlen; u ++) {
47 		buf[u] ^= tmp[u];
48 	}
49 }
50 
51 /* see inner.h */
52 uint32_t
53 br_rsa_oaep_unpad(const br_hash_class *dig,
54 	const void *label, size_t label_len,
55 	void *data, size_t *len)
56 {
57 	size_t u, k, hlen;
58 	unsigned char *buf;
59 	uint32_t r, s, zlen;
60 
61 	hlen = br_digest_size(dig);
62 	k = *len;
63 	buf = data;
64 
65 	/*
66 	 * There must be room for the padding.
67 	 */
68 	if (k < ((hlen << 1) + 2)) {
69 		return 0;
70 	}
71 
72 	/*
73 	 * Unmask the seed, then the DB value.
74 	 */
75 	br_mgf1_xor(buf + 1, hlen, dig, buf + 1 + hlen, k - hlen - 1);
76 	br_mgf1_xor(buf + 1 + hlen, k - hlen - 1, dig, buf + 1, hlen);
77 
78 	/*
79 	 * Hash the label and XOR it with the value in the array; if
80 	 * they are equal then these should yield only zeros.
81 	 */
82 	xor_hash_data(dig, buf + 1 + hlen, label, label_len);
83 
84 	/*
85 	 * At that point, if the padding was correct, when we should
86 	 * have: 0x00 || seed || 0x00 ... 0x00 0x01 || M
87 	 * Padding is valid as long as:
88 	 *  - There is at least hlen+1 leading bytes of value 0x00.
89 	 *  - There is at least one non-zero byte.
90 	 *  - The first (leftmost) non-zero byte has value 0x01.
91 	 *
92 	 * Ultimately, we may leak the resulting message length, i.e.
93 	 * the position of the byte of value 0x01, but we must take care
94 	 * to do so only if the number of zero bytes has been verified
95 	 * to be at least hlen+1.
96 	 *
97 	 * The loop below counts the number of bytes of value 0x00, and
98 	 * checks that the next byte has value 0x01, in constant-time.
99 	 *
100 	 *  - If the initial byte (before the seed) is not 0x00, then
101 	 *    r and s are set to 0, and stay there.
102 	 *  - Value r is 1 until the first non-zero byte is reached
103 	 *    (after the seed); it switches to 0 at that point.
104 	 *  - Value s is set to 1 if and only if the data encountered
105 	 *    at the time of the transition of r from 1 to 0 has value
106 	 *    exactly 0x01.
107 	 *  - Value zlen counts the number of leading bytes of value zero
108 	 *    (after the seed).
109 	 */
110 	r = 1 - ((buf[0] + 0xFF) >> 8);
111 	s = 0;
112 	zlen = 0;
113 	for (u = hlen + 1; u < k; u ++) {
114 		uint32_t w, nz;
115 
116 		w = buf[u];
117 
118 		/*
119 		 * nz == 1 only for the first non-zero byte.
120 		 */
121 		nz = r & ((w + 0xFF) >> 8);
122 		s |= nz & EQ(w, 0x01);
123 		r &= NOT(nz);
124 		zlen += r;
125 	}
126 
127 	/*
128 	 * Padding is correct only if s == 1, _and_ zlen >= hlen.
129 	 */
130 	s &= GE(zlen, (uint32_t)hlen);
131 
132 	/*
133 	 * At that point, padding was verified, and we are now allowed
134 	 * to make conditional jumps.
135 	 */
136 	if (s) {
137 		size_t plen;
138 
139 		plen = 2 + hlen + zlen;
140 		k -= plen;
141 		memmove(buf, buf + plen, k);
142 		*len = k;
143 	}
144 	return s;
145 }
146