xref: /freebsd/contrib/bearssl/src/rsa/rsa_pkcs1_sig_unpad.c (revision d0b2dbfa0ecf2bbc9709efc5e20baf8e4b44bbbf)
1 /*
2  * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining
5  * a copy of this software and associated documentation files (the
6  * "Software"), to deal in the Software without restriction, including
7  * without limitation the rights to use, copy, modify, merge, publish,
8  * distribute, sublicense, and/or sell copies of the Software, and to
9  * permit persons to whom the Software is furnished to do so, subject to
10  * the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be
13  * included in all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 
25 #include "inner.h"
26 
27 /* see bearssl_rsa.h */
28 uint32_t
29 br_rsa_pkcs1_sig_unpad(const unsigned char *sig, size_t sig_len,
30 	const unsigned char *hash_oid, size_t hash_len,
31 	unsigned char *hash_out)
32 {
33 	static const unsigned char pad1[] = {
34 		0x00, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
35 	};
36 
37 	unsigned char pad2[43];
38 	size_t u, x2, x3, pad_len, zlen;
39 
40 	if (sig_len < 11) {
41 		return 0;
42 	}
43 
44 	/*
45 	 * Expected format:
46 	 *  00 01 FF ... FF 00 30 x1 30 x2 06 x3 OID [ 05 00 ] 04 x4 HASH
47 	 *
48 	 * with the following rules:
49 	 *
50 	 *  -- Total length is that of the modulus and the signature
51 	 *     (this was already verified by br_rsa_i31_public()).
52 	 *
53 	 *  -- There are at least eight bytes of value 0xFF.
54 	 *
55 	 *  -- x4 is equal to the hash length (hash_len).
56 	 *
57 	 *  -- x3 is equal to the encoded OID value length (so x3 is the
58 	 *     first byte of hash_oid[]).
59 	 *
60 	 *  -- If the "05 00" is present, then x2 == x3 + 4; otherwise,
61 	 *     x2 == x3 + 2.
62 	 *
63 	 *  -- x1 == x2 + x4 + 4.
64 	 *
65 	 * So the total length after the last "FF" is either x3 + x4 + 11
66 	 * (with the "05 00") or x3 + x4 + 9 (without the "05 00").
67 	 */
68 
69 	/*
70 	 * Check the "00 01 FF .. FF 00" with at least eight 0xFF bytes.
71 	 * The comparison is valid because we made sure that the signature
72 	 * is at least 11 bytes long.
73 	 */
74 	if (memcmp(sig, pad1, sizeof pad1) != 0) {
75 		return 0;
76 	}
77 	for (u = sizeof pad1; u < sig_len; u ++) {
78 		if (sig[u] != 0xFF) {
79 			break;
80 		}
81 	}
82 
83 	/*
84 	 * Remaining length is sig_len - u bytes (including the 00 just
85 	 * after the last FF). This must be equal to one of the two
86 	 * possible values (depending on whether the "05 00" sequence is
87 	 * present or not).
88 	 */
89 	if (hash_oid == NULL) {
90 		if (sig_len - u != hash_len + 1 || sig[u] != 0x00) {
91 			return 0;
92 		}
93 	} else {
94 		x3 = hash_oid[0];
95 		pad_len = x3 + 9;
96 		memset(pad2, 0, pad_len);
97 		zlen = sig_len - u - hash_len;
98 		if (zlen == pad_len) {
99 			x2 = x3 + 2;
100 		} else if (zlen == pad_len + 2) {
101 			x2 = x3 + 4;
102 			pad_len = zlen;
103 			pad2[pad_len - 4] = 0x05;
104 		} else {
105 			return 0;
106 		}
107 		pad2[1] = 0x30;
108 		pad2[2] = x2 + hash_len + 4;
109 		pad2[3] = 0x30;
110 		pad2[4] = x2;
111 		pad2[5] = 0x06;
112 		memcpy(pad2 + 6, hash_oid, x3 + 1);
113 		pad2[pad_len - 2] = 0x04;
114 		pad2[pad_len - 1] = hash_len;
115 		if (memcmp(pad2, sig + u, pad_len) != 0) {
116 			return 0;
117 		}
118 	}
119 	memcpy(hash_out, sig + sig_len - hash_len, hash_len);
120 	return 1;
121 }
122