xref: /freebsd/contrib/libfido2/src/compress.c (revision 4f8f43b06ed07e96a250855488cc531799d5b78f)
1 /*
2  * Copyright (c) 2020-2022 Yubico AB. All rights reserved.
3  * Use of this source code is governed by a BSD-style
4  * license that can be found in the LICENSE file.
5  * SPDX-License-Identifier: BSD-2-Clause
6  */
7 
8 #include <zlib.h>
9 #include "fido.h"
10 
11 #define BOUND (1024UL * 1024UL)
12 
13 /* zlib inflate (raw + headers) */
14 static int
15 rfc1950_inflate(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
16 {
17 	u_long ilen, olen;
18 	int z;
19 
20 	memset(out, 0, sizeof(*out));
21 
22 	if (in->len > ULONG_MAX || (ilen = (u_long)in->len) > BOUND ||
23 	    origsiz > ULONG_MAX || (olen = (u_long)origsiz) > BOUND) {
24 		fido_log_debug("%s: in->len=%zu, origsiz=%zu", __func__,
25 		    in->len, origsiz);
26 		return FIDO_ERR_INVALID_ARGUMENT;
27 	}
28 
29 	if ((out->ptr = calloc(1, olen)) == NULL)
30 		return FIDO_ERR_INTERNAL;
31 	out->len = olen;
32 
33 	if ((z = uncompress(out->ptr, &olen, in->ptr, ilen)) != Z_OK ||
34 	    olen > SIZE_MAX || olen != out->len) {
35 		fido_log_debug("%s: uncompress: %d, olen=%lu, out->len=%zu",
36 		    __func__, z, olen, out->len);
37 		fido_blob_reset(out);
38 		return FIDO_ERR_COMPRESS;
39 	}
40 
41 	return FIDO_OK;
42 }
43 
44 /* raw inflate */
45 static int
46 rfc1951_inflate(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
47 {
48 	z_stream zs;
49 	u_int ilen, olen;
50 	int r, z;
51 
52 	memset(&zs, 0, sizeof(zs));
53 	memset(out, 0, sizeof(*out));
54 
55 	if (in->len > UINT_MAX || (ilen = (u_int)in->len) > BOUND ||
56 	    origsiz > UINT_MAX || (olen = (u_int)origsiz) > BOUND) {
57 		fido_log_debug("%s: in->len=%zu, origsiz=%zu", __func__,
58 		    in->len, origsiz);
59 		return FIDO_ERR_INVALID_ARGUMENT;
60 	}
61 	if ((z = inflateInit2(&zs, -MAX_WBITS)) != Z_OK) {
62 		fido_log_debug("%s: inflateInit2: %d", __func__, z);
63 		return FIDO_ERR_COMPRESS;
64 	}
65 
66 	if ((out->ptr = calloc(1, olen)) == NULL) {
67 		r = FIDO_ERR_INTERNAL;
68 		goto fail;
69 	}
70 	out->len = olen;
71 	zs.next_in = in->ptr;
72 	zs.avail_in = ilen;
73 	zs.next_out = out->ptr;
74 	zs.avail_out = olen;
75 
76 	if ((z = inflate(&zs, Z_FINISH)) != Z_STREAM_END) {
77 		fido_log_debug("%s: inflate: %d", __func__, z);
78 		r = FIDO_ERR_COMPRESS;
79 		goto fail;
80 	}
81 	if (zs.avail_out != 0) {
82 		fido_log_debug("%s: %u != 0", __func__, zs.avail_out);
83 		r = FIDO_ERR_COMPRESS;
84 		goto fail;
85 	}
86 
87 	r = FIDO_OK;
88 fail:
89 	if ((z = inflateEnd(&zs)) != Z_OK) {
90 		fido_log_debug("%s: inflateEnd: %d", __func__, z);
91 		r = FIDO_ERR_COMPRESS;
92 	}
93 	if (r != FIDO_OK)
94 		fido_blob_reset(out);
95 
96 	return r;
97 }
98 
99 /* raw deflate */
100 static int
101 rfc1951_deflate(fido_blob_t *out, const fido_blob_t *in)
102 {
103 	z_stream zs;
104 	u_int ilen, olen;
105 	int r, z;
106 
107 	memset(&zs, 0, sizeof(zs));
108 	memset(out, 0, sizeof(*out));
109 
110 	if (in->len > UINT_MAX || (ilen = (u_int)in->len) > BOUND) {
111 		fido_log_debug("%s: in->len=%zu", __func__, in->len);
112 		return FIDO_ERR_INVALID_ARGUMENT;
113 	}
114 	if ((z = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
115 	    -MAX_WBITS, 8, Z_DEFAULT_STRATEGY)) != Z_OK) {
116 		fido_log_debug("%s: deflateInit2: %d", __func__, z);
117 		return FIDO_ERR_COMPRESS;
118 	}
119 
120 	olen = BOUND;
121 	if ((out->ptr = calloc(1, olen)) == NULL) {
122 		r = FIDO_ERR_INTERNAL;
123 		goto fail;
124 	}
125 	out->len = olen;
126 	zs.next_in = in->ptr;
127 	zs.avail_in = ilen;
128 	zs.next_out = out->ptr;
129 	zs.avail_out = olen;
130 
131 	if ((z = deflate(&zs, Z_FINISH)) != Z_STREAM_END) {
132 		fido_log_debug("%s: inflate: %d", __func__, z);
133 		r = FIDO_ERR_COMPRESS;
134 		goto fail;
135 	}
136 	if (zs.avail_out >= out->len) {
137 		fido_log_debug("%s: %u > %zu", __func__, zs.avail_out,
138 		    out->len);
139 		r = FIDO_ERR_COMPRESS;
140 		goto fail;
141 	}
142 	out->len -= zs.avail_out;
143 
144 	r = FIDO_OK;
145 fail:
146 	if ((z = deflateEnd(&zs)) != Z_OK) {
147 		fido_log_debug("%s: deflateEnd: %d", __func__, z);
148 		r = FIDO_ERR_COMPRESS;
149 	}
150 	if (r != FIDO_OK)
151 		fido_blob_reset(out);
152 
153 	return r;
154 }
155 
156 int
157 fido_compress(fido_blob_t *out, const fido_blob_t *in)
158 {
159 	return rfc1951_deflate(out, in);
160 }
161 
162 int
163 fido_uncompress(fido_blob_t *out, const fido_blob_t *in, size_t origsiz)
164 {
165 	if (rfc1950_inflate(out, in, origsiz) == FIDO_OK)
166 		return FIDO_OK; /* backwards compat with libfido2 < 1.11 */
167 	return rfc1951_inflate(out, in, origsiz);
168 }
169