1 /*
2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
28 */
29
30 #include <sys/types.h>
31 #include <sys/errno.h>
32
33 #ifdef _KERNEL
34 #include <sys/libkern.h>
35 #include <sys/malloc.h>
36 #include <sys/sysctl.h>
37 #include <opencrypto/cryptodev.h>
38 #include <opencrypto/xform.h>
39 #endif
40
41 #include <sys/zio_crypt.h>
42 #include <sys/fs/zfs.h>
43 #include <sys/zio.h>
44
45 #include <sys/freebsd_crypto.h>
46
47 #define SHA512_HMAC_BLOCK_SIZE 128
48
49 static int crypt_sessions = 0;
50 SYSCTL_DECL(_vfs_zfs);
51 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
52 &crypt_sessions, 0, "Number of cryptographic sessions created");
53
54 void
crypto_mac_init(struct hmac_ctx * ctx,const crypto_key_t * c_key)55 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key)
56 {
57 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
58 k_opad[SHA512_HMAC_BLOCK_SIZE],
59 key[SHA512_HMAC_BLOCK_SIZE];
60 SHA512_CTX lctx;
61 int i;
62 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);
63
64 /*
65 * This code is based on the similar code in geom/eli/g_eli_hmac.c
66 */
67 memset(key, 0, sizeof (key));
68 if (c_key->ck_length == 0)
69 /* do nothing */;
70 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
71 memcpy(key, c_key->ck_data, cl_bytes);
72 else {
73 /*
74 * If key is longer than 128 bytes reset it to
75 * key = SHA512(key).
76 */
77 SHA512_Init(&lctx);
78 SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
79 SHA512_Final(key, &lctx);
80 }
81
82 /* XOR key with ipad and opad values. */
83 for (i = 0; i < sizeof (key); i++) {
84 k_ipad[i] = key[i] ^ 0x36;
85 k_opad[i] = key[i] ^ 0x5c;
86 }
87 memset(key, 0, sizeof (key));
88
89 /* Start inner SHA512. */
90 SHA512_Init(&ctx->innerctx);
91 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
92 memset(k_ipad, 0, sizeof (k_ipad));
93 /* Start outer SHA512. */
94 SHA512_Init(&ctx->outerctx);
95 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
96 memset(k_opad, 0, sizeof (k_opad));
97 }
98
99 void
crypto_mac_update(struct hmac_ctx * ctx,const void * data,size_t datasize)100 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize)
101 {
102 SHA512_Update(&ctx->innerctx, data, datasize);
103 }
104
105 void
crypto_mac_final(struct hmac_ctx * ctx,void * md,size_t mdsize)106 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize)
107 {
108 uint8_t digest[SHA512_DIGEST_LENGTH];
109
110 /* Complete inner hash */
111 SHA512_Final(digest, &ctx->innerctx);
112
113 /* Complete outer hash */
114 SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
115 SHA512_Final(digest, &ctx->outerctx);
116
117 memset(ctx, 0, sizeof (*ctx));
118 /* mdsize == 0 means "Give me the whole hash!" */
119 if (mdsize == 0)
120 mdsize = SHA512_DIGEST_LENGTH;
121 memcpy(md, digest, mdsize);
122 memset(digest, 0, sizeof (digest));
123 }
124
125 void
crypto_mac(const crypto_key_t * key,const void * in_data,size_t in_data_size,void * out_data,size_t out_data_size)126 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size,
127 void *out_data, size_t out_data_size)
128 {
129 struct hmac_ctx ctx;
130
131 crypto_mac_init(&ctx, key);
132 crypto_mac_update(&ctx, in_data, in_data_size);
133 crypto_mac_final(&ctx, out_data, out_data_size);
134 }
135
136 static int
freebsd_zfs_crypt_done(struct cryptop * crp)137 freebsd_zfs_crypt_done(struct cryptop *crp)
138 {
139 freebsd_crypt_session_t *ses;
140
141 ses = crp->crp_opaque;
142 mtx_lock(&ses->fs_lock);
143 ses->fs_done = true;
144 mtx_unlock(&ses->fs_lock);
145 wakeup(crp);
146 return (0);
147 }
148
149 static int
freebsd_zfs_crypt_done_sync(struct cryptop * crp)150 freebsd_zfs_crypt_done_sync(struct cryptop *crp)
151 {
152
153 return (0);
154 }
155
156 void
freebsd_crypt_freesession(freebsd_crypt_session_t * sess)157 freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
158 {
159 mtx_destroy(&sess->fs_lock);
160 crypto_freesession(sess->fs_sid);
161 memset(sess, 0, sizeof (*sess));
162 }
163
164 static int
zfs_crypto_dispatch(freebsd_crypt_session_t * session,struct cryptop * crp)165 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp)
166 {
167 int error;
168
169 crp->crp_opaque = session;
170 for (;;) {
171 #if __FreeBSD_version < 1400004
172 boolean_t async = ((crypto_ses2caps(crp->crp_session) &
173 CRYPTOCAP_F_SYNC) == 0);
174 #else
175 boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session);
176 #endif
177 crp->crp_callback = async ? freebsd_zfs_crypt_done :
178 freebsd_zfs_crypt_done_sync;
179 error = crypto_dispatch(crp);
180 if (error == 0) {
181 if (async) {
182 mtx_lock(&session->fs_lock);
183 while (session->fs_done == false) {
184 msleep(crp, &session->fs_lock, 0,
185 "zfs_crypto", 0);
186 }
187 mtx_unlock(&session->fs_lock);
188 }
189 error = crp->crp_etype;
190 }
191
192 if (error == ENOMEM) {
193 pause("zcrnomem", 1);
194 } else if (error != EAGAIN) {
195 break;
196 }
197 crp->crp_etype = 0;
198 crp->crp_flags &= ~CRYPTO_F_DONE;
199 session->fs_done = false;
200 }
201 return (error);
202 }
203 static void
freebsd_crypt_uio_debug_log(boolean_t encrypt,freebsd_crypt_session_t * input_sessionp,const struct zio_crypt_info * c_info,zfs_uio_t * data_uio,crypto_key_t * key,uint8_t * ivbuf,size_t datalen,size_t auth_len)204 freebsd_crypt_uio_debug_log(boolean_t encrypt,
205 freebsd_crypt_session_t *input_sessionp,
206 const struct zio_crypt_info *c_info,
207 zfs_uio_t *data_uio,
208 crypto_key_t *key,
209 uint8_t *ivbuf,
210 size_t datalen,
211 size_t auth_len)
212 {
213 #ifdef FCRYPTO_DEBUG
214 struct cryptodesc *crd;
215 uint8_t *p = NULL;
216 size_t total = 0;
217
218 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, "
219 "%p, %u, %u)\n",
220 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
221 c_info->ci_algname, c_info->ci_crypt_type,
222 (unsigned int)c_info->ci_keylen, c_info->ci_name,
223 data_uio, key->ck_data,
224 (unsigned int)key->ck_length,
225 ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
226 printf("\tkey = { ");
227 for (int i = 0; i < key->ck_length / 8; i++) {
228 uint8_t *b = (uint8_t *)key->ck_data;
229 printf("%02x ", b[i]);
230 }
231 printf("}\n");
232 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
233 printf("\tiovec #%d: <%p, %u>\n", i,
234 zfs_uio_iovbase(data_uio, i),
235 (unsigned int)zfs_uio_iovlen(data_uio, i));
236 total += zfs_uio_iovlen(data_uio, i);
237 }
238 zfs_uio_resid(data_uio) = total;
239 #endif
240 }
241 /*
242 * Create a new cryptographic session. This should
243 * happen every time the key changes (including when
244 * it's first loaded).
245 */
246 int
freebsd_crypt_newsession(freebsd_crypt_session_t * sessp,const struct zio_crypt_info * c_info,crypto_key_t * key)247 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
248 const struct zio_crypt_info *c_info, crypto_key_t *key)
249 {
250 struct crypto_session_params csp = {0};
251 int error = 0;
252
253 #ifdef FCRYPTO_DEBUG
254 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n",
255 __FUNCTION__, sessp,
256 c_info->ci_algname, c_info->ci_crypt_type,
257 (unsigned int)c_info->ci_keylen, c_info->ci_name,
258 key->ck_data, (unsigned int)key->ck_length);
259 printf("\tkey = { ");
260 for (int i = 0; i < key->ck_length / 8; i++) {
261 uint8_t *b = (uint8_t *)key->ck_data;
262 printf("%02x ", b[i]);
263 }
264 printf("}\n");
265 #endif
266 csp.csp_mode = CSP_MODE_AEAD;
267 csp.csp_cipher_key = key->ck_data;
268 csp.csp_cipher_klen = key->ck_length / 8;
269 switch (c_info->ci_crypt_type) {
270 case ZC_TYPE_GCM:
271 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
272 csp.csp_ivlen = AES_GCM_IV_LEN;
273 switch (key->ck_length/8) {
274 case AES_128_GMAC_KEY_LEN:
275 case AES_192_GMAC_KEY_LEN:
276 case AES_256_GMAC_KEY_LEN:
277 break;
278 default:
279 error = EINVAL;
280 goto bad;
281 }
282 break;
283 case ZC_TYPE_CCM:
284 csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
285 csp.csp_ivlen = AES_CCM_IV_LEN;
286 switch (key->ck_length/8) {
287 case AES_128_CBC_MAC_KEY_LEN:
288 case AES_192_CBC_MAC_KEY_LEN:
289 case AES_256_CBC_MAC_KEY_LEN:
290 break;
291 default:
292 error = EINVAL;
293 goto bad;
294 break;
295 }
296 break;
297 default:
298 error = ENOTSUP;
299 goto bad;
300 }
301
302 /*
303 * Disable the use of hardware drivers on FreeBSD 13 and later since
304 * common crypto offload drivers impose constraints on AES-GCM AAD
305 * lengths that make them unusable for ZFS, and we currently do not have
306 * a mechanism to fall back to a software driver for requests not
307 * handled by a hardware driver.
308 *
309 * On 12 we continue to permit the use of hardware drivers since
310 * CPU-accelerated drivers such as aesni(4) register themselves as
311 * hardware drivers.
312 */
313 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
314 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
315 NULL, MTX_DEF);
316 crypt_sessions++;
317 bad:
318 #ifdef FCRYPTO_DEBUG
319 if (error)
320 printf("%s: returning error %d\n", __FUNCTION__, error);
321 #endif
322 return (error);
323 }
324
325 int
freebsd_crypt_uio(boolean_t encrypt,freebsd_crypt_session_t * input_sessionp,const struct zio_crypt_info * c_info,zfs_uio_t * data_uio,crypto_key_t * key,uint8_t * ivbuf,size_t datalen,size_t auth_len)326 freebsd_crypt_uio(boolean_t encrypt,
327 freebsd_crypt_session_t *input_sessionp,
328 const struct zio_crypt_info *c_info,
329 zfs_uio_t *data_uio,
330 crypto_key_t *key,
331 uint8_t *ivbuf,
332 size_t datalen,
333 size_t auth_len)
334 {
335 struct cryptop *crp;
336 freebsd_crypt_session_t *session = NULL;
337 int error = 0;
338 size_t total = 0;
339
340 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
341 key, ivbuf, datalen, auth_len);
342 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
343 total += zfs_uio_iovlen(data_uio, i);
344 zfs_uio_resid(data_uio) = total;
345 if (input_sessionp == NULL) {
346 session = kmem_zalloc(sizeof (*session), KM_SLEEP);
347 error = freebsd_crypt_newsession(session, c_info, key);
348 if (error)
349 goto out;
350 } else
351 session = input_sessionp;
352
353 crp = crypto_getreq(session->fs_sid, M_WAITOK);
354 if (encrypt) {
355 crp->crp_op = CRYPTO_OP_ENCRYPT |
356 CRYPTO_OP_COMPUTE_DIGEST;
357 } else {
358 crp->crp_op = CRYPTO_OP_DECRYPT |
359 CRYPTO_OP_VERIFY_DIGEST;
360 }
361 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
362 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));
363
364 crp->crp_aad_start = 0;
365 crp->crp_aad_length = auth_len;
366 crp->crp_payload_start = auth_len;
367 crp->crp_payload_length = datalen;
368 crp->crp_digest_start = auth_len + datalen;
369
370 memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN);
371 error = zfs_crypto_dispatch(session, crp);
372 crypto_freereq(crp);
373 out:
374 #ifdef FCRYPTO_DEBUG
375 if (error)
376 printf("%s: returning error %d\n", __FUNCTION__, error);
377 #endif
378 if (input_sessionp == NULL) {
379 freebsd_crypt_freesession(session);
380 kmem_free(session, sizeof (*session));
381 }
382 return (error);
383 }
384