xref: /freebsd/sys/contrib/openzfs/module/os/linux/zfs/zio_crypt.c (revision 19261079b74319502c6ffa1249920079f0f69a72)
1 /*
2  * CDDL HEADER START
3  *
4  * This file and its contents are supplied under the terms of the
5  * Common Development and Distribution License ("CDDL"), version 1.0.
6  * You may only use this file in accordance with the terms of version
7  * 1.0 of the CDDL.
8  *
9  * A full copy of the text of the CDDL should have accompanied this
10  * source.  A copy of the CDDL is also available via the Internet at
11  * http://www.illumos.org/license/CDDL.
12  *
13  * CDDL HEADER END
14  */
15 
16 /*
17  * Copyright (c) 2017, Datto, Inc. All rights reserved.
18  */
19 
20 #include <sys/zio_crypt.h>
21 #include <sys/dmu.h>
22 #include <sys/dmu_objset.h>
23 #include <sys/dnode.h>
24 #include <sys/fs/zfs.h>
25 #include <sys/zio.h>
26 #include <sys/zil.h>
27 #include <sys/sha2.h>
28 #include <sys/hkdf.h>
29 #include <sys/qat.h>
30 
31 /*
32  * This file is responsible for handling all of the details of generating
33  * encryption parameters and performing encryption and authentication.
34  *
35  * BLOCK ENCRYPTION PARAMETERS:
36  * Encryption /Authentication Algorithm Suite (crypt):
37  * The encryption algorithm, mode, and key length we are going to use. We
38  * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
39  * keys. All authentication is currently done with SHA512-HMAC.
40  *
41  * Plaintext:
42  * The unencrypted data that we want to encrypt.
43  *
44  * Initialization Vector (IV):
45  * An initialization vector for the encryption algorithms. This is used to
46  * "tweak" the encryption algorithms so that two blocks of the same data are
47  * encrypted into different ciphertext outputs, thus obfuscating block patterns.
48  * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
49  * never reused with the same encryption key. This value is stored unencrypted
50  * and must simply be provided to the decryption function. We use a 96 bit IV
51  * (as recommended by NIST) for all block encryption. For non-dedup blocks we
52  * derive the IV randomly. The first 64 bits of the IV are stored in the second
53  * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
54  * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
55  * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
56  * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
57  * level 0 blocks is the number of allocated dnodes in that block. The on-disk
58  * format supports at most 2^15 slots per L0 dnode block, because the maximum
59  * block size is 16MB (2^24). In either case, for level 0 blocks this number
60  * will still be smaller than UINT32_MAX so it is safe to store the IV in the
61  * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
62  * for the dnode code.
63  *
64  * Master key:
65  * This is the most important secret data of an encrypted dataset. It is used
66  * along with the salt to generate that actual encryption keys via HKDF. We
67  * do not use the master key to directly encrypt any data because there are
68  * theoretical limits on how much data can actually be safely encrypted with
69  * any encryption mode. The master key is stored encrypted on disk with the
70  * user's wrapping key. Its length is determined by the encryption algorithm.
71  * For details on how this is stored see the block comment in dsl_crypt.c
72  *
73  * Salt:
74  * Used as an input to the HKDF function, along with the master key. We use a
75  * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
76  * can be used for encrypting many blocks, so we cache the current salt and the
77  * associated derived key in zio_crypt_t so we do not need to derive it again
78  * needlessly.
79  *
80  * Encryption Key:
81  * A secret binary key, generated from an HKDF function used to encrypt and
82  * decrypt data.
83  *
84  * Message Authentication Code (MAC)
85  * The MAC is an output of authenticated encryption modes such as AES-GCM and
86  * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
87  * data on disk and return garbage to the application. Effectively, it is a
88  * checksum that can not be reproduced by an attacker. We store the MAC in the
89  * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
90  * regular checksum of the ciphertext which can be used for scrubbing.
91  *
92  * OBJECT AUTHENTICATION:
93  * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
94  * they contain some info that always needs to be readable. To prevent this
95  * data from being altered, we authenticate this data using SHA512-HMAC. This
96  * will produce a MAC (similar to the one produced via encryption) which can
97  * be used to verify the object was not modified. HMACs do not require key
98  * rotation or IVs, so we can keep up to the full 3 copies of authenticated
99  * data.
100  *
101  * ZIL ENCRYPTION:
102  * ZIL blocks have their bp written to disk ahead of the associated data, so we
103  * cannot store the MAC there as we normally do. For these blocks the MAC is
104  * stored in the embedded checksum within the zil_chain_t header. The salt and
105  * IV are generated for the block on bp allocation instead of at encryption
106  * time. In addition, ZIL blocks have some pieces that must be left in plaintext
107  * for claiming even though all of the sensitive user data still needs to be
108  * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
109  * pieces of the block need to be encrypted. All data that is not encrypted is
110  * authenticated using the AAD mechanisms that the supported encryption modes
111  * provide for. In order to preserve the semantics of the ZIL for encrypted
112  * datasets, the ZIL is not protected at the objset level as described below.
113  *
114  * DNODE ENCRYPTION:
115  * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
116  * in plaintext for scrubbing and claiming, but the bonus buffers might contain
117  * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
118  * which pieces of the block need to be encrypted. For more details about
119  * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
120  *
121  * OBJECT SET AUTHENTICATION:
122  * Up to this point, everything we have encrypted and authenticated has been
123  * at level 0 (or -2 for the ZIL). If we did not do any further work the
124  * on-disk format would be susceptible to attacks that deleted or rearranged
125  * the order of level 0 blocks. Ideally, the cleanest solution would be to
126  * maintain a tree of authentication MACs going up the bp tree. However, this
127  * presents a problem for raw sends. Send files do not send information about
128  * indirect blocks so there would be no convenient way to transfer the MACs and
129  * they cannot be recalculated on the receive side without the master key which
130  * would defeat one of the purposes of raw sends in the first place. Instead,
131  * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
132  * from the level below. We also include some portable fields from blk_prop such
133  * as the lsize and compression algorithm to prevent the data from being
134  * misinterpreted.
135  *
136  * At the objset level, we maintain 2 separate 256 bit MACs in the
137  * objset_phys_t. The first one is "portable" and is the logical root of the
138  * MAC tree maintained in the metadnode's bps. The second, is "local" and is
139  * used as the root MAC for the user accounting objects, which are also not
140  * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
141  * of the send file. The useraccounting code ensures that the useraccounting
142  * info is not present upon a receive, so the local MAC can simply be cleared
143  * out at that time. For more info about objset_phys_t authentication, see
144  * zio_crypt_do_objset_hmacs().
145  *
146  * CONSIDERATIONS FOR DEDUP:
147  * In order for dedup to work, blocks that we want to dedup with one another
148  * need to use the same IV and encryption key, so that they will have the same
149  * ciphertext. Normally, one should never reuse an IV with the same encryption
150  * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
151  * blocks. In this case, however, since we are using the same plaintext as
152  * well all that we end up with is a duplicate of the original ciphertext we
153  * already had. As a result, an attacker with read access to the raw disk will
154  * be able to tell which blocks are the same but this information is given away
155  * by dedup anyway. In order to get the same IVs and encryption keys for
156  * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC
157  * here so that a reproducible checksum of the plaintext is never available to
158  * the attacker. The HMAC key is kept alongside the master key, encrypted on
159  * disk. The first 64 bits of the HMAC are used in place of the random salt, and
160  * the next 96 bits are used as the IV. As a result of this mechanism, dedup
161  * will only work within a clone family since encrypted dedup requires use of
162  * the same master and HMAC keys.
163  */
164 
165 /*
166  * After encrypting many blocks with the same key we may start to run up
167  * against the theoretical limits of how much data can securely be encrypted
168  * with a single key using the supported encryption modes. The most obvious
169  * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
170  * the more IVs we generate (which both GCM and CCM modes strictly forbid).
171  * This risk actually grows surprisingly quickly over time according to the
172  * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
173  * generated n IVs with a cryptographically secure RNG, the approximate
174  * probability p(n) of a collision is given as:
175  *
176  * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
177  *
178  * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
179  *
180  * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
181  * we must not write more than 398,065,730 blocks with the same encryption key.
182  * Therefore, we rotate our keys after 400,000,000 blocks have been written by
183  * generating a new random 64 bit salt for our HKDF encryption key generation
184  * function.
185  */
186 #define	ZFS_KEY_MAX_SALT_USES_DEFAULT	400000000
187 #define	ZFS_CURRENT_MAX_SALT_USES	\
188 	(MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
189 unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT;
190 
191 typedef struct blkptr_auth_buf {
192 	uint64_t bab_prop;			/* blk_prop - portable mask */
193 	uint8_t bab_mac[ZIO_DATA_MAC_LEN];	/* MAC from blk_cksum */
194 	uint64_t bab_pad;			/* reserved for future use */
195 } blkptr_auth_buf_t;
196 
197 zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
198 	{"",			ZC_TYPE_NONE,	0,	"inherit"},
199 	{"",			ZC_TYPE_NONE,	0,	"on"},
200 	{"",			ZC_TYPE_NONE,	0,	"off"},
201 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	16,	"aes-128-ccm"},
202 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	24,	"aes-192-ccm"},
203 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	32,	"aes-256-ccm"},
204 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	16,	"aes-128-gcm"},
205 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	24,	"aes-192-gcm"},
206 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	32,	"aes-256-gcm"}
207 };
208 
209 void
210 zio_crypt_key_destroy(zio_crypt_key_t *key)
211 {
212 	rw_destroy(&key->zk_salt_lock);
213 
214 	/* free crypto templates */
215 	crypto_destroy_ctx_template(key->zk_current_tmpl);
216 	crypto_destroy_ctx_template(key->zk_hmac_tmpl);
217 
218 	/* zero out sensitive data */
219 	bzero(key, sizeof (zio_crypt_key_t));
220 }
221 
222 int
223 zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
224 {
225 	int ret;
226 	crypto_mechanism_t mech;
227 	uint_t keydata_len;
228 
229 	ASSERT(key != NULL);
230 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
231 
232 	keydata_len = zio_crypt_table[crypt].ci_keylen;
233 	bzero(key, sizeof (zio_crypt_key_t));
234 
235 	/* fill keydata buffers and salt with random data */
236 	ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t));
237 	if (ret != 0)
238 		goto error;
239 
240 	ret = random_get_bytes(key->zk_master_keydata, keydata_len);
241 	if (ret != 0)
242 		goto error;
243 
244 	ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN);
245 	if (ret != 0)
246 		goto error;
247 
248 	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
249 	if (ret != 0)
250 		goto error;
251 
252 	/* derive the current key from the master key */
253 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
254 	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
255 	    keydata_len);
256 	if (ret != 0)
257 		goto error;
258 
259 	/* initialize keys for the ICP */
260 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
261 	key->zk_current_key.ck_data = key->zk_current_keydata;
262 	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
263 
264 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
265 	key->zk_hmac_key.ck_data = &key->zk_hmac_key;
266 	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
267 
268 	/*
269 	 * Initialize the crypto templates. It's ok if this fails because
270 	 * this is just an optimization.
271 	 */
272 	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
273 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
274 	    &key->zk_current_tmpl, KM_SLEEP);
275 	if (ret != CRYPTO_SUCCESS)
276 		key->zk_current_tmpl = NULL;
277 
278 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
279 	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
280 	    &key->zk_hmac_tmpl, KM_SLEEP);
281 	if (ret != CRYPTO_SUCCESS)
282 		key->zk_hmac_tmpl = NULL;
283 
284 	key->zk_crypt = crypt;
285 	key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION;
286 	key->zk_salt_count = 0;
287 	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
288 
289 	return (0);
290 
291 error:
292 	zio_crypt_key_destroy(key);
293 	return (ret);
294 }
295 
296 static int
297 zio_crypt_key_change_salt(zio_crypt_key_t *key)
298 {
299 	int ret = 0;
300 	uint8_t salt[ZIO_DATA_SALT_LEN];
301 	crypto_mechanism_t mech;
302 	uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen;
303 
304 	/* generate a new salt */
305 	ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN);
306 	if (ret != 0)
307 		goto error;
308 
309 	rw_enter(&key->zk_salt_lock, RW_WRITER);
310 
311 	/* someone beat us to the salt rotation, just unlock and return */
312 	if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES)
313 		goto out_unlock;
314 
315 	/* derive the current key from the master key and the new salt */
316 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
317 	    salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len);
318 	if (ret != 0)
319 		goto out_unlock;
320 
321 	/* assign the salt and reset the usage count */
322 	bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN);
323 	key->zk_salt_count = 0;
324 
325 	/* destroy the old context template and create the new one */
326 	crypto_destroy_ctx_template(key->zk_current_tmpl);
327 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
328 	    &key->zk_current_tmpl, KM_SLEEP);
329 	if (ret != CRYPTO_SUCCESS)
330 		key->zk_current_tmpl = NULL;
331 
332 	rw_exit(&key->zk_salt_lock);
333 
334 	return (0);
335 
336 out_unlock:
337 	rw_exit(&key->zk_salt_lock);
338 error:
339 	return (ret);
340 }
341 
342 /* See comment above zfs_key_max_salt_uses definition for details */
343 int
344 zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt)
345 {
346 	int ret;
347 	boolean_t salt_change;
348 
349 	rw_enter(&key->zk_salt_lock, RW_READER);
350 
351 	bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN);
352 	salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >=
353 	    ZFS_CURRENT_MAX_SALT_USES);
354 
355 	rw_exit(&key->zk_salt_lock);
356 
357 	if (salt_change) {
358 		ret = zio_crypt_key_change_salt(key);
359 		if (ret != 0)
360 			goto error;
361 	}
362 
363 	return (0);
364 
365 error:
366 	return (ret);
367 }
368 
369 /*
370  * This function handles all encryption and decryption in zfs. When
371  * encrypting it expects puio to reference the plaintext and cuio to
372  * reference the ciphertext. cuio must have enough space for the
373  * ciphertext + room for a MAC. datalen should be the length of the
374  * plaintext / ciphertext alone.
375  */
376 static int
377 zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
378     crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
379     zfs_uio_t *puio, zfs_uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
380 {
381 	int ret;
382 	crypto_data_t plaindata, cipherdata;
383 	CK_AES_CCM_PARAMS ccmp;
384 	CK_AES_GCM_PARAMS gcmp;
385 	crypto_mechanism_t mech;
386 	zio_crypt_info_t crypt_info;
387 	uint_t plain_full_len, maclen;
388 
389 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
390 	ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW);
391 
392 	/* lookup the encryption info */
393 	crypt_info = zio_crypt_table[crypt];
394 
395 	/* the mac will always be the last iovec_t in the cipher uio */
396 	maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len;
397 
398 	ASSERT(maclen <= ZIO_DATA_MAC_LEN);
399 
400 	/* setup encryption mechanism (same as crypt) */
401 	mech.cm_type = crypto_mech2id(crypt_info.ci_mechname);
402 
403 	/*
404 	 * Strangely, the ICP requires that plain_full_len must include
405 	 * the MAC length when decrypting, even though the UIO does not
406 	 * need to have the extra space allocated.
407 	 */
408 	if (encrypt) {
409 		plain_full_len = datalen;
410 	} else {
411 		plain_full_len = datalen + maclen;
412 	}
413 
414 	/*
415 	 * setup encryption params (currently only AES CCM and AES GCM
416 	 * are supported)
417 	 */
418 	if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) {
419 		ccmp.ulNonceSize = ZIO_DATA_IV_LEN;
420 		ccmp.ulAuthDataSize = auth_len;
421 		ccmp.authData = authbuf;
422 		ccmp.ulMACSize = maclen;
423 		ccmp.nonce = ivbuf;
424 		ccmp.ulDataSize = plain_full_len;
425 
426 		mech.cm_param = (char *)(&ccmp);
427 		mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
428 	} else {
429 		gcmp.ulIvLen = ZIO_DATA_IV_LEN;
430 		gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
431 		gcmp.ulAADLen = auth_len;
432 		gcmp.pAAD = authbuf;
433 		gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
434 		gcmp.pIv = ivbuf;
435 
436 		mech.cm_param = (char *)(&gcmp);
437 		mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
438 	}
439 
440 	/* populate the cipher and plain data structs. */
441 	plaindata.cd_format = CRYPTO_DATA_UIO;
442 	plaindata.cd_offset = 0;
443 	plaindata.cd_uio = puio;
444 	plaindata.cd_miscdata = NULL;
445 	plaindata.cd_length = plain_full_len;
446 
447 	cipherdata.cd_format = CRYPTO_DATA_UIO;
448 	cipherdata.cd_offset = 0;
449 	cipherdata.cd_uio = cuio;
450 	cipherdata.cd_miscdata = NULL;
451 	cipherdata.cd_length = datalen + maclen;
452 
453 	/* perform the actual encryption */
454 	if (encrypt) {
455 		ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata,
456 		    NULL);
457 		if (ret != CRYPTO_SUCCESS) {
458 			ret = SET_ERROR(EIO);
459 			goto error;
460 		}
461 	} else {
462 		ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata,
463 		    NULL);
464 		if (ret != CRYPTO_SUCCESS) {
465 			ASSERT3U(ret, ==, CRYPTO_INVALID_MAC);
466 			ret = SET_ERROR(ECKSUM);
467 			goto error;
468 		}
469 	}
470 
471 	return (0);
472 
473 error:
474 	return (ret);
475 }
476 
477 int
478 zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
479     uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
480 {
481 	int ret;
482 	zfs_uio_t puio, cuio;
483 	uint64_t aad[3];
484 	iovec_t plain_iovecs[2], cipher_iovecs[3];
485 	uint64_t crypt = key->zk_crypt;
486 	uint_t enc_len, keydata_len, aad_len;
487 
488 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
489 	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
490 
491 	keydata_len = zio_crypt_table[crypt].ci_keylen;
492 
493 	/* generate iv for wrapping the master and hmac key */
494 	ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN);
495 	if (ret != 0)
496 		goto error;
497 
498 	/* initialize zfs_uio_ts */
499 	plain_iovecs[0].iov_base = key->zk_master_keydata;
500 	plain_iovecs[0].iov_len = keydata_len;
501 	plain_iovecs[1].iov_base = key->zk_hmac_keydata;
502 	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
503 
504 	cipher_iovecs[0].iov_base = keydata_out;
505 	cipher_iovecs[0].iov_len = keydata_len;
506 	cipher_iovecs[1].iov_base = hmac_keydata_out;
507 	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
508 	cipher_iovecs[2].iov_base = mac;
509 	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
510 
511 	/*
512 	 * Although we don't support writing to the old format, we do
513 	 * support rewrapping the key so that the user can move and
514 	 * quarantine datasets on the old format.
515 	 */
516 	if (key->zk_version == 0) {
517 		aad_len = sizeof (uint64_t);
518 		aad[0] = LE_64(key->zk_guid);
519 	} else {
520 		ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
521 		aad_len = sizeof (uint64_t) * 3;
522 		aad[0] = LE_64(key->zk_guid);
523 		aad[1] = LE_64(crypt);
524 		aad[2] = LE_64(key->zk_version);
525 	}
526 
527 	enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN;
528 	puio.uio_iov = plain_iovecs;
529 	puio.uio_iovcnt = 2;
530 	puio.uio_segflg = UIO_SYSSPACE;
531 	cuio.uio_iov = cipher_iovecs;
532 	cuio.uio_iovcnt = 3;
533 	cuio.uio_segflg = UIO_SYSSPACE;
534 
535 	/* encrypt the keys and store the resulting ciphertext and mac */
536 	ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
537 	    &puio, &cuio, (uint8_t *)aad, aad_len);
538 	if (ret != 0)
539 		goto error;
540 
541 	return (0);
542 
543 error:
544 	return (ret);
545 }
546 
547 int
548 zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
549     uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
550     uint8_t *mac, zio_crypt_key_t *key)
551 {
552 	crypto_mechanism_t mech;
553 	zfs_uio_t puio, cuio;
554 	uint64_t aad[3];
555 	iovec_t plain_iovecs[2], cipher_iovecs[3];
556 	uint_t enc_len, keydata_len, aad_len;
557 	int ret;
558 
559 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
560 	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
561 
562 	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
563 
564 	keydata_len = zio_crypt_table[crypt].ci_keylen;
565 
566 	/* initialize zfs_uio_ts */
567 	plain_iovecs[0].iov_base = key->zk_master_keydata;
568 	plain_iovecs[0].iov_len = keydata_len;
569 	plain_iovecs[1].iov_base = key->zk_hmac_keydata;
570 	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
571 
572 	cipher_iovecs[0].iov_base = keydata;
573 	cipher_iovecs[0].iov_len = keydata_len;
574 	cipher_iovecs[1].iov_base = hmac_keydata;
575 	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
576 	cipher_iovecs[2].iov_base = mac;
577 	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
578 
579 	if (version == 0) {
580 		aad_len = sizeof (uint64_t);
581 		aad[0] = LE_64(guid);
582 	} else {
583 		ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
584 		aad_len = sizeof (uint64_t) * 3;
585 		aad[0] = LE_64(guid);
586 		aad[1] = LE_64(crypt);
587 		aad[2] = LE_64(version);
588 	}
589 
590 	enc_len = keydata_len + SHA512_HMAC_KEYLEN;
591 	puio.uio_iov = plain_iovecs;
592 	puio.uio_segflg = UIO_SYSSPACE;
593 	puio.uio_iovcnt = 2;
594 	cuio.uio_iov = cipher_iovecs;
595 	cuio.uio_iovcnt = 3;
596 	cuio.uio_segflg = UIO_SYSSPACE;
597 
598 	/* decrypt the keys and store the result in the output buffers */
599 	ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
600 	    &puio, &cuio, (uint8_t *)aad, aad_len);
601 	if (ret != 0)
602 		goto error;
603 
604 	/* generate a fresh salt */
605 	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
606 	if (ret != 0)
607 		goto error;
608 
609 	/* derive the current key from the master key */
610 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
611 	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
612 	    keydata_len);
613 	if (ret != 0)
614 		goto error;
615 
616 	/* initialize keys for ICP */
617 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
618 	key->zk_current_key.ck_data = key->zk_current_keydata;
619 	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
620 
621 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
622 	key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
623 	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
624 
625 	/*
626 	 * Initialize the crypto templates. It's ok if this fails because
627 	 * this is just an optimization.
628 	 */
629 	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
630 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
631 	    &key->zk_current_tmpl, KM_SLEEP);
632 	if (ret != CRYPTO_SUCCESS)
633 		key->zk_current_tmpl = NULL;
634 
635 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
636 	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
637 	    &key->zk_hmac_tmpl, KM_SLEEP);
638 	if (ret != CRYPTO_SUCCESS)
639 		key->zk_hmac_tmpl = NULL;
640 
641 	key->zk_crypt = crypt;
642 	key->zk_version = version;
643 	key->zk_guid = guid;
644 	key->zk_salt_count = 0;
645 
646 	return (0);
647 
648 error:
649 	zio_crypt_key_destroy(key);
650 	return (ret);
651 }
652 
653 int
654 zio_crypt_generate_iv(uint8_t *ivbuf)
655 {
656 	int ret;
657 
658 	/* randomly generate the IV */
659 	ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN);
660 	if (ret != 0)
661 		goto error;
662 
663 	return (0);
664 
665 error:
666 	bzero(ivbuf, ZIO_DATA_IV_LEN);
667 	return (ret);
668 }
669 
670 int
671 zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
672     uint8_t *digestbuf, uint_t digestlen)
673 {
674 	int ret;
675 	crypto_mechanism_t mech;
676 	crypto_data_t in_data, digest_data;
677 	uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
678 
679 	ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
680 
681 	/* initialize sha512-hmac mechanism and crypto data */
682 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
683 	mech.cm_param = NULL;
684 	mech.cm_param_len = 0;
685 
686 	/* initialize the crypto data */
687 	in_data.cd_format = CRYPTO_DATA_RAW;
688 	in_data.cd_offset = 0;
689 	in_data.cd_length = datalen;
690 	in_data.cd_raw.iov_base = (char *)data;
691 	in_data.cd_raw.iov_len = in_data.cd_length;
692 
693 	digest_data.cd_format = CRYPTO_DATA_RAW;
694 	digest_data.cd_offset = 0;
695 	digest_data.cd_length = SHA512_DIGEST_LENGTH;
696 	digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
697 	digest_data.cd_raw.iov_len = digest_data.cd_length;
698 
699 	/* generate the hmac */
700 	ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl,
701 	    &digest_data, NULL);
702 	if (ret != CRYPTO_SUCCESS) {
703 		ret = SET_ERROR(EIO);
704 		goto error;
705 	}
706 
707 	bcopy(raw_digestbuf, digestbuf, digestlen);
708 
709 	return (0);
710 
711 error:
712 	bzero(digestbuf, digestlen);
713 	return (ret);
714 }
715 
716 int
717 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
718     uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
719 {
720 	int ret;
721 	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
722 
723 	ret = zio_crypt_do_hmac(key, data, datalen,
724 	    digestbuf, SHA512_DIGEST_LENGTH);
725 	if (ret != 0)
726 		return (ret);
727 
728 	bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN);
729 	bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN);
730 
731 	return (0);
732 }
733 
734 /*
735  * The following functions are used to encode and decode encryption parameters
736  * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
737  * byte strings, which normally means that these strings would not need to deal
738  * with byteswapping at all. However, both blkptr_t and zil_header_t may be
739  * byteswapped by lower layers and so we must "undo" that byteswap here upon
740  * decoding and encoding in a non-native byteorder. These functions require
741  * that the byteorder bit is correct before being called.
742  */
743 void
744 zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv)
745 {
746 	uint64_t val64;
747 	uint32_t val32;
748 
749 	ASSERT(BP_IS_ENCRYPTED(bp));
750 
751 	if (!BP_SHOULD_BYTESWAP(bp)) {
752 		bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t));
753 		bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t));
754 		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
755 		BP_SET_IV2(bp, val32);
756 	} else {
757 		bcopy(salt, &val64, sizeof (uint64_t));
758 		bp->blk_dva[2].dva_word[0] = BSWAP_64(val64);
759 
760 		bcopy(iv, &val64, sizeof (uint64_t));
761 		bp->blk_dva[2].dva_word[1] = BSWAP_64(val64);
762 
763 		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
764 		BP_SET_IV2(bp, BSWAP_32(val32));
765 	}
766 }
767 
768 void
769 zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv)
770 {
771 	uint64_t val64;
772 	uint32_t val32;
773 
774 	ASSERT(BP_IS_PROTECTED(bp));
775 
776 	/* for convenience, so callers don't need to check */
777 	if (BP_IS_AUTHENTICATED(bp)) {
778 		bzero(salt, ZIO_DATA_SALT_LEN);
779 		bzero(iv, ZIO_DATA_IV_LEN);
780 		return;
781 	}
782 
783 	if (!BP_SHOULD_BYTESWAP(bp)) {
784 		bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t));
785 		bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t));
786 
787 		val32 = (uint32_t)BP_GET_IV2(bp);
788 		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
789 	} else {
790 		val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]);
791 		bcopy(&val64, salt, sizeof (uint64_t));
792 
793 		val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]);
794 		bcopy(&val64, iv, sizeof (uint64_t));
795 
796 		val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp));
797 		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
798 	}
799 }
800 
801 void
802 zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac)
803 {
804 	uint64_t val64;
805 
806 	ASSERT(BP_USES_CRYPT(bp));
807 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET);
808 
809 	if (!BP_SHOULD_BYTESWAP(bp)) {
810 		bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t));
811 		bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3],
812 		    sizeof (uint64_t));
813 	} else {
814 		bcopy(mac, &val64, sizeof (uint64_t));
815 		bp->blk_cksum.zc_word[2] = BSWAP_64(val64);
816 
817 		bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t));
818 		bp->blk_cksum.zc_word[3] = BSWAP_64(val64);
819 	}
820 }
821 
822 void
823 zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac)
824 {
825 	uint64_t val64;
826 
827 	ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp));
828 
829 	/* for convenience, so callers don't need to check */
830 	if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
831 		bzero(mac, ZIO_DATA_MAC_LEN);
832 		return;
833 	}
834 
835 	if (!BP_SHOULD_BYTESWAP(bp)) {
836 		bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t));
837 		bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t),
838 		    sizeof (uint64_t));
839 	} else {
840 		val64 = BSWAP_64(bp->blk_cksum.zc_word[2]);
841 		bcopy(&val64, mac, sizeof (uint64_t));
842 
843 		val64 = BSWAP_64(bp->blk_cksum.zc_word[3]);
844 		bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t));
845 	}
846 }
847 
848 void
849 zio_crypt_encode_mac_zil(void *data, uint8_t *mac)
850 {
851 	zil_chain_t *zilc = data;
852 
853 	bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t));
854 	bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3],
855 	    sizeof (uint64_t));
856 }
857 
858 void
859 zio_crypt_decode_mac_zil(const void *data, uint8_t *mac)
860 {
861 	/*
862 	 * The ZIL MAC is embedded in the block it protects, which will
863 	 * not have been byteswapped by the time this function has been called.
864 	 * As a result, we don't need to worry about byteswapping the MAC.
865 	 */
866 	const zil_chain_t *zilc = data;
867 
868 	bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t));
869 	bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t),
870 	    sizeof (uint64_t));
871 }
872 
873 /*
874  * This routine takes a block of dnodes (src_abd) and copies only the bonus
875  * buffers to the same offsets in the dst buffer. datalen should be the size
876  * of both the src_abd and the dst buffer (not just the length of the bonus
877  * buffers).
878  */
879 void
880 zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
881 {
882 	uint_t i, max_dnp = datalen >> DNODE_SHIFT;
883 	uint8_t *src;
884 	dnode_phys_t *dnp, *sdnp, *ddnp;
885 
886 	src = abd_borrow_buf_copy(src_abd, datalen);
887 
888 	sdnp = (dnode_phys_t *)src;
889 	ddnp = (dnode_phys_t *)dst;
890 
891 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
892 		dnp = &sdnp[i];
893 		if (dnp->dn_type != DMU_OT_NONE &&
894 		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
895 		    dnp->dn_bonuslen != 0) {
896 			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]),
897 			    DN_MAX_BONUS_LEN(dnp));
898 		}
899 	}
900 
901 	abd_return_buf(src_abd, src, datalen);
902 }
903 
904 /*
905  * This function decides what fields from blk_prop are included in
906  * the on-disk various MAC algorithms.
907  */
908 static void
909 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version)
910 {
911 	/*
912 	 * Version 0 did not properly zero out all non-portable fields
913 	 * as it should have done. We maintain this code so that we can
914 	 * do read-only imports of pools on this version.
915 	 */
916 	if (version == 0) {
917 		BP_SET_DEDUP(bp, 0);
918 		BP_SET_CHECKSUM(bp, 0);
919 		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
920 		return;
921 	}
922 
923 	ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
924 
925 	/*
926 	 * The hole_birth feature might set these fields even if this bp
927 	 * is a hole. We zero them out here to guarantee that raw sends
928 	 * will function with or without the feature.
929 	 */
930 	if (BP_IS_HOLE(bp)) {
931 		bp->blk_prop = 0ULL;
932 		return;
933 	}
934 
935 	/*
936 	 * At L0 we want to verify these fields to ensure that data blocks
937 	 * can not be reinterpreted. For instance, we do not want an attacker
938 	 * to trick us into returning raw lz4 compressed data to the user
939 	 * by modifying the compression bits. At higher levels, we cannot
940 	 * enforce this policy since raw sends do not convey any information
941 	 * about indirect blocks, so these values might be different on the
942 	 * receive side. Fortunately, this does not open any new attack
943 	 * vectors, since any alterations that can be made to a higher level
944 	 * bp must still verify the correct order of the layer below it.
945 	 */
946 	if (BP_GET_LEVEL(bp) != 0) {
947 		BP_SET_BYTEORDER(bp, 0);
948 		BP_SET_COMPRESS(bp, 0);
949 
950 		/*
951 		 * psize cannot be set to zero or it will trigger
952 		 * asserts, but the value doesn't really matter as
953 		 * long as it is constant.
954 		 */
955 		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
956 	}
957 
958 	BP_SET_DEDUP(bp, 0);
959 	BP_SET_CHECKSUM(bp, 0);
960 }
961 
962 static void
963 zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp,
964     blkptr_auth_buf_t *bab, uint_t *bab_len)
965 {
966 	blkptr_t tmpbp = *bp;
967 
968 	if (should_bswap)
969 		byteswap_uint64_array(&tmpbp, sizeof (blkptr_t));
970 
971 	ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp));
972 	ASSERT0(BP_IS_EMBEDDED(&tmpbp));
973 
974 	zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac);
975 
976 	/*
977 	 * We always MAC blk_prop in LE to ensure portability. This
978 	 * must be done after decoding the mac, since the endianness
979 	 * will get zero'd out here.
980 	 */
981 	zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version);
982 	bab->bab_prop = LE_64(tmpbp.blk_prop);
983 	bab->bab_pad = 0ULL;
984 
985 	/* version 0 did not include the padding */
986 	*bab_len = sizeof (blkptr_auth_buf_t);
987 	if (version == 0)
988 		*bab_len -= sizeof (uint64_t);
989 }
990 
991 static int
992 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version,
993     boolean_t should_bswap, blkptr_t *bp)
994 {
995 	int ret;
996 	uint_t bab_len;
997 	blkptr_auth_buf_t bab;
998 	crypto_data_t cd;
999 
1000 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1001 	cd.cd_format = CRYPTO_DATA_RAW;
1002 	cd.cd_offset = 0;
1003 	cd.cd_length = bab_len;
1004 	cd.cd_raw.iov_base = (char *)&bab;
1005 	cd.cd_raw.iov_len = cd.cd_length;
1006 
1007 	ret = crypto_mac_update(ctx, &cd, NULL);
1008 	if (ret != CRYPTO_SUCCESS) {
1009 		ret = SET_ERROR(EIO);
1010 		goto error;
1011 	}
1012 
1013 	return (0);
1014 
1015 error:
1016 	return (ret);
1017 }
1018 
1019 static void
1020 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version,
1021     boolean_t should_bswap, blkptr_t *bp)
1022 {
1023 	uint_t bab_len;
1024 	blkptr_auth_buf_t bab;
1025 
1026 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1027 	SHA2Update(ctx, &bab, bab_len);
1028 }
1029 
1030 static void
1031 zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version,
1032     boolean_t should_bswap, blkptr_t *bp)
1033 {
1034 	uint_t bab_len;
1035 	blkptr_auth_buf_t bab;
1036 
1037 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1038 	bcopy(&bab, *aadp, bab_len);
1039 	*aadp += bab_len;
1040 	*aad_len += bab_len;
1041 }
1042 
1043 static int
1044 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version,
1045     boolean_t should_bswap, dnode_phys_t *dnp)
1046 {
1047 	int ret, i;
1048 	dnode_phys_t *adnp, tmp_dncore;
1049 	size_t dn_core_size = offsetof(dnode_phys_t, dn_blkptr);
1050 	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1051 	crypto_data_t cd;
1052 
1053 	cd.cd_format = CRYPTO_DATA_RAW;
1054 	cd.cd_offset = 0;
1055 
1056 	/*
1057 	 * Authenticate the core dnode (masking out non-portable bits).
1058 	 * We only copy the first 64 bytes we operate on to avoid the overhead
1059 	 * of copying 512-64 unneeded bytes. The compiler seems to be fine
1060 	 * with that.
1061 	 */
1062 	bcopy(dnp, &tmp_dncore, dn_core_size);
1063 	adnp = &tmp_dncore;
1064 
1065 	if (le_bswap) {
1066 		adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec);
1067 		adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen);
1068 		adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid);
1069 		adnp->dn_used = BSWAP_64(adnp->dn_used);
1070 	}
1071 	adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1072 	adnp->dn_used = 0;
1073 
1074 	cd.cd_length = dn_core_size;
1075 	cd.cd_raw.iov_base = (char *)adnp;
1076 	cd.cd_raw.iov_len = cd.cd_length;
1077 
1078 	ret = crypto_mac_update(ctx, &cd, NULL);
1079 	if (ret != CRYPTO_SUCCESS) {
1080 		ret = SET_ERROR(EIO);
1081 		goto error;
1082 	}
1083 
1084 	for (i = 0; i < dnp->dn_nblkptr; i++) {
1085 		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1086 		    should_bswap, &dnp->dn_blkptr[i]);
1087 		if (ret != 0)
1088 			goto error;
1089 	}
1090 
1091 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1092 		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1093 		    should_bswap, DN_SPILL_BLKPTR(dnp));
1094 		if (ret != 0)
1095 			goto error;
1096 	}
1097 
1098 	return (0);
1099 
1100 error:
1101 	return (ret);
1102 }
1103 
1104 /*
1105  * objset_phys_t blocks introduce a number of exceptions to the normal
1106  * authentication process. objset_phys_t's contain 2 separate HMACS for
1107  * protecting the integrity of their data. The portable_mac protects the
1108  * metadnode. This MAC can be sent with a raw send and protects against
1109  * reordering of data within the metadnode. The local_mac protects the user
1110  * accounting objects which are not sent from one system to another.
1111  *
1112  * In addition, objset blocks are the only blocks that can be modified and
1113  * written to disk without the key loaded under certain circumstances. During
1114  * zil_claim() we need to be able to update the zil_header_t to complete
1115  * claiming log blocks and during raw receives we need to write out the
1116  * portable_mac from the send file. Both of these actions are possible
1117  * because these fields are not protected by either MAC so neither one will
1118  * need to modify the MACs without the key. However, when the modified blocks
1119  * are written out they will be byteswapped into the host machine's native
1120  * endianness which will modify fields protected by the MAC. As a result, MAC
1121  * calculation for objset blocks works slightly differently from other block
1122  * types. Where other block types MAC the data in whatever endianness is
1123  * written to disk, objset blocks always MAC little endian version of their
1124  * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1125  * and le_bswap indicates whether a byteswap is needed to get this block
1126  * into little endian format.
1127  */
1128 int
1129 zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
1130     boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac)
1131 {
1132 	int ret;
1133 	crypto_mechanism_t mech;
1134 	crypto_context_t ctx;
1135 	crypto_data_t cd;
1136 	objset_phys_t *osp = data;
1137 	uint64_t intval;
1138 	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1139 	uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
1140 	uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
1141 
1142 	/* initialize HMAC mechanism */
1143 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
1144 	mech.cm_param = NULL;
1145 	mech.cm_param_len = 0;
1146 
1147 	cd.cd_format = CRYPTO_DATA_RAW;
1148 	cd.cd_offset = 0;
1149 
1150 	/* calculate the portable MAC from the portable fields and metadnode */
1151 	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1152 	if (ret != CRYPTO_SUCCESS) {
1153 		ret = SET_ERROR(EIO);
1154 		goto error;
1155 	}
1156 
1157 	/* add in the os_type */
1158 	intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type);
1159 	cd.cd_length = sizeof (uint64_t);
1160 	cd.cd_raw.iov_base = (char *)&intval;
1161 	cd.cd_raw.iov_len = cd.cd_length;
1162 
1163 	ret = crypto_mac_update(ctx, &cd, NULL);
1164 	if (ret != CRYPTO_SUCCESS) {
1165 		ret = SET_ERROR(EIO);
1166 		goto error;
1167 	}
1168 
1169 	/* add in the portable os_flags */
1170 	intval = osp->os_flags;
1171 	if (should_bswap)
1172 		intval = BSWAP_64(intval);
1173 	intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1174 	if (!ZFS_HOST_BYTEORDER)
1175 		intval = BSWAP_64(intval);
1176 
1177 	cd.cd_length = sizeof (uint64_t);
1178 	cd.cd_raw.iov_base = (char *)&intval;
1179 	cd.cd_raw.iov_len = cd.cd_length;
1180 
1181 	ret = crypto_mac_update(ctx, &cd, NULL);
1182 	if (ret != CRYPTO_SUCCESS) {
1183 		ret = SET_ERROR(EIO);
1184 		goto error;
1185 	}
1186 
1187 	/* add in fields from the metadnode */
1188 	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1189 	    should_bswap, &osp->os_meta_dnode);
1190 	if (ret)
1191 		goto error;
1192 
1193 	/* store the final digest in a temporary buffer and copy what we need */
1194 	cd.cd_length = SHA512_DIGEST_LENGTH;
1195 	cd.cd_raw.iov_base = (char *)raw_portable_mac;
1196 	cd.cd_raw.iov_len = cd.cd_length;
1197 
1198 	ret = crypto_mac_final(ctx, &cd, NULL);
1199 	if (ret != CRYPTO_SUCCESS) {
1200 		ret = SET_ERROR(EIO);
1201 		goto error;
1202 	}
1203 
1204 	bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN);
1205 
1206 	/*
1207 	 * The local MAC protects the user, group and project accounting.
1208 	 * If these objects are not present, the local MAC is zeroed out.
1209 	 */
1210 	if ((datalen >= OBJSET_PHYS_SIZE_V3 &&
1211 	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1212 	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE &&
1213 	    osp->os_projectused_dnode.dn_type == DMU_OT_NONE) ||
1214 	    (datalen >= OBJSET_PHYS_SIZE_V2 &&
1215 	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1216 	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE) ||
1217 	    (datalen <= OBJSET_PHYS_SIZE_V1)) {
1218 		bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1219 		return (0);
1220 	}
1221 
1222 	/* calculate the local MAC from the userused and groupused dnodes */
1223 	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1224 	if (ret != CRYPTO_SUCCESS) {
1225 		ret = SET_ERROR(EIO);
1226 		goto error;
1227 	}
1228 
1229 	/* add in the non-portable os_flags */
1230 	intval = osp->os_flags;
1231 	if (should_bswap)
1232 		intval = BSWAP_64(intval);
1233 	intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1234 	if (!ZFS_HOST_BYTEORDER)
1235 		intval = BSWAP_64(intval);
1236 
1237 	cd.cd_length = sizeof (uint64_t);
1238 	cd.cd_raw.iov_base = (char *)&intval;
1239 	cd.cd_raw.iov_len = cd.cd_length;
1240 
1241 	ret = crypto_mac_update(ctx, &cd, NULL);
1242 	if (ret != CRYPTO_SUCCESS) {
1243 		ret = SET_ERROR(EIO);
1244 		goto error;
1245 	}
1246 
1247 	/* add in fields from the user accounting dnodes */
1248 	if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) {
1249 		ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1250 		    should_bswap, &osp->os_userused_dnode);
1251 		if (ret)
1252 			goto error;
1253 	}
1254 
1255 	if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) {
1256 		ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1257 		    should_bswap, &osp->os_groupused_dnode);
1258 		if (ret)
1259 			goto error;
1260 	}
1261 
1262 	if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE &&
1263 	    datalen >= OBJSET_PHYS_SIZE_V3) {
1264 		ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1265 		    should_bswap, &osp->os_projectused_dnode);
1266 		if (ret)
1267 			goto error;
1268 	}
1269 
1270 	/* store the final digest in a temporary buffer and copy what we need */
1271 	cd.cd_length = SHA512_DIGEST_LENGTH;
1272 	cd.cd_raw.iov_base = (char *)raw_local_mac;
1273 	cd.cd_raw.iov_len = cd.cd_length;
1274 
1275 	ret = crypto_mac_final(ctx, &cd, NULL);
1276 	if (ret != CRYPTO_SUCCESS) {
1277 		ret = SET_ERROR(EIO);
1278 		goto error;
1279 	}
1280 
1281 	bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN);
1282 
1283 	return (0);
1284 
1285 error:
1286 	bzero(portable_mac, ZIO_OBJSET_MAC_LEN);
1287 	bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1288 	return (ret);
1289 }
1290 
1291 static void
1292 zio_crypt_destroy_uio(zfs_uio_t *uio)
1293 {
1294 	if (uio->uio_iov)
1295 		kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t));
1296 }
1297 
1298 /*
1299  * This function parses an uncompressed indirect block and returns a checksum
1300  * of all the portable fields from all of the contained bps. The portable
1301  * fields are the MAC and all of the fields from blk_prop except for the dedup,
1302  * checksum, and psize bits. For an explanation of the purpose of this, see
1303  * the comment block on object set authentication.
1304  */
1305 static int
1306 zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf,
1307     uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum)
1308 {
1309 	blkptr_t *bp;
1310 	int i, epb = datalen >> SPA_BLKPTRSHIFT;
1311 	SHA2_CTX ctx;
1312 	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
1313 
1314 	/* checksum all of the MACs from the layer below */
1315 	SHA2Init(SHA512, &ctx);
1316 	for (i = 0, bp = buf; i < epb; i++, bp++) {
1317 		zio_crypt_bp_do_indrect_checksum_updates(&ctx, version,
1318 		    byteswap, bp);
1319 	}
1320 	SHA2Final(digestbuf, &ctx);
1321 
1322 	if (generate) {
1323 		bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN);
1324 		return (0);
1325 	}
1326 
1327 	if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0)
1328 		return (SET_ERROR(ECKSUM));
1329 
1330 	return (0);
1331 }
1332 
1333 int
1334 zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
1335     uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1336 {
1337 	int ret;
1338 
1339 	/*
1340 	 * Unfortunately, callers of this function will not always have
1341 	 * easy access to the on-disk format version. This info is
1342 	 * normally found in the DSL Crypto Key, but the checksum-of-MACs
1343 	 * is expected to be verifiable even when the key isn't loaded.
1344 	 * Here, instead of doing a ZAP lookup for the version for each
1345 	 * zio, we simply try both existing formats.
1346 	 */
1347 	ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf,
1348 	    datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum);
1349 	if (ret == ECKSUM) {
1350 		ASSERT(!generate);
1351 		ret = zio_crypt_do_indirect_mac_checksum_impl(generate,
1352 		    buf, datalen, 0, byteswap, cksum);
1353 	}
1354 
1355 	return (ret);
1356 }
1357 
1358 int
1359 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
1360     uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1361 {
1362 	int ret;
1363 	void *buf;
1364 
1365 	buf = abd_borrow_buf_copy(abd, datalen);
1366 	ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen,
1367 	    byteswap, cksum);
1368 	abd_return_buf(abd, buf, datalen);
1369 
1370 	return (ret);
1371 }
1372 
1373 /*
1374  * Special case handling routine for encrypting / decrypting ZIL blocks.
1375  * We do not check for the older ZIL chain because the encryption feature
1376  * was not available before the newer ZIL chain was introduced. The goal
1377  * here is to encrypt everything except the blkptr_t of a lr_write_t and
1378  * the zil_chain_t header. Everything that is not encrypted is authenticated.
1379  */
1380 static int
1381 zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
1382     uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio,
1383     zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len,
1384     boolean_t *no_crypt)
1385 {
1386 	int ret;
1387 	uint64_t txtype, lr_len;
1388 	uint_t nr_src, nr_dst, crypt_len;
1389 	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1390 	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1391 	uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
1392 	zil_chain_t *zilc;
1393 	lr_t *lr;
1394 	uint8_t *aadbuf = zio_buf_alloc(datalen);
1395 
1396 	/* cipherbuf always needs an extra iovec for the MAC */
1397 	if (encrypt) {
1398 		src = plainbuf;
1399 		dst = cipherbuf;
1400 		nr_src = 0;
1401 		nr_dst = 1;
1402 	} else {
1403 		src = cipherbuf;
1404 		dst = plainbuf;
1405 		nr_src = 1;
1406 		nr_dst = 0;
1407 	}
1408 	bzero(dst, datalen);
1409 
1410 	/* find the start and end record of the log block */
1411 	zilc = (zil_chain_t *)src;
1412 	slrp = src + sizeof (zil_chain_t);
1413 	aadp = aadbuf;
1414 	blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused);
1415 
1416 	/* calculate the number of encrypted iovecs we will need */
1417 	for (; slrp < blkend; slrp += lr_len) {
1418 		lr = (lr_t *)slrp;
1419 
1420 		if (!byteswap) {
1421 			txtype = lr->lrc_txtype;
1422 			lr_len = lr->lrc_reclen;
1423 		} else {
1424 			txtype = BSWAP_64(lr->lrc_txtype);
1425 			lr_len = BSWAP_64(lr->lrc_reclen);
1426 		}
1427 
1428 		nr_iovecs++;
1429 		if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t))
1430 			nr_iovecs++;
1431 	}
1432 
1433 	nr_src += nr_iovecs;
1434 	nr_dst += nr_iovecs;
1435 
1436 	/* allocate the iovec arrays */
1437 	if (nr_src != 0) {
1438 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1439 		if (src_iovecs == NULL) {
1440 			ret = SET_ERROR(ENOMEM);
1441 			goto error;
1442 		}
1443 	}
1444 
1445 	if (nr_dst != 0) {
1446 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1447 		if (dst_iovecs == NULL) {
1448 			ret = SET_ERROR(ENOMEM);
1449 			goto error;
1450 		}
1451 	}
1452 
1453 	/*
1454 	 * Copy the plain zil header over and authenticate everything except
1455 	 * the checksum that will store our MAC. If we are writing the data
1456 	 * the embedded checksum will not have been calculated yet, so we don't
1457 	 * authenticate that.
1458 	 */
1459 	bcopy(src, dst, sizeof (zil_chain_t));
1460 	bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t));
1461 	aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1462 	aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1463 
1464 	/* loop over records again, filling in iovecs */
1465 	nr_iovecs = 0;
1466 	slrp = src + sizeof (zil_chain_t);
1467 	dlrp = dst + sizeof (zil_chain_t);
1468 
1469 	for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) {
1470 		lr = (lr_t *)slrp;
1471 
1472 		if (!byteswap) {
1473 			txtype = lr->lrc_txtype;
1474 			lr_len = lr->lrc_reclen;
1475 		} else {
1476 			txtype = BSWAP_64(lr->lrc_txtype);
1477 			lr_len = BSWAP_64(lr->lrc_reclen);
1478 		}
1479 
1480 		/* copy the common lr_t */
1481 		bcopy(slrp, dlrp, sizeof (lr_t));
1482 		bcopy(slrp, aadp, sizeof (lr_t));
1483 		aadp += sizeof (lr_t);
1484 		aad_len += sizeof (lr_t);
1485 
1486 		ASSERT3P(src_iovecs, !=, NULL);
1487 		ASSERT3P(dst_iovecs, !=, NULL);
1488 
1489 		/*
1490 		 * If this is a TX_WRITE record we want to encrypt everything
1491 		 * except the bp if exists. If the bp does exist we want to
1492 		 * authenticate it.
1493 		 */
1494 		if (txtype == TX_WRITE) {
1495 			crypt_len = sizeof (lr_write_t) -
1496 			    sizeof (lr_t) - sizeof (blkptr_t);
1497 			src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t);
1498 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1499 			dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t);
1500 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1501 
1502 			/* copy the bp now since it will not be encrypted */
1503 			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1504 			    dlrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1505 			    sizeof (blkptr_t));
1506 			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1507 			    aadp, sizeof (blkptr_t));
1508 			aadp += sizeof (blkptr_t);
1509 			aad_len += sizeof (blkptr_t);
1510 			nr_iovecs++;
1511 			total_len += crypt_len;
1512 
1513 			if (lr_len != sizeof (lr_write_t)) {
1514 				crypt_len = lr_len - sizeof (lr_write_t);
1515 				src_iovecs[nr_iovecs].iov_base =
1516 				    slrp + sizeof (lr_write_t);
1517 				src_iovecs[nr_iovecs].iov_len = crypt_len;
1518 				dst_iovecs[nr_iovecs].iov_base =
1519 				    dlrp + sizeof (lr_write_t);
1520 				dst_iovecs[nr_iovecs].iov_len = crypt_len;
1521 				nr_iovecs++;
1522 				total_len += crypt_len;
1523 			}
1524 		} else {
1525 			crypt_len = lr_len - sizeof (lr_t);
1526 			src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t);
1527 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1528 			dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t);
1529 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1530 			nr_iovecs++;
1531 			total_len += crypt_len;
1532 		}
1533 	}
1534 
1535 	*no_crypt = (nr_iovecs == 0);
1536 	*enc_len = total_len;
1537 	*authbuf = aadbuf;
1538 	*auth_len = aad_len;
1539 
1540 	if (encrypt) {
1541 		puio->uio_iov = src_iovecs;
1542 		puio->uio_iovcnt = nr_src;
1543 		cuio->uio_iov = dst_iovecs;
1544 		cuio->uio_iovcnt = nr_dst;
1545 	} else {
1546 		puio->uio_iov = dst_iovecs;
1547 		puio->uio_iovcnt = nr_dst;
1548 		cuio->uio_iov = src_iovecs;
1549 		cuio->uio_iovcnt = nr_src;
1550 	}
1551 
1552 	return (0);
1553 
1554 error:
1555 	zio_buf_free(aadbuf, datalen);
1556 	if (src_iovecs != NULL)
1557 		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1558 	if (dst_iovecs != NULL)
1559 		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1560 
1561 	*enc_len = 0;
1562 	*authbuf = NULL;
1563 	*auth_len = 0;
1564 	*no_crypt = B_FALSE;
1565 	puio->uio_iov = NULL;
1566 	puio->uio_iovcnt = 0;
1567 	cuio->uio_iov = NULL;
1568 	cuio->uio_iovcnt = 0;
1569 	return (ret);
1570 }
1571 
1572 /*
1573  * Special case handling routine for encrypting / decrypting dnode blocks.
1574  */
1575 static int
1576 zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version,
1577     uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1578     zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1579     uint_t *auth_len, boolean_t *no_crypt)
1580 {
1581 	int ret;
1582 	uint_t nr_src, nr_dst, crypt_len;
1583 	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1584 	uint_t i, j, max_dnp = datalen >> DNODE_SHIFT;
1585 	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1586 	uint8_t *src, *dst, *aadp;
1587 	dnode_phys_t *dnp, *adnp, *sdnp, *ddnp;
1588 	uint8_t *aadbuf = zio_buf_alloc(datalen);
1589 
1590 	if (encrypt) {
1591 		src = plainbuf;
1592 		dst = cipherbuf;
1593 		nr_src = 0;
1594 		nr_dst = 1;
1595 	} else {
1596 		src = cipherbuf;
1597 		dst = plainbuf;
1598 		nr_src = 1;
1599 		nr_dst = 0;
1600 	}
1601 
1602 	sdnp = (dnode_phys_t *)src;
1603 	ddnp = (dnode_phys_t *)dst;
1604 	aadp = aadbuf;
1605 
1606 	/*
1607 	 * Count the number of iovecs we will need to do the encryption by
1608 	 * counting the number of bonus buffers that need to be encrypted.
1609 	 */
1610 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1611 		/*
1612 		 * This block may still be byteswapped. However, all of the
1613 		 * values we use are either uint8_t's (for which byteswapping
1614 		 * is a noop) or a * != 0 check, which will work regardless
1615 		 * of whether or not we byteswap.
1616 		 */
1617 		if (sdnp[i].dn_type != DMU_OT_NONE &&
1618 		    DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) &&
1619 		    sdnp[i].dn_bonuslen != 0) {
1620 			nr_iovecs++;
1621 		}
1622 	}
1623 
1624 	nr_src += nr_iovecs;
1625 	nr_dst += nr_iovecs;
1626 
1627 	if (nr_src != 0) {
1628 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1629 		if (src_iovecs == NULL) {
1630 			ret = SET_ERROR(ENOMEM);
1631 			goto error;
1632 		}
1633 	}
1634 
1635 	if (nr_dst != 0) {
1636 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1637 		if (dst_iovecs == NULL) {
1638 			ret = SET_ERROR(ENOMEM);
1639 			goto error;
1640 		}
1641 	}
1642 
1643 	nr_iovecs = 0;
1644 
1645 	/*
1646 	 * Iterate through the dnodes again, this time filling in the uios
1647 	 * we allocated earlier. We also concatenate any data we want to
1648 	 * authenticate onto aadbuf.
1649 	 */
1650 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1651 		dnp = &sdnp[i];
1652 
1653 		/* copy over the core fields and blkptrs (kept as plaintext) */
1654 		bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp);
1655 
1656 		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1657 			bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]),
1658 			    sizeof (blkptr_t));
1659 		}
1660 
1661 		/*
1662 		 * Handle authenticated data. We authenticate everything in
1663 		 * the dnode that can be brought over when we do a raw send.
1664 		 * This includes all of the core fields as well as the MACs
1665 		 * stored in the bp checksums and all of the portable bits
1666 		 * from blk_prop. We include the dnode padding here in case it
1667 		 * ever gets used in the future. Some dn_flags and dn_used are
1668 		 * not portable so we mask those out values out of the
1669 		 * authenticated data.
1670 		 */
1671 		crypt_len = offsetof(dnode_phys_t, dn_blkptr);
1672 		bcopy(dnp, aadp, crypt_len);
1673 		adnp = (dnode_phys_t *)aadp;
1674 		adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1675 		adnp->dn_used = 0;
1676 		aadp += crypt_len;
1677 		aad_len += crypt_len;
1678 
1679 		for (j = 0; j < dnp->dn_nblkptr; j++) {
1680 			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1681 			    version, byteswap, &dnp->dn_blkptr[j]);
1682 		}
1683 
1684 		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1685 			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1686 			    version, byteswap, DN_SPILL_BLKPTR(dnp));
1687 		}
1688 
1689 		/*
1690 		 * If this bonus buffer needs to be encrypted, we prepare an
1691 		 * iovec_t. The encryption / decryption functions will fill
1692 		 * this in for us with the encrypted or decrypted data.
1693 		 * Otherwise we add the bonus buffer to the authenticated
1694 		 * data buffer and copy it over to the destination. The
1695 		 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1696 		 * we can guarantee alignment with the AES block size
1697 		 * (128 bits).
1698 		 */
1699 		crypt_len = DN_MAX_BONUS_LEN(dnp);
1700 		if (dnp->dn_type != DMU_OT_NONE &&
1701 		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
1702 		    dnp->dn_bonuslen != 0) {
1703 			ASSERT3U(nr_iovecs, <, nr_src);
1704 			ASSERT3U(nr_iovecs, <, nr_dst);
1705 			ASSERT3P(src_iovecs, !=, NULL);
1706 			ASSERT3P(dst_iovecs, !=, NULL);
1707 			src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
1708 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1709 			dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
1710 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1711 
1712 			nr_iovecs++;
1713 			total_len += crypt_len;
1714 		} else {
1715 			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len);
1716 			bcopy(DN_BONUS(dnp), aadp, crypt_len);
1717 			aadp += crypt_len;
1718 			aad_len += crypt_len;
1719 		}
1720 	}
1721 
1722 	*no_crypt = (nr_iovecs == 0);
1723 	*enc_len = total_len;
1724 	*authbuf = aadbuf;
1725 	*auth_len = aad_len;
1726 
1727 	if (encrypt) {
1728 		puio->uio_iov = src_iovecs;
1729 		puio->uio_iovcnt = nr_src;
1730 		cuio->uio_iov = dst_iovecs;
1731 		cuio->uio_iovcnt = nr_dst;
1732 	} else {
1733 		puio->uio_iov = dst_iovecs;
1734 		puio->uio_iovcnt = nr_dst;
1735 		cuio->uio_iov = src_iovecs;
1736 		cuio->uio_iovcnt = nr_src;
1737 	}
1738 
1739 	return (0);
1740 
1741 error:
1742 	zio_buf_free(aadbuf, datalen);
1743 	if (src_iovecs != NULL)
1744 		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1745 	if (dst_iovecs != NULL)
1746 		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1747 
1748 	*enc_len = 0;
1749 	*authbuf = NULL;
1750 	*auth_len = 0;
1751 	*no_crypt = B_FALSE;
1752 	puio->uio_iov = NULL;
1753 	puio->uio_iovcnt = 0;
1754 	cuio->uio_iov = NULL;
1755 	cuio->uio_iovcnt = 0;
1756 	return (ret);
1757 }
1758 
1759 static int
1760 zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf,
1761     uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio,
1762     uint_t *enc_len)
1763 {
1764 	int ret;
1765 	uint_t nr_plain = 1, nr_cipher = 2;
1766 	iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL;
1767 
1768 	/* allocate the iovecs for the plain and cipher data */
1769 	plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t),
1770 	    KM_SLEEP);
1771 	if (!plain_iovecs) {
1772 		ret = SET_ERROR(ENOMEM);
1773 		goto error;
1774 	}
1775 
1776 	cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t),
1777 	    KM_SLEEP);
1778 	if (!cipher_iovecs) {
1779 		ret = SET_ERROR(ENOMEM);
1780 		goto error;
1781 	}
1782 
1783 	plain_iovecs[0].iov_base = plainbuf;
1784 	plain_iovecs[0].iov_len = datalen;
1785 	cipher_iovecs[0].iov_base = cipherbuf;
1786 	cipher_iovecs[0].iov_len = datalen;
1787 
1788 	*enc_len = datalen;
1789 	puio->uio_iov = plain_iovecs;
1790 	puio->uio_iovcnt = nr_plain;
1791 	cuio->uio_iov = cipher_iovecs;
1792 	cuio->uio_iovcnt = nr_cipher;
1793 
1794 	return (0);
1795 
1796 error:
1797 	if (plain_iovecs != NULL)
1798 		kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t));
1799 	if (cipher_iovecs != NULL)
1800 		kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t));
1801 
1802 	*enc_len = 0;
1803 	puio->uio_iov = NULL;
1804 	puio->uio_iovcnt = 0;
1805 	cuio->uio_iov = NULL;
1806 	cuio->uio_iovcnt = 0;
1807 	return (ret);
1808 }
1809 
1810 /*
1811  * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1812  * that they can be used for encryption and decryption by zio_do_crypt_uio().
1813  * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1814  * requiring special handling to parse out pieces that are to be encrypted. The
1815  * authbuf is used by these special cases to store additional authenticated
1816  * data (AAD) for the encryption modes.
1817  */
1818 static int
1819 zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot,
1820     uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1821     uint8_t *mac, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len,
1822     uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt)
1823 {
1824 	int ret;
1825 	iovec_t *mac_iov;
1826 
1827 	ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE);
1828 
1829 	/* route to handler */
1830 	switch (ot) {
1831 	case DMU_OT_INTENT_LOG:
1832 		ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf,
1833 		    datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len,
1834 		    no_crypt);
1835 		break;
1836 	case DMU_OT_DNODE:
1837 		ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf,
1838 		    cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf,
1839 		    auth_len, no_crypt);
1840 		break;
1841 	default:
1842 		ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf,
1843 		    datalen, puio, cuio, enc_len);
1844 		*authbuf = NULL;
1845 		*auth_len = 0;
1846 		*no_crypt = B_FALSE;
1847 		break;
1848 	}
1849 
1850 	if (ret != 0)
1851 		goto error;
1852 
1853 	/* populate the uios */
1854 	puio->uio_segflg = UIO_SYSSPACE;
1855 	cuio->uio_segflg = UIO_SYSSPACE;
1856 
1857 	mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]);
1858 	mac_iov->iov_base = mac;
1859 	mac_iov->iov_len = ZIO_DATA_MAC_LEN;
1860 
1861 	return (0);
1862 
1863 error:
1864 	return (ret);
1865 }
1866 
1867 /*
1868  * Primary encryption / decryption entrypoint for zio data.
1869  */
1870 int
1871 zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
1872     dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
1873     uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
1874     boolean_t *no_crypt)
1875 {
1876 	int ret;
1877 	boolean_t locked = B_FALSE;
1878 	uint64_t crypt = key->zk_crypt;
1879 	uint_t keydata_len = zio_crypt_table[crypt].ci_keylen;
1880 	uint_t enc_len, auth_len;
1881 	zfs_uio_t puio, cuio;
1882 	uint8_t enc_keydata[MASTER_KEY_MAX_LEN];
1883 	crypto_key_t tmp_ckey, *ckey = NULL;
1884 	crypto_ctx_template_t tmpl;
1885 	uint8_t *authbuf = NULL;
1886 
1887 	/*
1888 	 * If the needed key is the current one, just use it. Otherwise we
1889 	 * need to generate a temporary one from the given salt + master key.
1890 	 * If we are encrypting, we must return a copy of the current salt
1891 	 * so that it can be stored in the blkptr_t.
1892 	 */
1893 	rw_enter(&key->zk_salt_lock, RW_READER);
1894 	locked = B_TRUE;
1895 
1896 	if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) {
1897 		ckey = &key->zk_current_key;
1898 		tmpl = key->zk_current_tmpl;
1899 	} else {
1900 		rw_exit(&key->zk_salt_lock);
1901 		locked = B_FALSE;
1902 
1903 		ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
1904 		    salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len);
1905 		if (ret != 0)
1906 			goto error;
1907 
1908 		tmp_ckey.ck_format = CRYPTO_KEY_RAW;
1909 		tmp_ckey.ck_data = enc_keydata;
1910 		tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
1911 
1912 		ckey = &tmp_ckey;
1913 		tmpl = NULL;
1914 	}
1915 
1916 	/*
1917 	 * Attempt to use QAT acceleration if we can. We currently don't
1918 	 * do this for metadnode and ZIL blocks, since they have a much
1919 	 * more involved buffer layout and the qat_crypt() function only
1920 	 * works in-place.
1921 	 */
1922 	if (qat_crypt_use_accel(datalen) &&
1923 	    ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) {
1924 		uint8_t *srcbuf, *dstbuf;
1925 
1926 		if (encrypt) {
1927 			srcbuf = plainbuf;
1928 			dstbuf = cipherbuf;
1929 		} else {
1930 			srcbuf = cipherbuf;
1931 			dstbuf = plainbuf;
1932 		}
1933 
1934 		ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf,
1935 		    dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen);
1936 		if (ret == CPA_STATUS_SUCCESS) {
1937 			if (locked) {
1938 				rw_exit(&key->zk_salt_lock);
1939 				locked = B_FALSE;
1940 			}
1941 
1942 			return (0);
1943 		}
1944 		/* If the hardware implementation fails fall back to software */
1945 	}
1946 
1947 	bzero(&puio, sizeof (zfs_uio_t));
1948 	bzero(&cuio, sizeof (zfs_uio_t));
1949 
1950 	/* create uios for encryption */
1951 	ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf,
1952 	    cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len,
1953 	    &authbuf, &auth_len, no_crypt);
1954 	if (ret != 0)
1955 		goto error;
1956 
1957 	/* perform the encryption / decryption in software */
1958 	ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
1959 	    &puio, &cuio, authbuf, auth_len);
1960 	if (ret != 0)
1961 		goto error;
1962 
1963 	if (locked) {
1964 		rw_exit(&key->zk_salt_lock);
1965 		locked = B_FALSE;
1966 	}
1967 
1968 	if (authbuf != NULL)
1969 		zio_buf_free(authbuf, datalen);
1970 	if (ckey == &tmp_ckey)
1971 		bzero(enc_keydata, keydata_len);
1972 	zio_crypt_destroy_uio(&puio);
1973 	zio_crypt_destroy_uio(&cuio);
1974 
1975 	return (0);
1976 
1977 error:
1978 	if (locked)
1979 		rw_exit(&key->zk_salt_lock);
1980 	if (authbuf != NULL)
1981 		zio_buf_free(authbuf, datalen);
1982 	if (ckey == &tmp_ckey)
1983 		bzero(enc_keydata, keydata_len);
1984 	zio_crypt_destroy_uio(&puio);
1985 	zio_crypt_destroy_uio(&cuio);
1986 
1987 	return (ret);
1988 }
1989 
1990 /*
1991  * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1992  * linear buffers.
1993  */
1994 int
1995 zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
1996     boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
1997     uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
1998 {
1999 	int ret;
2000 	void *ptmp, *ctmp;
2001 
2002 	if (encrypt) {
2003 		ptmp = abd_borrow_buf_copy(pabd, datalen);
2004 		ctmp = abd_borrow_buf(cabd, datalen);
2005 	} else {
2006 		ptmp = abd_borrow_buf(pabd, datalen);
2007 		ctmp = abd_borrow_buf_copy(cabd, datalen);
2008 	}
2009 
2010 	ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
2011 	    datalen, ptmp, ctmp, no_crypt);
2012 	if (ret != 0)
2013 		goto error;
2014 
2015 	if (encrypt) {
2016 		abd_return_buf(pabd, ptmp, datalen);
2017 		abd_return_buf_copy(cabd, ctmp, datalen);
2018 	} else {
2019 		abd_return_buf_copy(pabd, ptmp, datalen);
2020 		abd_return_buf(cabd, ctmp, datalen);
2021 	}
2022 
2023 	return (0);
2024 
2025 error:
2026 	if (encrypt) {
2027 		abd_return_buf(pabd, ptmp, datalen);
2028 		abd_return_buf_copy(cabd, ctmp, datalen);
2029 	} else {
2030 		abd_return_buf_copy(pabd, ptmp, datalen);
2031 		abd_return_buf(cabd, ctmp, datalen);
2032 	}
2033 
2034 	return (ret);
2035 }
2036 
2037 #if defined(_KERNEL)
2038 /* BEGIN CSTYLED */
2039 module_param(zfs_key_max_salt_uses, ulong, 0644);
2040 MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value "
2041 	"can be used for generating encryption keys before it is rotated");
2042 /* END CSTYLED */
2043 #endif
2044