xref: /illumos-gate/usr/src/uts/common/fs/zfs/zio_crypt.c (revision 0d48c3fdfa345b461d90342d8a96a773f386cb76)
1 /*
2  * CDDL HEADER START
3  *
4  * This file and its contents are supplied under the terms of the
5  * Common Development and Distribution License ("CDDL"), version 1.0.
6  * You may only use this file in accordance with the terms of version
7  * 1.0 of the CDDL.
8  *
9  * A full copy of the text of the CDDL should have accompanied this
10  * source.  A copy of the CDDL is also available via the Internet at
11  * http://www.illumos.org/license/CDDL.
12  *
13  * CDDL HEADER END
14  */
15 
16 /*
17  * Copyright (c) 2017, Datto, Inc. All rights reserved.
18  */
19 
20 #include <sys/zio_crypt.h>
21 #include <sys/dmu.h>
22 #include <sys/dmu_objset.h>
23 #include <sys/dnode.h>
24 #include <sys/fs/zfs.h>
25 #include <sys/zio.h>
26 #include <sys/zil.h>
27 #include <sys/sha2.h>
28 #include <sys/hkdf.h>
29 
30 /*
31  * This file is responsible for handling all of the details of generating
32  * encryption parameters and performing encryption and authentication.
33  *
34  * BLOCK ENCRYPTION PARAMETERS:
35  * Encryption /Authentication Algorithm Suite (crypt):
36  * The encryption algorithm, mode, and key length we are going to use. We
37  * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
38  * keys. All authentication is currently done with SHA512-HMAC.
39  *
40  * Plaintext:
41  * The unencrypted data that we want to encrypt.
42  *
43  * Initialization Vector (IV):
44  * An initialization vector for the encryption algorithms. This is used to
45  * "tweak" the encryption algorithms so that two blocks of the same data are
46  * encrypted into different ciphertext outputs, thus obfuscating block patterns.
47  * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
48  * never reused with the same encryption key. This value is stored unencrypted
49  * and must simply be provided to the decryption function. We use a 96 bit IV
50  * (as recommended by NIST) for all block encryption. For non-dedup blocks we
51  * derive the IV randomly. The first 64 bits of the IV are stored in the second
52  * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
53  * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
54  * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
55  * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
56  * level 0 blocks is the number of allocated dnodes in that block. The on-disk
57  * format supports at most 2^15 slots per L0 dnode block, because the maximum
58  * block size is 16MB (2^24). In either case, for level 0 blocks this number
59  * will still be smaller than UINT32_MAX so it is safe to store the IV in the
60  * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
61  * for the dnode code.
62  *
63  * Master key:
64  * This is the most important secret data of an encrypted dataset. It is used
65  * along with the salt to generate that actual encryption keys via HKDF. We
66  * do not use the master key to directly encrypt any data because there are
67  * theoretical limits on how much data can actually be safely encrypted with
68  * any encryption mode. The master key is stored encrypted on disk with the
69  * user's wrapping key. Its length is determined by the encryption algorithm.
70  * For details on how this is stored see the block comment in dsl_crypt.c
71  *
72  * Salt:
73  * Used as an input to the HKDF function, along with the master key. We use a
74  * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
75  * can be used for encrypting many blocks, so we cache the current salt and the
76  * associated derived key in zio_crypt_t so we do not need to derive it again
77  * needlessly.
78  *
79  * Encryption Key:
80  * A secret binary key, generated from an HKDF function used to encrypt and
81  * decrypt data.
82  *
83  * Message Authenication Code (MAC)
84  * The MAC is an output of authenticated encryption modes such as AES-GCM and
85  * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
86  * data on disk and return garbage to the application. Effectively, it is a
87  * checksum that can not be reproduced by an attacker. We store the MAC in the
88  * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
89  * regular checksum of the ciphertext which can be used for scrubbing.
90  *
91  * OBJECT AUTHENTICATION:
92  * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
93  * they contain some info that always needs to be readable. To prevent this
94  * data from being altered, we authenticate this data using SHA512-HMAC. This
95  * will produce a MAC (similar to the one produced via encryption) which can
96  * be used to verify the object was not modified. HMACs do not require key
97  * rotation or IVs, so we can keep up to the full 3 copies of authenticated
98  * data.
99  *
100  * ZIL ENCRYPTION:
101  * ZIL blocks have their bp written to disk ahead of the associated data, so we
102  * cannot store the MAC there as we normally do. For these blocks the MAC is
103  * stored in the embedded checksum within the zil_chain_t header. The salt and
104  * IV are generated for the block on bp allocation instead of at encryption
105  * time. In addition, ZIL blocks have some pieces that must be left in plaintext
106  * for claiming even though all of the sensitive user data still needs to be
107  * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
108  * pieces of the block need to be encrypted. All data that is not encrypted is
109  * authenticated using the AAD mechanisms that the supported encryption modes
110  * provide for. In order to preserve the semantics of the ZIL for encrypted
111  * datasets, the ZIL is not protected at the objset level as described below.
112  *
113  * DNODE ENCRYPTION:
114  * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
115  * in plaintext for scrubbing and claiming, but the bonus buffers might contain
116  * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
117  * which pieces of the block need to be encrypted. For more details about
118  * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
119  *
120  * OBJECT SET AUTHENTICATION:
121  * Up to this point, everything we have encrypted and authenticated has been
122  * at level 0 (or -2 for the ZIL). If we did not do any further work the
123  * on-disk format would be susceptible to attacks that deleted or rearrannged
124  * the order of level 0 blocks. Ideally, the cleanest solution would be to
125  * maintain a tree of authentication MACs going up the bp tree. However, this
126  * presents a problem for raw sends. Send files do not send information about
127  * indirect blocks so there would be no convenient way to transfer the MACs and
128  * they cannot be recalculated on the receive side without the master key which
129  * would defeat one of the purposes of raw sends in the first place. Instead,
130  * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
131  * from the level below. We also include some portable fields from blk_prop such
132  * as the lsize and compression algorithm to prevent the data from being
133  * misinterpretted.
134  *
135  * At the objset level, we maintain 2 seperate 256 bit MACs in the
136  * objset_phys_t. The first one is "portable" and is the logical root of the
137  * MAC tree maintianed in the metadnode's bps. The second, is "local" and is
138  * used as the root MAC for the user accounting objects, which are also not
139  * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
140  * of the send file. The useraccounting code ensures that the useraccounting
141  * info is not present upon a receive, so the local MAC can simply be cleared
142  * out at that time. For more info about objset_phys_t authentication, see
143  * zio_crypt_do_objset_hmacs().
144  *
145  * CONSIDERATIONS FOR DEDUP:
146  * In order for dedup to work, blocks that we want to dedup with one another
147  * need to use the same IV and encryption key, so that they will have the same
148  * ciphertext. Normally, one should never reuse an IV with the same encryption
149  * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
150  * blocks. In this case, however, since we are using the same plaindata as
151  * well all that we end up with is a duplicate of the original ciphertext we
152  * already had. As a result, an attacker with read access to the raw disk will
153  * be able to tell which blocks are the same but this information is given away
154  * by dedup anyway. In order to get the same IVs and encryption keys for
155  * equivalent blocks of data we use an HMAC of the plaindata. We use an HMAC
156  * here so that a reproducible checksum of the plaindata is never available to
157  * the attacker. The HMAC key is kept alongside the master key, encrypted on
158  * disk. The first 64 bits of the HMAC are used in place of the random salt, and
159  * the next 96 bits are used as the IV. As a result of this mechanism, dedup
160  * will only work within a clone family since encrypted dedup requires use of
161  * the same master and HMAC keys.
162  */
163 
164 /*
165  * After encrypting many blocks with the same key we may start to run up
166  * against the theoretical limits of how much data can securely be encrypted
167  * with a single key using the supported encryption modes. The most obvious
168  * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
169  * the more IVs we generate (which both GCM and CCM modes strictly forbid).
170  * This risk actually grows surprisingly quickly over time according to the
171  * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
172  * generated n IVs with a cryptographically secure RNG, the approximate
173  * probability p(n) of a collision is given as:
174  *
175  * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
176  *
177  * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
178  *
179  * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
180  * we must not write more than 398,065,730 blocks with the same encryption key.
181  * Therefore, we rotate our keys after 400,000,000 blocks have been written by
182  * generating a new random 64 bit salt for our HKDF encryption key generation
183  * function.
184  */
185 #define	ZFS_KEY_MAX_SALT_USES_DEFAULT	400000000
186 #define	ZFS_CURRENT_MAX_SALT_USES	\
187 	(MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
188 unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT;
189 
190 /*
191  * Set to a nonzero value to cause zio_do_crypt_uio() to fail 1/this many
192  * calls, to test decryption error handling code paths.
193  */
194 uint64_t zio_decrypt_fail_fraction = 0;
195 
196 typedef struct blkptr_auth_buf {
197 	uint64_t bab_prop;			/* blk_prop - portable mask */
198 	uint8_t bab_mac[ZIO_DATA_MAC_LEN];	/* MAC from blk_cksum */
199 	uint64_t bab_pad;			/* reserved for future use */
200 } blkptr_auth_buf_t;
201 
202 zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
203 	{"",			ZC_TYPE_NONE,	0,	"inherit"},
204 	{"",			ZC_TYPE_NONE,	0,	"on"},
205 	{"",			ZC_TYPE_NONE,	0,	"off"},
206 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	16,	"aes-128-ccm"},
207 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	24,	"aes-192-ccm"},
208 	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	32,	"aes-256-ccm"},
209 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	16,	"aes-128-gcm"},
210 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	24,	"aes-192-gcm"},
211 	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	32,	"aes-256-gcm"}
212 };
213 
214 void
zio_crypt_key_destroy(zio_crypt_key_t * key)215 zio_crypt_key_destroy(zio_crypt_key_t *key)
216 {
217 	rw_destroy(&key->zk_salt_lock);
218 
219 	/* free crypto templates */
220 	crypto_destroy_ctx_template(key->zk_current_tmpl);
221 	crypto_destroy_ctx_template(key->zk_hmac_tmpl);
222 
223 	/* zero out sensitive data */
224 	bzero(key, sizeof (zio_crypt_key_t));
225 }
226 
227 int
zio_crypt_key_init(uint64_t crypt,zio_crypt_key_t * key)228 zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
229 {
230 	int ret;
231 	crypto_mechanism_t mech;
232 	uint_t keydata_len;
233 
234 	ASSERT(key != NULL);
235 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
236 
237 	keydata_len = zio_crypt_table[crypt].ci_keylen;
238 	bzero(key, sizeof (zio_crypt_key_t));
239 
240 	/* fill keydata buffers and salt with random data */
241 	ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t));
242 	if (ret != 0)
243 		goto error;
244 
245 	ret = random_get_bytes(key->zk_master_keydata, keydata_len);
246 	if (ret != 0)
247 		goto error;
248 
249 	ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN);
250 	if (ret != 0)
251 		goto error;
252 
253 	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
254 	if (ret != 0)
255 		goto error;
256 
257 	/* derive the current key from the master key */
258 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
259 	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
260 	    keydata_len);
261 	if (ret != 0)
262 		goto error;
263 
264 	/* initialize keys for the ICP */
265 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
266 	key->zk_current_key.ck_data = key->zk_current_keydata;
267 	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
268 
269 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
270 	key->zk_hmac_key.ck_data = &key->zk_hmac_key;
271 	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
272 
273 	/*
274 	 * Initialize the crypto templates. It's ok if this fails because
275 	 * this is just an optimization.
276 	 */
277 	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
278 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
279 	    &key->zk_current_tmpl, KM_SLEEP);
280 	if (ret != CRYPTO_SUCCESS)
281 		key->zk_current_tmpl = NULL;
282 
283 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
284 	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
285 	    &key->zk_hmac_tmpl, KM_SLEEP);
286 	if (ret != CRYPTO_SUCCESS)
287 		key->zk_hmac_tmpl = NULL;
288 
289 	key->zk_crypt = crypt;
290 	key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION;
291 	key->zk_salt_count = 0;
292 	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
293 
294 	return (0);
295 
296 error:
297 	zio_crypt_key_destroy(key);
298 	return (ret);
299 }
300 
301 static int
zio_crypt_key_change_salt(zio_crypt_key_t * key)302 zio_crypt_key_change_salt(zio_crypt_key_t *key)
303 {
304 	int ret = 0;
305 	uint8_t salt[ZIO_DATA_SALT_LEN];
306 	crypto_mechanism_t mech;
307 	uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen;
308 
309 	/* generate a new salt */
310 	ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN);
311 	if (ret != 0)
312 		goto error;
313 
314 	rw_enter(&key->zk_salt_lock, RW_WRITER);
315 
316 	/* someone beat us to the salt rotation, just unlock and return */
317 	if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES)
318 		goto out_unlock;
319 
320 	/* derive the current key from the master key and the new salt */
321 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
322 	    salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len);
323 	if (ret != 0)
324 		goto out_unlock;
325 
326 	/* assign the salt and reset the usage count */
327 	bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN);
328 	key->zk_salt_count = 0;
329 
330 	/* destroy the old context template and create the new one */
331 	crypto_destroy_ctx_template(key->zk_current_tmpl);
332 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
333 	    &key->zk_current_tmpl, KM_SLEEP);
334 	if (ret != CRYPTO_SUCCESS)
335 		key->zk_current_tmpl = NULL;
336 
337 	rw_exit(&key->zk_salt_lock);
338 
339 	return (0);
340 
341 out_unlock:
342 	rw_exit(&key->zk_salt_lock);
343 error:
344 	return (ret);
345 }
346 
347 /* See comment above zfs_key_max_salt_uses definition for details */
348 int
zio_crypt_key_get_salt(zio_crypt_key_t * key,uint8_t * salt)349 zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt)
350 {
351 	int ret;
352 	boolean_t salt_change;
353 
354 	rw_enter(&key->zk_salt_lock, RW_READER);
355 
356 	bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN);
357 	salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >=
358 	    ZFS_CURRENT_MAX_SALT_USES);
359 
360 	rw_exit(&key->zk_salt_lock);
361 
362 	if (salt_change) {
363 		ret = zio_crypt_key_change_salt(key);
364 		if (ret != 0)
365 			goto error;
366 	}
367 
368 	return (0);
369 
370 error:
371 	return (ret);
372 }
373 
374 void *failed_decrypt_buf;
375 int failed_decrypt_size;
376 
377 /*
378  * This function handles all encryption and decryption in zfs. When
379  * encrypting it expects puio to reference the plaintext and cuio to
380  * reference the cphertext. cuio must have enough space for the
381  * ciphertext + room for a MAC. datalen should be the length of the
382  * plaintext / ciphertext alone.
383  */
384 /* ARGSUSED */
385 static int
zio_do_crypt_uio(boolean_t encrypt,uint64_t crypt,crypto_key_t * key,crypto_ctx_template_t tmpl,uint8_t * ivbuf,uint_t datalen,uio_t * puio,uio_t * cuio,uint8_t * authbuf,uint_t auth_len)386 zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
387     crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
388     uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
389 {
390 	int ret;
391 	crypto_data_t plaindata, cipherdata;
392 	CK_AES_CCM_PARAMS ccmp;
393 	CK_AES_GCM_PARAMS gcmp;
394 	crypto_mechanism_t mech;
395 	zio_crypt_info_t crypt_info;
396 	uint_t plain_full_len, maclen;
397 
398 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
399 	ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW);
400 
401 	/* lookup the encryption info */
402 	crypt_info = zio_crypt_table[crypt];
403 
404 	/* the mac will always be the last iovec_t in the cipher uio */
405 	maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len;
406 
407 	ASSERT(maclen <= ZIO_DATA_MAC_LEN);
408 
409 	/* setup encryption mechanism (same as crypt) */
410 	mech.cm_type = crypto_mech2id(crypt_info.ci_mechname);
411 
412 	/*
413 	 * Strangely, the ICP requires that plain_full_len must include
414 	 * the MAC length when decrypting, even though the UIO does not
415 	 * need to have the extra space allocated.
416 	 */
417 	if (encrypt) {
418 		plain_full_len = datalen;
419 	} else {
420 		plain_full_len = datalen + maclen;
421 	}
422 
423 	/*
424 	 * setup encryption params (currently only AES CCM and AES GCM
425 	 * are supported)
426 	 */
427 	if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) {
428 		ccmp.ulNonceSize = ZIO_DATA_IV_LEN;
429 		ccmp.ulAuthDataSize = auth_len;
430 		ccmp.authData = authbuf;
431 		ccmp.ulMACSize = maclen;
432 		ccmp.nonce = ivbuf;
433 		ccmp.ulDataSize = plain_full_len;
434 
435 		mech.cm_param = (char *)(&ccmp);
436 		mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
437 	} else {
438 		gcmp.ulIvLen = ZIO_DATA_IV_LEN;
439 		gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
440 		gcmp.ulAADLen = auth_len;
441 		gcmp.pAAD = authbuf;
442 		gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
443 		gcmp.pIv = ivbuf;
444 
445 		mech.cm_param = (char *)(&gcmp);
446 		mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
447 	}
448 
449 	/* populate the cipher and plain data structs. */
450 	plaindata.cd_format = CRYPTO_DATA_UIO;
451 	plaindata.cd_offset = 0;
452 	plaindata.cd_uio = puio;
453 	plaindata.cd_miscdata = NULL;
454 	plaindata.cd_length = plain_full_len;
455 
456 	cipherdata.cd_format = CRYPTO_DATA_UIO;
457 	cipherdata.cd_offset = 0;
458 	cipherdata.cd_uio = cuio;
459 	cipherdata.cd_miscdata = NULL;
460 	cipherdata.cd_length = datalen + maclen;
461 
462 	/* perform the actual encryption */
463 	if (encrypt) {
464 		ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata,
465 		    NULL);
466 		if (ret != CRYPTO_SUCCESS) {
467 			ret = SET_ERROR(EIO);
468 			goto error;
469 		}
470 	} else {
471 		if (zio_decrypt_fail_fraction != 0 &&
472 		    spa_get_random(zio_decrypt_fail_fraction) == 0) {
473 			ret = CRYPTO_INVALID_MAC;
474 		} else {
475 			ret = crypto_decrypt(&mech, &cipherdata,
476 			    key, tmpl, &plaindata, NULL);
477 		}
478 		if (ret != CRYPTO_SUCCESS) {
479 			ASSERT3U(ret, ==, CRYPTO_INVALID_MAC);
480 			ret = SET_ERROR(ECKSUM);
481 			goto error;
482 		}
483 	}
484 
485 	return (0);
486 
487 error:
488 	return (ret);
489 }
490 
491 int
zio_crypt_key_wrap(crypto_key_t * cwkey,zio_crypt_key_t * key,uint8_t * iv,uint8_t * mac,uint8_t * keydata_out,uint8_t * hmac_keydata_out)492 zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
493     uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
494 {
495 	int ret;
496 	uio_t puio, cuio;
497 	uint64_t aad[3];
498 	iovec_t plain_iovecs[2], cipher_iovecs[3];
499 	uint64_t crypt = key->zk_crypt;
500 	uint_t enc_len, keydata_len, aad_len;
501 
502 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
503 	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
504 
505 	keydata_len = zio_crypt_table[crypt].ci_keylen;
506 
507 	/* generate iv for wrapping the master and hmac key */
508 	ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN);
509 	if (ret != 0)
510 		goto error;
511 
512 	/* initialize uio_ts */
513 	plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
514 	plain_iovecs[0].iov_len = keydata_len;
515 	plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
516 	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
517 
518 	cipher_iovecs[0].iov_base = (char *)keydata_out;
519 	cipher_iovecs[0].iov_len = keydata_len;
520 	cipher_iovecs[1].iov_base = (char *)hmac_keydata_out;
521 	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
522 	cipher_iovecs[2].iov_base = (char *)mac;
523 	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
524 
525 	/*
526 	 * Although we don't support writing to the old format, we do
527 	 * support rewrapping the key so that the user can move and
528 	 * quarantine datasets on the old format.
529 	 */
530 	if (key->zk_version == 0) {
531 		aad_len = sizeof (uint64_t);
532 		aad[0] = LE_64(key->zk_guid);
533 	} else {
534 		ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
535 		aad_len = sizeof (uint64_t) * 3;
536 		aad[0] = LE_64(key->zk_guid);
537 		aad[1] = LE_64(crypt);
538 		aad[2] = LE_64(key->zk_version);
539 	}
540 
541 	enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN;
542 	puio.uio_iov = plain_iovecs;
543 	puio.uio_iovcnt = 2;
544 	puio.uio_segflg = UIO_SYSSPACE;
545 	cuio.uio_iov = cipher_iovecs;
546 	cuio.uio_iovcnt = 3;
547 	cuio.uio_segflg = UIO_SYSSPACE;
548 
549 	/* encrypt the keys and store the resulting ciphertext and mac */
550 	ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
551 	    &puio, &cuio, (uint8_t *)aad, aad_len);
552 	if (ret != 0)
553 		goto error;
554 
555 	return (0);
556 
557 error:
558 	return (ret);
559 }
560 
561 int
zio_crypt_key_unwrap(crypto_key_t * cwkey,uint64_t crypt,uint64_t version,uint64_t guid,uint8_t * keydata,uint8_t * hmac_keydata,uint8_t * iv,uint8_t * mac,zio_crypt_key_t * key)562 zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
563     uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
564     uint8_t *mac, zio_crypt_key_t *key)
565 {
566 	int ret;
567 	crypto_mechanism_t mech;
568 	uio_t puio, cuio;
569 	uint64_t aad[3];
570 	iovec_t plain_iovecs[2], cipher_iovecs[3];
571 	uint_t enc_len, keydata_len, aad_len;
572 
573 	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
574 	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
575 
576 	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
577 	keydata_len = zio_crypt_table[crypt].ci_keylen;
578 
579 	/* initialize uio_ts */
580 	plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
581 	plain_iovecs[0].iov_len = keydata_len;
582 	plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
583 	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
584 
585 	cipher_iovecs[0].iov_base = (char *)keydata;
586 	cipher_iovecs[0].iov_len = keydata_len;
587 	cipher_iovecs[1].iov_base = (char *)hmac_keydata;
588 	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
589 	cipher_iovecs[2].iov_base = (char *)mac;
590 	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
591 
592 	if (version == 0) {
593 		aad_len = sizeof (uint64_t);
594 		aad[0] = LE_64(guid);
595 	} else {
596 		ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
597 		aad_len = sizeof (uint64_t) * 3;
598 		aad[0] = LE_64(guid);
599 		aad[1] = LE_64(crypt);
600 		aad[2] = LE_64(version);
601 	}
602 
603 	enc_len = keydata_len + SHA512_HMAC_KEYLEN;
604 	puio.uio_iov = plain_iovecs;
605 	puio.uio_segflg = UIO_SYSSPACE;
606 	puio.uio_iovcnt = 2;
607 	cuio.uio_iov = cipher_iovecs;
608 	cuio.uio_iovcnt = 3;
609 	cuio.uio_segflg = UIO_SYSSPACE;
610 
611 	/* decrypt the keys and store the result in the output buffers */
612 	ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
613 	    &puio, &cuio, (uint8_t *)aad, aad_len);
614 	if (ret != 0)
615 		goto error;
616 
617 	/* generate a fresh salt */
618 	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
619 	if (ret != 0)
620 		goto error;
621 
622 	/* derive the current key from the master key */
623 	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
624 	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
625 	    keydata_len);
626 	if (ret != 0)
627 		goto error;
628 
629 	/* initialize keys for ICP */
630 	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
631 	key->zk_current_key.ck_data = key->zk_current_keydata;
632 	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
633 
634 	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
635 	key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
636 	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
637 
638 	/*
639 	 * Initialize the crypto templates. It's ok if this fails because
640 	 * this is just an optimization.
641 	 */
642 	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
643 	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
644 	    &key->zk_current_tmpl, KM_SLEEP);
645 	if (ret != CRYPTO_SUCCESS)
646 		key->zk_current_tmpl = NULL;
647 
648 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
649 	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
650 	    &key->zk_hmac_tmpl, KM_SLEEP);
651 	if (ret != CRYPTO_SUCCESS)
652 		key->zk_hmac_tmpl = NULL;
653 
654 	key->zk_crypt = crypt;
655 	key->zk_version = version;
656 	key->zk_guid = guid;
657 	key->zk_salt_count = 0;
658 
659 	return (0);
660 
661 error:
662 	zio_crypt_key_destroy(key);
663 	return (ret);
664 }
665 
666 int
zio_crypt_generate_iv(uint8_t * ivbuf)667 zio_crypt_generate_iv(uint8_t *ivbuf)
668 {
669 	int ret;
670 
671 	/* randomly generate the IV */
672 	ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN);
673 	if (ret != 0)
674 		goto error;
675 
676 	return (0);
677 
678 error:
679 	bzero(ivbuf, ZIO_DATA_IV_LEN);
680 	return (ret);
681 }
682 
683 int
zio_crypt_do_hmac(zio_crypt_key_t * key,uint8_t * data,uint_t datalen,uint8_t * digestbuf,uint_t digestlen)684 zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
685     uint8_t *digestbuf, uint_t digestlen)
686 {
687 	int ret;
688 	crypto_mechanism_t mech;
689 	crypto_data_t in_data, digest_data;
690 	uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
691 
692 	ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
693 
694 	/* initialize sha512-hmac mechanism and crypto data */
695 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
696 	mech.cm_param = NULL;
697 	mech.cm_param_len = 0;
698 
699 	/* initialize the crypto data */
700 	in_data.cd_format = CRYPTO_DATA_RAW;
701 	in_data.cd_offset = 0;
702 	in_data.cd_length = datalen;
703 	in_data.cd_raw.iov_base = (char *)data;
704 	in_data.cd_raw.iov_len = in_data.cd_length;
705 
706 	digest_data.cd_format = CRYPTO_DATA_RAW;
707 	digest_data.cd_offset = 0;
708 	digest_data.cd_length = SHA512_DIGEST_LENGTH;
709 	digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
710 	digest_data.cd_raw.iov_len = digest_data.cd_length;
711 
712 	/* generate the hmac */
713 	ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl,
714 	    &digest_data, NULL);
715 	if (ret != CRYPTO_SUCCESS) {
716 		ret = SET_ERROR(EIO);
717 		goto error;
718 	}
719 
720 	bcopy(raw_digestbuf, digestbuf, digestlen);
721 
722 	return (0);
723 
724 error:
725 	bzero(digestbuf, digestlen);
726 	return (ret);
727 }
728 
729 int
zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t * key,uint8_t * data,uint_t datalen,uint8_t * ivbuf,uint8_t * salt)730 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
731     uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
732 {
733 	int ret;
734 	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
735 
736 	ret = zio_crypt_do_hmac(key, data, datalen,
737 	    digestbuf, SHA512_DIGEST_LENGTH);
738 	if (ret != 0)
739 		return (ret);
740 
741 	bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN);
742 	bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN);
743 
744 	return (0);
745 }
746 
747 /*
748  * The following functions are used to encode and decode encryption parameters
749  * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
750  * byte strings, which normally means that these strings would not need to deal
751  * with byteswapping at all. However, both blkptr_t and zil_header_t may be
752  * byteswapped by lower layers and so we must "undo" that byteswap here upon
753  * decoding and encoding in a non-native byteorder. These functions require
754  * that the byteorder bit is correct before being called.
755  */
756 void
zio_crypt_encode_params_bp(blkptr_t * bp,uint8_t * salt,uint8_t * iv)757 zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv)
758 {
759 	uint64_t val64;
760 	uint32_t val32;
761 
762 	ASSERT(BP_IS_ENCRYPTED(bp));
763 
764 	if (!BP_SHOULD_BYTESWAP(bp)) {
765 		bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t));
766 		bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t));
767 		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
768 		BP_SET_IV2(bp, val32);
769 	} else {
770 		bcopy(salt, &val64, sizeof (uint64_t));
771 		bp->blk_dva[2].dva_word[0] = BSWAP_64(val64);
772 
773 		bcopy(iv, &val64, sizeof (uint64_t));
774 		bp->blk_dva[2].dva_word[1] = BSWAP_64(val64);
775 
776 		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
777 		BP_SET_IV2(bp, BSWAP_32(val32));
778 	}
779 }
780 
781 void
zio_crypt_decode_params_bp(const blkptr_t * bp,uint8_t * salt,uint8_t * iv)782 zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv)
783 {
784 	uint64_t val64;
785 	uint32_t val32;
786 
787 	ASSERT(BP_IS_PROTECTED(bp));
788 
789 	/* for convenience, so callers don't need to check */
790 	if (BP_IS_AUTHENTICATED(bp)) {
791 		bzero(salt, ZIO_DATA_SALT_LEN);
792 		bzero(iv, ZIO_DATA_IV_LEN);
793 		return;
794 	}
795 
796 	if (!BP_SHOULD_BYTESWAP(bp)) {
797 		bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t));
798 		bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t));
799 
800 		val32 = (uint32_t)BP_GET_IV2(bp);
801 		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
802 	} else {
803 		val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]);
804 		bcopy(&val64, salt, sizeof (uint64_t));
805 
806 		val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]);
807 		bcopy(&val64, iv, sizeof (uint64_t));
808 
809 		val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp));
810 		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
811 	}
812 }
813 
814 void
zio_crypt_encode_mac_bp(blkptr_t * bp,uint8_t * mac)815 zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac)
816 {
817 	uint64_t val64;
818 
819 	ASSERT(BP_USES_CRYPT(bp));
820 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET);
821 
822 	if (!BP_SHOULD_BYTESWAP(bp)) {
823 		bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t));
824 		bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3],
825 		    sizeof (uint64_t));
826 	} else {
827 		bcopy(mac, &val64, sizeof (uint64_t));
828 		bp->blk_cksum.zc_word[2] = BSWAP_64(val64);
829 
830 		bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t));
831 		bp->blk_cksum.zc_word[3] = BSWAP_64(val64);
832 	}
833 }
834 
835 void
zio_crypt_decode_mac_bp(const blkptr_t * bp,uint8_t * mac)836 zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac)
837 {
838 	uint64_t val64;
839 
840 	ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp));
841 
842 	/* for convenience, so callers don't need to check */
843 	if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
844 		bzero(mac, ZIO_DATA_MAC_LEN);
845 		return;
846 	}
847 
848 	if (!BP_SHOULD_BYTESWAP(bp)) {
849 		bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t));
850 		bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t),
851 		    sizeof (uint64_t));
852 	} else {
853 		val64 = BSWAP_64(bp->blk_cksum.zc_word[2]);
854 		bcopy(&val64, mac, sizeof (uint64_t));
855 
856 		val64 = BSWAP_64(bp->blk_cksum.zc_word[3]);
857 		bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t));
858 	}
859 }
860 
861 void
zio_crypt_encode_mac_zil(void * data,uint8_t * mac)862 zio_crypt_encode_mac_zil(void *data, uint8_t *mac)
863 {
864 	zil_chain_t *zilc = data;
865 
866 	bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t));
867 	bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3],
868 	    sizeof (uint64_t));
869 }
870 
871 void
zio_crypt_decode_mac_zil(const void * data,uint8_t * mac)872 zio_crypt_decode_mac_zil(const void *data, uint8_t *mac)
873 {
874 	/*
875 	 * The ZIL MAC is embedded in the block it protects, which will
876 	 * not have been byteswapped by the time this function has been called.
877 	 * As a result, we don't need to worry about byteswapping the MAC.
878 	 */
879 	const zil_chain_t *zilc = data;
880 
881 	bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t));
882 	bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t),
883 	    sizeof (uint64_t));
884 }
885 
886 /*
887  * This routine takes a block of dnodes (src_abd) and copies only the bonus
888  * buffers to the same offsets in the dst buffer. datalen should be the size
889  * of both the src_abd and the dst buffer (not just the length of the bonus
890  * buffers).
891  */
892 void
zio_crypt_copy_dnode_bonus(abd_t * src_abd,uint8_t * dst,uint_t datalen)893 zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
894 {
895 	uint_t i, max_dnp = datalen >> DNODE_SHIFT;
896 	uint8_t *src;
897 	dnode_phys_t *dnp, *sdnp, *ddnp;
898 
899 	src = abd_borrow_buf_copy(src_abd, datalen);
900 
901 	sdnp = (dnode_phys_t *)src;
902 	ddnp = (dnode_phys_t *)dst;
903 
904 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
905 		dnp = &sdnp[i];
906 		if (dnp->dn_type != DMU_OT_NONE &&
907 		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
908 		    dnp->dn_bonuslen != 0) {
909 			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]),
910 			    DN_MAX_BONUS_LEN(dnp));
911 		}
912 	}
913 
914 	abd_return_buf(src_abd, src, datalen);
915 }
916 
917 /*
918  * This function decides what fields from blk_prop are included in
919  * the on-disk various MAC algorithms.
920  */
921 static void
zio_crypt_bp_zero_nonportable_blkprop(blkptr_t * bp,uint64_t version)922 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version)
923 {
924 	/*
925 	 * Version 0 did not properly zero out all non-portable fields
926 	 * as it should have done. We maintain this code so that we can
927 	 * do read-only imports of pools on this version.
928 	 */
929 	if (version == 0) {
930 		BP_SET_DEDUP(bp, 0);
931 		BP_SET_CHECKSUM(bp, 0);
932 		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
933 		return;
934 	}
935 
936 	ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
937 
938 	/*
939 	 * The hole_birth feature might set these fields even if this bp
940 	 * is a hole. We zero them out here to guarantee that raw sends
941 	 * will function with or without the feature.
942 	 */
943 	if (BP_IS_HOLE(bp)) {
944 		bp->blk_prop = 0ULL;
945 		return;
946 	}
947 
948 	/*
949 	 * At L0 we want to verify these fields to ensure that data blocks
950 	 * can not be reinterpretted. For instance, we do not want an attacker
951 	 * to trick us into returning raw lz4 compressed data to the user
952 	 * by modifying the compression bits. At higher levels, we cannot
953 	 * enforce this policy since raw sends do not convey any information
954 	 * about indirect blocks, so these values might be different on the
955 	 * receive side. Fortunately, this does not open any new attack
956 	 * vectors, since any alterations that can be made to a higher level
957 	 * bp must still verify the correct order of the layer below it.
958 	 */
959 	if (BP_GET_LEVEL(bp) != 0) {
960 		BP_SET_BYTEORDER(bp, 0);
961 		BP_SET_COMPRESS(bp, 0);
962 
963 		/*
964 		 * psize cannot be set to zero or it will trigger
965 		 * asserts, but the value doesn't really matter as
966 		 * long as it is constant.
967 		 */
968 		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
969 	}
970 
971 	BP_SET_DEDUP(bp, 0);
972 	BP_SET_CHECKSUM(bp, 0);
973 }
974 
975 static void
zio_crypt_bp_auth_init(uint64_t version,boolean_t should_bswap,blkptr_t * bp,blkptr_auth_buf_t * bab,uint_t * bab_len)976 zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp,
977     blkptr_auth_buf_t *bab, uint_t *bab_len)
978 {
979 	blkptr_t tmpbp = *bp;
980 
981 	if (should_bswap)
982 		byteswap_uint64_array(&tmpbp, sizeof (blkptr_t));
983 
984 	ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp));
985 	ASSERT0(BP_IS_EMBEDDED(&tmpbp));
986 
987 	zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac);
988 
989 	/*
990 	 * We always MAC blk_prop in LE to ensure portability. This
991 	 * must be done after decoding the mac, since the endianness
992 	 * will get zero'd out here.
993 	 */
994 	zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version);
995 	bab->bab_prop = LE_64(tmpbp.blk_prop);
996 	bab->bab_pad = 0ULL;
997 
998 	/* version 0 did not include the padding */
999 	*bab_len = sizeof (blkptr_auth_buf_t);
1000 	if (version == 0)
1001 		*bab_len -= sizeof (uint64_t);
1002 }
1003 
1004 static int
zio_crypt_bp_do_hmac_updates(crypto_context_t ctx,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1005 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version,
1006     boolean_t should_bswap, blkptr_t *bp)
1007 {
1008 	int ret;
1009 	uint_t bab_len;
1010 	blkptr_auth_buf_t bab;
1011 	crypto_data_t cd;
1012 
1013 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1014 	cd.cd_format = CRYPTO_DATA_RAW;
1015 	cd.cd_offset = 0;
1016 	cd.cd_length = bab_len;
1017 	cd.cd_raw.iov_base = (char *)&bab;
1018 	cd.cd_raw.iov_len = cd.cd_length;
1019 
1020 	ret = crypto_mac_update(ctx, &cd, NULL);
1021 	if (ret != CRYPTO_SUCCESS) {
1022 		ret = SET_ERROR(EIO);
1023 		goto error;
1024 	}
1025 
1026 	return (0);
1027 
1028 error:
1029 	return (ret);
1030 }
1031 
1032 static void
zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX * ctx,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1033 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version,
1034     boolean_t should_bswap, blkptr_t *bp)
1035 {
1036 	uint_t bab_len;
1037 	blkptr_auth_buf_t bab;
1038 
1039 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1040 	SHA2Update(ctx, &bab, bab_len);
1041 }
1042 
1043 static void
zio_crypt_bp_do_aad_updates(uint8_t ** aadp,uint_t * aad_len,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1044 zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version,
1045     boolean_t should_bswap, blkptr_t *bp)
1046 {
1047 	uint_t bab_len;
1048 	blkptr_auth_buf_t bab;
1049 
1050 	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1051 	bcopy(&bab, *aadp, bab_len);
1052 	*aadp += bab_len;
1053 	*aad_len += bab_len;
1054 }
1055 
1056 static int
zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx,uint64_t version,boolean_t should_bswap,dnode_phys_t * dnp)1057 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version,
1058     boolean_t should_bswap, dnode_phys_t *dnp)
1059 {
1060 	int ret, i;
1061 	dnode_phys_t *adnp;
1062 	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1063 	crypto_data_t cd;
1064 	uint8_t tmp_dncore[sizeof (dnode_phys_t)];
1065 
1066 	adnp = (dnode_phys_t *)tmp_dncore;
1067 	cd.cd_format = CRYPTO_DATA_RAW;
1068 	cd.cd_offset = 0;
1069 	cd.cd_length = offsetof(dnode_phys_t, dn_blkptr);
1070 	cd.cd_raw.iov_base = (char *)adnp;
1071 	cd.cd_raw.iov_len = cd.cd_length;
1072 
1073 	/* authenticate the core dnode (masking out non-portable bits) */
1074 	bcopy(dnp, tmp_dncore, cd.cd_length);
1075 	if (le_bswap) {
1076 		adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec);
1077 		adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen);
1078 		adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid);
1079 		adnp->dn_used = BSWAP_64(adnp->dn_used);
1080 	}
1081 	adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1082 	adnp->dn_used = 0;
1083 
1084 	ret = crypto_mac_update(ctx, &cd, NULL);
1085 	if (ret != CRYPTO_SUCCESS) {
1086 		ret = SET_ERROR(EIO);
1087 		goto error;
1088 	}
1089 
1090 	for (i = 0; i < dnp->dn_nblkptr; i++) {
1091 		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1092 		    should_bswap, &dnp->dn_blkptr[i]);
1093 		if (ret != 0)
1094 			goto error;
1095 	}
1096 
1097 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1098 		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1099 		    should_bswap, DN_SPILL_BLKPTR(dnp));
1100 		if (ret != 0)
1101 			goto error;
1102 	}
1103 
1104 	return (0);
1105 
1106 error:
1107 	return (ret);
1108 }
1109 
1110 /*
1111  * objset_phys_t blocks introduce a number of exceptions to the normal
1112  * authentication process. objset_phys_t's contain 2 seperate HMACS for
1113  * protecting the integrity of their data. The portable_mac protects the
1114  * the metadnode. This MAC can be sent with a raw send and protects against
1115  * reordering of data within the metadnode. The local_mac protects the user
1116  * accounting objects which are not sent from one system to another.
1117  *
1118  * In addition, objset blocks are the only blocks that can be modified and
1119  * written to disk without the key loaded under certain circumstances. During
1120  * zil_claim() we need to be able to update the zil_header_t to complete
1121  * claiming log blocks and during raw receives we need to write out the
1122  * portable_mac from the send file. Both of these actions are possible
1123  * because these fields are not protected by either MAC so neither one will
1124  * need to modify the MACs without the key. However, when the modified blocks
1125  * are written out they will be byteswapped into the host machine's native
1126  * endianness which will modify fields protected by the MAC. As a result, MAC
1127  * calculation for objset blocks works slightly differently from other block
1128  * types. Where other block types MAC the data in whatever endianness is
1129  * written to disk, objset blocks always MAC little endian version of their
1130  * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1131  * and le_bswap indicates whether a byteswap is needed to get this block
1132  * into little endian format.
1133  */
1134 /* ARGSUSED */
1135 int
zio_crypt_do_objset_hmacs(zio_crypt_key_t * key,void * data,uint_t datalen,boolean_t should_bswap,uint8_t * portable_mac,uint8_t * local_mac)1136 zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
1137     boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac)
1138 {
1139 	int ret;
1140 	crypto_mechanism_t mech;
1141 	crypto_context_t ctx;
1142 	crypto_data_t cd;
1143 	objset_phys_t *osp = data;
1144 	uint64_t intval;
1145 	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1146 	uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
1147 	uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
1148 
1149 	/* initialize HMAC mechanism */
1150 	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
1151 	mech.cm_param = NULL;
1152 	mech.cm_param_len = 0;
1153 
1154 	cd.cd_format = CRYPTO_DATA_RAW;
1155 	cd.cd_offset = 0;
1156 
1157 	/* calculate the portable MAC from the portable fields and metadnode */
1158 	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1159 	if (ret != CRYPTO_SUCCESS) {
1160 		ret = SET_ERROR(EIO);
1161 		goto error;
1162 	}
1163 
1164 	/* add in the os_type */
1165 	intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type);
1166 	cd.cd_length = sizeof (uint64_t);
1167 	cd.cd_raw.iov_base = (char *)&intval;
1168 	cd.cd_raw.iov_len = cd.cd_length;
1169 
1170 	ret = crypto_mac_update(ctx, &cd, NULL);
1171 	if (ret != CRYPTO_SUCCESS) {
1172 		ret = SET_ERROR(EIO);
1173 		goto error;
1174 	}
1175 
1176 	/* add in the portable os_flags */
1177 	intval = osp->os_flags;
1178 	if (should_bswap)
1179 		intval = BSWAP_64(intval);
1180 	intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1181 	/* CONSTCOND */
1182 	if (!ZFS_HOST_BYTEORDER)
1183 		intval = BSWAP_64(intval);
1184 
1185 	cd.cd_length = sizeof (uint64_t);
1186 	cd.cd_raw.iov_base = (char *)&intval;
1187 	cd.cd_raw.iov_len = cd.cd_length;
1188 
1189 	ret = crypto_mac_update(ctx, &cd, NULL);
1190 	if (ret != CRYPTO_SUCCESS) {
1191 		ret = SET_ERROR(EIO);
1192 		goto error;
1193 	}
1194 
1195 	/* add in fields from the metadnode */
1196 	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1197 	    should_bswap, &osp->os_meta_dnode);
1198 	if (ret)
1199 		goto error;
1200 
1201 	/* store the final digest in a temporary buffer and copy what we need */
1202 	cd.cd_length = SHA512_DIGEST_LENGTH;
1203 	cd.cd_raw.iov_base = (char *)raw_portable_mac;
1204 	cd.cd_raw.iov_len = cd.cd_length;
1205 
1206 	ret = crypto_mac_final(ctx, &cd, NULL);
1207 	if (ret != CRYPTO_SUCCESS) {
1208 		ret = SET_ERROR(EIO);
1209 		goto error;
1210 	}
1211 
1212 	bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN);
1213 
1214 	/*
1215 	 * The local MAC protects the user, group and project accounting.
1216 	 * If these objects are not present, the local MAC is zeroed out.
1217 	 */
1218 	if ((datalen >= OBJSET_PHYS_SIZE_V3 &&
1219 	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1220 	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE &&
1221 	    osp->os_projectused_dnode.dn_type == DMU_OT_NONE) ||
1222 	    (datalen >= OBJSET_PHYS_SIZE_V2 &&
1223 	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1224 	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE) ||
1225 	    (datalen <= OBJSET_PHYS_SIZE_V1)) {
1226 		bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1227 		return (0);
1228 	}
1229 
1230 	/* calculate the local MAC from the userused and groupused dnodes */
1231 	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1232 	if (ret != CRYPTO_SUCCESS) {
1233 		ret = SET_ERROR(EIO);
1234 		goto error;
1235 	}
1236 
1237 	/* add in the non-portable os_flags */
1238 	intval = osp->os_flags;
1239 	if (should_bswap)
1240 		intval = BSWAP_64(intval);
1241 	intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1242 	/* CONSTCOND */
1243 	if (!ZFS_HOST_BYTEORDER)
1244 		intval = BSWAP_64(intval);
1245 
1246 	cd.cd_length = sizeof (uint64_t);
1247 	cd.cd_raw.iov_base = (char *)&intval;
1248 	cd.cd_raw.iov_len = cd.cd_length;
1249 
1250 	ret = crypto_mac_update(ctx, &cd, NULL);
1251 	if (ret != CRYPTO_SUCCESS) {
1252 		ret = SET_ERROR(EIO);
1253 		goto error;
1254 	}
1255 
1256 	/* add in fields from the user accounting dnodes */
1257 	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1258 	    should_bswap, &osp->os_userused_dnode);
1259 	if (ret)
1260 		goto error;
1261 
1262 	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1263 	    should_bswap, &osp->os_groupused_dnode);
1264 	if (ret)
1265 		goto error;
1266 
1267 	/* store the final digest in a temporary buffer and copy what we need */
1268 	cd.cd_length = SHA512_DIGEST_LENGTH;
1269 	cd.cd_raw.iov_base = (char *)raw_local_mac;
1270 	cd.cd_raw.iov_len = cd.cd_length;
1271 
1272 	ret = crypto_mac_final(ctx, &cd, NULL);
1273 	if (ret != CRYPTO_SUCCESS) {
1274 		ret = SET_ERROR(EIO);
1275 		goto error;
1276 	}
1277 
1278 	bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN);
1279 
1280 	return (0);
1281 
1282 error:
1283 	bzero(portable_mac, ZIO_OBJSET_MAC_LEN);
1284 	bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1285 	return (ret);
1286 }
1287 
1288 static void
zio_crypt_destroy_uio(uio_t * uio)1289 zio_crypt_destroy_uio(uio_t *uio)
1290 {
1291 	if (uio->uio_iov)
1292 		kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t));
1293 }
1294 
1295 /*
1296  * This function parses an uncompressed indirect block and returns a checksum
1297  * of all the portable fields from all of the contained bps. The portable
1298  * fields are the MAC and all of the fields from blk_prop except for the dedup,
1299  * checksum, and psize bits. For an explanation of the purpose of this, see
1300  * the comment block on object set authentication.
1301  */
1302 static int
zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate,void * buf,uint_t datalen,uint64_t version,boolean_t byteswap,uint8_t * cksum)1303 zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf,
1304     uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum)
1305 {
1306 	blkptr_t *bp;
1307 	int i, epb = datalen >> SPA_BLKPTRSHIFT;
1308 	SHA2_CTX ctx;
1309 	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
1310 
1311 	/* checksum all of the MACs from the layer below */
1312 	SHA2Init(SHA512, &ctx);
1313 	for (i = 0, bp = buf; i < epb; i++, bp++) {
1314 		zio_crypt_bp_do_indrect_checksum_updates(&ctx, version,
1315 		    byteswap, bp);
1316 	}
1317 	SHA2Final(digestbuf, &ctx);
1318 
1319 	if (generate) {
1320 		bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN);
1321 		return (0);
1322 	}
1323 
1324 	if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0)
1325 		return (SET_ERROR(ECKSUM));
1326 
1327 	return (0);
1328 }
1329 
1330 int
zio_crypt_do_indirect_mac_checksum(boolean_t generate,void * buf,uint_t datalen,boolean_t byteswap,uint8_t * cksum)1331 zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
1332     uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1333 {
1334 	int ret;
1335 
1336 	/*
1337 	 * Unfortunately, callers of this function will not always have
1338 	 * easy access to the on-disk format version. This info is
1339 	 * normally found in the DSL Crypto Key, but the checksum-of-MACs
1340 	 * is expected to be verifiable even when the key isn't loaded.
1341 	 * Here, instead of doing a ZAP lookup for the version for each
1342 	 * zio, we simply try both existing formats.
1343 	 */
1344 	ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf,
1345 	    datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum);
1346 	if (ret == ECKSUM) {
1347 		ASSERT(!generate);
1348 		ret = zio_crypt_do_indirect_mac_checksum_impl(generate,
1349 		    buf, datalen, 0, byteswap, cksum);
1350 	}
1351 
1352 	return (ret);
1353 }
1354 
1355 int
zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate,abd_t * abd,uint_t datalen,boolean_t byteswap,uint8_t * cksum)1356 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
1357     uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1358 {
1359 	int ret;
1360 	void *buf;
1361 
1362 	buf = abd_borrow_buf_copy(abd, datalen);
1363 	ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen,
1364 	    byteswap, cksum);
1365 	abd_return_buf(abd, buf, datalen);
1366 
1367 	return (ret);
1368 }
1369 
1370 /*
1371  * Special case handling routine for encrypting / decrypting ZIL blocks.
1372  * We do not check for the older ZIL chain because the encryption feature
1373  * was not available before the newer ZIL chain was introduced. The goal
1374  * here is to encrypt everything except the blkptr_t of a lr_write_t and
1375  * the zil_chain_t header. Everything that is not encrypted is authenticated.
1376  */
1377 
1378 /* ARGSUSED */
1379 static int
zio_crypt_init_uios_zil(boolean_t encrypt,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1380 zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
1381     uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio,
1382     uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len,
1383     boolean_t *no_crypt)
1384 {
1385 	int ret;
1386 	uint64_t txtype, lr_len;
1387 	uint_t nr_src, nr_dst, crypt_len;
1388 	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1389 	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1390 	uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
1391 	zil_chain_t *zilc;
1392 	lr_t *lr;
1393 	uint8_t *aadbuf = zio_buf_alloc(datalen);
1394 
1395 	/* cipherbuf always needs an extra iovec for the MAC */
1396 	if (encrypt) {
1397 		src = plainbuf;
1398 		dst = cipherbuf;
1399 		nr_src = 0;
1400 		nr_dst = 1;
1401 	} else {
1402 		src = cipherbuf;
1403 		dst = plainbuf;
1404 		nr_src = 1;
1405 		nr_dst = 0;
1406 	}
1407 
1408 	/* find the start and end record of the log block */
1409 	zilc = (zil_chain_t *)src;
1410 	slrp = src + sizeof (zil_chain_t);
1411 	aadp = aadbuf;
1412 	blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused);
1413 
1414 	/* calculate the number of encrypted iovecs we will need */
1415 	for (; slrp < blkend; slrp += lr_len) {
1416 		lr = (lr_t *)slrp;
1417 
1418 		if (!byteswap) {
1419 			txtype = lr->lrc_txtype;
1420 			lr_len = lr->lrc_reclen;
1421 		} else {
1422 			txtype = BSWAP_64(lr->lrc_txtype);
1423 			lr_len = BSWAP_64(lr->lrc_reclen);
1424 		}
1425 
1426 		nr_iovecs++;
1427 		if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t))
1428 			nr_iovecs++;
1429 	}
1430 
1431 	nr_src += nr_iovecs;
1432 	nr_dst += nr_iovecs;
1433 
1434 	/* allocate the iovec arrays */
1435 	if (nr_src != 0) {
1436 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1437 		if (src_iovecs == NULL) {
1438 			ret = SET_ERROR(ENOMEM);
1439 			goto error;
1440 		}
1441 	}
1442 
1443 	if (nr_dst != 0) {
1444 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1445 		if (dst_iovecs == NULL) {
1446 			ret = SET_ERROR(ENOMEM);
1447 			goto error;
1448 		}
1449 	}
1450 
1451 	/*
1452 	 * Copy the plain zil header over and authenticate everything except
1453 	 * the checksum that will store our MAC. If we are writing the data
1454 	 * the embedded checksum will not have been calculated yet, so we don't
1455 	 * authenticate that.
1456 	 */
1457 	bcopy(src, dst, sizeof (zil_chain_t));
1458 	bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t));
1459 	aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1460 	aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1461 
1462 	/* loop over records again, filling in iovecs */
1463 	nr_iovecs = 0;
1464 	slrp = src + sizeof (zil_chain_t);
1465 	dlrp = dst + sizeof (zil_chain_t);
1466 
1467 	for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) {
1468 		lr = (lr_t *)slrp;
1469 
1470 		if (!byteswap) {
1471 			txtype = lr->lrc_txtype;
1472 			lr_len = lr->lrc_reclen;
1473 		} else {
1474 			txtype = BSWAP_64(lr->lrc_txtype);
1475 			lr_len = BSWAP_64(lr->lrc_reclen);
1476 		}
1477 
1478 		/* copy the common lr_t */
1479 		bcopy(slrp, dlrp, sizeof (lr_t));
1480 		bcopy(slrp, aadp, sizeof (lr_t));
1481 		aadp += sizeof (lr_t);
1482 		aad_len += sizeof (lr_t);
1483 
1484 		ASSERT3P(src_iovecs, !=, NULL);
1485 		ASSERT3P(dst_iovecs, !=, NULL);
1486 
1487 		/*
1488 		 * If this is a TX_WRITE record we want to encrypt everything
1489 		 * except the bp if exists. If the bp does exist we want to
1490 		 * authenticate it.
1491 		 */
1492 		if (txtype == TX_WRITE) {
1493 			crypt_len = sizeof (lr_write_t) -
1494 			    sizeof (lr_t) - sizeof (blkptr_t);
1495 			src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1496 			    sizeof (lr_t);
1497 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1498 			dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1499 			    sizeof (lr_t);
1500 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1501 
1502 			/* copy the bp now since it will not be encrypted */
1503 			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1504 			    dlrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1505 			    sizeof (blkptr_t));
1506 			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1507 			    aadp, sizeof (blkptr_t));
1508 			aadp += sizeof (blkptr_t);
1509 			aad_len += sizeof (blkptr_t);
1510 			nr_iovecs++;
1511 			total_len += crypt_len;
1512 
1513 			if (lr_len != sizeof (lr_write_t)) {
1514 				crypt_len = lr_len - sizeof (lr_write_t);
1515 				src_iovecs[nr_iovecs].iov_base = (char *)
1516 				    slrp + sizeof (lr_write_t);
1517 				src_iovecs[nr_iovecs].iov_len = crypt_len;
1518 				dst_iovecs[nr_iovecs].iov_base = (char *)
1519 				    dlrp + sizeof (lr_write_t);
1520 				dst_iovecs[nr_iovecs].iov_len = crypt_len;
1521 				nr_iovecs++;
1522 				total_len += crypt_len;
1523 			}
1524 		} else {
1525 			crypt_len = lr_len - sizeof (lr_t);
1526 			src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1527 			    sizeof (lr_t);
1528 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1529 			dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1530 			    sizeof (lr_t);
1531 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1532 			nr_iovecs++;
1533 			total_len += crypt_len;
1534 		}
1535 	}
1536 
1537 	*no_crypt = (nr_iovecs == 0);
1538 	*enc_len = total_len;
1539 	*authbuf = aadbuf;
1540 	*auth_len = aad_len;
1541 
1542 	if (encrypt) {
1543 		puio->uio_iov = src_iovecs;
1544 		puio->uio_iovcnt = nr_src;
1545 		cuio->uio_iov = dst_iovecs;
1546 		cuio->uio_iovcnt = nr_dst;
1547 	} else {
1548 		puio->uio_iov = dst_iovecs;
1549 		puio->uio_iovcnt = nr_dst;
1550 		cuio->uio_iov = src_iovecs;
1551 		cuio->uio_iovcnt = nr_src;
1552 	}
1553 
1554 	return (0);
1555 
1556 error:
1557 	zio_buf_free(aadbuf, datalen);
1558 	if (src_iovecs != NULL)
1559 		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1560 	if (dst_iovecs != NULL)
1561 		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1562 
1563 	*enc_len = 0;
1564 	*authbuf = NULL;
1565 	*auth_len = 0;
1566 	*no_crypt = B_FALSE;
1567 	puio->uio_iov = NULL;
1568 	puio->uio_iovcnt = 0;
1569 	cuio->uio_iov = NULL;
1570 	cuio->uio_iovcnt = 0;
1571 	return (ret);
1572 }
1573 
1574 /*
1575  * Special case handling routine for encrypting / decrypting dnode blocks.
1576  */
1577 static int
zio_crypt_init_uios_dnode(boolean_t encrypt,uint64_t version,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1578 zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version,
1579     uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1580     uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1581     uint_t *auth_len, boolean_t *no_crypt)
1582 {
1583 	int ret;
1584 	uint_t nr_src, nr_dst, crypt_len;
1585 	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1586 	uint_t i, j, max_dnp = datalen >> DNODE_SHIFT;
1587 	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1588 	uint8_t *src, *dst, *aadp;
1589 	dnode_phys_t *dnp, *adnp, *sdnp, *ddnp;
1590 	uint8_t *aadbuf = zio_buf_alloc(datalen);
1591 
1592 	if (encrypt) {
1593 		src = plainbuf;
1594 		dst = cipherbuf;
1595 		nr_src = 0;
1596 		nr_dst = 1;
1597 	} else {
1598 		src = cipherbuf;
1599 		dst = plainbuf;
1600 		nr_src = 1;
1601 		nr_dst = 0;
1602 	}
1603 
1604 	sdnp = (dnode_phys_t *)src;
1605 	ddnp = (dnode_phys_t *)dst;
1606 	aadp = aadbuf;
1607 
1608 	/*
1609 	 * Count the number of iovecs we will need to do the encryption by
1610 	 * counting the number of bonus buffers that need to be encrypted.
1611 	 */
1612 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1613 		/*
1614 		 * This block may still be byteswapped. However, all of the
1615 		 * values we use are either uint8_t's (for which byteswapping
1616 		 * is a noop) or a * != 0 check, which will work regardless
1617 		 * of whether or not we byteswap.
1618 		 */
1619 		if (sdnp[i].dn_type != DMU_OT_NONE &&
1620 		    DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) &&
1621 		    sdnp[i].dn_bonuslen != 0) {
1622 			nr_iovecs++;
1623 		}
1624 	}
1625 
1626 	nr_src += nr_iovecs;
1627 	nr_dst += nr_iovecs;
1628 
1629 	if (nr_src != 0) {
1630 		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1631 		if (src_iovecs == NULL) {
1632 			ret = SET_ERROR(ENOMEM);
1633 			goto error;
1634 		}
1635 	}
1636 
1637 	if (nr_dst != 0) {
1638 		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1639 		if (dst_iovecs == NULL) {
1640 			ret = SET_ERROR(ENOMEM);
1641 			goto error;
1642 		}
1643 	}
1644 
1645 	nr_iovecs = 0;
1646 
1647 	/*
1648 	 * Iterate through the dnodes again, this time filling in the uios
1649 	 * we allocated earlier. We also concatenate any data we want to
1650 	 * authenticate onto aadbuf.
1651 	 */
1652 	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1653 		dnp = &sdnp[i];
1654 		/* copy over the core fields and blkptrs (kept as plaintext) */
1655 		bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp);
1656 		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1657 			bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]),
1658 			    sizeof (blkptr_t));
1659 		}
1660 
1661 		/*
1662 		 * Handle authenticated data. We authenticate everything in
1663 		 * the dnode that can be brought over when we do a raw send.
1664 		 * This includes all of the core fields as well as the MACs
1665 		 * stored in the bp checksums and all of the portable bits
1666 		 * from blk_prop. We include the dnode padding here in case it
1667 		 * ever gets used in the future. Some dn_flags and dn_used are
1668 		 * not portable so we mask those out values out of the
1669 		 * authenticated data.
1670 		 */
1671 		crypt_len = offsetof(dnode_phys_t, dn_blkptr);
1672 		bcopy(dnp, aadp, crypt_len);
1673 		adnp = (dnode_phys_t *)aadp;
1674 		adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1675 		adnp->dn_used = 0;
1676 		aadp += crypt_len;
1677 		aad_len += crypt_len;
1678 
1679 		for (j = 0; j < dnp->dn_nblkptr; j++) {
1680 			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1681 			    version, byteswap, &dnp->dn_blkptr[j]);
1682 		}
1683 
1684 		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1685 			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1686 			    version, byteswap, DN_SPILL_BLKPTR(dnp));
1687 		}
1688 
1689 		/*
1690 		 * If this bonus buffer needs to be encrypted, we prepare an
1691 		 * iovec_t. The encryption / decryption functions will fill
1692 		 * this in for us with the encrypted or decrypted data.
1693 		 * Otherwise we add the bonus buffer to the authenticated
1694 		 * data buffer and copy it over to the destination. The
1695 		 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1696 		 * we can guarantee alignment with the AES block size
1697 		 * (128 bits).
1698 		 */
1699 		crypt_len = DN_MAX_BONUS_LEN(dnp);
1700 		if (dnp->dn_type != DMU_OT_NONE &&
1701 		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
1702 		    dnp->dn_bonuslen != 0) {
1703 			ASSERT3U(nr_iovecs, <, nr_src);
1704 			ASSERT3U(nr_iovecs, <, nr_dst);
1705 			ASSERT3P(src_iovecs, !=, NULL);
1706 			ASSERT3P(dst_iovecs, !=, NULL);
1707 			src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
1708 			src_iovecs[nr_iovecs].iov_len = crypt_len;
1709 			dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
1710 			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1711 
1712 			nr_iovecs++;
1713 			total_len += crypt_len;
1714 		} else {
1715 			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len);
1716 			bcopy(DN_BONUS(dnp), aadp, crypt_len);
1717 			aadp += crypt_len;
1718 			aad_len += crypt_len;
1719 		}
1720 	}
1721 
1722 	*no_crypt = (nr_iovecs == 0);
1723 	*enc_len = total_len;
1724 	*authbuf = aadbuf;
1725 	*auth_len = aad_len;
1726 
1727 	if (encrypt) {
1728 		puio->uio_iov = src_iovecs;
1729 		puio->uio_iovcnt = nr_src;
1730 		cuio->uio_iov = dst_iovecs;
1731 		cuio->uio_iovcnt = nr_dst;
1732 	} else {
1733 		puio->uio_iov = dst_iovecs;
1734 		puio->uio_iovcnt = nr_dst;
1735 		cuio->uio_iov = src_iovecs;
1736 		cuio->uio_iovcnt = nr_src;
1737 	}
1738 
1739 	return (0);
1740 
1741 error:
1742 	zio_buf_free(aadbuf, datalen);
1743 	if (src_iovecs != NULL)
1744 		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1745 	if (dst_iovecs != NULL)
1746 		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1747 
1748 	*enc_len = 0;
1749 	*authbuf = NULL;
1750 	*auth_len = 0;
1751 	*no_crypt = B_FALSE;
1752 	puio->uio_iov = NULL;
1753 	puio->uio_iovcnt = 0;
1754 	cuio->uio_iov = NULL;
1755 	cuio->uio_iovcnt = 0;
1756 	return (ret);
1757 }
1758 
1759 /* ARGSUSED */
1760 static int
zio_crypt_init_uios_normal(boolean_t encrypt,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,uio_t * puio,uio_t * cuio,uint_t * enc_len)1761 zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf,
1762     uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio,
1763     uint_t *enc_len)
1764 {
1765 	int ret;
1766 	uint_t nr_plain = 1, nr_cipher = 2;
1767 	iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL;
1768 
1769 	/* allocate the iovecs for the plain and cipher data */
1770 	plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t),
1771 	    KM_SLEEP);
1772 	if (!plain_iovecs) {
1773 		ret = SET_ERROR(ENOMEM);
1774 		goto error;
1775 	}
1776 
1777 	cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t),
1778 	    KM_SLEEP);
1779 	if (!cipher_iovecs) {
1780 		ret = SET_ERROR(ENOMEM);
1781 		goto error;
1782 	}
1783 
1784 	plain_iovecs[0].iov_base = (void *)plainbuf;
1785 	plain_iovecs[0].iov_len = datalen;
1786 	cipher_iovecs[0].iov_base = (void *)cipherbuf;
1787 	cipher_iovecs[0].iov_len = datalen;
1788 
1789 	*enc_len = datalen;
1790 	puio->uio_iov = plain_iovecs;
1791 	puio->uio_iovcnt = nr_plain;
1792 	cuio->uio_iov = cipher_iovecs;
1793 	cuio->uio_iovcnt = nr_cipher;
1794 
1795 	return (0);
1796 
1797 error:
1798 	if (plain_iovecs != NULL)
1799 		kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t));
1800 	if (cipher_iovecs != NULL)
1801 		kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t));
1802 
1803 	*enc_len = 0;
1804 	puio->uio_iov = NULL;
1805 	puio->uio_iovcnt = 0;
1806 	cuio->uio_iov = NULL;
1807 	cuio->uio_iovcnt = 0;
1808 	return (ret);
1809 }
1810 
1811 /*
1812  * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1813  * that they can be used for encryption and decryption by zio_do_crypt_uio().
1814  * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1815  * requiring special handling to parse out pieces that are to be encrypted. The
1816  * authbuf is used by these special cases to store additional authenticated
1817  * data (AAD) for the encryption modes.
1818  */
1819 /* ARGSUSED */
1820 static int
zio_crypt_init_uios(boolean_t encrypt,uint64_t version,dmu_object_type_t ot,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uint8_t * mac,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1821 zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot,
1822     uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1823     uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1824     uint_t *auth_len, boolean_t *no_crypt)
1825 {
1826 	int ret;
1827 	iovec_t *mac_iov;
1828 
1829 	ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE);
1830 
1831 	/* route to handler */
1832 	switch (ot) {
1833 	case DMU_OT_INTENT_LOG:
1834 		ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf,
1835 		    datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len,
1836 		    no_crypt);
1837 		break;
1838 	case DMU_OT_DNODE:
1839 		ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf,
1840 		    cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf,
1841 		    auth_len, no_crypt);
1842 		break;
1843 	default:
1844 		ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf,
1845 		    datalen, puio, cuio, enc_len);
1846 		*authbuf = NULL;
1847 		*auth_len = 0;
1848 		*no_crypt = B_FALSE;
1849 		break;
1850 	}
1851 
1852 	if (ret != 0)
1853 		goto error;
1854 
1855 	/* populate the uios */
1856 	puio->uio_segflg = UIO_SYSSPACE;
1857 	cuio->uio_segflg = UIO_SYSSPACE;
1858 
1859 	mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]);
1860 	mac_iov->iov_base = (void *)mac;
1861 	mac_iov->iov_len = ZIO_DATA_MAC_LEN;
1862 
1863 	return (0);
1864 
1865 error:
1866 	return (ret);
1867 }
1868 
1869 /*
1870  * Primary encryption / decryption entrypoint for zio data.
1871  */
1872 int
zio_do_crypt_data(boolean_t encrypt,zio_crypt_key_t * key,dmu_object_type_t ot,boolean_t byteswap,uint8_t * salt,uint8_t * iv,uint8_t * mac,uint_t datalen,uint8_t * plainbuf,uint8_t * cipherbuf,boolean_t * no_crypt)1873 zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
1874     dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
1875     uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
1876     boolean_t *no_crypt)
1877 {
1878 	int ret;
1879 	boolean_t locked = B_FALSE;
1880 	uint64_t crypt = key->zk_crypt;
1881 	uint_t keydata_len = zio_crypt_table[crypt].ci_keylen;
1882 	uint_t enc_len, auth_len;
1883 	uio_t puio, cuio;
1884 	uint8_t enc_keydata[MASTER_KEY_MAX_LEN];
1885 	crypto_key_t tmp_ckey, *ckey = NULL;
1886 	crypto_ctx_template_t tmpl;
1887 	uint8_t *authbuf = NULL;
1888 
1889 	bzero(&puio, sizeof (uio_t));
1890 	bzero(&cuio, sizeof (uio_t));
1891 
1892 	/* create uios for encryption */
1893 	ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf,
1894 	    cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len,
1895 	    &authbuf, &auth_len, no_crypt);
1896 	if (ret != 0)
1897 		return (ret);
1898 
1899 	/*
1900 	 * If the needed key is the current one, just use it. Otherwise we
1901 	 * need to generate a temporary one from the given salt + master key.
1902 	 * If we are encrypting, we must return a copy of the current salt
1903 	 * so that it can be stored in the blkptr_t.
1904 	 */
1905 	rw_enter(&key->zk_salt_lock, RW_READER);
1906 	locked = B_TRUE;
1907 
1908 	if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) {
1909 		ckey = &key->zk_current_key;
1910 		tmpl = key->zk_current_tmpl;
1911 	} else {
1912 		rw_exit(&key->zk_salt_lock);
1913 		locked = B_FALSE;
1914 
1915 		ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
1916 		    salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len);
1917 		if (ret != 0)
1918 			goto error;
1919 
1920 		tmp_ckey.ck_format = CRYPTO_KEY_RAW;
1921 		tmp_ckey.ck_data = enc_keydata;
1922 		tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
1923 
1924 		ckey = &tmp_ckey;
1925 		tmpl = NULL;
1926 	}
1927 
1928 	/* perform the encryption / decryption */
1929 	ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
1930 	    &puio, &cuio, authbuf, auth_len);
1931 	if (ret != 0)
1932 		goto error;
1933 
1934 	if (locked) {
1935 		rw_exit(&key->zk_salt_lock);
1936 		locked = B_FALSE;
1937 	}
1938 
1939 	if (authbuf != NULL)
1940 		zio_buf_free(authbuf, datalen);
1941 	if (ckey == &tmp_ckey)
1942 		bzero(enc_keydata, keydata_len);
1943 	zio_crypt_destroy_uio(&puio);
1944 	zio_crypt_destroy_uio(&cuio);
1945 
1946 	return (0);
1947 
1948 error:
1949 	if (!encrypt) {
1950 		if (failed_decrypt_buf != NULL)
1951 			kmem_free(failed_decrypt_buf, failed_decrypt_size);
1952 		failed_decrypt_buf = kmem_alloc(datalen, KM_SLEEP);
1953 		failed_decrypt_size = datalen;
1954 		bcopy(cipherbuf, failed_decrypt_buf, datalen);
1955 	}
1956 	if (locked)
1957 		rw_exit(&key->zk_salt_lock);
1958 	if (authbuf != NULL)
1959 		zio_buf_free(authbuf, datalen);
1960 	if (ckey == &tmp_ckey)
1961 		bzero(enc_keydata, keydata_len);
1962 	zio_crypt_destroy_uio(&puio);
1963 	zio_crypt_destroy_uio(&cuio);
1964 
1965 	return (ret);
1966 }
1967 
1968 /*
1969  * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1970  * linear buffers.
1971  */
1972 int
zio_do_crypt_abd(boolean_t encrypt,zio_crypt_key_t * key,dmu_object_type_t ot,boolean_t byteswap,uint8_t * salt,uint8_t * iv,uint8_t * mac,uint_t datalen,abd_t * pabd,abd_t * cabd,boolean_t * no_crypt)1973 zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
1974     boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
1975     uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
1976 {
1977 	int ret;
1978 	void *ptmp, *ctmp;
1979 
1980 	if (encrypt) {
1981 		ptmp = abd_borrow_buf_copy(pabd, datalen);
1982 		ctmp = abd_borrow_buf(cabd, datalen);
1983 	} else {
1984 		ptmp = abd_borrow_buf(pabd, datalen);
1985 		ctmp = abd_borrow_buf_copy(cabd, datalen);
1986 	}
1987 
1988 	ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
1989 	    datalen, ptmp, ctmp, no_crypt);
1990 	if (ret != 0)
1991 		goto error;
1992 
1993 	if (encrypt) {
1994 		abd_return_buf(pabd, ptmp, datalen);
1995 		abd_return_buf_copy(cabd, ctmp, datalen);
1996 	} else {
1997 		abd_return_buf_copy(pabd, ptmp, datalen);
1998 		abd_return_buf(cabd, ctmp, datalen);
1999 	}
2000 
2001 	return (0);
2002 
2003 error:
2004 	if (encrypt) {
2005 		abd_return_buf(pabd, ptmp, datalen);
2006 		abd_return_buf_copy(cabd, ctmp, datalen);
2007 	} else {
2008 		abd_return_buf_copy(pabd, ptmp, datalen);
2009 		abd_return_buf(cabd, ctmp, datalen);
2010 	}
2011 
2012 	return (ret);
2013 }
2014