xref: /linux/fs/bcachefs/checksum.c (revision 3c40841cdcf374d661ff14e7a707681342565166)
11c6fdbd8SKent Overstreet // SPDX-License-Identifier: GPL-2.0
21c6fdbd8SKent Overstreet #include "bcachefs.h"
31c6fdbd8SKent Overstreet #include "checksum.h"
4d4bf5eecSKent Overstreet #include "errcode.h"
51c6fdbd8SKent Overstreet #include "super.h"
61c6fdbd8SKent Overstreet #include "super-io.h"
71c6fdbd8SKent Overstreet 
81c6fdbd8SKent Overstreet #include <linux/crc32c.h>
91c6fdbd8SKent Overstreet #include <linux/crypto.h>
1041e63382Sjpsollie #include <linux/xxhash.h>
111c6fdbd8SKent Overstreet #include <linux/key.h>
121c6fdbd8SKent Overstreet #include <linux/random.h>
131c6fdbd8SKent Overstreet #include <linux/scatterlist.h>
141c6fdbd8SKent Overstreet #include <crypto/algapi.h>
151c6fdbd8SKent Overstreet #include <crypto/chacha.h>
161c6fdbd8SKent Overstreet #include <crypto/hash.h>
171c6fdbd8SKent Overstreet #include <crypto/poly1305.h>
181c6fdbd8SKent Overstreet #include <crypto/skcipher.h>
191c6fdbd8SKent Overstreet #include <keys/user-type.h>
201c6fdbd8SKent Overstreet 
2180ff5d18Sjpsollie /*
2280ff5d18Sjpsollie  * bch2_checksum state is an abstraction of the checksum state calculated over different pages.
2380ff5d18Sjpsollie  * it features page merging without having the checksum algorithm lose its state.
2480ff5d18Sjpsollie  * for native checksum aglorithms (like crc), a default seed value will do.
2580ff5d18Sjpsollie  * for hash-like algorithms, a state needs to be stored
2680ff5d18Sjpsollie  */
2780ff5d18Sjpsollie 
2880ff5d18Sjpsollie struct bch2_checksum_state {
2980ff5d18Sjpsollie 	union {
3080ff5d18Sjpsollie 		u64 seed;
3141e63382Sjpsollie 		struct xxh64_state h64state;
3280ff5d18Sjpsollie 	};
3380ff5d18Sjpsollie 	unsigned int type;
3480ff5d18Sjpsollie };
3580ff5d18Sjpsollie 
3680ff5d18Sjpsollie static void bch2_checksum_init(struct bch2_checksum_state *state)
371c6fdbd8SKent Overstreet {
3880ff5d18Sjpsollie 	switch (state->type) {
396404dcc9SKent Overstreet 	case BCH_CSUM_none:
406404dcc9SKent Overstreet 	case BCH_CSUM_crc32c:
416404dcc9SKent Overstreet 	case BCH_CSUM_crc64:
4280ff5d18Sjpsollie 		state->seed = 0;
4380ff5d18Sjpsollie 		break;
446404dcc9SKent Overstreet 	case BCH_CSUM_crc32c_nonzero:
4580ff5d18Sjpsollie 		state->seed = U32_MAX;
4680ff5d18Sjpsollie 		break;
476404dcc9SKent Overstreet 	case BCH_CSUM_crc64_nonzero:
4880ff5d18Sjpsollie 		state->seed = U64_MAX;
4980ff5d18Sjpsollie 		break;
506404dcc9SKent Overstreet 	case BCH_CSUM_xxhash:
5141e63382Sjpsollie 		xxh64_reset(&state->h64state, 0);
5241e63382Sjpsollie 		break;
531c6fdbd8SKent Overstreet 	default:
541c6fdbd8SKent Overstreet 		BUG();
551c6fdbd8SKent Overstreet 	}
561c6fdbd8SKent Overstreet }
571c6fdbd8SKent Overstreet 
5880ff5d18Sjpsollie static u64 bch2_checksum_final(const struct bch2_checksum_state *state)
591c6fdbd8SKent Overstreet {
6080ff5d18Sjpsollie 	switch (state->type) {
616404dcc9SKent Overstreet 	case BCH_CSUM_none:
626404dcc9SKent Overstreet 	case BCH_CSUM_crc32c:
636404dcc9SKent Overstreet 	case BCH_CSUM_crc64:
6480ff5d18Sjpsollie 		return state->seed;
656404dcc9SKent Overstreet 	case BCH_CSUM_crc32c_nonzero:
6680ff5d18Sjpsollie 		return state->seed ^ U32_MAX;
676404dcc9SKent Overstreet 	case BCH_CSUM_crc64_nonzero:
6880ff5d18Sjpsollie 		return state->seed ^ U64_MAX;
696404dcc9SKent Overstreet 	case BCH_CSUM_xxhash:
7041e63382Sjpsollie 		return xxh64_digest(&state->h64state);
711c6fdbd8SKent Overstreet 	default:
721c6fdbd8SKent Overstreet 		BUG();
731c6fdbd8SKent Overstreet 	}
741c6fdbd8SKent Overstreet }
751c6fdbd8SKent Overstreet 
7680ff5d18Sjpsollie static void bch2_checksum_update(struct bch2_checksum_state *state, const void *data, size_t len)
771c6fdbd8SKent Overstreet {
7880ff5d18Sjpsollie 	switch (state->type) {
796404dcc9SKent Overstreet 	case BCH_CSUM_none:
8080ff5d18Sjpsollie 		return;
816404dcc9SKent Overstreet 	case BCH_CSUM_crc32c_nonzero:
826404dcc9SKent Overstreet 	case BCH_CSUM_crc32c:
8380ff5d18Sjpsollie 		state->seed = crc32c(state->seed, data, len);
8480ff5d18Sjpsollie 		break;
856404dcc9SKent Overstreet 	case BCH_CSUM_crc64_nonzero:
866404dcc9SKent Overstreet 	case BCH_CSUM_crc64:
8780ff5d18Sjpsollie 		state->seed = crc64_be(state->seed, data, len);
8880ff5d18Sjpsollie 		break;
896404dcc9SKent Overstreet 	case BCH_CSUM_xxhash:
9041e63382Sjpsollie 		xxh64_update(&state->h64state, data, len);
9141e63382Sjpsollie 		break;
921c6fdbd8SKent Overstreet 	default:
931c6fdbd8SKent Overstreet 		BUG();
941c6fdbd8SKent Overstreet 	}
951c6fdbd8SKent Overstreet }
961c6fdbd8SKent Overstreet 
97a9de137bSKent Overstreet static inline int do_encrypt_sg(struct crypto_sync_skcipher *tfm,
981c6fdbd8SKent Overstreet 				struct nonce nonce,
991c6fdbd8SKent Overstreet 				struct scatterlist *sg, size_t len)
1001c6fdbd8SKent Overstreet {
1011c6fdbd8SKent Overstreet 	SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
1021c6fdbd8SKent Overstreet 	int ret;
1031c6fdbd8SKent Overstreet 
1041c6fdbd8SKent Overstreet 	skcipher_request_set_sync_tfm(req, tfm);
1051c6fdbd8SKent Overstreet 	skcipher_request_set_crypt(req, sg, sg, len, nonce.d);
1061c6fdbd8SKent Overstreet 
1071c6fdbd8SKent Overstreet 	ret = crypto_skcipher_encrypt(req);
108a9de137bSKent Overstreet 	if (ret)
109a9de137bSKent Overstreet 		pr_err("got error %i from crypto_skcipher_encrypt()", ret);
110a9de137bSKent Overstreet 
111a9de137bSKent Overstreet 	return ret;
1121c6fdbd8SKent Overstreet }
1131c6fdbd8SKent Overstreet 
114a9de137bSKent Overstreet static inline int do_encrypt(struct crypto_sync_skcipher *tfm,
1151c6fdbd8SKent Overstreet 			      struct nonce nonce,
1161c6fdbd8SKent Overstreet 			      void *buf, size_t len)
1171c6fdbd8SKent Overstreet {
118636d4eefSKent Overstreet 	if (!is_vmalloc_addr(buf)) {
1191c6fdbd8SKent Overstreet 		struct scatterlist sg;
1201c6fdbd8SKent Overstreet 
121c346def9SKent Overstreet 		sg_init_table(&sg, 1);
122c346def9SKent Overstreet 		sg_set_page(&sg,
123c346def9SKent Overstreet 			    is_vmalloc_addr(buf)
124c346def9SKent Overstreet 			    ? vmalloc_to_page(buf)
125c346def9SKent Overstreet 			    : virt_to_page(buf),
126c346def9SKent Overstreet 			    len, offset_in_page(buf));
127a9de137bSKent Overstreet 		return do_encrypt_sg(tfm, nonce, &sg, len);
128636d4eefSKent Overstreet 	} else {
129636d4eefSKent Overstreet 		unsigned pages = buf_pages(buf, len);
130636d4eefSKent Overstreet 		struct scatterlist *sg;
131636d4eefSKent Overstreet 		size_t orig_len = len;
132636d4eefSKent Overstreet 		int ret, i;
133636d4eefSKent Overstreet 
1343e3e02e6SKent Overstreet 		sg = kmalloc_array(pages, sizeof(*sg), GFP_KERNEL);
135636d4eefSKent Overstreet 		if (!sg)
13665d48e35SKent Overstreet 			return -BCH_ERR_ENOMEM_do_encrypt;
137636d4eefSKent Overstreet 
138636d4eefSKent Overstreet 		sg_init_table(sg, pages);
139636d4eefSKent Overstreet 
140636d4eefSKent Overstreet 		for (i = 0; i < pages; i++) {
141636d4eefSKent Overstreet 			unsigned offset = offset_in_page(buf);
14253eda6f7SNathan Chancellor 			unsigned pg_len = min_t(size_t, len, PAGE_SIZE - offset);
143636d4eefSKent Overstreet 
144636d4eefSKent Overstreet 			sg_set_page(sg + i, vmalloc_to_page(buf), pg_len, offset);
145636d4eefSKent Overstreet 			buf += pg_len;
146636d4eefSKent Overstreet 			len -= pg_len;
147636d4eefSKent Overstreet 		}
148636d4eefSKent Overstreet 
149636d4eefSKent Overstreet 		ret = do_encrypt_sg(tfm, nonce, sg, orig_len);
150636d4eefSKent Overstreet 		kfree(sg);
151636d4eefSKent Overstreet 		return ret;
152636d4eefSKent Overstreet 	}
1531c6fdbd8SKent Overstreet }
1541c6fdbd8SKent Overstreet 
1551c6fdbd8SKent Overstreet int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce,
1561c6fdbd8SKent Overstreet 			    void *buf, size_t len)
1571c6fdbd8SKent Overstreet {
1581c6fdbd8SKent Overstreet 	struct crypto_sync_skcipher *chacha20 =
1591c6fdbd8SKent Overstreet 		crypto_alloc_sync_skcipher("chacha20", 0, 0);
1601c6fdbd8SKent Overstreet 	int ret;
1611c6fdbd8SKent Overstreet 
16275e0c478SKent Overstreet 	ret = PTR_ERR_OR_ZERO(chacha20);
16375e0c478SKent Overstreet 	if (ret) {
16475e0c478SKent Overstreet 		pr_err("error requesting chacha20 cipher: %s", bch2_err_str(ret));
16575e0c478SKent Overstreet 		return ret;
1661c6fdbd8SKent Overstreet 	}
1671c6fdbd8SKent Overstreet 
1681c6fdbd8SKent Overstreet 	ret = crypto_skcipher_setkey(&chacha20->base,
1691c6fdbd8SKent Overstreet 				     (void *) key, sizeof(*key));
1701c6fdbd8SKent Overstreet 	if (ret) {
17175e0c478SKent Overstreet 		pr_err("error from crypto_skcipher_setkey(): %s", bch2_err_str(ret));
1721c6fdbd8SKent Overstreet 		goto err;
1731c6fdbd8SKent Overstreet 	}
1741c6fdbd8SKent Overstreet 
175a9de137bSKent Overstreet 	ret = do_encrypt(chacha20, nonce, buf, len);
1761c6fdbd8SKent Overstreet err:
1771c6fdbd8SKent Overstreet 	crypto_free_sync_skcipher(chacha20);
1781c6fdbd8SKent Overstreet 	return ret;
1791c6fdbd8SKent Overstreet }
1801c6fdbd8SKent Overstreet 
181a9de137bSKent Overstreet static int gen_poly_key(struct bch_fs *c, struct shash_desc *desc,
1821c6fdbd8SKent Overstreet 			struct nonce nonce)
1831c6fdbd8SKent Overstreet {
1841c6fdbd8SKent Overstreet 	u8 key[POLY1305_KEY_SIZE];
185a9de137bSKent Overstreet 	int ret;
1861c6fdbd8SKent Overstreet 
1871c6fdbd8SKent Overstreet 	nonce.d[3] ^= BCH_NONCE_POLY;
1881c6fdbd8SKent Overstreet 
1891c6fdbd8SKent Overstreet 	memset(key, 0, sizeof(key));
190a9de137bSKent Overstreet 	ret = do_encrypt(c->chacha20, nonce, key, sizeof(key));
191a9de137bSKent Overstreet 	if (ret)
192a9de137bSKent Overstreet 		return ret;
1931c6fdbd8SKent Overstreet 
1941c6fdbd8SKent Overstreet 	desc->tfm = c->poly1305;
1951c6fdbd8SKent Overstreet 	crypto_shash_init(desc);
1961c6fdbd8SKent Overstreet 	crypto_shash_update(desc, key, sizeof(key));
197a9de137bSKent Overstreet 	return 0;
1981c6fdbd8SKent Overstreet }
1991c6fdbd8SKent Overstreet 
2001c6fdbd8SKent Overstreet struct bch_csum bch2_checksum(struct bch_fs *c, unsigned type,
2011c6fdbd8SKent Overstreet 			      struct nonce nonce, const void *data, size_t len)
2021c6fdbd8SKent Overstreet {
2031c6fdbd8SKent Overstreet 	switch (type) {
2046404dcc9SKent Overstreet 	case BCH_CSUM_none:
2056404dcc9SKent Overstreet 	case BCH_CSUM_crc32c_nonzero:
2066404dcc9SKent Overstreet 	case BCH_CSUM_crc64_nonzero:
2076404dcc9SKent Overstreet 	case BCH_CSUM_crc32c:
2086404dcc9SKent Overstreet 	case BCH_CSUM_xxhash:
2096404dcc9SKent Overstreet 	case BCH_CSUM_crc64: {
21080ff5d18Sjpsollie 		struct bch2_checksum_state state;
2111c6fdbd8SKent Overstreet 
21280ff5d18Sjpsollie 		state.type = type;
2131c6fdbd8SKent Overstreet 
21480ff5d18Sjpsollie 		bch2_checksum_init(&state);
21580ff5d18Sjpsollie 		bch2_checksum_update(&state, data, len);
21680ff5d18Sjpsollie 
21780ff5d18Sjpsollie 		return (struct bch_csum) { .lo = cpu_to_le64(bch2_checksum_final(&state)) };
2181c6fdbd8SKent Overstreet 	}
2191c6fdbd8SKent Overstreet 
2206404dcc9SKent Overstreet 	case BCH_CSUM_chacha20_poly1305_80:
2216404dcc9SKent Overstreet 	case BCH_CSUM_chacha20_poly1305_128: {
2221c6fdbd8SKent Overstreet 		SHASH_DESC_ON_STACK(desc, c->poly1305);
2231c6fdbd8SKent Overstreet 		u8 digest[POLY1305_DIGEST_SIZE];
2241c6fdbd8SKent Overstreet 		struct bch_csum ret = { 0 };
2251c6fdbd8SKent Overstreet 
2261c6fdbd8SKent Overstreet 		gen_poly_key(c, desc, nonce);
2271c6fdbd8SKent Overstreet 
2281c6fdbd8SKent Overstreet 		crypto_shash_update(desc, data, len);
2291c6fdbd8SKent Overstreet 		crypto_shash_final(desc, digest);
2301c6fdbd8SKent Overstreet 
2311c6fdbd8SKent Overstreet 		memcpy(&ret, digest, bch_crc_bytes[type]);
2321c6fdbd8SKent Overstreet 		return ret;
2331c6fdbd8SKent Overstreet 	}
2341c6fdbd8SKent Overstreet 	default:
2351c6fdbd8SKent Overstreet 		BUG();
2361c6fdbd8SKent Overstreet 	}
2371c6fdbd8SKent Overstreet }
2381c6fdbd8SKent Overstreet 
239a9de137bSKent Overstreet int bch2_encrypt(struct bch_fs *c, unsigned type,
2401c6fdbd8SKent Overstreet 		  struct nonce nonce, void *data, size_t len)
2411c6fdbd8SKent Overstreet {
2421c6fdbd8SKent Overstreet 	if (!bch2_csum_type_is_encryption(type))
243a9de137bSKent Overstreet 		return 0;
2441c6fdbd8SKent Overstreet 
245a9de137bSKent Overstreet 	return do_encrypt(c->chacha20, nonce, data, len);
2461c6fdbd8SKent Overstreet }
2471c6fdbd8SKent Overstreet 
2481c6fdbd8SKent Overstreet static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type,
2491c6fdbd8SKent Overstreet 					   struct nonce nonce, struct bio *bio,
2501c6fdbd8SKent Overstreet 					   struct bvec_iter *iter)
2511c6fdbd8SKent Overstreet {
2521c6fdbd8SKent Overstreet 	struct bio_vec bv;
2531c6fdbd8SKent Overstreet 
2541c6fdbd8SKent Overstreet 	switch (type) {
2556404dcc9SKent Overstreet 	case BCH_CSUM_none:
2561c6fdbd8SKent Overstreet 		return (struct bch_csum) { 0 };
2576404dcc9SKent Overstreet 	case BCH_CSUM_crc32c_nonzero:
2586404dcc9SKent Overstreet 	case BCH_CSUM_crc64_nonzero:
2596404dcc9SKent Overstreet 	case BCH_CSUM_crc32c:
2606404dcc9SKent Overstreet 	case BCH_CSUM_xxhash:
2616404dcc9SKent Overstreet 	case BCH_CSUM_crc64: {
26280ff5d18Sjpsollie 		struct bch2_checksum_state state;
26380ff5d18Sjpsollie 
26480ff5d18Sjpsollie 		state.type = type;
26580ff5d18Sjpsollie 		bch2_checksum_init(&state);
2661c6fdbd8SKent Overstreet 
2671c6fdbd8SKent Overstreet #ifdef CONFIG_HIGHMEM
2681c6fdbd8SKent Overstreet 		__bio_for_each_segment(bv, bio, *iter, *iter) {
2691e81f89bSKent Overstreet 			void *p = kmap_local_page(bv.bv_page) + bv.bv_offset;
2701e81f89bSKent Overstreet 
27180ff5d18Sjpsollie 			bch2_checksum_update(&state, p, bv.bv_len);
2721e81f89bSKent Overstreet 			kunmap_local(p);
2731c6fdbd8SKent Overstreet 		}
2741c6fdbd8SKent Overstreet #else
2750fd7263eSKent Overstreet 		__bio_for_each_bvec(bv, bio, *iter, *iter)
27680ff5d18Sjpsollie 			bch2_checksum_update(&state, page_address(bv.bv_page) + bv.bv_offset,
2771c6fdbd8SKent Overstreet 				bv.bv_len);
2781c6fdbd8SKent Overstreet #endif
27980ff5d18Sjpsollie 		return (struct bch_csum) { .lo = cpu_to_le64(bch2_checksum_final(&state)) };
2801c6fdbd8SKent Overstreet 	}
2811c6fdbd8SKent Overstreet 
2826404dcc9SKent Overstreet 	case BCH_CSUM_chacha20_poly1305_80:
2836404dcc9SKent Overstreet 	case BCH_CSUM_chacha20_poly1305_128: {
2841c6fdbd8SKent Overstreet 		SHASH_DESC_ON_STACK(desc, c->poly1305);
2851c6fdbd8SKent Overstreet 		u8 digest[POLY1305_DIGEST_SIZE];
2861c6fdbd8SKent Overstreet 		struct bch_csum ret = { 0 };
2871c6fdbd8SKent Overstreet 
2881c6fdbd8SKent Overstreet 		gen_poly_key(c, desc, nonce);
2891c6fdbd8SKent Overstreet 
2901c6fdbd8SKent Overstreet #ifdef CONFIG_HIGHMEM
2911c6fdbd8SKent Overstreet 		__bio_for_each_segment(bv, bio, *iter, *iter) {
2921e81f89bSKent Overstreet 			void *p = kmap_local_page(bv.bv_page) + bv.bv_offset;
2931c6fdbd8SKent Overstreet 
2941c6fdbd8SKent Overstreet 			crypto_shash_update(desc, p, bv.bv_len);
2951e81f89bSKent Overstreet 			kunmap_local(p);
2961c6fdbd8SKent Overstreet 		}
2971c6fdbd8SKent Overstreet #else
2980fd7263eSKent Overstreet 		__bio_for_each_bvec(bv, bio, *iter, *iter)
2991c6fdbd8SKent Overstreet 			crypto_shash_update(desc,
3001c6fdbd8SKent Overstreet 				page_address(bv.bv_page) + bv.bv_offset,
3011c6fdbd8SKent Overstreet 				bv.bv_len);
3021c6fdbd8SKent Overstreet #endif
3031c6fdbd8SKent Overstreet 		crypto_shash_final(desc, digest);
3041c6fdbd8SKent Overstreet 
3051c6fdbd8SKent Overstreet 		memcpy(&ret, digest, bch_crc_bytes[type]);
3061c6fdbd8SKent Overstreet 		return ret;
3071c6fdbd8SKent Overstreet 	}
3081c6fdbd8SKent Overstreet 	default:
3091c6fdbd8SKent Overstreet 		BUG();
3101c6fdbd8SKent Overstreet 	}
3111c6fdbd8SKent Overstreet }
3121c6fdbd8SKent Overstreet 
3131c6fdbd8SKent Overstreet struct bch_csum bch2_checksum_bio(struct bch_fs *c, unsigned type,
3141c6fdbd8SKent Overstreet 				  struct nonce nonce, struct bio *bio)
3151c6fdbd8SKent Overstreet {
3161c6fdbd8SKent Overstreet 	struct bvec_iter iter = bio->bi_iter;
3171c6fdbd8SKent Overstreet 
3181c6fdbd8SKent Overstreet 	return __bch2_checksum_bio(c, type, nonce, bio, &iter);
3191c6fdbd8SKent Overstreet }
3201c6fdbd8SKent Overstreet 
3210cc455b3SKent Overstreet int __bch2_encrypt_bio(struct bch_fs *c, unsigned type,
3221c6fdbd8SKent Overstreet 		     struct nonce nonce, struct bio *bio)
3231c6fdbd8SKent Overstreet {
3241c6fdbd8SKent Overstreet 	struct bio_vec bv;
3251c6fdbd8SKent Overstreet 	struct bvec_iter iter;
3261c6fdbd8SKent Overstreet 	struct scatterlist sgl[16], *sg = sgl;
3271c6fdbd8SKent Overstreet 	size_t bytes = 0;
328a9de137bSKent Overstreet 	int ret = 0;
3291c6fdbd8SKent Overstreet 
3301c6fdbd8SKent Overstreet 	if (!bch2_csum_type_is_encryption(type))
331a9de137bSKent Overstreet 		return 0;
3321c6fdbd8SKent Overstreet 
3331c6fdbd8SKent Overstreet 	sg_init_table(sgl, ARRAY_SIZE(sgl));
3341c6fdbd8SKent Overstreet 
3351c6fdbd8SKent Overstreet 	bio_for_each_segment(bv, bio, iter) {
3361c6fdbd8SKent Overstreet 		if (sg == sgl + ARRAY_SIZE(sgl)) {
3371c6fdbd8SKent Overstreet 			sg_mark_end(sg - 1);
338a9de137bSKent Overstreet 
339a9de137bSKent Overstreet 			ret = do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
340a9de137bSKent Overstreet 			if (ret)
341a9de137bSKent Overstreet 				return ret;
3421c6fdbd8SKent Overstreet 
3431c6fdbd8SKent Overstreet 			nonce = nonce_add(nonce, bytes);
3441c6fdbd8SKent Overstreet 			bytes = 0;
3451c6fdbd8SKent Overstreet 
3461c6fdbd8SKent Overstreet 			sg_init_table(sgl, ARRAY_SIZE(sgl));
3471c6fdbd8SKent Overstreet 			sg = sgl;
3481c6fdbd8SKent Overstreet 		}
3491c6fdbd8SKent Overstreet 
3501c6fdbd8SKent Overstreet 		sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset);
3511c6fdbd8SKent Overstreet 		bytes += bv.bv_len;
3521c6fdbd8SKent Overstreet 	}
3531c6fdbd8SKent Overstreet 
3541c6fdbd8SKent Overstreet 	sg_mark_end(sg - 1);
355a9de137bSKent Overstreet 	return do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
3561c6fdbd8SKent Overstreet }
3571c6fdbd8SKent Overstreet 
3586009b4e5SKent Overstreet struct bch_csum bch2_checksum_merge(unsigned type, struct bch_csum a,
3591c6fdbd8SKent Overstreet 				    struct bch_csum b, size_t b_len)
3601c6fdbd8SKent Overstreet {
36180ff5d18Sjpsollie 	struct bch2_checksum_state state;
36280ff5d18Sjpsollie 
36380ff5d18Sjpsollie 	state.type = type;
36480ff5d18Sjpsollie 	bch2_checksum_init(&state);
365*3c40841cSBrian Foster 	state.seed = le64_to_cpu(a.lo);
36680ff5d18Sjpsollie 
3671c6fdbd8SKent Overstreet 	BUG_ON(!bch2_checksum_mergeable(type));
3681c6fdbd8SKent Overstreet 
3691c6fdbd8SKent Overstreet 	while (b_len) {
37096dea3d5SKent Overstreet 		unsigned page_len = min_t(unsigned, b_len, PAGE_SIZE);
3711c6fdbd8SKent Overstreet 
37280ff5d18Sjpsollie 		bch2_checksum_update(&state,
37396dea3d5SKent Overstreet 				page_address(ZERO_PAGE(0)), page_len);
37496dea3d5SKent Overstreet 		b_len -= page_len;
3751c6fdbd8SKent Overstreet 	}
376*3c40841cSBrian Foster 	a.lo = cpu_to_le64(bch2_checksum_final(&state));
3771c6fdbd8SKent Overstreet 	a.lo ^= b.lo;
3781c6fdbd8SKent Overstreet 	a.hi ^= b.hi;
3791c6fdbd8SKent Overstreet 	return a;
3801c6fdbd8SKent Overstreet }
3811c6fdbd8SKent Overstreet 
3821c6fdbd8SKent Overstreet int bch2_rechecksum_bio(struct bch_fs *c, struct bio *bio,
3831c6fdbd8SKent Overstreet 			struct bversion version,
3841c6fdbd8SKent Overstreet 			struct bch_extent_crc_unpacked crc_old,
3851c6fdbd8SKent Overstreet 			struct bch_extent_crc_unpacked *crc_a,
3861c6fdbd8SKent Overstreet 			struct bch_extent_crc_unpacked *crc_b,
3871c6fdbd8SKent Overstreet 			unsigned len_a, unsigned len_b,
3881c6fdbd8SKent Overstreet 			unsigned new_csum_type)
3891c6fdbd8SKent Overstreet {
3901c6fdbd8SKent Overstreet 	struct bvec_iter iter = bio->bi_iter;
3911c6fdbd8SKent Overstreet 	struct nonce nonce = extent_nonce(version, crc_old);
3921c6fdbd8SKent Overstreet 	struct bch_csum merged = { 0 };
3931c6fdbd8SKent Overstreet 	struct crc_split {
3941c6fdbd8SKent Overstreet 		struct bch_extent_crc_unpacked	*crc;
3951c6fdbd8SKent Overstreet 		unsigned			len;
3961c6fdbd8SKent Overstreet 		unsigned			csum_type;
3971c6fdbd8SKent Overstreet 		struct bch_csum			csum;
3981c6fdbd8SKent Overstreet 	} splits[3] = {
39996dea3d5SKent Overstreet 		{ crc_a, len_a, new_csum_type, { 0 }},
40096dea3d5SKent Overstreet 		{ crc_b, len_b, new_csum_type, { 0 } },
40196dea3d5SKent Overstreet 		{ NULL,	 bio_sectors(bio) - len_a - len_b, new_csum_type, { 0 } },
4021c6fdbd8SKent Overstreet 	}, *i;
4031c6fdbd8SKent Overstreet 	bool mergeable = crc_old.csum_type == new_csum_type &&
4041c6fdbd8SKent Overstreet 		bch2_checksum_mergeable(new_csum_type);
4051c6fdbd8SKent Overstreet 	unsigned crc_nonce = crc_old.nonce;
4061c6fdbd8SKent Overstreet 
4071c6fdbd8SKent Overstreet 	BUG_ON(len_a + len_b > bio_sectors(bio));
4081c6fdbd8SKent Overstreet 	BUG_ON(crc_old.uncompressed_size != bio_sectors(bio));
409ab05de4cSKent Overstreet 	BUG_ON(crc_is_compressed(crc_old));
4101c6fdbd8SKent Overstreet 	BUG_ON(bch2_csum_type_is_encryption(crc_old.csum_type) !=
4111c6fdbd8SKent Overstreet 	       bch2_csum_type_is_encryption(new_csum_type));
4121c6fdbd8SKent Overstreet 
4131c6fdbd8SKent Overstreet 	for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
4141c6fdbd8SKent Overstreet 		iter.bi_size = i->len << 9;
4151c6fdbd8SKent Overstreet 		if (mergeable || i->crc)
4161c6fdbd8SKent Overstreet 			i->csum = __bch2_checksum_bio(c, i->csum_type,
4171c6fdbd8SKent Overstreet 						      nonce, bio, &iter);
4181c6fdbd8SKent Overstreet 		else
4191c6fdbd8SKent Overstreet 			bio_advance_iter(bio, &iter, i->len << 9);
4201c6fdbd8SKent Overstreet 		nonce = nonce_add(nonce, i->len << 9);
4211c6fdbd8SKent Overstreet 	}
4221c6fdbd8SKent Overstreet 
4231c6fdbd8SKent Overstreet 	if (mergeable)
4241c6fdbd8SKent Overstreet 		for (i = splits; i < splits + ARRAY_SIZE(splits); i++)
4251c6fdbd8SKent Overstreet 			merged = bch2_checksum_merge(new_csum_type, merged,
4261c6fdbd8SKent Overstreet 						     i->csum, i->len << 9);
4271c6fdbd8SKent Overstreet 	else
4281c6fdbd8SKent Overstreet 		merged = bch2_checksum_bio(c, crc_old.csum_type,
4291c6fdbd8SKent Overstreet 				extent_nonce(version, crc_old), bio);
4301c6fdbd8SKent Overstreet 
431970a5096SKent Overstreet 	if (bch2_crc_cmp(merged, crc_old.csum) && !c->opts.no_data_io) {
4321e81f89bSKent Overstreet 		bch_err(c, "checksum error in %s() (memory corruption or bug?)\n"
43323189da9SKent Overstreet 			"expected %0llx:%0llx got %0llx:%0llx (old type %s new type %s)",
4341e81f89bSKent Overstreet 			__func__,
43523189da9SKent Overstreet 			crc_old.csum.hi,
43623189da9SKent Overstreet 			crc_old.csum.lo,
43723189da9SKent Overstreet 			merged.hi,
43823189da9SKent Overstreet 			merged.lo,
43923189da9SKent Overstreet 			bch2_csum_types[crc_old.csum_type],
44023189da9SKent Overstreet 			bch2_csum_types[new_csum_type]);
4411c6fdbd8SKent Overstreet 		return -EIO;
44223189da9SKent Overstreet 	}
4431c6fdbd8SKent Overstreet 
4441c6fdbd8SKent Overstreet 	for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
4451c6fdbd8SKent Overstreet 		if (i->crc)
4461c6fdbd8SKent Overstreet 			*i->crc = (struct bch_extent_crc_unpacked) {
4471c6fdbd8SKent Overstreet 				.csum_type		= i->csum_type,
448ab05de4cSKent Overstreet 				.compression_type	= crc_old.compression_type,
4491c6fdbd8SKent Overstreet 				.compressed_size	= i->len,
4501c6fdbd8SKent Overstreet 				.uncompressed_size	= i->len,
4511c6fdbd8SKent Overstreet 				.offset			= 0,
4521c6fdbd8SKent Overstreet 				.live_size		= i->len,
4531c6fdbd8SKent Overstreet 				.nonce			= crc_nonce,
4541c6fdbd8SKent Overstreet 				.csum			= i->csum,
4551c6fdbd8SKent Overstreet 			};
4561c6fdbd8SKent Overstreet 
4571c6fdbd8SKent Overstreet 		if (bch2_csum_type_is_encryption(new_csum_type))
4581c6fdbd8SKent Overstreet 			crc_nonce += i->len;
4591c6fdbd8SKent Overstreet 	}
4601c6fdbd8SKent Overstreet 
4611c6fdbd8SKent Overstreet 	return 0;
4621c6fdbd8SKent Overstreet }
4631c6fdbd8SKent Overstreet 
4640ec39856SKent Overstreet /* BCH_SB_FIELD_crypt: */
4650ec39856SKent Overstreet 
4660ec39856SKent Overstreet static int bch2_sb_crypt_validate(struct bch_sb *sb,
4670ec39856SKent Overstreet 				  struct bch_sb_field *f,
4680ec39856SKent Overstreet 				  struct printbuf *err)
4690ec39856SKent Overstreet {
4700ec39856SKent Overstreet 	struct bch_sb_field_crypt *crypt = field_to_type(f, crypt);
4710ec39856SKent Overstreet 
4720ec39856SKent Overstreet 	if (vstruct_bytes(&crypt->field) < sizeof(*crypt)) {
4730ec39856SKent Overstreet 		prt_printf(err, "wrong size (got %zu should be %zu)",
4740ec39856SKent Overstreet 		       vstruct_bytes(&crypt->field), sizeof(*crypt));
4750ec39856SKent Overstreet 		return -BCH_ERR_invalid_sb_crypt;
4760ec39856SKent Overstreet 	}
4770ec39856SKent Overstreet 
4780ec39856SKent Overstreet 	if (BCH_CRYPT_KDF_TYPE(crypt)) {
4790ec39856SKent Overstreet 		prt_printf(err, "bad kdf type %llu", BCH_CRYPT_KDF_TYPE(crypt));
4800ec39856SKent Overstreet 		return -BCH_ERR_invalid_sb_crypt;
4810ec39856SKent Overstreet 	}
4820ec39856SKent Overstreet 
4830ec39856SKent Overstreet 	return 0;
4840ec39856SKent Overstreet }
4850ec39856SKent Overstreet 
4860ec39856SKent Overstreet static void bch2_sb_crypt_to_text(struct printbuf *out, struct bch_sb *sb,
4870ec39856SKent Overstreet 				  struct bch_sb_field *f)
4880ec39856SKent Overstreet {
4890ec39856SKent Overstreet 	struct bch_sb_field_crypt *crypt = field_to_type(f, crypt);
4900ec39856SKent Overstreet 
4910ec39856SKent Overstreet 	prt_printf(out, "KFD:               %llu", BCH_CRYPT_KDF_TYPE(crypt));
4920ec39856SKent Overstreet 	prt_newline(out);
4930ec39856SKent Overstreet 	prt_printf(out, "scrypt n:          %llu", BCH_KDF_SCRYPT_N(crypt));
4940ec39856SKent Overstreet 	prt_newline(out);
4950ec39856SKent Overstreet 	prt_printf(out, "scrypt r:          %llu", BCH_KDF_SCRYPT_R(crypt));
4960ec39856SKent Overstreet 	prt_newline(out);
4970ec39856SKent Overstreet 	prt_printf(out, "scrypt p:          %llu", BCH_KDF_SCRYPT_P(crypt));
4980ec39856SKent Overstreet 	prt_newline(out);
4990ec39856SKent Overstreet }
5000ec39856SKent Overstreet 
5010ec39856SKent Overstreet const struct bch_sb_field_ops bch_sb_field_ops_crypt = {
5020ec39856SKent Overstreet 	.validate	= bch2_sb_crypt_validate,
5030ec39856SKent Overstreet 	.to_text	= bch2_sb_crypt_to_text,
5040ec39856SKent Overstreet };
5050ec39856SKent Overstreet 
5061c6fdbd8SKent Overstreet #ifdef __KERNEL__
50703ea3962SKent Overstreet static int __bch2_request_key(char *key_description, struct bch_key *key)
5081c6fdbd8SKent Overstreet {
5091c6fdbd8SKent Overstreet 	struct key *keyring_key;
5101c6fdbd8SKent Overstreet 	const struct user_key_payload *ukp;
5111c6fdbd8SKent Overstreet 	int ret;
5121c6fdbd8SKent Overstreet 
513232697abSKent Overstreet 	keyring_key = request_key(&key_type_user, key_description, NULL);
5141c6fdbd8SKent Overstreet 	if (IS_ERR(keyring_key))
5151c6fdbd8SKent Overstreet 		return PTR_ERR(keyring_key);
5161c6fdbd8SKent Overstreet 
5171c6fdbd8SKent Overstreet 	down_read(&keyring_key->sem);
5181c6fdbd8SKent Overstreet 	ukp = dereference_key_locked(keyring_key);
5191c6fdbd8SKent Overstreet 	if (ukp->datalen == sizeof(*key)) {
5201c6fdbd8SKent Overstreet 		memcpy(key, ukp->data, ukp->datalen);
5211c6fdbd8SKent Overstreet 		ret = 0;
5221c6fdbd8SKent Overstreet 	} else {
5231c6fdbd8SKent Overstreet 		ret = -EINVAL;
5241c6fdbd8SKent Overstreet 	}
5251c6fdbd8SKent Overstreet 	up_read(&keyring_key->sem);
5261c6fdbd8SKent Overstreet 	key_put(keyring_key);
5271c6fdbd8SKent Overstreet 
5281c6fdbd8SKent Overstreet 	return ret;
5291c6fdbd8SKent Overstreet }
5301c6fdbd8SKent Overstreet #else
5311c6fdbd8SKent Overstreet #include <keyutils.h>
5321c6fdbd8SKent Overstreet 
53303ea3962SKent Overstreet static int __bch2_request_key(char *key_description, struct bch_key *key)
5341c6fdbd8SKent Overstreet {
5351c6fdbd8SKent Overstreet 	key_serial_t key_id;
5361c6fdbd8SKent Overstreet 
5371c6fdbd8SKent Overstreet 	key_id = request_key("user", key_description, NULL,
5381ee608c6SKent Overstreet 			     KEY_SPEC_SESSION_KEYRING);
5391ee608c6SKent Overstreet 	if (key_id >= 0)
5401ee608c6SKent Overstreet 		goto got_key;
5411ee608c6SKent Overstreet 
5421ee608c6SKent Overstreet 	key_id = request_key("user", key_description, NULL,
5431c6fdbd8SKent Overstreet 			     KEY_SPEC_USER_KEYRING);
5441ee608c6SKent Overstreet 	if (key_id >= 0)
5451ee608c6SKent Overstreet 		goto got_key;
5461ee608c6SKent Overstreet 
5471ee608c6SKent Overstreet 	key_id = request_key("user", key_description, NULL,
5481ee608c6SKent Overstreet 			     KEY_SPEC_USER_SESSION_KEYRING);
5491ee608c6SKent Overstreet 	if (key_id >= 0)
5501ee608c6SKent Overstreet 		goto got_key;
5511ee608c6SKent Overstreet 
5521c6fdbd8SKent Overstreet 	return -errno;
5531ee608c6SKent Overstreet got_key:
5541c6fdbd8SKent Overstreet 
5551c6fdbd8SKent Overstreet 	if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key))
5561c6fdbd8SKent Overstreet 		return -1;
5571c6fdbd8SKent Overstreet 
5581c6fdbd8SKent Overstreet 	return 0;
5591c6fdbd8SKent Overstreet }
5601ee608c6SKent Overstreet 
5611ee608c6SKent Overstreet #include "../crypto.h"
5621c6fdbd8SKent Overstreet #endif
5631c6fdbd8SKent Overstreet 
56403ea3962SKent Overstreet int bch2_request_key(struct bch_sb *sb, struct bch_key *key)
56503ea3962SKent Overstreet {
566401ec4dbSKent Overstreet 	struct printbuf key_description = PRINTBUF;
567401ec4dbSKent Overstreet 	int ret;
56803ea3962SKent Overstreet 
569401ec4dbSKent Overstreet 	prt_printf(&key_description, "bcachefs:");
570401ec4dbSKent Overstreet 	pr_uuid(&key_description, sb->user_uuid.b);
57103ea3962SKent Overstreet 
572401ec4dbSKent Overstreet 	ret = __bch2_request_key(key_description.buf, key);
573401ec4dbSKent Overstreet 	printbuf_exit(&key_description);
5741ee608c6SKent Overstreet 
5751ee608c6SKent Overstreet #ifndef __KERNEL__
5761ee608c6SKent Overstreet 	if (ret) {
5771ee608c6SKent Overstreet 		char *passphrase = read_passphrase("Enter passphrase: ");
5781ee608c6SKent Overstreet 		struct bch_encrypted_key sb_key;
5791ee608c6SKent Overstreet 
5801ee608c6SKent Overstreet 		bch2_passphrase_check(sb, passphrase,
5811ee608c6SKent Overstreet 				      key, &sb_key);
5821ee608c6SKent Overstreet 		ret = 0;
5831ee608c6SKent Overstreet 	}
5841ee608c6SKent Overstreet #endif
5851ee608c6SKent Overstreet 
5861ee608c6SKent Overstreet 	/* stash with memfd, pass memfd fd to mount */
5871ee608c6SKent Overstreet 
588401ec4dbSKent Overstreet 	return ret;
58903ea3962SKent Overstreet }
59003ea3962SKent Overstreet 
591793a06d9SKent Overstreet #ifndef __KERNEL__
592793a06d9SKent Overstreet int bch2_revoke_key(struct bch_sb *sb)
593793a06d9SKent Overstreet {
594793a06d9SKent Overstreet 	key_serial_t key_id;
595793a06d9SKent Overstreet 	struct printbuf key_description = PRINTBUF;
596793a06d9SKent Overstreet 
597793a06d9SKent Overstreet 	prt_printf(&key_description, "bcachefs:");
598793a06d9SKent Overstreet 	pr_uuid(&key_description, sb->user_uuid.b);
599793a06d9SKent Overstreet 
600793a06d9SKent Overstreet 	key_id = request_key("user", key_description.buf, NULL, KEY_SPEC_USER_KEYRING);
601793a06d9SKent Overstreet 	printbuf_exit(&key_description);
602793a06d9SKent Overstreet 	if (key_id < 0)
603793a06d9SKent Overstreet 		return errno;
604793a06d9SKent Overstreet 
605793a06d9SKent Overstreet 	keyctl_revoke(key_id);
606793a06d9SKent Overstreet 
607793a06d9SKent Overstreet 	return 0;
608793a06d9SKent Overstreet }
609793a06d9SKent Overstreet #endif
610793a06d9SKent Overstreet 
6111c6fdbd8SKent Overstreet int bch2_decrypt_sb_key(struct bch_fs *c,
6121c6fdbd8SKent Overstreet 			struct bch_sb_field_crypt *crypt,
6131c6fdbd8SKent Overstreet 			struct bch_key *key)
6141c6fdbd8SKent Overstreet {
6151c6fdbd8SKent Overstreet 	struct bch_encrypted_key sb_key = crypt->key;
6161c6fdbd8SKent Overstreet 	struct bch_key user_key;
6171c6fdbd8SKent Overstreet 	int ret = 0;
6181c6fdbd8SKent Overstreet 
6191c6fdbd8SKent Overstreet 	/* is key encrypted? */
6201c6fdbd8SKent Overstreet 	if (!bch2_key_is_encrypted(&sb_key))
6211c6fdbd8SKent Overstreet 		goto out;
6221c6fdbd8SKent Overstreet 
6231c6fdbd8SKent Overstreet 	ret = bch2_request_key(c->disk_sb.sb, &user_key);
6241c6fdbd8SKent Overstreet 	if (ret) {
625d4bf5eecSKent Overstreet 		bch_err(c, "error requesting encryption key: %s", bch2_err_str(ret));
6261c6fdbd8SKent Overstreet 		goto err;
6271c6fdbd8SKent Overstreet 	}
6281c6fdbd8SKent Overstreet 
6291c6fdbd8SKent Overstreet 	/* decrypt real key: */
6301c6fdbd8SKent Overstreet 	ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
6311c6fdbd8SKent Overstreet 				      &sb_key, sizeof(sb_key));
6321c6fdbd8SKent Overstreet 	if (ret)
6331c6fdbd8SKent Overstreet 		goto err;
6341c6fdbd8SKent Overstreet 
6351c6fdbd8SKent Overstreet 	if (bch2_key_is_encrypted(&sb_key)) {
6361c6fdbd8SKent Overstreet 		bch_err(c, "incorrect encryption key");
6371c6fdbd8SKent Overstreet 		ret = -EINVAL;
6381c6fdbd8SKent Overstreet 		goto err;
6391c6fdbd8SKent Overstreet 	}
6401c6fdbd8SKent Overstreet out:
6411c6fdbd8SKent Overstreet 	*key = sb_key.key;
6421c6fdbd8SKent Overstreet err:
6431c6fdbd8SKent Overstreet 	memzero_explicit(&sb_key, sizeof(sb_key));
6441c6fdbd8SKent Overstreet 	memzero_explicit(&user_key, sizeof(user_key));
6451c6fdbd8SKent Overstreet 	return ret;
6461c6fdbd8SKent Overstreet }
6471c6fdbd8SKent Overstreet 
6481c6fdbd8SKent Overstreet static int bch2_alloc_ciphers(struct bch_fs *c)
6491c6fdbd8SKent Overstreet {
650d4bf5eecSKent Overstreet 	int ret;
651d4bf5eecSKent Overstreet 
6521c6fdbd8SKent Overstreet 	if (!c->chacha20)
6531c6fdbd8SKent Overstreet 		c->chacha20 = crypto_alloc_sync_skcipher("chacha20", 0, 0);
654d4bf5eecSKent Overstreet 	ret = PTR_ERR_OR_ZERO(c->chacha20);
655d4bf5eecSKent Overstreet 
656d4bf5eecSKent Overstreet 	if (ret) {
657d4bf5eecSKent Overstreet 		bch_err(c, "error requesting chacha20 module: %s", bch2_err_str(ret));
658d4bf5eecSKent Overstreet 		return ret;
6591c6fdbd8SKent Overstreet 	}
6601c6fdbd8SKent Overstreet 
6611c6fdbd8SKent Overstreet 	if (!c->poly1305)
6621c6fdbd8SKent Overstreet 		c->poly1305 = crypto_alloc_shash("poly1305", 0, 0);
663d4bf5eecSKent Overstreet 	ret = PTR_ERR_OR_ZERO(c->poly1305);
664d4bf5eecSKent Overstreet 
665d4bf5eecSKent Overstreet 	if (ret) {
666d4bf5eecSKent Overstreet 		bch_err(c, "error requesting poly1305 module: %s", bch2_err_str(ret));
667d4bf5eecSKent Overstreet 		return ret;
6681c6fdbd8SKent Overstreet 	}
6691c6fdbd8SKent Overstreet 
6701c6fdbd8SKent Overstreet 	return 0;
6711c6fdbd8SKent Overstreet }
6721c6fdbd8SKent Overstreet 
6731c6fdbd8SKent Overstreet int bch2_disable_encryption(struct bch_fs *c)
6741c6fdbd8SKent Overstreet {
6751c6fdbd8SKent Overstreet 	struct bch_sb_field_crypt *crypt;
6761c6fdbd8SKent Overstreet 	struct bch_key key;
6771c6fdbd8SKent Overstreet 	int ret = -EINVAL;
6781c6fdbd8SKent Overstreet 
6791c6fdbd8SKent Overstreet 	mutex_lock(&c->sb_lock);
6801c6fdbd8SKent Overstreet 
6811c6fdbd8SKent Overstreet 	crypt = bch2_sb_get_crypt(c->disk_sb.sb);
6821c6fdbd8SKent Overstreet 	if (!crypt)
6831c6fdbd8SKent Overstreet 		goto out;
6841c6fdbd8SKent Overstreet 
6851c6fdbd8SKent Overstreet 	/* is key encrypted? */
6861c6fdbd8SKent Overstreet 	ret = 0;
6871c6fdbd8SKent Overstreet 	if (bch2_key_is_encrypted(&crypt->key))
6881c6fdbd8SKent Overstreet 		goto out;
6891c6fdbd8SKent Overstreet 
6901c6fdbd8SKent Overstreet 	ret = bch2_decrypt_sb_key(c, crypt, &key);
6911c6fdbd8SKent Overstreet 	if (ret)
6921c6fdbd8SKent Overstreet 		goto out;
6931c6fdbd8SKent Overstreet 
69473bd774dSKent Overstreet 	crypt->key.magic	= cpu_to_le64(BCH_KEY_MAGIC);
6951c6fdbd8SKent Overstreet 	crypt->key.key		= key;
6961c6fdbd8SKent Overstreet 
6971c6fdbd8SKent Overstreet 	SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb.sb, 0);
6981c6fdbd8SKent Overstreet 	bch2_write_super(c);
6991c6fdbd8SKent Overstreet out:
7001c6fdbd8SKent Overstreet 	mutex_unlock(&c->sb_lock);
7011c6fdbd8SKent Overstreet 
7021c6fdbd8SKent Overstreet 	return ret;
7031c6fdbd8SKent Overstreet }
7041c6fdbd8SKent Overstreet 
7051c6fdbd8SKent Overstreet int bch2_enable_encryption(struct bch_fs *c, bool keyed)
7061c6fdbd8SKent Overstreet {
7071c6fdbd8SKent Overstreet 	struct bch_encrypted_key key;
7081c6fdbd8SKent Overstreet 	struct bch_key user_key;
7091c6fdbd8SKent Overstreet 	struct bch_sb_field_crypt *crypt;
7101c6fdbd8SKent Overstreet 	int ret = -EINVAL;
7111c6fdbd8SKent Overstreet 
7121c6fdbd8SKent Overstreet 	mutex_lock(&c->sb_lock);
7131c6fdbd8SKent Overstreet 
7141c6fdbd8SKent Overstreet 	/* Do we already have an encryption key? */
7151c6fdbd8SKent Overstreet 	if (bch2_sb_get_crypt(c->disk_sb.sb))
7161c6fdbd8SKent Overstreet 		goto err;
7171c6fdbd8SKent Overstreet 
7181c6fdbd8SKent Overstreet 	ret = bch2_alloc_ciphers(c);
7191c6fdbd8SKent Overstreet 	if (ret)
7201c6fdbd8SKent Overstreet 		goto err;
7211c6fdbd8SKent Overstreet 
72273bd774dSKent Overstreet 	key.magic = cpu_to_le64(BCH_KEY_MAGIC);
7231c6fdbd8SKent Overstreet 	get_random_bytes(&key.key, sizeof(key.key));
7241c6fdbd8SKent Overstreet 
7251c6fdbd8SKent Overstreet 	if (keyed) {
7261c6fdbd8SKent Overstreet 		ret = bch2_request_key(c->disk_sb.sb, &user_key);
7271c6fdbd8SKent Overstreet 		if (ret) {
728d4bf5eecSKent Overstreet 			bch_err(c, "error requesting encryption key: %s", bch2_err_str(ret));
7291c6fdbd8SKent Overstreet 			goto err;
7301c6fdbd8SKent Overstreet 		}
7311c6fdbd8SKent Overstreet 
7321c6fdbd8SKent Overstreet 		ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
7331c6fdbd8SKent Overstreet 					      &key, sizeof(key));
7341c6fdbd8SKent Overstreet 		if (ret)
7351c6fdbd8SKent Overstreet 			goto err;
7361c6fdbd8SKent Overstreet 	}
7371c6fdbd8SKent Overstreet 
7381c6fdbd8SKent Overstreet 	ret = crypto_skcipher_setkey(&c->chacha20->base,
7391c6fdbd8SKent Overstreet 			(void *) &key.key, sizeof(key.key));
7401c6fdbd8SKent Overstreet 	if (ret)
7411c6fdbd8SKent Overstreet 		goto err;
7421c6fdbd8SKent Overstreet 
7431c6fdbd8SKent Overstreet 	crypt = bch2_sb_resize_crypt(&c->disk_sb, sizeof(*crypt) / sizeof(u64));
7441c6fdbd8SKent Overstreet 	if (!crypt) {
74565d48e35SKent Overstreet 		ret = -BCH_ERR_ENOSPC_sb_crypt;
7461c6fdbd8SKent Overstreet 		goto err;
7471c6fdbd8SKent Overstreet 	}
7481c6fdbd8SKent Overstreet 
7491c6fdbd8SKent Overstreet 	crypt->key = key;
7501c6fdbd8SKent Overstreet 
7511c6fdbd8SKent Overstreet 	/* write superblock */
7521c6fdbd8SKent Overstreet 	SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb.sb, 1);
7531c6fdbd8SKent Overstreet 	bch2_write_super(c);
7541c6fdbd8SKent Overstreet err:
7551c6fdbd8SKent Overstreet 	mutex_unlock(&c->sb_lock);
7561c6fdbd8SKent Overstreet 	memzero_explicit(&user_key, sizeof(user_key));
7571c6fdbd8SKent Overstreet 	memzero_explicit(&key, sizeof(key));
7581c6fdbd8SKent Overstreet 	return ret;
7591c6fdbd8SKent Overstreet }
7601c6fdbd8SKent Overstreet 
7611c6fdbd8SKent Overstreet void bch2_fs_encryption_exit(struct bch_fs *c)
7621c6fdbd8SKent Overstreet {
7631c6fdbd8SKent Overstreet 	if (!IS_ERR_OR_NULL(c->poly1305))
7641c6fdbd8SKent Overstreet 		crypto_free_shash(c->poly1305);
7651c6fdbd8SKent Overstreet 	if (!IS_ERR_OR_NULL(c->chacha20))
7661c6fdbd8SKent Overstreet 		crypto_free_sync_skcipher(c->chacha20);
7671c6fdbd8SKent Overstreet 	if (!IS_ERR_OR_NULL(c->sha256))
7681c6fdbd8SKent Overstreet 		crypto_free_shash(c->sha256);
7691c6fdbd8SKent Overstreet }
7701c6fdbd8SKent Overstreet 
7711c6fdbd8SKent Overstreet int bch2_fs_encryption_init(struct bch_fs *c)
7721c6fdbd8SKent Overstreet {
7731c6fdbd8SKent Overstreet 	struct bch_sb_field_crypt *crypt;
7741c6fdbd8SKent Overstreet 	struct bch_key key;
7751c6fdbd8SKent Overstreet 	int ret = 0;
7761c6fdbd8SKent Overstreet 
7771c6fdbd8SKent Overstreet 	c->sha256 = crypto_alloc_shash("sha256", 0, 0);
778d4bf5eecSKent Overstreet 	ret = PTR_ERR_OR_ZERO(c->sha256);
779d4bf5eecSKent Overstreet 	if (ret) {
780d4bf5eecSKent Overstreet 		bch_err(c, "error requesting sha256 module: %s", bch2_err_str(ret));
7811c6fdbd8SKent Overstreet 		goto out;
7821c6fdbd8SKent Overstreet 	}
7831c6fdbd8SKent Overstreet 
7841c6fdbd8SKent Overstreet 	crypt = bch2_sb_get_crypt(c->disk_sb.sb);
7851c6fdbd8SKent Overstreet 	if (!crypt)
7861c6fdbd8SKent Overstreet 		goto out;
7871c6fdbd8SKent Overstreet 
7881c6fdbd8SKent Overstreet 	ret = bch2_alloc_ciphers(c);
7891c6fdbd8SKent Overstreet 	if (ret)
7901c6fdbd8SKent Overstreet 		goto out;
7911c6fdbd8SKent Overstreet 
7921c6fdbd8SKent Overstreet 	ret = bch2_decrypt_sb_key(c, crypt, &key);
7931c6fdbd8SKent Overstreet 	if (ret)
7941c6fdbd8SKent Overstreet 		goto out;
7951c6fdbd8SKent Overstreet 
7961c6fdbd8SKent Overstreet 	ret = crypto_skcipher_setkey(&c->chacha20->base,
7971c6fdbd8SKent Overstreet 			(void *) &key.key, sizeof(key.key));
7981c6fdbd8SKent Overstreet 	if (ret)
7991c6fdbd8SKent Overstreet 		goto out;
8001c6fdbd8SKent Overstreet out:
8011c6fdbd8SKent Overstreet 	memzero_explicit(&key, sizeof(key));
8021c6fdbd8SKent Overstreet 	return ret;
8031c6fdbd8SKent Overstreet }
804