1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 Google LLC 4 */ 5 6 /* 7 * Refer to Documentation/block/inline-encryption.rst for detailed explanation. 8 */ 9 10 #define pr_fmt(fmt) "blk-crypto-fallback: " fmt 11 12 #include <crypto/skcipher.h> 13 #include <linux/blk-crypto.h> 14 #include <linux/blk-crypto-profile.h> 15 #include <linux/blkdev.h> 16 #include <linux/crypto.h> 17 #include <linux/mempool.h> 18 #include <linux/module.h> 19 #include <linux/random.h> 20 #include <linux/scatterlist.h> 21 22 #include "blk-cgroup.h" 23 #include "blk-crypto-internal.h" 24 25 static unsigned int num_prealloc_bounce_pg = 32; 26 module_param(num_prealloc_bounce_pg, uint, 0); 27 MODULE_PARM_DESC(num_prealloc_bounce_pg, 28 "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); 29 30 static unsigned int blk_crypto_num_keyslots = 100; 31 module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); 32 MODULE_PARM_DESC(num_keyslots, 33 "Number of keyslots for the blk-crypto crypto API fallback"); 34 35 static unsigned int num_prealloc_fallback_crypt_ctxs = 128; 36 module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); 37 MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, 38 "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); 39 40 struct bio_fallback_crypt_ctx { 41 struct bio_crypt_ctx crypt_ctx; 42 /* 43 * Copy of the bvec_iter when this bio was submitted. 44 * We only want to en/decrypt the part of the bio as described by the 45 * bvec_iter upon submission because bio might be split before being 46 * resubmitted 47 */ 48 struct bvec_iter crypt_iter; 49 union { 50 struct { 51 struct work_struct work; 52 struct bio *bio; 53 }; 54 struct { 55 void *bi_private_orig; 56 bio_end_io_t *bi_end_io_orig; 57 }; 58 }; 59 }; 60 61 static struct kmem_cache *bio_fallback_crypt_ctx_cache; 62 static mempool_t *bio_fallback_crypt_ctx_pool; 63 64 /* 65 * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate 66 * all of a mode's tfms when that mode starts being used. Since each mode may 67 * need all the keyslots at some point, each mode needs its own tfm for each 68 * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to 69 * match the behavior of real inline encryption hardware (which only supports a 70 * single encryption context per keyslot), we only allow one tfm per keyslot to 71 * be used at a time - the rest of the unused tfms have their keys cleared. 72 */ 73 static DEFINE_MUTEX(tfms_init_lock); 74 static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; 75 76 static struct blk_crypto_fallback_keyslot { 77 enum blk_crypto_mode_num crypto_mode; 78 struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; 79 } *blk_crypto_keyslots; 80 81 static struct blk_crypto_profile *blk_crypto_fallback_profile; 82 static struct workqueue_struct *blk_crypto_wq; 83 static mempool_t *blk_crypto_bounce_page_pool; 84 static struct bio_set crypto_bio_split; 85 86 /* 87 * This is the key we set when evicting a keyslot. This *should* be the all 0's 88 * key, but AES-XTS rejects that key, so we use some random bytes instead. 89 */ 90 static u8 blank_key[BLK_CRYPTO_MAX_RAW_KEY_SIZE]; 91 92 static void blk_crypto_fallback_evict_keyslot(unsigned int slot) 93 { 94 struct blk_crypto_fallback_keyslot *slotp = &blk_crypto_keyslots[slot]; 95 enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; 96 int err; 97 98 WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); 99 100 /* Clear the key in the skcipher */ 101 err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, 102 blk_crypto_modes[crypto_mode].keysize); 103 WARN_ON(err); 104 slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; 105 } 106 107 static int 108 blk_crypto_fallback_keyslot_program(struct blk_crypto_profile *profile, 109 const struct blk_crypto_key *key, 110 unsigned int slot) 111 { 112 struct blk_crypto_fallback_keyslot *slotp = &blk_crypto_keyslots[slot]; 113 const enum blk_crypto_mode_num crypto_mode = 114 key->crypto_cfg.crypto_mode; 115 int err; 116 117 if (crypto_mode != slotp->crypto_mode && 118 slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) 119 blk_crypto_fallback_evict_keyslot(slot); 120 121 slotp->crypto_mode = crypto_mode; 122 err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, 123 key->size); 124 if (err) { 125 blk_crypto_fallback_evict_keyslot(slot); 126 return err; 127 } 128 return 0; 129 } 130 131 static int blk_crypto_fallback_keyslot_evict(struct blk_crypto_profile *profile, 132 const struct blk_crypto_key *key, 133 unsigned int slot) 134 { 135 blk_crypto_fallback_evict_keyslot(slot); 136 return 0; 137 } 138 139 static const struct blk_crypto_ll_ops blk_crypto_fallback_ll_ops = { 140 .keyslot_program = blk_crypto_fallback_keyslot_program, 141 .keyslot_evict = blk_crypto_fallback_keyslot_evict, 142 }; 143 144 static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) 145 { 146 struct bio *src_bio = enc_bio->bi_private; 147 int i; 148 149 for (i = 0; i < enc_bio->bi_vcnt; i++) 150 mempool_free(enc_bio->bi_io_vec[i].bv_page, 151 blk_crypto_bounce_page_pool); 152 153 src_bio->bi_status = enc_bio->bi_status; 154 155 bio_uninit(enc_bio); 156 kfree(enc_bio); 157 bio_endio(src_bio); 158 } 159 160 static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src) 161 { 162 unsigned int nr_segs = bio_segments(bio_src); 163 struct bvec_iter iter; 164 struct bio_vec bv; 165 struct bio *bio; 166 167 bio = bio_kmalloc(nr_segs, GFP_NOIO); 168 if (!bio) 169 return NULL; 170 bio_init_inline(bio, bio_src->bi_bdev, nr_segs, bio_src->bi_opf); 171 if (bio_flagged(bio_src, BIO_REMAPPED)) 172 bio_set_flag(bio, BIO_REMAPPED); 173 bio->bi_ioprio = bio_src->bi_ioprio; 174 bio->bi_write_hint = bio_src->bi_write_hint; 175 bio->bi_write_stream = bio_src->bi_write_stream; 176 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; 177 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; 178 179 bio_for_each_segment(bv, bio_src, iter) 180 bio->bi_io_vec[bio->bi_vcnt++] = bv; 181 182 bio_clone_blkg_association(bio, bio_src); 183 184 return bio; 185 } 186 187 static bool 188 blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot, 189 struct skcipher_request **ciph_req_ret, 190 struct crypto_wait *wait) 191 { 192 struct skcipher_request *ciph_req; 193 const struct blk_crypto_fallback_keyslot *slotp; 194 int keyslot_idx = blk_crypto_keyslot_index(slot); 195 196 slotp = &blk_crypto_keyslots[keyslot_idx]; 197 ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], 198 GFP_NOIO); 199 if (!ciph_req) 200 return false; 201 202 skcipher_request_set_callback(ciph_req, 203 CRYPTO_TFM_REQ_MAY_BACKLOG | 204 CRYPTO_TFM_REQ_MAY_SLEEP, 205 crypto_req_done, wait); 206 *ciph_req_ret = ciph_req; 207 208 return true; 209 } 210 211 static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) 212 { 213 struct bio *bio = *bio_ptr; 214 unsigned int i = 0; 215 unsigned int num_sectors = 0; 216 struct bio_vec bv; 217 struct bvec_iter iter; 218 219 bio_for_each_segment(bv, bio, iter) { 220 num_sectors += bv.bv_len >> SECTOR_SHIFT; 221 if (++i == BIO_MAX_VECS) 222 break; 223 } 224 225 if (num_sectors < bio_sectors(bio)) { 226 bio = bio_submit_split_bioset(bio, num_sectors, 227 &crypto_bio_split); 228 if (!bio) 229 return false; 230 231 *bio_ptr = bio; 232 } 233 234 return true; 235 } 236 237 union blk_crypto_iv { 238 __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; 239 u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; 240 }; 241 242 static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], 243 union blk_crypto_iv *iv) 244 { 245 int i; 246 247 for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) 248 iv->dun[i] = cpu_to_le64(dun[i]); 249 } 250 251 /* 252 * The crypto API fallback's encryption routine. 253 * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, 254 * and replace *bio_ptr with the bounce bio. May split input bio if it's too 255 * large. Returns true on success. Returns false and sets bio->bi_status on 256 * error. 257 */ 258 static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) 259 { 260 struct bio *src_bio, *enc_bio; 261 struct bio_crypt_ctx *bc; 262 struct blk_crypto_keyslot *slot; 263 int data_unit_size; 264 struct skcipher_request *ciph_req = NULL; 265 DECLARE_CRYPTO_WAIT(wait); 266 u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; 267 struct scatterlist src, dst; 268 union blk_crypto_iv iv; 269 unsigned int i, j; 270 bool ret = false; 271 blk_status_t blk_st; 272 273 /* Split the bio if it's too big for single page bvec */ 274 if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr)) 275 return false; 276 277 src_bio = *bio_ptr; 278 bc = src_bio->bi_crypt_context; 279 data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; 280 281 /* Allocate bounce bio for encryption */ 282 enc_bio = blk_crypto_fallback_clone_bio(src_bio); 283 if (!enc_bio) { 284 src_bio->bi_status = BLK_STS_RESOURCE; 285 return false; 286 } 287 288 /* 289 * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for 290 * this bio's algorithm and key. 291 */ 292 blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, 293 bc->bc_key, &slot); 294 if (blk_st != BLK_STS_OK) { 295 src_bio->bi_status = blk_st; 296 goto out_put_enc_bio; 297 } 298 299 /* and then allocate an skcipher_request for it */ 300 if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { 301 src_bio->bi_status = BLK_STS_RESOURCE; 302 goto out_release_keyslot; 303 } 304 305 memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); 306 sg_init_table(&src, 1); 307 sg_init_table(&dst, 1); 308 309 skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, 310 iv.bytes); 311 312 /* Encrypt each page in the bounce bio */ 313 for (i = 0; i < enc_bio->bi_vcnt; i++) { 314 struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; 315 struct page *plaintext_page = enc_bvec->bv_page; 316 struct page *ciphertext_page = 317 mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); 318 319 enc_bvec->bv_page = ciphertext_page; 320 321 if (!ciphertext_page) { 322 src_bio->bi_status = BLK_STS_RESOURCE; 323 goto out_free_bounce_pages; 324 } 325 326 sg_set_page(&src, plaintext_page, data_unit_size, 327 enc_bvec->bv_offset); 328 sg_set_page(&dst, ciphertext_page, data_unit_size, 329 enc_bvec->bv_offset); 330 331 /* Encrypt each data unit in this page */ 332 for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { 333 blk_crypto_dun_to_iv(curr_dun, &iv); 334 if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req), 335 &wait)) { 336 i++; 337 src_bio->bi_status = BLK_STS_IOERR; 338 goto out_free_bounce_pages; 339 } 340 bio_crypt_dun_increment(curr_dun, 1); 341 src.offset += data_unit_size; 342 dst.offset += data_unit_size; 343 } 344 } 345 346 enc_bio->bi_private = src_bio; 347 enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio; 348 *bio_ptr = enc_bio; 349 ret = true; 350 351 enc_bio = NULL; 352 goto out_free_ciph_req; 353 354 out_free_bounce_pages: 355 while (i > 0) 356 mempool_free(enc_bio->bi_io_vec[--i].bv_page, 357 blk_crypto_bounce_page_pool); 358 out_free_ciph_req: 359 skcipher_request_free(ciph_req); 360 out_release_keyslot: 361 blk_crypto_put_keyslot(slot); 362 out_put_enc_bio: 363 if (enc_bio) 364 bio_uninit(enc_bio); 365 kfree(enc_bio); 366 return ret; 367 } 368 369 /* 370 * The crypto API fallback's main decryption routine. 371 * Decrypts input bio in place, and calls bio_endio on the bio. 372 */ 373 static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) 374 { 375 struct bio_fallback_crypt_ctx *f_ctx = 376 container_of(work, struct bio_fallback_crypt_ctx, work); 377 struct bio *bio = f_ctx->bio; 378 struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; 379 struct blk_crypto_keyslot *slot; 380 struct skcipher_request *ciph_req = NULL; 381 DECLARE_CRYPTO_WAIT(wait); 382 u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; 383 union blk_crypto_iv iv; 384 struct scatterlist sg; 385 struct bio_vec bv; 386 struct bvec_iter iter; 387 const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; 388 unsigned int i; 389 blk_status_t blk_st; 390 391 /* 392 * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for 393 * this bio's algorithm and key. 394 */ 395 blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, 396 bc->bc_key, &slot); 397 if (blk_st != BLK_STS_OK) { 398 bio->bi_status = blk_st; 399 goto out_no_keyslot; 400 } 401 402 /* and then allocate an skcipher_request for it */ 403 if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { 404 bio->bi_status = BLK_STS_RESOURCE; 405 goto out; 406 } 407 408 memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); 409 sg_init_table(&sg, 1); 410 skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, 411 iv.bytes); 412 413 /* Decrypt each segment in the bio */ 414 __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { 415 struct page *page = bv.bv_page; 416 417 sg_set_page(&sg, page, data_unit_size, bv.bv_offset); 418 419 /* Decrypt each data unit in the segment */ 420 for (i = 0; i < bv.bv_len; i += data_unit_size) { 421 blk_crypto_dun_to_iv(curr_dun, &iv); 422 if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), 423 &wait)) { 424 bio->bi_status = BLK_STS_IOERR; 425 goto out; 426 } 427 bio_crypt_dun_increment(curr_dun, 1); 428 sg.offset += data_unit_size; 429 } 430 } 431 432 out: 433 skcipher_request_free(ciph_req); 434 blk_crypto_put_keyslot(slot); 435 out_no_keyslot: 436 mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); 437 bio_endio(bio); 438 } 439 440 /** 441 * blk_crypto_fallback_decrypt_endio - queue bio for fallback decryption 442 * 443 * @bio: the bio to queue 444 * 445 * Restore bi_private and bi_end_io, and queue the bio for decryption into a 446 * workqueue, since this function will be called from an atomic context. 447 */ 448 static void blk_crypto_fallback_decrypt_endio(struct bio *bio) 449 { 450 struct bio_fallback_crypt_ctx *f_ctx = bio->bi_private; 451 452 bio->bi_private = f_ctx->bi_private_orig; 453 bio->bi_end_io = f_ctx->bi_end_io_orig; 454 455 /* If there was an IO error, don't queue for decrypt. */ 456 if (bio->bi_status) { 457 mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); 458 bio_endio(bio); 459 return; 460 } 461 462 INIT_WORK(&f_ctx->work, blk_crypto_fallback_decrypt_bio); 463 f_ctx->bio = bio; 464 queue_work(blk_crypto_wq, &f_ctx->work); 465 } 466 467 /** 468 * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption 469 * 470 * @bio_ptr: pointer to the bio to prepare 471 * 472 * If bio is doing a WRITE operation, this splits the bio into two parts if it's 473 * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a 474 * bounce bio for the first part, encrypts it, and updates bio_ptr to point to 475 * the bounce bio. 476 * 477 * For a READ operation, we mark the bio for decryption by using bi_private and 478 * bi_end_io. 479 * 480 * In either case, this function will make the bio look like a regular bio (i.e. 481 * as if no encryption context was ever specified) for the purposes of the rest 482 * of the stack except for blk-integrity (blk-integrity and blk-crypto are not 483 * currently supported together). 484 * 485 * Return: true on success. Sets bio->bi_status and returns false on error. 486 */ 487 bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) 488 { 489 struct bio *bio = *bio_ptr; 490 struct bio_crypt_ctx *bc = bio->bi_crypt_context; 491 struct bio_fallback_crypt_ctx *f_ctx; 492 493 if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) { 494 /* User didn't call blk_crypto_start_using_key() first */ 495 bio->bi_status = BLK_STS_IOERR; 496 return false; 497 } 498 499 if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile, 500 &bc->bc_key->crypto_cfg)) { 501 bio->bi_status = BLK_STS_NOTSUPP; 502 return false; 503 } 504 505 if (bio_data_dir(bio) == WRITE) 506 return blk_crypto_fallback_encrypt_bio(bio_ptr); 507 508 /* 509 * bio READ case: Set up a f_ctx in the bio's bi_private and set the 510 * bi_end_io appropriately to trigger decryption when the bio is ended. 511 */ 512 f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); 513 f_ctx->crypt_ctx = *bc; 514 f_ctx->crypt_iter = bio->bi_iter; 515 f_ctx->bi_private_orig = bio->bi_private; 516 f_ctx->bi_end_io_orig = bio->bi_end_io; 517 bio->bi_private = (void *)f_ctx; 518 bio->bi_end_io = blk_crypto_fallback_decrypt_endio; 519 bio_crypt_free_ctx(bio); 520 521 return true; 522 } 523 524 int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) 525 { 526 return __blk_crypto_evict_key(blk_crypto_fallback_profile, key); 527 } 528 529 static bool blk_crypto_fallback_inited; 530 static int blk_crypto_fallback_init(void) 531 { 532 int i; 533 int err; 534 535 if (blk_crypto_fallback_inited) 536 return 0; 537 538 get_random_bytes(blank_key, sizeof(blank_key)); 539 540 err = bioset_init(&crypto_bio_split, 64, 0, 0); 541 if (err) 542 goto out; 543 544 /* Dynamic allocation is needed because of lockdep_register_key(). */ 545 blk_crypto_fallback_profile = 546 kzalloc(sizeof(*blk_crypto_fallback_profile), GFP_KERNEL); 547 if (!blk_crypto_fallback_profile) { 548 err = -ENOMEM; 549 goto fail_free_bioset; 550 } 551 552 err = blk_crypto_profile_init(blk_crypto_fallback_profile, 553 blk_crypto_num_keyslots); 554 if (err) 555 goto fail_free_profile; 556 err = -ENOMEM; 557 558 blk_crypto_fallback_profile->ll_ops = blk_crypto_fallback_ll_ops; 559 blk_crypto_fallback_profile->max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE; 560 blk_crypto_fallback_profile->key_types_supported = BLK_CRYPTO_KEY_TYPE_RAW; 561 562 /* All blk-crypto modes have a crypto API fallback. */ 563 for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) 564 blk_crypto_fallback_profile->modes_supported[i] = 0xFFFFFFFF; 565 blk_crypto_fallback_profile->modes_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; 566 567 blk_crypto_wq = alloc_workqueue("blk_crypto_wq", 568 WQ_UNBOUND | WQ_HIGHPRI | 569 WQ_MEM_RECLAIM, num_online_cpus()); 570 if (!blk_crypto_wq) 571 goto fail_destroy_profile; 572 573 blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, 574 sizeof(blk_crypto_keyslots[0]), 575 GFP_KERNEL); 576 if (!blk_crypto_keyslots) 577 goto fail_free_wq; 578 579 blk_crypto_bounce_page_pool = 580 mempool_create_page_pool(num_prealloc_bounce_pg, 0); 581 if (!blk_crypto_bounce_page_pool) 582 goto fail_free_keyslots; 583 584 bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); 585 if (!bio_fallback_crypt_ctx_cache) 586 goto fail_free_bounce_page_pool; 587 588 bio_fallback_crypt_ctx_pool = 589 mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, 590 bio_fallback_crypt_ctx_cache); 591 if (!bio_fallback_crypt_ctx_pool) 592 goto fail_free_crypt_ctx_cache; 593 594 blk_crypto_fallback_inited = true; 595 596 return 0; 597 fail_free_crypt_ctx_cache: 598 kmem_cache_destroy(bio_fallback_crypt_ctx_cache); 599 fail_free_bounce_page_pool: 600 mempool_destroy(blk_crypto_bounce_page_pool); 601 fail_free_keyslots: 602 kfree(blk_crypto_keyslots); 603 fail_free_wq: 604 destroy_workqueue(blk_crypto_wq); 605 fail_destroy_profile: 606 blk_crypto_profile_destroy(blk_crypto_fallback_profile); 607 fail_free_profile: 608 kfree(blk_crypto_fallback_profile); 609 fail_free_bioset: 610 bioset_exit(&crypto_bio_split); 611 out: 612 return err; 613 } 614 615 /* 616 * Prepare blk-crypto-fallback for the specified crypto mode. 617 * Returns -ENOPKG if the needed crypto API support is missing. 618 */ 619 int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) 620 { 621 const char *cipher_str = blk_crypto_modes[mode_num].cipher_str; 622 struct blk_crypto_fallback_keyslot *slotp; 623 unsigned int i; 624 int err = 0; 625 626 /* 627 * Fast path 628 * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] 629 * for each i are visible before we try to access them. 630 */ 631 if (likely(smp_load_acquire(&tfms_inited[mode_num]))) 632 return 0; 633 634 mutex_lock(&tfms_init_lock); 635 if (tfms_inited[mode_num]) 636 goto out; 637 638 err = blk_crypto_fallback_init(); 639 if (err) 640 goto out; 641 642 for (i = 0; i < blk_crypto_num_keyslots; i++) { 643 slotp = &blk_crypto_keyslots[i]; 644 slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); 645 if (IS_ERR(slotp->tfms[mode_num])) { 646 err = PTR_ERR(slotp->tfms[mode_num]); 647 if (err == -ENOENT) { 648 pr_warn_once("Missing crypto API support for \"%s\"\n", 649 cipher_str); 650 err = -ENOPKG; 651 } 652 slotp->tfms[mode_num] = NULL; 653 goto out_free_tfms; 654 } 655 656 crypto_skcipher_set_flags(slotp->tfms[mode_num], 657 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); 658 } 659 660 /* 661 * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] 662 * for each i are visible before we set tfms_inited[mode_num]. 663 */ 664 smp_store_release(&tfms_inited[mode_num], true); 665 goto out; 666 667 out_free_tfms: 668 for (i = 0; i < blk_crypto_num_keyslots; i++) { 669 slotp = &blk_crypto_keyslots[i]; 670 crypto_free_skcipher(slotp->tfms[mode_num]); 671 slotp->tfms[mode_num] = NULL; 672 } 673 out: 674 mutex_unlock(&tfms_init_lock); 675 return err; 676 } 677