1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 Google LLC 4 */ 5 6 /** 7 * DOC: blk-crypto profiles 8 * 9 * 'struct blk_crypto_profile' contains all generic inline encryption-related 10 * state for a particular inline encryption device. blk_crypto_profile serves 11 * as the way that drivers for inline encryption hardware expose their crypto 12 * capabilities and certain functions (e.g., functions to program and evict 13 * keys) to upper layers. Device drivers that want to support inline encryption 14 * construct a crypto profile, then associate it with the disk's request_queue. 15 * 16 * If the device has keyslots, then its blk_crypto_profile also handles managing 17 * these keyslots in a device-independent way, using the driver-provided 18 * functions to program and evict keys as needed. This includes keeping track 19 * of which key and how many I/O requests are using each keyslot, getting 20 * keyslots for I/O requests, and handling key eviction requests. 21 * 22 * For more information, see Documentation/block/inline-encryption.rst. 23 */ 24 25 #define pr_fmt(fmt) "blk-crypto: " fmt 26 27 #include <linux/blk-crypto-profile.h> 28 #include <linux/device.h> 29 #include <linux/atomic.h> 30 #include <linux/mutex.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/wait.h> 33 #include <linux/blkdev.h> 34 #include <linux/blk-integrity.h> 35 #include "blk-crypto-internal.h" 36 37 struct blk_crypto_keyslot { 38 atomic_t slot_refs; 39 struct list_head idle_slot_node; 40 struct hlist_node hash_node; 41 const struct blk_crypto_key *key; 42 struct blk_crypto_profile *profile; 43 }; 44 45 static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile) 46 { 47 /* 48 * Calling into the driver requires profile->lock held and the device 49 * resumed. But we must resume the device first, since that can acquire 50 * and release profile->lock via blk_crypto_reprogram_all_keys(). 51 */ 52 if (profile->dev) 53 pm_runtime_get_sync(profile->dev); 54 down_write(&profile->lock); 55 } 56 57 static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile) 58 { 59 up_write(&profile->lock); 60 if (profile->dev) 61 pm_runtime_put_sync(profile->dev); 62 } 63 64 /** 65 * blk_crypto_profile_init() - Initialize a blk_crypto_profile 66 * @profile: the blk_crypto_profile to initialize 67 * @num_slots: the number of keyslots 68 * 69 * Storage drivers must call this when starting to set up a blk_crypto_profile, 70 * before filling in additional fields. 71 * 72 * Return: 0 on success, or else a negative error code. 73 */ 74 int blk_crypto_profile_init(struct blk_crypto_profile *profile, 75 unsigned int num_slots) 76 { 77 unsigned int slot; 78 unsigned int i; 79 unsigned int slot_hashtable_size; 80 81 memset(profile, 0, sizeof(*profile)); 82 83 /* 84 * profile->lock of an underlying device can nest inside profile->lock 85 * of a device-mapper device, so use a dynamic lock class to avoid 86 * false-positive lockdep reports. 87 */ 88 lockdep_register_key(&profile->lockdep_key); 89 __init_rwsem(&profile->lock, "&profile->lock", &profile->lockdep_key); 90 91 if (num_slots == 0) 92 return 0; 93 94 /* Initialize keyslot management data. */ 95 96 profile->slots = kvzalloc_objs(profile->slots[0], num_slots); 97 if (!profile->slots) 98 goto err_destroy; 99 100 profile->num_slots = num_slots; 101 102 init_waitqueue_head(&profile->idle_slots_wait_queue); 103 INIT_LIST_HEAD(&profile->idle_slots); 104 105 for (slot = 0; slot < num_slots; slot++) { 106 profile->slots[slot].profile = profile; 107 list_add_tail(&profile->slots[slot].idle_slot_node, 108 &profile->idle_slots); 109 } 110 111 spin_lock_init(&profile->idle_slots_lock); 112 113 slot_hashtable_size = roundup_pow_of_two(num_slots); 114 /* 115 * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2 116 * buckets. This only makes a difference when there is only 1 keyslot. 117 */ 118 if (slot_hashtable_size < 2) 119 slot_hashtable_size = 2; 120 121 profile->log_slot_ht_size = ilog2(slot_hashtable_size); 122 profile->slot_hashtable = 123 kvmalloc_objs(profile->slot_hashtable[0], slot_hashtable_size, 124 GFP_KERNEL); 125 if (!profile->slot_hashtable) 126 goto err_destroy; 127 for (i = 0; i < slot_hashtable_size; i++) 128 INIT_HLIST_HEAD(&profile->slot_hashtable[i]); 129 130 return 0; 131 132 err_destroy: 133 blk_crypto_profile_destroy(profile); 134 return -ENOMEM; 135 } 136 EXPORT_SYMBOL_GPL(blk_crypto_profile_init); 137 138 static void blk_crypto_profile_destroy_callback(void *profile) 139 { 140 blk_crypto_profile_destroy(profile); 141 } 142 143 /** 144 * devm_blk_crypto_profile_init() - Resource-managed blk_crypto_profile_init() 145 * @dev: the device which owns the blk_crypto_profile 146 * @profile: the blk_crypto_profile to initialize 147 * @num_slots: the number of keyslots 148 * 149 * Like blk_crypto_profile_init(), but causes blk_crypto_profile_destroy() to be 150 * called automatically on driver detach. 151 * 152 * Return: 0 on success, or else a negative error code. 153 */ 154 int devm_blk_crypto_profile_init(struct device *dev, 155 struct blk_crypto_profile *profile, 156 unsigned int num_slots) 157 { 158 int err = blk_crypto_profile_init(profile, num_slots); 159 160 if (err) 161 return err; 162 163 return devm_add_action_or_reset(dev, 164 blk_crypto_profile_destroy_callback, 165 profile); 166 } 167 EXPORT_SYMBOL_GPL(devm_blk_crypto_profile_init); 168 169 static inline struct hlist_head * 170 blk_crypto_hash_bucket_for_key(struct blk_crypto_profile *profile, 171 const struct blk_crypto_key *key) 172 { 173 return &profile->slot_hashtable[ 174 hash_ptr(key, profile->log_slot_ht_size)]; 175 } 176 177 static void 178 blk_crypto_remove_slot_from_lru_list(struct blk_crypto_keyslot *slot) 179 { 180 struct blk_crypto_profile *profile = slot->profile; 181 unsigned long flags; 182 183 spin_lock_irqsave(&profile->idle_slots_lock, flags); 184 list_del(&slot->idle_slot_node); 185 spin_unlock_irqrestore(&profile->idle_slots_lock, flags); 186 } 187 188 static struct blk_crypto_keyslot * 189 blk_crypto_find_keyslot(struct blk_crypto_profile *profile, 190 const struct blk_crypto_key *key) 191 { 192 const struct hlist_head *head = 193 blk_crypto_hash_bucket_for_key(profile, key); 194 struct blk_crypto_keyslot *slotp; 195 196 hlist_for_each_entry(slotp, head, hash_node) { 197 if (slotp->key == key) 198 return slotp; 199 } 200 return NULL; 201 } 202 203 static struct blk_crypto_keyslot * 204 blk_crypto_find_and_grab_keyslot(struct blk_crypto_profile *profile, 205 const struct blk_crypto_key *key) 206 { 207 struct blk_crypto_keyslot *slot; 208 209 slot = blk_crypto_find_keyslot(profile, key); 210 if (!slot) 211 return NULL; 212 if (atomic_inc_return(&slot->slot_refs) == 1) { 213 /* Took first reference to this slot; remove it from LRU list */ 214 blk_crypto_remove_slot_from_lru_list(slot); 215 } 216 return slot; 217 } 218 219 /** 220 * blk_crypto_keyslot_index() - Get the index of a keyslot 221 * @slot: a keyslot that blk_crypto_get_keyslot() returned 222 * 223 * Return: the 0-based index of the keyslot within the device's keyslots. 224 */ 225 unsigned int blk_crypto_keyslot_index(struct blk_crypto_keyslot *slot) 226 { 227 return slot - slot->profile->slots; 228 } 229 EXPORT_SYMBOL_GPL(blk_crypto_keyslot_index); 230 231 /** 232 * blk_crypto_get_keyslot() - Get a keyslot for a key, if needed. 233 * @profile: the crypto profile of the device the key will be used on 234 * @key: the key that will be used 235 * @slot_ptr: If a keyslot is allocated, an opaque pointer to the keyslot struct 236 * will be stored here. blk_crypto_put_keyslot() must be called 237 * later to release it. Otherwise, NULL will be stored here. 238 * 239 * If the device has keyslots, this gets a keyslot that's been programmed with 240 * the specified key. If the key is already in a slot, this reuses it; 241 * otherwise this waits for a slot to become idle and programs the key into it. 242 * 243 * Context: Process context. Takes and releases profile->lock. 244 * Return: BLK_STS_OK on success, meaning that either a keyslot was allocated or 245 * one wasn't needed; or a blk_status_t error on failure. 246 */ 247 blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile, 248 const struct blk_crypto_key *key, 249 struct blk_crypto_keyslot **slot_ptr) 250 { 251 struct blk_crypto_keyslot *slot; 252 int slot_idx; 253 int err; 254 255 *slot_ptr = NULL; 256 257 /* 258 * If the device has no concept of "keyslots", then there is no need to 259 * get one. 260 */ 261 if (profile->num_slots == 0) 262 return BLK_STS_OK; 263 264 down_read(&profile->lock); 265 slot = blk_crypto_find_and_grab_keyslot(profile, key); 266 up_read(&profile->lock); 267 if (slot) 268 goto success; 269 270 for (;;) { 271 blk_crypto_hw_enter(profile); 272 slot = blk_crypto_find_and_grab_keyslot(profile, key); 273 if (slot) { 274 blk_crypto_hw_exit(profile); 275 goto success; 276 } 277 278 /* 279 * If we're here, that means there wasn't a slot that was 280 * already programmed with the key. So try to program it. 281 */ 282 if (!list_empty(&profile->idle_slots)) 283 break; 284 285 blk_crypto_hw_exit(profile); 286 wait_event(profile->idle_slots_wait_queue, 287 !list_empty(&profile->idle_slots)); 288 } 289 290 slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot, 291 idle_slot_node); 292 slot_idx = blk_crypto_keyslot_index(slot); 293 294 err = profile->ll_ops.keyslot_program(profile, key, slot_idx); 295 if (err) { 296 wake_up(&profile->idle_slots_wait_queue); 297 blk_crypto_hw_exit(profile); 298 return errno_to_blk_status(err); 299 } 300 301 /* Move this slot to the hash list for the new key. */ 302 if (slot->key) 303 hlist_del(&slot->hash_node); 304 slot->key = key; 305 hlist_add_head(&slot->hash_node, 306 blk_crypto_hash_bucket_for_key(profile, key)); 307 308 atomic_set(&slot->slot_refs, 1); 309 310 blk_crypto_remove_slot_from_lru_list(slot); 311 312 blk_crypto_hw_exit(profile); 313 success: 314 *slot_ptr = slot; 315 return BLK_STS_OK; 316 } 317 318 /** 319 * blk_crypto_put_keyslot() - Release a reference to a keyslot 320 * @slot: The keyslot to release the reference of 321 * 322 * Context: Any context. 323 */ 324 void blk_crypto_put_keyslot(struct blk_crypto_keyslot *slot) 325 { 326 struct blk_crypto_profile *profile = slot->profile; 327 unsigned long flags; 328 329 if (atomic_dec_and_lock_irqsave(&slot->slot_refs, 330 &profile->idle_slots_lock, flags)) { 331 list_add_tail(&slot->idle_slot_node, &profile->idle_slots); 332 spin_unlock_irqrestore(&profile->idle_slots_lock, flags); 333 wake_up(&profile->idle_slots_wait_queue); 334 } 335 } 336 337 /** 338 * __blk_crypto_cfg_supported() - Check whether the given crypto profile 339 * supports the given crypto configuration. 340 * @profile: the crypto profile to check 341 * @cfg: the crypto configuration to check for 342 * 343 * Return: %true if @profile supports the given @cfg. 344 */ 345 bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile, 346 const struct blk_crypto_config *cfg) 347 { 348 if (!profile) 349 return false; 350 if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size)) 351 return false; 352 if (profile->max_dun_bytes_supported < cfg->dun_bytes) 353 return false; 354 if (!(profile->key_types_supported & cfg->key_type)) 355 return false; 356 return true; 357 } 358 359 /* 360 * This is an internal function that evicts a key from an inline encryption 361 * device that can be either a real device or the blk-crypto-fallback "device". 362 * It is used only by blk_crypto_evict_key(); see that function for details. 363 */ 364 int __blk_crypto_evict_key(struct blk_crypto_profile *profile, 365 const struct blk_crypto_key *key) 366 { 367 struct blk_crypto_keyslot *slot; 368 int err; 369 370 if (profile->num_slots == 0) { 371 if (profile->ll_ops.keyslot_evict) { 372 blk_crypto_hw_enter(profile); 373 err = profile->ll_ops.keyslot_evict(profile, key, -1); 374 blk_crypto_hw_exit(profile); 375 return err; 376 } 377 return 0; 378 } 379 380 blk_crypto_hw_enter(profile); 381 slot = blk_crypto_find_keyslot(profile, key); 382 if (!slot) { 383 /* 384 * Not an error, since a key not in use by I/O is not guaranteed 385 * to be in a keyslot. There can be more keys than keyslots. 386 */ 387 err = 0; 388 goto out; 389 } 390 391 if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) { 392 /* BUG: key is still in use by I/O */ 393 err = -EBUSY; 394 goto out_remove; 395 } 396 err = profile->ll_ops.keyslot_evict(profile, key, 397 blk_crypto_keyslot_index(slot)); 398 out_remove: 399 /* 400 * Callers free the key even on error, so unlink the key from the hash 401 * table and clear slot->key even on error. 402 */ 403 hlist_del(&slot->hash_node); 404 slot->key = NULL; 405 out: 406 blk_crypto_hw_exit(profile); 407 return err; 408 } 409 410 /** 411 * blk_crypto_reprogram_all_keys() - Re-program all keyslots. 412 * @profile: The crypto profile 413 * 414 * Re-program all keyslots that are supposed to have a key programmed. This is 415 * intended only for use by drivers for hardware that loses its keys on reset. 416 * 417 * Context: Process context. Takes and releases profile->lock. 418 */ 419 void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile) 420 { 421 unsigned int slot; 422 423 if (profile->num_slots == 0) 424 return; 425 426 /* This is for device initialization, so don't resume the device */ 427 down_write(&profile->lock); 428 for (slot = 0; slot < profile->num_slots; slot++) { 429 const struct blk_crypto_key *key = profile->slots[slot].key; 430 int err; 431 432 if (!key) 433 continue; 434 435 err = profile->ll_ops.keyslot_program(profile, key, slot); 436 WARN_ON(err); 437 } 438 up_write(&profile->lock); 439 } 440 EXPORT_SYMBOL_GPL(blk_crypto_reprogram_all_keys); 441 442 void blk_crypto_profile_destroy(struct blk_crypto_profile *profile) 443 { 444 if (!profile) 445 return; 446 lockdep_unregister_key(&profile->lockdep_key); 447 kvfree(profile->slot_hashtable); 448 kvfree_sensitive(profile->slots, 449 sizeof(profile->slots[0]) * profile->num_slots); 450 memzero_explicit(profile, sizeof(*profile)); 451 } 452 EXPORT_SYMBOL_GPL(blk_crypto_profile_destroy); 453 454 bool blk_crypto_register(struct blk_crypto_profile *profile, 455 struct request_queue *q) 456 { 457 if (blk_integrity_queue_supports_integrity(q)) { 458 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n"); 459 return false; 460 } 461 q->crypto_profile = profile; 462 return true; 463 } 464 EXPORT_SYMBOL_GPL(blk_crypto_register); 465 466 /** 467 * blk_crypto_derive_sw_secret() - Derive software secret from wrapped key 468 * @bdev: a block device that supports hardware-wrapped keys 469 * @eph_key: a hardware-wrapped key in ephemerally-wrapped form 470 * @eph_key_size: size of @eph_key in bytes 471 * @sw_secret: (output) the software secret 472 * 473 * Given a hardware-wrapped key in ephemerally-wrapped form (the same form that 474 * it is used for I/O), ask the hardware to derive the secret which software can 475 * use for cryptographic tasks other than inline encryption. This secret is 476 * guaranteed to be cryptographically isolated from the inline encryption key, 477 * i.e. derived with a different KDF context. 478 * 479 * Return: 0 on success, -EOPNOTSUPP if the block device doesn't support 480 * hardware-wrapped keys, -EBADMSG if the key isn't a valid 481 * ephemerally-wrapped key, or another -errno code. 482 */ 483 int blk_crypto_derive_sw_secret(struct block_device *bdev, 484 const u8 *eph_key, size_t eph_key_size, 485 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) 486 { 487 struct blk_crypto_profile *profile = 488 bdev_get_queue(bdev)->crypto_profile; 489 int err; 490 491 if (!profile) 492 return -EOPNOTSUPP; 493 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 494 return -EOPNOTSUPP; 495 if (!profile->ll_ops.derive_sw_secret) 496 return -EOPNOTSUPP; 497 blk_crypto_hw_enter(profile); 498 err = profile->ll_ops.derive_sw_secret(profile, eph_key, eph_key_size, 499 sw_secret); 500 blk_crypto_hw_exit(profile); 501 return err; 502 } 503 EXPORT_SYMBOL_GPL(blk_crypto_derive_sw_secret); 504 505 int blk_crypto_import_key(struct blk_crypto_profile *profile, 506 const u8 *raw_key, size_t raw_key_size, 507 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 508 { 509 int ret; 510 511 if (!profile) 512 return -EOPNOTSUPP; 513 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 514 return -EOPNOTSUPP; 515 if (!profile->ll_ops.import_key) 516 return -EOPNOTSUPP; 517 blk_crypto_hw_enter(profile); 518 ret = profile->ll_ops.import_key(profile, raw_key, raw_key_size, 519 lt_key); 520 blk_crypto_hw_exit(profile); 521 return ret; 522 } 523 EXPORT_SYMBOL_GPL(blk_crypto_import_key); 524 525 int blk_crypto_generate_key(struct blk_crypto_profile *profile, 526 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 527 { 528 int ret; 529 530 if (!profile) 531 return -EOPNOTSUPP; 532 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 533 return -EOPNOTSUPP; 534 if (!profile->ll_ops.generate_key) 535 return -EOPNOTSUPP; 536 blk_crypto_hw_enter(profile); 537 ret = profile->ll_ops.generate_key(profile, lt_key); 538 blk_crypto_hw_exit(profile); 539 return ret; 540 } 541 EXPORT_SYMBOL_GPL(blk_crypto_generate_key); 542 543 int blk_crypto_prepare_key(struct blk_crypto_profile *profile, 544 const u8 *lt_key, size_t lt_key_size, 545 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 546 { 547 int ret; 548 549 if (!profile) 550 return -EOPNOTSUPP; 551 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 552 return -EOPNOTSUPP; 553 if (!profile->ll_ops.prepare_key) 554 return -EOPNOTSUPP; 555 blk_crypto_hw_enter(profile); 556 ret = profile->ll_ops.prepare_key(profile, lt_key, lt_key_size, 557 eph_key); 558 blk_crypto_hw_exit(profile); 559 return ret; 560 } 561 EXPORT_SYMBOL_GPL(blk_crypto_prepare_key); 562 563 /** 564 * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities 565 * by child device 566 * @parent: the crypto profile for the parent device 567 * @child: the crypto profile for the child device, or NULL 568 * 569 * This clears all crypto capabilities in @parent that aren't set in @child. If 570 * @child is NULL, then this clears all parent capabilities. 571 * 572 * Only use this when setting up the crypto profile for a layered device, before 573 * it's been exposed yet. 574 */ 575 void blk_crypto_intersect_capabilities(struct blk_crypto_profile *parent, 576 const struct blk_crypto_profile *child) 577 { 578 if (child) { 579 unsigned int i; 580 581 parent->max_dun_bytes_supported = 582 min(parent->max_dun_bytes_supported, 583 child->max_dun_bytes_supported); 584 for (i = 0; i < ARRAY_SIZE(child->modes_supported); i++) 585 parent->modes_supported[i] &= child->modes_supported[i]; 586 parent->key_types_supported &= child->key_types_supported; 587 } else { 588 parent->max_dun_bytes_supported = 0; 589 memset(parent->modes_supported, 0, 590 sizeof(parent->modes_supported)); 591 parent->key_types_supported = 0; 592 } 593 } 594 EXPORT_SYMBOL_GPL(blk_crypto_intersect_capabilities); 595 596 /** 597 * blk_crypto_has_capabilities() - Check whether @target supports at least all 598 * the crypto capabilities that @reference does. 599 * @target: the target profile 600 * @reference: the reference profile 601 * 602 * Return: %true if @target supports all the crypto capabilities of @reference. 603 */ 604 bool blk_crypto_has_capabilities(const struct blk_crypto_profile *target, 605 const struct blk_crypto_profile *reference) 606 { 607 int i; 608 609 if (!reference) 610 return true; 611 612 if (!target) 613 return false; 614 615 for (i = 0; i < ARRAY_SIZE(target->modes_supported); i++) { 616 if (reference->modes_supported[i] & ~target->modes_supported[i]) 617 return false; 618 } 619 620 if (reference->max_dun_bytes_supported > 621 target->max_dun_bytes_supported) 622 return false; 623 624 if (reference->key_types_supported & ~target->key_types_supported) 625 return false; 626 627 return true; 628 } 629 EXPORT_SYMBOL_GPL(blk_crypto_has_capabilities); 630 631 /** 632 * blk_crypto_update_capabilities() - Update the capabilities of a crypto 633 * profile to match those of another crypto 634 * profile. 635 * @dst: The crypto profile whose capabilities to update. 636 * @src: The crypto profile whose capabilities this function will update @dst's 637 * capabilities to. 638 * 639 * Blk-crypto requires that crypto capabilities that were 640 * advertised when a bio was created continue to be supported by the 641 * device until that bio is ended. This is turn means that a device cannot 642 * shrink its advertised crypto capabilities without any explicit 643 * synchronization with upper layers. So if there's no such explicit 644 * synchronization, @src must support all the crypto capabilities that 645 * @dst does (i.e. we need blk_crypto_has_capabilities(@src, @dst)). 646 * 647 * Note also that as long as the crypto capabilities are being expanded, the 648 * order of updates becoming visible is not important because it's alright 649 * for blk-crypto to see stale values - they only cause blk-crypto to 650 * believe that a crypto capability isn't supported when it actually is (which 651 * might result in blk-crypto-fallback being used if available, or the bio being 652 * failed). 653 */ 654 void blk_crypto_update_capabilities(struct blk_crypto_profile *dst, 655 const struct blk_crypto_profile *src) 656 { 657 memcpy(dst->modes_supported, src->modes_supported, 658 sizeof(dst->modes_supported)); 659 660 dst->max_dun_bytes_supported = src->max_dun_bytes_supported; 661 dst->key_types_supported = src->key_types_supported; 662 } 663 EXPORT_SYMBOL_GPL(blk_crypto_update_capabilities); 664