1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 Google LLC 4 */ 5 6 /** 7 * DOC: blk-crypto profiles 8 * 9 * 'struct blk_crypto_profile' contains all generic inline encryption-related 10 * state for a particular inline encryption device. blk_crypto_profile serves 11 * as the way that drivers for inline encryption hardware expose their crypto 12 * capabilities and certain functions (e.g., functions to program and evict 13 * keys) to upper layers. Device drivers that want to support inline encryption 14 * construct a crypto profile, then associate it with the disk's request_queue. 15 * 16 * If the device has keyslots, then its blk_crypto_profile also handles managing 17 * these keyslots in a device-independent way, using the driver-provided 18 * functions to program and evict keys as needed. This includes keeping track 19 * of which key and how many I/O requests are using each keyslot, getting 20 * keyslots for I/O requests, and handling key eviction requests. 21 * 22 * For more information, see Documentation/block/inline-encryption.rst. 23 */ 24 25 #define pr_fmt(fmt) "blk-crypto: " fmt 26 27 #include <linux/blk-crypto-profile.h> 28 #include <linux/device.h> 29 #include <linux/atomic.h> 30 #include <linux/mutex.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/wait.h> 33 #include <linux/blkdev.h> 34 #include <linux/blk-integrity.h> 35 #include "blk-crypto-internal.h" 36 37 struct blk_crypto_keyslot { 38 atomic_t slot_refs; 39 struct list_head idle_slot_node; 40 struct hlist_node hash_node; 41 const struct blk_crypto_key *key; 42 struct blk_crypto_profile *profile; 43 }; 44 45 static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile) 46 __acquires(&profile->lock) 47 { 48 /* 49 * Calling into the driver requires profile->lock held and the device 50 * resumed. But we must resume the device first, since that can acquire 51 * and release profile->lock via blk_crypto_reprogram_all_keys(). 52 */ 53 if (profile->dev) 54 pm_runtime_get_sync(profile->dev); 55 down_write(&profile->lock); 56 } 57 58 static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile) 59 __releases(&profile->lock) 60 { 61 up_write(&profile->lock); 62 if (profile->dev) 63 pm_runtime_put_sync(profile->dev); 64 } 65 66 /** 67 * blk_crypto_profile_init() - Initialize a blk_crypto_profile 68 * @profile: the blk_crypto_profile to initialize 69 * @num_slots: the number of keyslots 70 * 71 * Storage drivers must call this when starting to set up a blk_crypto_profile, 72 * before filling in additional fields. 73 * 74 * Return: 0 on success, or else a negative error code. 75 */ 76 int blk_crypto_profile_init(struct blk_crypto_profile *profile, 77 unsigned int num_slots) 78 { 79 unsigned int slot; 80 unsigned int i; 81 unsigned int slot_hashtable_size; 82 83 memset(profile, 0, sizeof(*profile)); 84 85 /* 86 * profile->lock of an underlying device can nest inside profile->lock 87 * of a device-mapper device, so use a dynamic lock class to avoid 88 * false-positive lockdep reports. 89 */ 90 lockdep_register_key(&profile->lockdep_key); 91 __init_rwsem(&profile->lock, "&profile->lock", &profile->lockdep_key); 92 93 if (num_slots == 0) 94 return 0; 95 96 /* Initialize keyslot management data. */ 97 98 profile->slots = kvzalloc_objs(profile->slots[0], num_slots); 99 if (!profile->slots) 100 goto err_destroy; 101 102 profile->num_slots = num_slots; 103 104 init_waitqueue_head(&profile->idle_slots_wait_queue); 105 INIT_LIST_HEAD(&profile->idle_slots); 106 107 for (slot = 0; slot < num_slots; slot++) { 108 profile->slots[slot].profile = profile; 109 list_add_tail(&profile->slots[slot].idle_slot_node, 110 &profile->idle_slots); 111 } 112 113 spin_lock_init(&profile->idle_slots_lock); 114 115 slot_hashtable_size = roundup_pow_of_two(num_slots); 116 /* 117 * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2 118 * buckets. This only makes a difference when there is only 1 keyslot. 119 */ 120 if (slot_hashtable_size < 2) 121 slot_hashtable_size = 2; 122 123 profile->log_slot_ht_size = ilog2(slot_hashtable_size); 124 profile->slot_hashtable = 125 kvmalloc_objs(profile->slot_hashtable[0], slot_hashtable_size); 126 if (!profile->slot_hashtable) 127 goto err_destroy; 128 for (i = 0; i < slot_hashtable_size; i++) 129 INIT_HLIST_HEAD(&profile->slot_hashtable[i]); 130 131 return 0; 132 133 err_destroy: 134 blk_crypto_profile_destroy(profile); 135 return -ENOMEM; 136 } 137 EXPORT_SYMBOL_GPL(blk_crypto_profile_init); 138 139 static void blk_crypto_profile_destroy_callback(void *profile) 140 { 141 blk_crypto_profile_destroy(profile); 142 } 143 144 /** 145 * devm_blk_crypto_profile_init() - Resource-managed blk_crypto_profile_init() 146 * @dev: the device which owns the blk_crypto_profile 147 * @profile: the blk_crypto_profile to initialize 148 * @num_slots: the number of keyslots 149 * 150 * Like blk_crypto_profile_init(), but causes blk_crypto_profile_destroy() to be 151 * called automatically on driver detach. 152 * 153 * Return: 0 on success, or else a negative error code. 154 */ 155 int devm_blk_crypto_profile_init(struct device *dev, 156 struct blk_crypto_profile *profile, 157 unsigned int num_slots) 158 { 159 int err = blk_crypto_profile_init(profile, num_slots); 160 161 if (err) 162 return err; 163 164 return devm_add_action_or_reset(dev, 165 blk_crypto_profile_destroy_callback, 166 profile); 167 } 168 EXPORT_SYMBOL_GPL(devm_blk_crypto_profile_init); 169 170 static inline struct hlist_head * 171 blk_crypto_hash_bucket_for_key(struct blk_crypto_profile *profile, 172 const struct blk_crypto_key *key) 173 { 174 return &profile->slot_hashtable[ 175 hash_ptr(key, profile->log_slot_ht_size)]; 176 } 177 178 static void 179 blk_crypto_remove_slot_from_lru_list(struct blk_crypto_keyslot *slot) 180 { 181 struct blk_crypto_profile *profile = slot->profile; 182 unsigned long flags; 183 184 spin_lock_irqsave(&profile->idle_slots_lock, flags); 185 list_del(&slot->idle_slot_node); 186 spin_unlock_irqrestore(&profile->idle_slots_lock, flags); 187 } 188 189 static struct blk_crypto_keyslot * 190 blk_crypto_find_keyslot(struct blk_crypto_profile *profile, 191 const struct blk_crypto_key *key) 192 { 193 const struct hlist_head *head = 194 blk_crypto_hash_bucket_for_key(profile, key); 195 struct blk_crypto_keyslot *slotp; 196 197 hlist_for_each_entry(slotp, head, hash_node) { 198 if (slotp->key == key) 199 return slotp; 200 } 201 return NULL; 202 } 203 204 static struct blk_crypto_keyslot * 205 blk_crypto_find_and_grab_keyslot(struct blk_crypto_profile *profile, 206 const struct blk_crypto_key *key) 207 { 208 struct blk_crypto_keyslot *slot; 209 210 slot = blk_crypto_find_keyslot(profile, key); 211 if (!slot) 212 return NULL; 213 if (atomic_inc_return(&slot->slot_refs) == 1) { 214 /* Took first reference to this slot; remove it from LRU list */ 215 blk_crypto_remove_slot_from_lru_list(slot); 216 } 217 return slot; 218 } 219 220 /** 221 * blk_crypto_keyslot_index() - Get the index of a keyslot 222 * @slot: a keyslot that blk_crypto_get_keyslot() returned 223 * 224 * Return: the 0-based index of the keyslot within the device's keyslots. 225 */ 226 unsigned int blk_crypto_keyslot_index(struct blk_crypto_keyslot *slot) 227 { 228 return slot - slot->profile->slots; 229 } 230 EXPORT_SYMBOL_GPL(blk_crypto_keyslot_index); 231 232 /** 233 * blk_crypto_get_keyslot() - Get a keyslot for a key, if needed. 234 * @profile: the crypto profile of the device the key will be used on 235 * @key: the key that will be used 236 * @slot_ptr: If a keyslot is allocated, an opaque pointer to the keyslot struct 237 * will be stored here. blk_crypto_put_keyslot() must be called 238 * later to release it. Otherwise, NULL will be stored here. 239 * 240 * If the device has keyslots, this gets a keyslot that's been programmed with 241 * the specified key. If the key is already in a slot, this reuses it; 242 * otherwise this waits for a slot to become idle and programs the key into it. 243 * 244 * Context: Process context. Takes and releases profile->lock. 245 * Return: BLK_STS_OK on success, meaning that either a keyslot was allocated or 246 * one wasn't needed; or a blk_status_t error on failure. 247 */ 248 blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile, 249 const struct blk_crypto_key *key, 250 struct blk_crypto_keyslot **slot_ptr) 251 { 252 struct blk_crypto_keyslot *slot; 253 int slot_idx; 254 int err; 255 256 *slot_ptr = NULL; 257 258 /* 259 * If the device has no concept of "keyslots", then there is no need to 260 * get one. 261 */ 262 if (profile->num_slots == 0) 263 return BLK_STS_OK; 264 265 down_read(&profile->lock); 266 slot = blk_crypto_find_and_grab_keyslot(profile, key); 267 up_read(&profile->lock); 268 if (slot) 269 goto success; 270 271 for (;;) { 272 blk_crypto_hw_enter(profile); 273 slot = blk_crypto_find_and_grab_keyslot(profile, key); 274 if (slot) { 275 blk_crypto_hw_exit(profile); 276 goto success; 277 } 278 279 /* 280 * If we're here, that means there wasn't a slot that was 281 * already programmed with the key. So try to program it. 282 */ 283 if (!list_empty(&profile->idle_slots)) 284 break; 285 286 blk_crypto_hw_exit(profile); 287 wait_event(profile->idle_slots_wait_queue, 288 !list_empty(&profile->idle_slots)); 289 } 290 291 slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot, 292 idle_slot_node); 293 slot_idx = blk_crypto_keyslot_index(slot); 294 295 err = profile->ll_ops.keyslot_program(profile, key, slot_idx); 296 if (err) { 297 wake_up(&profile->idle_slots_wait_queue); 298 blk_crypto_hw_exit(profile); 299 return errno_to_blk_status(err); 300 } 301 302 /* Move this slot to the hash list for the new key. */ 303 if (slot->key) 304 hlist_del(&slot->hash_node); 305 slot->key = key; 306 hlist_add_head(&slot->hash_node, 307 blk_crypto_hash_bucket_for_key(profile, key)); 308 309 atomic_set(&slot->slot_refs, 1); 310 311 blk_crypto_remove_slot_from_lru_list(slot); 312 313 blk_crypto_hw_exit(profile); 314 success: 315 *slot_ptr = slot; 316 return BLK_STS_OK; 317 } 318 319 /** 320 * blk_crypto_put_keyslot() - Release a reference to a keyslot 321 * @slot: The keyslot to release the reference of 322 * 323 * Context: Any context. 324 */ 325 void blk_crypto_put_keyslot(struct blk_crypto_keyslot *slot) 326 { 327 struct blk_crypto_profile *profile = slot->profile; 328 unsigned long flags; 329 330 if (atomic_dec_and_lock_irqsave(&slot->slot_refs, 331 &profile->idle_slots_lock, flags)) { 332 list_add_tail(&slot->idle_slot_node, &profile->idle_slots); 333 spin_unlock_irqrestore(&profile->idle_slots_lock, flags); 334 wake_up(&profile->idle_slots_wait_queue); 335 } 336 } 337 338 /* 339 * This is an internal function that evicts a key from an inline encryption 340 * device that can be either a real device or the blk-crypto-fallback "device". 341 * It is used only by blk_crypto_evict_key(); see that function for details. 342 */ 343 int __blk_crypto_evict_key(struct blk_crypto_profile *profile, 344 const struct blk_crypto_key *key) 345 { 346 struct blk_crypto_keyslot *slot; 347 int err; 348 349 if (profile->num_slots == 0) { 350 if (profile->ll_ops.keyslot_evict) { 351 blk_crypto_hw_enter(profile); 352 err = profile->ll_ops.keyslot_evict(profile, key, -1); 353 blk_crypto_hw_exit(profile); 354 return err; 355 } 356 return 0; 357 } 358 359 blk_crypto_hw_enter(profile); 360 slot = blk_crypto_find_keyslot(profile, key); 361 if (!slot) { 362 /* 363 * Not an error, since a key not in use by I/O is not guaranteed 364 * to be in a keyslot. There can be more keys than keyslots. 365 */ 366 err = 0; 367 goto out; 368 } 369 370 if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) { 371 /* BUG: key is still in use by I/O */ 372 err = -EBUSY; 373 goto out_remove; 374 } 375 err = profile->ll_ops.keyslot_evict(profile, key, 376 blk_crypto_keyslot_index(slot)); 377 out_remove: 378 /* 379 * Callers free the key even on error, so unlink the key from the hash 380 * table and clear slot->key even on error. 381 */ 382 hlist_del(&slot->hash_node); 383 slot->key = NULL; 384 out: 385 blk_crypto_hw_exit(profile); 386 return err; 387 } 388 389 /** 390 * blk_crypto_reprogram_all_keys() - Re-program all keyslots. 391 * @profile: The crypto profile 392 * 393 * Re-program all keyslots that are supposed to have a key programmed. This is 394 * intended only for use by drivers for hardware that loses its keys on reset. 395 * 396 * Context: Process context. Takes and releases profile->lock. 397 */ 398 void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile) 399 { 400 unsigned int slot; 401 402 if (profile->num_slots == 0) 403 return; 404 405 /* This is for device initialization, so don't resume the device */ 406 down_write(&profile->lock); 407 for (slot = 0; slot < profile->num_slots; slot++) { 408 const struct blk_crypto_key *key = profile->slots[slot].key; 409 int err; 410 411 if (!key) 412 continue; 413 414 err = profile->ll_ops.keyslot_program(profile, key, slot); 415 WARN_ON(err); 416 } 417 up_write(&profile->lock); 418 } 419 EXPORT_SYMBOL_GPL(blk_crypto_reprogram_all_keys); 420 421 void blk_crypto_profile_destroy(struct blk_crypto_profile *profile) 422 { 423 if (!profile) 424 return; 425 lockdep_unregister_key(&profile->lockdep_key); 426 kvfree(profile->slot_hashtable); 427 kvfree_sensitive(profile->slots, 428 sizeof(profile->slots[0]) * profile->num_slots); 429 memzero_explicit(profile, sizeof(*profile)); 430 } 431 EXPORT_SYMBOL_GPL(blk_crypto_profile_destroy); 432 433 bool blk_crypto_register(struct blk_crypto_profile *profile, 434 struct request_queue *q) 435 { 436 if (blk_integrity_queue_supports_integrity(q)) { 437 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n"); 438 return false; 439 } 440 q->crypto_profile = profile; 441 return true; 442 } 443 EXPORT_SYMBOL_GPL(blk_crypto_register); 444 445 /** 446 * blk_crypto_derive_sw_secret() - Derive software secret from wrapped key 447 * @bdev: a block device that supports hardware-wrapped keys 448 * @eph_key: a hardware-wrapped key in ephemerally-wrapped form 449 * @eph_key_size: size of @eph_key in bytes 450 * @sw_secret: (output) the software secret 451 * 452 * Given a hardware-wrapped key in ephemerally-wrapped form (the same form that 453 * it is used for I/O), ask the hardware to derive the secret which software can 454 * use for cryptographic tasks other than inline encryption. This secret is 455 * guaranteed to be cryptographically isolated from the inline encryption key, 456 * i.e. derived with a different KDF context. 457 * 458 * Return: 0 on success, -EOPNOTSUPP if the block device doesn't support 459 * hardware-wrapped keys, -EBADMSG if the key isn't a valid 460 * ephemerally-wrapped key, or another -errno code. 461 */ 462 int blk_crypto_derive_sw_secret(struct block_device *bdev, 463 const u8 *eph_key, size_t eph_key_size, 464 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) 465 { 466 struct blk_crypto_profile *profile = 467 bdev_get_queue(bdev)->crypto_profile; 468 int err; 469 470 if (!profile) 471 return -EOPNOTSUPP; 472 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 473 return -EOPNOTSUPP; 474 if (!profile->ll_ops.derive_sw_secret) 475 return -EOPNOTSUPP; 476 blk_crypto_hw_enter(profile); 477 err = profile->ll_ops.derive_sw_secret(profile, eph_key, eph_key_size, 478 sw_secret); 479 blk_crypto_hw_exit(profile); 480 return err; 481 } 482 EXPORT_SYMBOL_GPL(blk_crypto_derive_sw_secret); 483 484 int blk_crypto_import_key(struct blk_crypto_profile *profile, 485 const u8 *raw_key, size_t raw_key_size, 486 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 487 { 488 int ret; 489 490 if (!profile) 491 return -EOPNOTSUPP; 492 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 493 return -EOPNOTSUPP; 494 if (!profile->ll_ops.import_key) 495 return -EOPNOTSUPP; 496 blk_crypto_hw_enter(profile); 497 ret = profile->ll_ops.import_key(profile, raw_key, raw_key_size, 498 lt_key); 499 blk_crypto_hw_exit(profile); 500 return ret; 501 } 502 EXPORT_SYMBOL_GPL(blk_crypto_import_key); 503 504 int blk_crypto_generate_key(struct blk_crypto_profile *profile, 505 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 506 { 507 int ret; 508 509 if (!profile) 510 return -EOPNOTSUPP; 511 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 512 return -EOPNOTSUPP; 513 if (!profile->ll_ops.generate_key) 514 return -EOPNOTSUPP; 515 blk_crypto_hw_enter(profile); 516 ret = profile->ll_ops.generate_key(profile, lt_key); 517 blk_crypto_hw_exit(profile); 518 return ret; 519 } 520 EXPORT_SYMBOL_GPL(blk_crypto_generate_key); 521 522 int blk_crypto_prepare_key(struct blk_crypto_profile *profile, 523 const u8 *lt_key, size_t lt_key_size, 524 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) 525 { 526 int ret; 527 528 if (!profile) 529 return -EOPNOTSUPP; 530 if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) 531 return -EOPNOTSUPP; 532 if (!profile->ll_ops.prepare_key) 533 return -EOPNOTSUPP; 534 blk_crypto_hw_enter(profile); 535 ret = profile->ll_ops.prepare_key(profile, lt_key, lt_key_size, 536 eph_key); 537 blk_crypto_hw_exit(profile); 538 return ret; 539 } 540 EXPORT_SYMBOL_GPL(blk_crypto_prepare_key); 541 542 /** 543 * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities 544 * by child device 545 * @parent: the crypto profile for the parent device 546 * @child: the crypto profile for the child device, or NULL 547 * 548 * This clears all crypto capabilities in @parent that aren't set in @child. If 549 * @child is NULL, then this clears all parent capabilities. 550 * 551 * Only use this when setting up the crypto profile for a layered device, before 552 * it's been exposed yet. 553 */ 554 void blk_crypto_intersect_capabilities(struct blk_crypto_profile *parent, 555 const struct blk_crypto_profile *child) 556 { 557 if (child) { 558 unsigned int i; 559 560 parent->max_dun_bytes_supported = 561 min(parent->max_dun_bytes_supported, 562 child->max_dun_bytes_supported); 563 for (i = 0; i < ARRAY_SIZE(child->modes_supported); i++) 564 parent->modes_supported[i] &= child->modes_supported[i]; 565 parent->key_types_supported &= child->key_types_supported; 566 } else { 567 parent->max_dun_bytes_supported = 0; 568 memset(parent->modes_supported, 0, 569 sizeof(parent->modes_supported)); 570 parent->key_types_supported = 0; 571 } 572 } 573 EXPORT_SYMBOL_GPL(blk_crypto_intersect_capabilities); 574 575 /** 576 * blk_crypto_has_capabilities() - Check whether @target supports at least all 577 * the crypto capabilities that @reference does. 578 * @target: the target profile 579 * @reference: the reference profile 580 * 581 * Return: %true if @target supports all the crypto capabilities of @reference. 582 */ 583 bool blk_crypto_has_capabilities(const struct blk_crypto_profile *target, 584 const struct blk_crypto_profile *reference) 585 { 586 int i; 587 588 if (!reference) 589 return true; 590 591 if (!target) 592 return false; 593 594 for (i = 0; i < ARRAY_SIZE(target->modes_supported); i++) { 595 if (reference->modes_supported[i] & ~target->modes_supported[i]) 596 return false; 597 } 598 599 if (reference->max_dun_bytes_supported > 600 target->max_dun_bytes_supported) 601 return false; 602 603 if (reference->key_types_supported & ~target->key_types_supported) 604 return false; 605 606 return true; 607 } 608 EXPORT_SYMBOL_GPL(blk_crypto_has_capabilities); 609 610 /** 611 * blk_crypto_update_capabilities() - Update the capabilities of a crypto 612 * profile to match those of another crypto 613 * profile. 614 * @dst: The crypto profile whose capabilities to update. 615 * @src: The crypto profile whose capabilities this function will update @dst's 616 * capabilities to. 617 * 618 * Blk-crypto requires that crypto capabilities that were 619 * advertised when a bio was created continue to be supported by the 620 * device until that bio is ended. This is turn means that a device cannot 621 * shrink its advertised crypto capabilities without any explicit 622 * synchronization with upper layers. So if there's no such explicit 623 * synchronization, @src must support all the crypto capabilities that 624 * @dst does (i.e. we need blk_crypto_has_capabilities(@src, @dst)). 625 * 626 * Note also that as long as the crypto capabilities are being expanded, the 627 * order of updates becoming visible is not important because it's alright 628 * for blk-crypto to see stale values - they only cause blk-crypto to 629 * believe that a crypto capability isn't supported when it actually is (which 630 * might result in blk-crypto-fallback being used if available, or the bio being 631 * failed). 632 */ 633 void blk_crypto_update_capabilities(struct blk_crypto_profile *dst, 634 const struct blk_crypto_profile *src) 635 { 636 memcpy(dst->modes_supported, src->modes_supported, 637 sizeof(dst->modes_supported)); 638 639 dst->max_dun_bytes_supported = src->max_dun_bytes_supported; 640 dst->key_types_supported = src->key_types_supported; 641 } 642 EXPORT_SYMBOL_GPL(blk_crypto_update_capabilities); 643