1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2024 Google LLC 4 */ 5 6 #include <linux/blk-crypto.h> 7 #include <linux/ctype.h> 8 #include <linux/device-mapper.h> 9 #include <linux/hex.h> 10 #include <linux/module.h> 11 #include <keys/user-type.h> 12 13 #define DM_MSG_PREFIX "inlinecrypt" 14 15 static const struct dm_inlinecrypt_cipher { 16 const char *name; 17 enum blk_crypto_mode_num mode_num; 18 } dm_inlinecrypt_ciphers[] = { 19 { 20 .name = "aes-xts-plain64", 21 .mode_num = BLK_ENCRYPTION_MODE_AES_256_XTS, 22 }, 23 }; 24 25 /** 26 * struct inlinecrypt_ctx - private data of an inlinecrypt target 27 * @dev: the underlying device 28 * @start: starting sector of the range of @dev which this target actually maps. 29 * For this purpose a "sector" is 512 bytes. 30 * @cipher_string: the name of the encryption algorithm being used 31 * @key_size: size of the encryption key in bytes 32 * @iv_offset: starting offset for IVs. IVs are generated as if the target were 33 * preceded by @iv_offset 512-byte sectors. 34 * @sector_size: crypto sector size in bytes (usually 4096) 35 * @sector_bits: log2(sector_size) 36 * @key: the encryption key to use 37 * @max_dun: the maximum DUN that may be used (computed from other params) 38 */ 39 struct inlinecrypt_ctx { 40 struct dm_dev *dev; 41 sector_t start; 42 const char *cipher_string; 43 unsigned int key_size; 44 u64 iv_offset; 45 unsigned int sector_size; 46 unsigned int sector_bits; 47 struct blk_crypto_key key; 48 u64 max_dun; 49 }; 50 51 static const struct dm_inlinecrypt_cipher * 52 lookup_cipher(const char *cipher_string) 53 { 54 int i; 55 56 for (i = 0; i < ARRAY_SIZE(dm_inlinecrypt_ciphers); i++) { 57 if (strcmp(cipher_string, dm_inlinecrypt_ciphers[i].name) == 0) 58 return &dm_inlinecrypt_ciphers[i]; 59 } 60 return NULL; 61 } 62 63 static void inlinecrypt_dtr(struct dm_target *ti) 64 { 65 struct inlinecrypt_ctx *ctx = ti->private; 66 67 if (ctx->dev) { 68 if (ctx->key.size) 69 blk_crypto_evict_key(ctx->dev->bdev, &ctx->key); 70 dm_put_device(ti, ctx->dev); 71 } 72 kfree_sensitive(ctx->cipher_string); 73 kfree_sensitive(ctx); 74 } 75 76 #ifdef CONFIG_KEYS 77 78 static bool contains_whitespace(const char *str) 79 { 80 while (*str) 81 if (isspace(*str++)) 82 return true; 83 return false; 84 } 85 86 static int set_key_user(struct key *key, char *bin_key, 87 const unsigned int bin_key_size) 88 { 89 const struct user_key_payload *ukp; 90 91 ukp = user_key_payload_locked(key); 92 if (!ukp) 93 return -EKEYREVOKED; 94 95 if (bin_key_size != ukp->datalen) 96 return -EINVAL; 97 98 memcpy(bin_key, ukp->data, bin_key_size); 99 100 return 0; 101 } 102 103 static int inlinecrypt_get_keyring_key(const char *key_string, u8 *bin_key, 104 const unsigned int bin_key_size) 105 { 106 char *key_desc; 107 int ret; 108 struct key_type *type; 109 struct key *key; 110 int (*set_key)(struct key *key, char *bin_key, 111 const unsigned int bin_key_size); 112 113 /* 114 * Reject key_string with whitespace. dm core currently lacks code for 115 * proper whitespace escaping in arguments on DM_TABLE_STATUS path. 116 */ 117 if (contains_whitespace(key_string)) { 118 DMERR("whitespace chars not allowed in key string"); 119 return -EINVAL; 120 } 121 122 /* look for next ':' separating key_type from key_description */ 123 key_desc = strchr(key_string, ':'); 124 if (!key_desc || key_desc == key_string || !strlen(key_desc + 1)) 125 return -EINVAL; 126 127 if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) { 128 type = &key_type_logon; 129 set_key = set_key_user; 130 } else { 131 return -EINVAL; 132 } 133 134 key = request_key(type, key_desc + 1, NULL); 135 if (IS_ERR(key)) 136 return PTR_ERR(key); 137 138 down_read(&key->sem); 139 140 ret = set_key(key, (char *)bin_key, bin_key_size); 141 142 up_read(&key->sem); 143 key_put(key); 144 145 return ret; 146 } 147 148 static int get_key_size(char **key_string) 149 { 150 char *colon, dummy; 151 int ret; 152 153 if (*key_string[0] != ':') { 154 ret = strlen(*key_string); 155 156 if (ret > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE 157 || ret % 2 158 || !ret) { 159 DMERR("Invalid keysize"); 160 return -EINVAL; 161 } 162 return ret >> 1; 163 } 164 165 /* look for next ':' in key string */ 166 colon = strpbrk(*key_string + 1, ":"); 167 if (!colon) 168 return -EINVAL; 169 170 if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':') 171 return -EINVAL; 172 173 /* remaining key string should be :<logon|user>:<key_desc> */ 174 *key_string = colon; 175 176 return ret; 177 } 178 179 #else 180 181 static int inlinecrypt_get_keyring_key(const char *key_string, u8 *bin_key, 182 const unsigned int bin_key_size) 183 { 184 return -EINVAL; 185 } 186 187 static int get_key_size(char **key_string) 188 { 189 int key_hex_size = strlen(*key_string); 190 191 if (*key_string[0] == ':') 192 return -EINVAL; 193 194 if (key_hex_size > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE 195 || key_hex_size % 2 196 || !key_hex_size) { 197 DMERR("Invalid keysize"); 198 return -EINVAL; 199 } 200 201 return key_hex_size >> 1; 202 } 203 204 #endif /* CONFIG_KEYS */ 205 206 static int inlinecrypt_get_key(const char *key_string, 207 u8 key[BLK_CRYPTO_MAX_ANY_KEY_SIZE], 208 const unsigned int key_size) 209 { 210 int ret = 0; 211 212 if (key_size > BLK_CRYPTO_MAX_ANY_KEY_SIZE) { 213 DMERR("Invalid keysize"); 214 return -EINVAL; 215 } 216 217 /* ':' means the key is in kernel keyring, short-circuit normal key processing */ 218 if (key_string[0] == ':') { 219 /* key string should be :<logon|user>:<key_desc> */ 220 ret = inlinecrypt_get_keyring_key(key_string + 1, key, key_size); 221 goto out; 222 } 223 224 if (hex2bin(key, key_string, key_size) != 0) 225 ret = -EINVAL; 226 227 out: 228 return ret; 229 } 230 231 static int inlinecrypt_ctr_optional(struct dm_target *ti, 232 unsigned int argc, char **argv) 233 { 234 struct inlinecrypt_ctx *ctx = ti->private; 235 struct dm_arg_set as; 236 static const struct dm_arg _args[] = { 237 {0, 3, "Invalid number of feature args"}, 238 }; 239 unsigned int opt_params; 240 const char *opt_string; 241 bool iv_large_sectors = false; 242 char dummy; 243 int err; 244 245 as.argc = argc; 246 as.argv = argv; 247 248 err = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 249 if (err) 250 return err; 251 252 while (opt_params--) { 253 opt_string = dm_shift_arg(&as); 254 if (!opt_string) { 255 ti->error = "Not enough feature arguments"; 256 return -EINVAL; 257 } 258 if (!strcmp(opt_string, "allow_discards")) { 259 ti->num_discard_bios = 1; 260 } else if (sscanf(opt_string, "sector_size:%u%c", 261 &ctx->sector_size, &dummy) == 1) { 262 if (ctx->sector_size < SECTOR_SIZE || 263 ctx->sector_size > 4096 || 264 !is_power_of_2(ctx->sector_size)) { 265 ti->error = "Invalid sector_size"; 266 return -EINVAL; 267 } 268 } else if (!strcmp(opt_string, "iv_large_sectors")) { 269 iv_large_sectors = true; 270 } else { 271 ti->error = "Invalid feature arguments"; 272 return -EINVAL; 273 } 274 } 275 276 /* dm-inlinecrypt doesn't implement iv_large_sectors=false. */ 277 if (ctx->sector_size != SECTOR_SIZE && !iv_large_sectors) { 278 ti->error = "iv_large_sectors must be specified"; 279 return -EINVAL; 280 } 281 282 return 0; 283 } 284 285 /* 286 * Construct an inlinecrypt mapping: 287 * <cipher> [<key>|:<key_size>:<logon>:<key_description>] <iv_offset> <dev_path> <start> 288 * 289 * This syntax matches dm-crypt's, but the set of supported functionality has 290 * been stripped down. 291 */ 292 static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 293 { 294 struct inlinecrypt_ctx *ctx; 295 const struct dm_inlinecrypt_cipher *cipher; 296 u8 raw_key[BLK_CRYPTO_MAX_ANY_KEY_SIZE]; 297 unsigned int dun_bytes; 298 unsigned long long tmpll; 299 char dummy; 300 int err; 301 302 if (argc < 5) { 303 ti->error = "Not enough arguments"; 304 return -EINVAL; 305 } 306 307 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 308 if (!ctx) { 309 ti->error = "Out of memory"; 310 return -ENOMEM; 311 } 312 ti->private = ctx; 313 314 /* <cipher> */ 315 ctx->cipher_string = kstrdup(argv[0], GFP_KERNEL); 316 if (!ctx->cipher_string) { 317 ti->error = "Out of memory"; 318 err = -ENOMEM; 319 goto bad; 320 } 321 cipher = lookup_cipher(ctx->cipher_string); 322 if (!cipher) { 323 ti->error = "Unsupported cipher"; 324 err = -EINVAL; 325 goto bad; 326 } 327 328 /* <key> */ 329 err = get_key_size(&argv[1]); 330 if (err < 0) { 331 ti->error = "Cannot parse key size"; 332 return -EINVAL; 333 } 334 ctx->key_size = err; 335 336 err = inlinecrypt_get_key(argv[1], raw_key, ctx->key_size); 337 if (err) { 338 ti->error = "Malformed key string"; 339 goto bad; 340 } 341 342 /* <iv_offset> */ 343 if (sscanf(argv[2], "%llu%c", &ctx->iv_offset, &dummy) != 1) { 344 ti->error = "Invalid iv_offset sector"; 345 err = -EINVAL; 346 goto bad; 347 } 348 349 /* <dev_path> */ 350 err = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), 351 &ctx->dev); 352 if (err) { 353 ti->error = "Device lookup failed"; 354 goto bad; 355 } 356 357 /* <start> */ 358 if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || 359 tmpll != (sector_t)tmpll) { 360 ti->error = "Invalid start sector"; 361 err = -EINVAL; 362 goto bad; 363 } 364 ctx->start = tmpll; 365 366 /* optional arguments */ 367 ctx->sector_size = SECTOR_SIZE; 368 if (argc > 5) { 369 err = inlinecrypt_ctr_optional(ti, argc - 5, &argv[5]); 370 if (err) 371 goto bad; 372 } 373 ctx->sector_bits = ilog2(ctx->sector_size); 374 if (ti->len & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) { 375 ti->error = "Device size is not a multiple of sector_size"; 376 err = -EINVAL; 377 goto bad; 378 } 379 if (ctx->iv_offset & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) { 380 ti->error = "Wrong alignment of iv_offset sector"; 381 err = -EINVAL; 382 } 383 384 ctx->max_dun = (ctx->iv_offset + ti->len - 1) >> 385 (ctx->sector_bits - SECTOR_SHIFT); 386 dun_bytes = DIV_ROUND_UP(fls64(ctx->max_dun), 8); 387 388 err = blk_crypto_init_key(&ctx->key, raw_key, ctx->key_size, 389 BLK_CRYPTO_KEY_TYPE_RAW, 390 cipher->mode_num, dun_bytes, 391 ctx->sector_size); 392 if (err) { 393 ti->error = "Error initializing blk-crypto key"; 394 goto bad; 395 } 396 397 err = blk_crypto_start_using_key(ctx->dev->bdev, &ctx->key); 398 if (err) { 399 ti->error = "Error starting to use blk-crypto"; 400 goto bad; 401 } 402 403 ti->num_flush_bios = 1; 404 405 err = 0; 406 goto out; 407 408 bad: 409 inlinecrypt_dtr(ti); 410 out: 411 memzero_explicit(raw_key, sizeof(raw_key)); 412 return err; 413 } 414 415 static int inlinecrypt_map(struct dm_target *ti, struct bio *bio) 416 { 417 const struct inlinecrypt_ctx *ctx = ti->private; 418 sector_t sector_in_target; 419 u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE] = {}; 420 421 bio_set_dev(bio, ctx->dev->bdev); 422 423 /* 424 * If the bio is a device-level request which doesn't target a specific 425 * sector, there's nothing more to do. 426 */ 427 if (bio_sectors(bio) == 0) 428 return DM_MAPIO_REMAPPED; 429 430 /* 431 * The bio should never have an encryption context already, since 432 * dm-inlinecrypt doesn't pass through any inline encryption 433 * capabilities to the layer above it. 434 */ 435 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) 436 return DM_MAPIO_KILL; 437 438 /* Map the bio's sector to the underlying device. (512-byte sectors) */ 439 sector_in_target = dm_target_offset(ti, bio->bi_iter.bi_sector); 440 bio->bi_iter.bi_sector = ctx->start + sector_in_target; 441 /* 442 * If the bio doesn't have any data (e.g. if it's a DISCARD request), 443 * there's nothing more to do. 444 */ 445 if (!bio_has_data(bio)) 446 return DM_MAPIO_REMAPPED; 447 448 /* Calculate the DUN and enforce data-unit (crypto sector) alignment. */ 449 dun[0] = ctx->iv_offset + sector_in_target; /* 512-byte sectors */ 450 if (dun[0] & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) 451 return DM_MAPIO_KILL; 452 dun[0] >>= ctx->sector_bits - SECTOR_SHIFT; /* crypto sectors */ 453 454 /* 455 * This check isn't necessary as we should have calculated max_dun 456 * correctly, but be safe. 457 */ 458 if (WARN_ON_ONCE(dun[0] > ctx->max_dun)) 459 return DM_MAPIO_KILL; 460 461 bio_crypt_set_ctx(bio, &ctx->key, dun, GFP_NOIO); 462 463 /* 464 * Since we've added an encryption context to the bio and 465 * blk-crypto-fallback may be needed to process it, it's necessary to 466 * use the fallback-aware bio submission code rather than 467 * unconditionally returning DM_MAPIO_REMAPPED. 468 * 469 * To get the correct accounting for a dm target in the case where 470 * __blk_crypto_submit_bio() doesn't take ownership of the bio (returns 471 * true), call __blk_crypto_submit_bio() directly and return 472 * DM_MAPIO_REMAPPED in that case, rather than relying on 473 * blk_crypto_submit_bio() which calls submit_bio() in that case. 474 * 475 * TODO: blk-crypto fallback write slow-path currently double-accounts 476 * IO in vmstat, as encrypted bios are submitted via submit_bio(). 477 * This does not affect data correctness. Consider fixing this if 478 * a cleaner accounting model for derived bios is introduced. 479 */ 480 if (__blk_crypto_submit_bio(bio)) 481 return DM_MAPIO_REMAPPED; 482 return DM_MAPIO_SUBMITTED; 483 } 484 485 static void inlinecrypt_status(struct dm_target *ti, status_type_t type, 486 unsigned int status_flags, char *result, 487 unsigned int maxlen) 488 { 489 const struct inlinecrypt_ctx *ctx = ti->private; 490 unsigned int sz = 0; 491 int num_feature_args = 0; 492 493 switch (type) { 494 case STATUSTYPE_INFO: 495 case STATUSTYPE_IMA: 496 result[0] = '\0'; 497 break; 498 499 case STATUSTYPE_TABLE: 500 /* 501 * Warning: like dm-crypt, dm-inlinecrypt includes the key in 502 * the returned table. Userspace is responsible for redacting 503 * the key when needed. 504 */ 505 DMEMIT("%s %*phN %llu %s %llu", ctx->cipher_string, 506 ctx->key.size, ctx->key.bytes, ctx->iv_offset, 507 ctx->dev->name, ctx->start); 508 num_feature_args += !!ti->num_discard_bios; 509 if (ctx->sector_size != SECTOR_SIZE) 510 num_feature_args += 2; 511 if (num_feature_args != 0) { 512 DMEMIT(" %d", num_feature_args); 513 if (ti->num_discard_bios) 514 DMEMIT(" allow_discards"); 515 if (ctx->sector_size != SECTOR_SIZE) { 516 DMEMIT(" sector_size:%u", ctx->sector_size); 517 DMEMIT(" iv_large_sectors"); 518 } 519 } 520 break; 521 } 522 } 523 524 static int inlinecrypt_prepare_ioctl(struct dm_target *ti, 525 struct block_device **bdev, unsigned int cmd, 526 unsigned long arg, bool *forward) 527 { 528 const struct inlinecrypt_ctx *ctx = ti->private; 529 const struct dm_dev *dev = ctx->dev; 530 531 *bdev = dev->bdev; 532 533 /* Only pass ioctls through if the device sizes match exactly. */ 534 return ctx->start != 0 || ti->len != bdev_nr_sectors(dev->bdev); 535 } 536 537 static int inlinecrypt_iterate_devices(struct dm_target *ti, 538 iterate_devices_callout_fn fn, 539 void *data) 540 { 541 const struct inlinecrypt_ctx *ctx = ti->private; 542 543 return fn(ti, ctx->dev, ctx->start, ti->len, data); 544 } 545 546 #ifdef CONFIG_BLK_DEV_ZONED 547 static int inlinecrypt_report_zones(struct dm_target *ti, 548 struct dm_report_zones_args *args, 549 unsigned int nr_zones) 550 { 551 const struct inlinecrypt_ctx *ctx = ti->private; 552 553 return dm_report_zones(ctx->dev->bdev, ctx->start, 554 ctx->start + dm_target_offset(ti, args->next_sector), 555 args, nr_zones); 556 } 557 #else 558 #define inlinecrypt_report_zones NULL 559 #endif 560 561 static void inlinecrypt_io_hints(struct dm_target *ti, 562 struct queue_limits *limits) 563 { 564 const struct inlinecrypt_ctx *ctx = ti->private; 565 const unsigned int sector_size = ctx->sector_size; 566 567 limits->logical_block_size = 568 max_t(unsigned int, limits->logical_block_size, sector_size); 569 limits->physical_block_size = 570 max_t(unsigned int, limits->physical_block_size, sector_size); 571 limits->io_min = max_t(unsigned int, limits->io_min, sector_size); 572 limits->dma_alignment = limits->logical_block_size - 1; 573 } 574 575 static struct target_type inlinecrypt_target = { 576 .name = "inlinecrypt", 577 .version = {1, 0, 0}, 578 /* 579 * Do not set DM_TARGET_PASSES_CRYPTO, since dm-inlinecrypt consumes the 580 * crypto capability itself. 581 */ 582 .features = DM_TARGET_ZONED_HM, 583 .module = THIS_MODULE, 584 .ctr = inlinecrypt_ctr, 585 .dtr = inlinecrypt_dtr, 586 .map = inlinecrypt_map, 587 .status = inlinecrypt_status, 588 .prepare_ioctl = inlinecrypt_prepare_ioctl, 589 .iterate_devices = inlinecrypt_iterate_devices, 590 .report_zones = inlinecrypt_report_zones, 591 .io_hints = inlinecrypt_io_hints, 592 }; 593 594 module_dm(inlinecrypt); 595 596 MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); 597 MODULE_AUTHOR("Linlin Zhang <linlin.zhang@oss.qualcomm.com>"); 598 MODULE_DESCRIPTION(DM_NAME " target for inline encryption"); 599 MODULE_LICENSE("GPL"); 600