1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. 4 * 5 * Author: Mikulas Patocka <mpatocka@redhat.com> 6 * 7 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors 8 * 9 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set 10 * default prefetch value. Data are read in "prefetch_cluster" chunks from the 11 * hash device. Setting this greatly improves performance when data and hash 12 * are on the same disk on different partitions on devices with poor random 13 * access behavior. 14 */ 15 16 #include "dm-verity.h" 17 #include "dm-verity-fec.h" 18 #include "dm-verity-verify-sig.h" 19 #include "dm-audit.h" 20 #include <linux/hex.h> 21 #include <linux/module.h> 22 #include <linux/reboot.h> 23 #include <linux/string.h> 24 #include <linux/jump_label.h> 25 #include <linux/security.h> 26 27 #define DM_MSG_PREFIX "verity" 28 29 #define DM_VERITY_ENV_LENGTH 46 30 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" 31 32 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 33 #define DM_VERITY_USE_BH_DEFAULT_BYTES 8192 34 35 #define DM_VERITY_MAX_CORRUPTED_ERRS 100 36 37 #define DM_VERITY_OPT_LOGGING "ignore_corruption" 38 #define DM_VERITY_OPT_RESTART "restart_on_corruption" 39 #define DM_VERITY_OPT_PANIC "panic_on_corruption" 40 #define DM_VERITY_OPT_ERROR_RESTART "restart_on_error" 41 #define DM_VERITY_OPT_ERROR_PANIC "panic_on_error" 42 #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" 43 #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" 44 #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" 45 46 #define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \ 47 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) 48 49 static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; 50 51 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); 52 53 static unsigned int dm_verity_use_bh_bytes[4] = { 54 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_NONE 55 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_RT 56 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_BE 57 0 // IOPRIO_CLASS_IDLE 58 }; 59 60 module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644); 61 62 static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); 63 64 struct dm_verity_prefetch_work { 65 struct work_struct work; 66 struct dm_verity *v; 67 unsigned short ioprio; 68 sector_t block; 69 unsigned int n_blocks; 70 }; 71 72 /* 73 * Auxiliary structure appended to each dm-bufio buffer. If the value 74 * hash_verified is nonzero, hash of the block has been verified. 75 * 76 * The variable hash_verified is set to 0 when allocating the buffer, then 77 * it can be changed to 1 and it is never reset to 0 again. 78 * 79 * There is no lock around this value, a race condition can at worst cause 80 * that multiple processes verify the hash of the same buffer simultaneously 81 * and write 1 to hash_verified simultaneously. 82 * This condition is harmless, so we don't need locking. 83 */ 84 struct buffer_aux { 85 int hash_verified; 86 }; 87 88 /* 89 * Initialize struct buffer_aux for a freshly created buffer. 90 */ 91 static void dm_bufio_alloc_callback(struct dm_buffer *buf) 92 { 93 struct buffer_aux *aux = dm_bufio_get_aux_data(buf); 94 95 aux->hash_verified = 0; 96 } 97 98 /* 99 * Translate input sector number to the sector number on the target device. 100 */ 101 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) 102 { 103 return dm_target_offset(v->ti, bi_sector); 104 } 105 106 /* 107 * Return hash position of a specified block at a specified tree level 108 * (0 is the lowest level). 109 * The lowest "hash_per_block_bits"-bits of the result denote hash position 110 * inside a hash block. The remaining bits denote location of the hash block. 111 */ 112 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, 113 int level) 114 { 115 return block >> (level * v->hash_per_block_bits); 116 } 117 118 int verity_hash(struct dm_verity *v, struct dm_verity_io *io, 119 const u8 *data, size_t len, u8 *digest) 120 { 121 struct shash_desc *desc; 122 int r; 123 124 if (likely(v->use_sha256_lib)) { 125 struct sha256_ctx *ctx = &io->hash_ctx.sha256; 126 127 /* 128 * Fast path using SHA-256 library. This is enabled only for 129 * verity version 1, where the salt is at the beginning. 130 */ 131 *ctx = *v->initial_hashstate.sha256; 132 sha256_update(ctx, data, len); 133 sha256_final(ctx, digest); 134 return 0; 135 } 136 137 desc = &io->hash_ctx.shash; 138 desc->tfm = v->shash_tfm; 139 if (unlikely(v->initial_hashstate.shash == NULL)) { 140 /* Version 0: salt at end */ 141 r = crypto_shash_init(desc) ?: 142 crypto_shash_update(desc, data, len) ?: 143 crypto_shash_update(desc, v->salt, v->salt_size) ?: 144 crypto_shash_final(desc, digest); 145 } else { 146 /* Version 1: salt at beginning */ 147 r = crypto_shash_import(desc, v->initial_hashstate.shash) ?: 148 crypto_shash_finup(desc, data, len, digest); 149 } 150 if (unlikely(r)) 151 DMERR("Error hashing block: %d", r); 152 return r; 153 } 154 155 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, 156 sector_t *hash_block, unsigned int *offset) 157 { 158 sector_t position = verity_position_at_level(v, block, level); 159 unsigned int idx; 160 161 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); 162 163 if (!offset) 164 return; 165 166 idx = position & ((1 << v->hash_per_block_bits) - 1); 167 if (!v->version) 168 *offset = idx * v->digest_size; 169 else 170 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); 171 } 172 173 /* 174 * Handle verification errors. 175 */ 176 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, 177 unsigned long long block) 178 { 179 char verity_env[DM_VERITY_ENV_LENGTH]; 180 char *envp[] = { verity_env, NULL }; 181 const char *type_str = ""; 182 struct mapped_device *md = dm_table_get_md(v->ti->table); 183 int ce; 184 185 /* Corruption should be visible in device status in all modes */ 186 v->hash_failed = true; 187 188 ce = atomic_read(&v->corrupted_errs); 189 do { 190 if (ce >= DM_VERITY_MAX_CORRUPTED_ERRS) 191 goto out; 192 } while (!atomic_try_cmpxchg(&v->corrupted_errs, &ce, ce + 1)); 193 194 switch (type) { 195 case DM_VERITY_BLOCK_TYPE_DATA: 196 type_str = "data"; 197 break; 198 case DM_VERITY_BLOCK_TYPE_METADATA: 199 type_str = "metadata"; 200 break; 201 default: 202 BUG(); 203 } 204 205 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, 206 type_str, block); 207 208 if (ce + 1 == DM_VERITY_MAX_CORRUPTED_ERRS) { 209 DMERR("%s: reached maximum errors", v->data_dev->name); 210 dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); 211 } 212 213 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", 214 DM_VERITY_ENV_VAR_NAME, type, block); 215 216 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); 217 218 out: 219 if (v->mode == DM_VERITY_MODE_LOGGING) 220 return 0; 221 222 if (v->mode == DM_VERITY_MODE_RESTART) 223 kernel_restart("dm-verity device corrupted"); 224 225 if (v->mode == DM_VERITY_MODE_PANIC) 226 panic("dm-verity device corrupted"); 227 228 return 1; 229 } 230 231 /* 232 * Verify hash of a metadata block pertaining to the specified data block 233 * ("block" argument) at a specified level ("level" argument). 234 * 235 * On successful return, want_digest contains the hash value for a lower tree 236 * level or for the data block (if we're at the lowest level). 237 * 238 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. 239 * If "skip_unverified" is false, unverified buffer is hashed and verified 240 * against current value of want_digest. 241 */ 242 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 243 sector_t block, int level, bool skip_unverified, 244 u8 *want_digest) 245 { 246 struct dm_buffer *buf; 247 struct buffer_aux *aux; 248 u8 *data; 249 int r; 250 sector_t hash_block; 251 unsigned int offset; 252 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 253 254 verity_hash_at_level(v, block, level, &hash_block, &offset); 255 256 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 257 data = dm_bufio_get(v->bufio, hash_block, &buf); 258 if (IS_ERR_OR_NULL(data)) { 259 /* 260 * In softirq and the hash was not in the bufio cache. 261 * Return early and resume execution from a kworker to 262 * read the hash from disk. 263 */ 264 return -EAGAIN; 265 } 266 } else { 267 data = dm_bufio_read_with_ioprio(v->bufio, hash_block, 268 &buf, bio->bi_ioprio); 269 } 270 271 if (IS_ERR(data)) { 272 if (skip_unverified) 273 return 1; 274 r = PTR_ERR(data); 275 data = dm_bufio_new(v->bufio, hash_block, &buf); 276 if (IS_ERR(data)) 277 return r; 278 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, 279 want_digest, hash_block, data) == 0) { 280 aux = dm_bufio_get_aux_data(buf); 281 aux->hash_verified = 1; 282 goto release_ok; 283 } else { 284 dm_bufio_release(buf); 285 dm_bufio_forget(v->bufio, hash_block); 286 return r; 287 } 288 } 289 290 aux = dm_bufio_get_aux_data(buf); 291 292 if (!aux->hash_verified) { 293 if (skip_unverified) { 294 r = 1; 295 goto release_ret_r; 296 } 297 298 r = verity_hash(v, io, data, 1 << v->hash_dev_block_bits, 299 io->tmp_digest); 300 if (unlikely(r < 0)) 301 goto release_ret_r; 302 303 if (likely(memcmp(io->tmp_digest, want_digest, 304 v->digest_size) == 0)) 305 aux->hash_verified = 1; 306 else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 307 /* 308 * Error handling code (FEC included) cannot be run in a 309 * softirq since it may sleep, so fallback to a kworker. 310 */ 311 r = -EAGAIN; 312 goto release_ret_r; 313 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, 314 want_digest, hash_block, data) == 0) 315 aux->hash_verified = 1; 316 else if (verity_handle_err(v, 317 DM_VERITY_BLOCK_TYPE_METADATA, 318 hash_block)) { 319 struct bio *bio; 320 io->had_mismatch = true; 321 bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 322 dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio, 323 block, 0); 324 r = -EIO; 325 goto release_ret_r; 326 } 327 } 328 329 release_ok: 330 data += offset; 331 memcpy(want_digest, data, v->digest_size); 332 r = 0; 333 334 release_ret_r: 335 dm_bufio_release(buf); 336 return r; 337 } 338 339 /* 340 * Find a hash for a given block, write it to digest and verify the integrity 341 * of the hash tree if necessary. 342 */ 343 int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, 344 sector_t block, u8 *digest, bool *is_zero) 345 { 346 int r = 0, i; 347 348 if (likely(v->levels)) { 349 /* 350 * First, we try to get the requested hash for 351 * the current block. If the hash block itself is 352 * verified, zero is returned. If it isn't, this 353 * function returns 1 and we fall back to whole 354 * chain verification. 355 */ 356 r = verity_verify_level(v, io, block, 0, true, digest); 357 if (likely(r <= 0)) 358 goto out; 359 } 360 361 memcpy(digest, v->root_digest, v->digest_size); 362 363 for (i = v->levels - 1; i >= 0; i--) { 364 r = verity_verify_level(v, io, block, i, false, digest); 365 if (unlikely(r)) 366 goto out; 367 } 368 out: 369 if (!r && v->zero_digest) 370 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); 371 else 372 *is_zero = false; 373 374 return r; 375 } 376 377 static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, 378 const u8 *want_digest, sector_t cur_block, 379 u8 *dest) 380 { 381 struct page *page; 382 void *buffer; 383 int r; 384 struct dm_io_request io_req; 385 struct dm_io_region io_loc; 386 387 page = mempool_alloc(&v->recheck_pool, GFP_NOIO); 388 buffer = page_to_virt(page); 389 390 io_req.bi_opf = REQ_OP_READ; 391 io_req.mem.type = DM_IO_KMEM; 392 io_req.mem.ptr.addr = buffer; 393 io_req.notify.fn = NULL; 394 io_req.client = v->io; 395 io_loc.bdev = v->data_dev->bdev; 396 io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); 397 io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); 398 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 399 if (unlikely(r)) 400 goto free_ret; 401 402 r = verity_hash(v, io, buffer, 1 << v->data_dev_block_bits, 403 io->tmp_digest); 404 if (unlikely(r)) 405 goto free_ret; 406 407 if (memcmp(io->tmp_digest, want_digest, v->digest_size)) { 408 r = -EIO; 409 goto free_ret; 410 } 411 412 memcpy(dest, buffer, 1 << v->data_dev_block_bits); 413 r = 0; 414 free_ret: 415 mempool_free(page, &v->recheck_pool); 416 417 return r; 418 } 419 420 static int verity_handle_data_hash_mismatch(struct dm_verity *v, 421 struct dm_verity_io *io, 422 struct bio *bio, 423 struct pending_block *block) 424 { 425 const u8 *want_digest = block->want_digest; 426 sector_t blkno = block->blkno; 427 u8 *data = block->data; 428 429 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 430 /* 431 * Error handling code (FEC included) cannot be run in a 432 * softirq since it may sleep, so fallback to a kworker. 433 */ 434 return -EAGAIN; 435 } 436 if (verity_recheck(v, io, want_digest, blkno, data) == 0) { 437 if (v->validated_blocks) 438 set_bit(blkno, v->validated_blocks); 439 return 0; 440 } 441 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, want_digest, 442 blkno, data) == 0) 443 return 0; 444 if (bio->bi_status) 445 return -EIO; /* Error correction failed; Just return error */ 446 447 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, blkno)) { 448 io->had_mismatch = true; 449 dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", bio, blkno, 0); 450 return -EIO; 451 } 452 return 0; 453 } 454 455 static void verity_clear_pending_blocks(struct dm_verity_io *io) 456 { 457 int i; 458 459 for (i = io->num_pending - 1; i >= 0; i--) { 460 kunmap_local(io->pending_blocks[i].data); 461 io->pending_blocks[i].data = NULL; 462 } 463 io->num_pending = 0; 464 } 465 466 static int verity_verify_pending_blocks(struct dm_verity *v, 467 struct dm_verity_io *io, 468 struct bio *bio) 469 { 470 const unsigned int block_size = 1 << v->data_dev_block_bits; 471 int i, r; 472 473 if (io->num_pending == 2) { 474 /* num_pending == 2 implies that the algorithm is SHA-256 */ 475 sha256_finup_2x(v->initial_hashstate.sha256, 476 io->pending_blocks[0].data, 477 io->pending_blocks[1].data, block_size, 478 io->pending_blocks[0].real_digest, 479 io->pending_blocks[1].real_digest); 480 } else { 481 for (i = 0; i < io->num_pending; i++) { 482 r = verity_hash(v, io, io->pending_blocks[i].data, 483 block_size, 484 io->pending_blocks[i].real_digest); 485 if (unlikely(r)) 486 return r; 487 } 488 } 489 490 for (i = 0; i < io->num_pending; i++) { 491 struct pending_block *block = &io->pending_blocks[i]; 492 493 if (likely(memcmp(block->real_digest, block->want_digest, 494 v->digest_size) == 0)) { 495 if (v->validated_blocks) 496 set_bit(block->blkno, v->validated_blocks); 497 } else { 498 r = verity_handle_data_hash_mismatch(v, io, bio, block); 499 if (unlikely(r)) 500 return r; 501 } 502 } 503 verity_clear_pending_blocks(io); 504 return 0; 505 } 506 507 /* 508 * Verify one "dm_verity_io" structure. 509 */ 510 static int verity_verify_io(struct dm_verity_io *io) 511 { 512 struct dm_verity *v = io->v; 513 const unsigned int block_size = 1 << v->data_dev_block_bits; 514 const int max_pending = v->use_sha256_finup_2x ? 2 : 1; 515 struct bvec_iter iter_copy; 516 struct bvec_iter *iter; 517 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 518 unsigned int b; 519 int r; 520 521 io->num_pending = 0; 522 523 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 524 /* 525 * Copy the iterator in case we need to restart verification in 526 * a kworker. 527 */ 528 iter_copy = io->iter; 529 iter = &iter_copy; 530 } else 531 iter = &io->iter; 532 533 for (b = 0; b < io->n_blocks; 534 b++, bio_advance_iter_single(bio, iter, block_size)) { 535 sector_t blkno = io->block + b; 536 struct pending_block *block; 537 bool is_zero; 538 struct bio_vec bv; 539 void *data; 540 541 if (v->validated_blocks && bio->bi_status == BLK_STS_OK && 542 likely(test_bit(blkno, v->validated_blocks))) 543 continue; 544 545 block = &io->pending_blocks[io->num_pending]; 546 547 r = verity_hash_for_block(v, io, blkno, block->want_digest, 548 &is_zero); 549 if (unlikely(r < 0)) 550 goto error; 551 552 bv = bio_iter_iovec(bio, *iter); 553 if (unlikely(bv.bv_len < block_size)) { 554 /* 555 * Data block spans pages. This should not happen, 556 * since dm-verity sets dma_alignment to the data block 557 * size minus 1, and dm-verity also doesn't allow the 558 * data block size to be greater than PAGE_SIZE. 559 */ 560 DMERR_LIMIT("unaligned io (data block spans pages)"); 561 r = -EIO; 562 goto error; 563 } 564 565 data = bvec_kmap_local(&bv); 566 567 if (is_zero) { 568 /* 569 * If we expect a zero block, don't validate, just 570 * return zeros. 571 */ 572 memset(data, 0, block_size); 573 kunmap_local(data); 574 continue; 575 } 576 block->data = data; 577 block->blkno = blkno; 578 if (++io->num_pending == max_pending) { 579 r = verity_verify_pending_blocks(v, io, bio); 580 if (unlikely(r)) 581 goto error; 582 } 583 } 584 585 if (io->num_pending) { 586 r = verity_verify_pending_blocks(v, io, bio); 587 if (unlikely(r)) 588 goto error; 589 } 590 591 return 0; 592 593 error: 594 verity_clear_pending_blocks(io); 595 return r; 596 } 597 598 /* 599 * Skip verity work in response to I/O error when system is shutting down. 600 */ 601 static inline bool verity_is_system_shutting_down(void) 602 { 603 return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF 604 || system_state == SYSTEM_RESTART; 605 } 606 607 static void restart_io_error(struct work_struct *w) 608 { 609 kernel_restart("dm-verity device has I/O error"); 610 } 611 612 /* 613 * End one "io" structure with a given error. 614 */ 615 static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) 616 { 617 struct dm_verity *v = io->v; 618 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 619 620 bio->bi_end_io = io->orig_bi_end_io; 621 bio->bi_status = status; 622 623 verity_fec_finish_io(io); 624 625 if (unlikely(status != BLK_STS_OK) && 626 unlikely(!(bio->bi_opf & REQ_RAHEAD)) && 627 !io->had_mismatch && 628 !verity_is_system_shutting_down()) { 629 if (v->error_mode == DM_VERITY_MODE_PANIC) { 630 panic("dm-verity device has I/O error"); 631 } 632 if (v->error_mode == DM_VERITY_MODE_RESTART) { 633 static DECLARE_WORK(restart_work, restart_io_error); 634 queue_work(v->verify_wq, &restart_work); 635 /* 636 * We deliberately don't call bio_endio here, because 637 * the machine will be restarted anyway. 638 */ 639 return; 640 } 641 } 642 643 bio_endio(bio); 644 } 645 646 static void verity_work(struct work_struct *w) 647 { 648 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 649 650 io->in_bh = false; 651 652 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); 653 } 654 655 static void verity_bh_work(struct work_struct *w) 656 { 657 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 658 int err; 659 660 io->in_bh = true; 661 err = verity_verify_io(io); 662 if (err == -EAGAIN || err == -ENOMEM) { 663 /* fallback to retrying in a kworker */ 664 INIT_WORK(&io->work, verity_work); 665 queue_work(io->v->verify_wq, &io->work); 666 return; 667 } 668 669 verity_finish_io(io, errno_to_blk_status(err)); 670 } 671 672 static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio) 673 { 674 return ioprio <= IOPRIO_CLASS_IDLE && 675 bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]) && 676 !need_resched(); 677 } 678 679 static void verity_end_io(struct bio *bio) 680 { 681 struct dm_verity_io *io = bio->bi_private; 682 unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio); 683 unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits; 684 685 if (bio->bi_status && 686 (!verity_fec_is_enabled(io->v) || 687 verity_is_system_shutting_down() || 688 (bio->bi_opf & REQ_RAHEAD))) { 689 verity_finish_io(io, bio->bi_status); 690 return; 691 } 692 693 if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq && 694 verity_use_bh(bytes, ioprio)) { 695 if (in_hardirq() || irqs_disabled()) { 696 INIT_WORK(&io->work, verity_bh_work); 697 queue_work(system_bh_wq, &io->work); 698 } else { 699 verity_bh_work(&io->work); 700 } 701 } else { 702 INIT_WORK(&io->work, verity_work); 703 queue_work(io->v->verify_wq, &io->work); 704 } 705 } 706 707 /* 708 * Prefetch buffers for the specified io. 709 * The root buffer is not prefetched, it is assumed that it will be cached 710 * all the time. 711 */ 712 static void verity_prefetch_io(struct work_struct *work) 713 { 714 struct dm_verity_prefetch_work *pw = 715 container_of(work, struct dm_verity_prefetch_work, work); 716 struct dm_verity *v = pw->v; 717 int i; 718 719 for (i = v->levels - 2; i >= 0; i--) { 720 sector_t hash_block_start; 721 sector_t hash_block_end; 722 723 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); 724 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); 725 726 if (!i) { 727 unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); 728 729 cluster >>= v->data_dev_block_bits; 730 if (unlikely(!cluster)) 731 goto no_prefetch_cluster; 732 733 if (unlikely(cluster & (cluster - 1))) 734 cluster = 1 << __fls(cluster); 735 736 hash_block_start &= ~(sector_t)(cluster - 1); 737 hash_block_end |= cluster - 1; 738 if (unlikely(hash_block_end >= v->hash_end)) 739 hash_block_end = v->hash_end - 1; 740 } 741 no_prefetch_cluster: 742 dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start, 743 hash_block_end - hash_block_start + 1, 744 pw->ioprio); 745 } 746 747 kfree(pw); 748 } 749 750 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, 751 unsigned short ioprio) 752 { 753 sector_t block = io->block; 754 unsigned int n_blocks = io->n_blocks; 755 struct dm_verity_prefetch_work *pw; 756 757 if (v->validated_blocks) { 758 while (n_blocks && test_bit(block, v->validated_blocks)) { 759 block++; 760 n_blocks--; 761 } 762 while (n_blocks && test_bit(block + n_blocks - 1, 763 v->validated_blocks)) 764 n_blocks--; 765 if (!n_blocks) 766 return; 767 } 768 769 pw = kmalloc_obj(struct dm_verity_prefetch_work, 770 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 771 772 if (!pw) 773 return; 774 775 INIT_WORK(&pw->work, verity_prefetch_io); 776 pw->v = v; 777 pw->block = block; 778 pw->n_blocks = n_blocks; 779 pw->ioprio = ioprio; 780 queue_work(v->verify_wq, &pw->work); 781 } 782 783 /* 784 * Bio map function. It allocates dm_verity_io structure and bio vector and 785 * fills them. Then it issues prefetches and the I/O. 786 */ 787 static int verity_map(struct dm_target *ti, struct bio *bio) 788 { 789 struct dm_verity *v = ti->private; 790 struct dm_verity_io *io; 791 792 bio_set_dev(bio, v->data_dev->bdev); 793 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); 794 795 if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & 796 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { 797 DMERR_LIMIT("unaligned io"); 798 return DM_MAPIO_KILL; 799 } 800 801 if (bio_end_sector(bio) >> 802 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { 803 DMERR_LIMIT("io out of range"); 804 return DM_MAPIO_KILL; 805 } 806 807 if (bio_data_dir(bio) == WRITE) 808 return DM_MAPIO_KILL; 809 810 io = dm_per_bio_data(bio, ti->per_io_data_size); 811 io->v = v; 812 io->orig_bi_end_io = bio->bi_end_io; 813 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); 814 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; 815 io->had_mismatch = false; 816 817 bio->bi_end_io = verity_end_io; 818 bio->bi_private = io; 819 io->iter = bio->bi_iter; 820 821 verity_fec_init_io(io); 822 823 verity_submit_prefetch(v, io, bio->bi_ioprio); 824 825 submit_bio_noacct(bio); 826 827 return DM_MAPIO_SUBMITTED; 828 } 829 830 static void verity_postsuspend(struct dm_target *ti) 831 { 832 struct dm_verity *v = ti->private; 833 flush_workqueue(v->verify_wq); 834 dm_bufio_client_reset(v->bufio); 835 } 836 837 /* 838 * Status: V (valid) or C (corruption found) 839 */ 840 static void verity_status(struct dm_target *ti, status_type_t type, 841 unsigned int status_flags, char *result, unsigned int maxlen) 842 { 843 struct dm_verity *v = ti->private; 844 unsigned int args = 0; 845 unsigned int sz = 0; 846 unsigned int x; 847 848 switch (type) { 849 case STATUSTYPE_INFO: 850 DMEMIT("%c", v->hash_failed ? 'C' : 'V'); 851 if (verity_fec_is_enabled(v)) 852 DMEMIT(" %lld", atomic64_read(&v->fec->corrected)); 853 else 854 DMEMIT(" -"); 855 break; 856 case STATUSTYPE_TABLE: 857 DMEMIT("%u %s %s %u %u %llu %llu %s ", 858 v->version, 859 v->data_dev->name, 860 v->hash_dev->name, 861 1 << v->data_dev_block_bits, 862 1 << v->hash_dev_block_bits, 863 (unsigned long long)v->data_blocks, 864 (unsigned long long)v->hash_start, 865 v->alg_name 866 ); 867 for (x = 0; x < v->digest_size; x++) 868 DMEMIT("%02x", v->root_digest[x]); 869 DMEMIT(" "); 870 if (!v->salt_size) 871 DMEMIT("-"); 872 else 873 for (x = 0; x < v->salt_size; x++) 874 DMEMIT("%02x", v->salt[x]); 875 if (v->mode != DM_VERITY_MODE_EIO) 876 args++; 877 if (v->error_mode != DM_VERITY_MODE_EIO) 878 args++; 879 if (verity_fec_is_enabled(v)) 880 args += DM_VERITY_OPTS_FEC; 881 if (v->zero_digest) 882 args++; 883 if (v->validated_blocks) 884 args++; 885 if (v->use_bh_wq) 886 args++; 887 if (v->signature_key_desc) 888 args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; 889 if (!args) 890 return; 891 DMEMIT(" %u", args); 892 if (v->mode != DM_VERITY_MODE_EIO) { 893 DMEMIT(" "); 894 switch (v->mode) { 895 case DM_VERITY_MODE_LOGGING: 896 DMEMIT(DM_VERITY_OPT_LOGGING); 897 break; 898 case DM_VERITY_MODE_RESTART: 899 DMEMIT(DM_VERITY_OPT_RESTART); 900 break; 901 case DM_VERITY_MODE_PANIC: 902 DMEMIT(DM_VERITY_OPT_PANIC); 903 break; 904 default: 905 BUG(); 906 } 907 } 908 if (v->error_mode != DM_VERITY_MODE_EIO) { 909 DMEMIT(" "); 910 switch (v->error_mode) { 911 case DM_VERITY_MODE_RESTART: 912 DMEMIT(DM_VERITY_OPT_ERROR_RESTART); 913 break; 914 case DM_VERITY_MODE_PANIC: 915 DMEMIT(DM_VERITY_OPT_ERROR_PANIC); 916 break; 917 default: 918 BUG(); 919 } 920 } 921 if (v->zero_digest) 922 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); 923 if (v->validated_blocks) 924 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); 925 if (v->use_bh_wq) 926 DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); 927 sz = verity_fec_status_table(v, sz, result, maxlen); 928 if (v->signature_key_desc) 929 DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY 930 " %s", v->signature_key_desc); 931 break; 932 933 case STATUSTYPE_IMA: 934 DMEMIT_TARGET_NAME_VERSION(ti->type); 935 DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); 936 DMEMIT(",verity_version=%u", v->version); 937 DMEMIT(",data_device_name=%s", v->data_dev->name); 938 DMEMIT(",hash_device_name=%s", v->hash_dev->name); 939 DMEMIT(",verity_algorithm=%s", v->alg_name); 940 941 DMEMIT(",root_digest="); 942 for (x = 0; x < v->digest_size; x++) 943 DMEMIT("%02x", v->root_digest[x]); 944 945 DMEMIT(",salt="); 946 if (!v->salt_size) 947 DMEMIT("-"); 948 else 949 for (x = 0; x < v->salt_size; x++) 950 DMEMIT("%02x", v->salt[x]); 951 952 DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); 953 DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); 954 if (v->signature_key_desc) 955 DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); 956 957 if (v->mode != DM_VERITY_MODE_EIO) { 958 DMEMIT(",verity_mode="); 959 switch (v->mode) { 960 case DM_VERITY_MODE_LOGGING: 961 DMEMIT(DM_VERITY_OPT_LOGGING); 962 break; 963 case DM_VERITY_MODE_RESTART: 964 DMEMIT(DM_VERITY_OPT_RESTART); 965 break; 966 case DM_VERITY_MODE_PANIC: 967 DMEMIT(DM_VERITY_OPT_PANIC); 968 break; 969 default: 970 DMEMIT("invalid"); 971 } 972 } 973 if (v->error_mode != DM_VERITY_MODE_EIO) { 974 DMEMIT(",verity_error_mode="); 975 switch (v->error_mode) { 976 case DM_VERITY_MODE_RESTART: 977 DMEMIT(DM_VERITY_OPT_ERROR_RESTART); 978 break; 979 case DM_VERITY_MODE_PANIC: 980 DMEMIT(DM_VERITY_OPT_ERROR_PANIC); 981 break; 982 default: 983 DMEMIT("invalid"); 984 } 985 } 986 DMEMIT(";"); 987 break; 988 } 989 } 990 991 static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev, 992 unsigned int cmd, unsigned long arg, 993 bool *forward) 994 { 995 struct dm_verity *v = ti->private; 996 997 *bdev = v->data_dev->bdev; 998 999 if (ti->len != bdev_nr_sectors(v->data_dev->bdev)) 1000 return 1; 1001 return 0; 1002 } 1003 1004 static int verity_iterate_devices(struct dm_target *ti, 1005 iterate_devices_callout_fn fn, void *data) 1006 { 1007 struct dm_verity *v = ti->private; 1008 1009 return fn(ti, v->data_dev, 0, ti->len, data); 1010 } 1011 1012 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) 1013 { 1014 struct dm_verity *v = ti->private; 1015 1016 dm_stack_bs_limits(limits, 1 << v->data_dev_block_bits); 1017 1018 /* 1019 * Similar to what dm-crypt does, opt dm-verity out of support for 1020 * direct I/O that is aligned to less than the traditional direct I/O 1021 * alignment requirement of logical_block_size. This prevents dm-verity 1022 * data blocks from crossing pages, eliminating various edge cases. 1023 */ 1024 limits->dma_alignment = limits->logical_block_size - 1; 1025 } 1026 1027 #ifdef CONFIG_SECURITY 1028 1029 static int verity_init_sig(struct dm_verity *v, const void *sig, 1030 size_t sig_size) 1031 { 1032 v->sig_size = sig_size; 1033 1034 if (sig) { 1035 v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL); 1036 if (!v->root_digest_sig) 1037 return -ENOMEM; 1038 } 1039 1040 return 0; 1041 } 1042 1043 static void verity_free_sig(struct dm_verity *v) 1044 { 1045 kfree(v->root_digest_sig); 1046 } 1047 1048 #else 1049 1050 static inline int verity_init_sig(struct dm_verity *v, const void *sig, 1051 size_t sig_size) 1052 { 1053 return 0; 1054 } 1055 1056 static inline void verity_free_sig(struct dm_verity *v) 1057 { 1058 } 1059 1060 #endif /* CONFIG_SECURITY */ 1061 1062 static void verity_dtr(struct dm_target *ti) 1063 { 1064 struct dm_verity *v = ti->private; 1065 1066 if (v->verify_wq) 1067 destroy_workqueue(v->verify_wq); 1068 1069 mempool_exit(&v->recheck_pool); 1070 if (v->io) 1071 dm_io_client_destroy(v->io); 1072 1073 if (v->bufio) 1074 dm_bufio_client_destroy(v->bufio); 1075 1076 kvfree(v->validated_blocks); 1077 kfree(v->salt); 1078 kfree(v->initial_hashstate.shash); 1079 kfree(v->root_digest); 1080 kfree(v->zero_digest); 1081 verity_free_sig(v); 1082 1083 crypto_free_shash(v->shash_tfm); 1084 1085 kfree(v->alg_name); 1086 1087 if (v->hash_dev) 1088 dm_put_device(ti, v->hash_dev); 1089 1090 if (v->data_dev) 1091 dm_put_device(ti, v->data_dev); 1092 1093 verity_fec_dtr(v); 1094 1095 kfree(v->signature_key_desc); 1096 1097 if (v->use_bh_wq) 1098 static_branch_dec(&use_bh_wq_enabled); 1099 1100 kfree(v); 1101 1102 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); 1103 } 1104 1105 static int verity_alloc_most_once(struct dm_verity *v) 1106 { 1107 struct dm_target *ti = v->ti; 1108 1109 if (v->validated_blocks) 1110 return 0; 1111 1112 /* the bitset can only handle INT_MAX blocks */ 1113 if (v->data_blocks > INT_MAX) { 1114 ti->error = "device too large to use check_at_most_once"; 1115 return -E2BIG; 1116 } 1117 1118 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), 1119 sizeof(unsigned long), 1120 GFP_KERNEL); 1121 if (!v->validated_blocks) { 1122 ti->error = "failed to allocate bitset for check_at_most_once"; 1123 return -ENOMEM; 1124 } 1125 1126 return 0; 1127 } 1128 1129 static int verity_alloc_zero_digest(struct dm_verity *v) 1130 { 1131 int r = -ENOMEM; 1132 struct dm_verity_io *io; 1133 u8 *zero_data; 1134 1135 if (v->zero_digest) 1136 return 0; 1137 1138 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); 1139 1140 if (!v->zero_digest) 1141 return r; 1142 1143 io = kmalloc(v->ti->per_io_data_size, GFP_KERNEL); 1144 1145 if (!io) 1146 return r; /* verity_dtr will free zero_digest */ 1147 1148 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); 1149 1150 if (!zero_data) 1151 goto out; 1152 1153 r = verity_hash(v, io, zero_data, 1 << v->data_dev_block_bits, 1154 v->zero_digest); 1155 1156 out: 1157 kfree(io); 1158 kfree(zero_data); 1159 1160 return r; 1161 } 1162 1163 static inline bool verity_is_verity_mode(const char *arg_name) 1164 { 1165 return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) || 1166 !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) || 1167 !strcasecmp(arg_name, DM_VERITY_OPT_PANIC)); 1168 } 1169 1170 static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) 1171 { 1172 if (v->mode) 1173 return -EINVAL; 1174 1175 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) 1176 v->mode = DM_VERITY_MODE_LOGGING; 1177 else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) 1178 v->mode = DM_VERITY_MODE_RESTART; 1179 else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC)) 1180 v->mode = DM_VERITY_MODE_PANIC; 1181 1182 return 0; 1183 } 1184 1185 static inline bool verity_is_verity_error_mode(const char *arg_name) 1186 { 1187 return (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART) || 1188 !strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC)); 1189 } 1190 1191 static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name) 1192 { 1193 if (v->error_mode) 1194 return -EINVAL; 1195 1196 if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART)) 1197 v->error_mode = DM_VERITY_MODE_RESTART; 1198 else if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC)) 1199 v->error_mode = DM_VERITY_MODE_PANIC; 1200 1201 return 0; 1202 } 1203 1204 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 1205 struct dm_verity_sig_opts *verify_args, 1206 bool only_modifier_opts) 1207 { 1208 int r = 0; 1209 unsigned int argc; 1210 struct dm_target *ti = v->ti; 1211 const char *arg_name; 1212 1213 static const struct dm_arg _args[] = { 1214 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, 1215 }; 1216 1217 r = dm_read_arg_group(_args, as, &argc, &ti->error); 1218 if (r) 1219 return -EINVAL; 1220 1221 if (!argc) 1222 return 0; 1223 1224 do { 1225 arg_name = dm_shift_arg(as); 1226 argc--; 1227 1228 if (verity_is_verity_mode(arg_name)) { 1229 if (only_modifier_opts) 1230 continue; 1231 r = verity_parse_verity_mode(v, arg_name); 1232 if (r) { 1233 ti->error = "Conflicting error handling parameters"; 1234 return r; 1235 } 1236 continue; 1237 1238 } else if (verity_is_verity_error_mode(arg_name)) { 1239 if (only_modifier_opts) 1240 continue; 1241 r = verity_parse_verity_error_mode(v, arg_name); 1242 if (r) { 1243 ti->error = "Conflicting error handling parameters"; 1244 return r; 1245 } 1246 continue; 1247 1248 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { 1249 if (only_modifier_opts) 1250 continue; 1251 r = verity_alloc_zero_digest(v); 1252 if (r) { 1253 ti->error = "Cannot allocate zero digest"; 1254 return r; 1255 } 1256 continue; 1257 1258 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { 1259 if (only_modifier_opts) 1260 continue; 1261 r = verity_alloc_most_once(v); 1262 if (r) 1263 return r; 1264 continue; 1265 1266 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { 1267 if (v->use_bh_wq) 1268 continue; 1269 v->use_bh_wq = true; 1270 static_branch_inc(&use_bh_wq_enabled); 1271 continue; 1272 1273 } else if (verity_is_fec_opt_arg(arg_name)) { 1274 if (only_modifier_opts) 1275 continue; 1276 r = verity_fec_parse_opt_args(as, v, &argc, arg_name); 1277 if (r) 1278 return r; 1279 continue; 1280 1281 } else if (verity_verify_is_sig_opt_arg(arg_name)) { 1282 if (only_modifier_opts) 1283 continue; 1284 r = verity_verify_sig_parse_opt_args(as, v, 1285 verify_args, 1286 &argc, arg_name); 1287 if (r) 1288 return r; 1289 continue; 1290 1291 } else if (only_modifier_opts) { 1292 /* 1293 * Ignore unrecognized opt, could easily be an extra 1294 * argument to an option whose parsing was skipped. 1295 * Normal parsing (@only_modifier_opts=false) will 1296 * properly parse all options (and their extra args). 1297 */ 1298 continue; 1299 } 1300 1301 DMERR("Unrecognized verity feature request: %s", arg_name); 1302 ti->error = "Unrecognized verity feature request"; 1303 return -EINVAL; 1304 } while (argc && !r); 1305 1306 return r; 1307 } 1308 1309 static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name) 1310 { 1311 struct dm_target *ti = v->ti; 1312 struct crypto_shash *shash; 1313 1314 v->alg_name = kstrdup(alg_name, GFP_KERNEL); 1315 if (!v->alg_name) { 1316 ti->error = "Cannot allocate algorithm name"; 1317 return -ENOMEM; 1318 } 1319 1320 shash = crypto_alloc_shash(alg_name, 0, 0); 1321 if (IS_ERR(shash)) { 1322 ti->error = "Cannot initialize hash function"; 1323 return PTR_ERR(shash); 1324 } 1325 v->shash_tfm = shash; 1326 v->digest_size = crypto_shash_digestsize(shash); 1327 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { 1328 ti->error = "Digest size too big"; 1329 return -EINVAL; 1330 } 1331 if (likely(v->version && strcmp(alg_name, "sha256") == 0)) { 1332 /* 1333 * Fast path: use the library API for reduced overhead and 1334 * interleaved hashing support. 1335 */ 1336 v->use_sha256_lib = true; 1337 if (sha256_finup_2x_is_optimized()) 1338 v->use_sha256_finup_2x = true; 1339 ti->per_io_data_size = 1340 offsetofend(struct dm_verity_io, hash_ctx.sha256); 1341 } else { 1342 /* Fallback case: use the generic crypto API. */ 1343 ti->per_io_data_size = 1344 offsetofend(struct dm_verity_io, hash_ctx.shash) + 1345 crypto_shash_descsize(shash); 1346 } 1347 return 0; 1348 } 1349 1350 static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg) 1351 { 1352 struct dm_target *ti = v->ti; 1353 1354 if (strcmp(arg, "-") != 0) { 1355 v->salt_size = strlen(arg) / 2; 1356 v->salt = kmalloc(v->salt_size, GFP_KERNEL); 1357 if (!v->salt) { 1358 ti->error = "Cannot allocate salt"; 1359 return -ENOMEM; 1360 } 1361 if (strlen(arg) != v->salt_size * 2 || 1362 hex2bin(v->salt, arg, v->salt_size)) { 1363 ti->error = "Invalid salt"; 1364 return -EINVAL; 1365 } 1366 } 1367 if (likely(v->use_sha256_lib)) { 1368 /* Implies version 1: salt at beginning */ 1369 v->initial_hashstate.sha256 = 1370 kmalloc_obj(struct sha256_ctx); 1371 if (!v->initial_hashstate.sha256) { 1372 ti->error = "Cannot allocate initial hash state"; 1373 return -ENOMEM; 1374 } 1375 sha256_init(v->initial_hashstate.sha256); 1376 sha256_update(v->initial_hashstate.sha256, 1377 v->salt, v->salt_size); 1378 } else if (v->version) { /* Version 1: salt at beginning */ 1379 SHASH_DESC_ON_STACK(desc, v->shash_tfm); 1380 int r; 1381 1382 /* 1383 * Compute the pre-salted hash state that can be passed to 1384 * crypto_shash_import() for each block later. 1385 */ 1386 v->initial_hashstate.shash = kmalloc( 1387 crypto_shash_statesize(v->shash_tfm), GFP_KERNEL); 1388 if (!v->initial_hashstate.shash) { 1389 ti->error = "Cannot allocate initial hash state"; 1390 return -ENOMEM; 1391 } 1392 desc->tfm = v->shash_tfm; 1393 r = crypto_shash_init(desc) ?: 1394 crypto_shash_update(desc, v->salt, v->salt_size) ?: 1395 crypto_shash_export(desc, v->initial_hashstate.shash); 1396 if (r) { 1397 ti->error = "Cannot set up initial hash state"; 1398 return r; 1399 } 1400 } 1401 return 0; 1402 } 1403 1404 /* 1405 * Target parameters: 1406 * <version> The current format is version 1. 1407 * Vsn 0 is compatible with original Chromium OS releases. 1408 * <data device> 1409 * <hash device> 1410 * <data block size> 1411 * <hash block size> 1412 * <the number of data blocks> 1413 * <hash start block> 1414 * <algorithm> 1415 * <digest> 1416 * <salt> Hex string or "-" if no salt. 1417 */ 1418 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1419 { 1420 struct dm_verity *v; 1421 struct dm_verity_sig_opts verify_args = {0}; 1422 struct dm_arg_set as; 1423 unsigned int num; 1424 unsigned long long num_ll; 1425 int r; 1426 int i; 1427 sector_t hash_position; 1428 char dummy; 1429 char *root_hash_digest_to_validate; 1430 1431 v = kzalloc_obj(struct dm_verity); 1432 if (!v) { 1433 ti->error = "Cannot allocate verity structure"; 1434 return -ENOMEM; 1435 } 1436 ti->private = v; 1437 v->ti = ti; 1438 1439 r = verity_fec_ctr_alloc(v); 1440 if (r) 1441 goto bad; 1442 1443 if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) { 1444 ti->error = "Device must be readonly"; 1445 r = -EINVAL; 1446 goto bad; 1447 } 1448 1449 if (argc < 10) { 1450 ti->error = "Not enough arguments"; 1451 r = -EINVAL; 1452 goto bad; 1453 } 1454 1455 /* Parse optional parameters that modify primary args */ 1456 if (argc > 10) { 1457 as.argc = argc - 10; 1458 as.argv = argv + 10; 1459 r = verity_parse_opt_args(&as, v, &verify_args, true); 1460 if (r < 0) 1461 goto bad; 1462 } 1463 1464 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || 1465 num > 1) { 1466 ti->error = "Invalid version"; 1467 r = -EINVAL; 1468 goto bad; 1469 } 1470 v->version = num; 1471 1472 r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev); 1473 if (r) { 1474 ti->error = "Data device lookup failed"; 1475 goto bad; 1476 } 1477 1478 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev); 1479 if (r) { 1480 ti->error = "Hash device lookup failed"; 1481 goto bad; 1482 } 1483 1484 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || 1485 !num || (num & (num - 1)) || 1486 num < bdev_logical_block_size(v->data_dev->bdev) || 1487 num > PAGE_SIZE) { 1488 ti->error = "Invalid data device block size"; 1489 r = -EINVAL; 1490 goto bad; 1491 } 1492 v->data_dev_block_bits = __ffs(num); 1493 1494 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || 1495 !num || (num & (num - 1)) || 1496 num < bdev_logical_block_size(v->hash_dev->bdev) || 1497 num > INT_MAX) { 1498 ti->error = "Invalid hash device block size"; 1499 r = -EINVAL; 1500 goto bad; 1501 } 1502 v->hash_dev_block_bits = __ffs(num); 1503 1504 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || 1505 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) 1506 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1507 ti->error = "Invalid data blocks"; 1508 r = -EINVAL; 1509 goto bad; 1510 } 1511 v->data_blocks = num_ll; 1512 1513 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { 1514 ti->error = "Data device is too small"; 1515 r = -EINVAL; 1516 goto bad; 1517 } 1518 1519 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || 1520 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) 1521 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1522 ti->error = "Invalid hash start"; 1523 r = -EINVAL; 1524 goto bad; 1525 } 1526 v->hash_start = num_ll; 1527 1528 r = verity_setup_hash_alg(v, argv[7]); 1529 if (r) 1530 goto bad; 1531 1532 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); 1533 if (!v->root_digest) { 1534 ti->error = "Cannot allocate root digest"; 1535 r = -ENOMEM; 1536 goto bad; 1537 } 1538 if (strlen(argv[8]) != v->digest_size * 2 || 1539 hex2bin(v->root_digest, argv[8], v->digest_size)) { 1540 ti->error = "Invalid root digest"; 1541 r = -EINVAL; 1542 goto bad; 1543 } 1544 root_hash_digest_to_validate = argv[8]; 1545 1546 r = verity_setup_salt_and_hashstate(v, argv[9]); 1547 if (r) 1548 goto bad; 1549 1550 argv += 10; 1551 argc -= 10; 1552 1553 /* Optional parameters */ 1554 if (argc) { 1555 as.argc = argc; 1556 as.argv = argv; 1557 r = verity_parse_opt_args(&as, v, &verify_args, false); 1558 if (r < 0) 1559 goto bad; 1560 } 1561 1562 /* Root hash signature is an optional parameter */ 1563 r = verity_verify_root_hash(root_hash_digest_to_validate, 1564 strlen(root_hash_digest_to_validate), 1565 verify_args.sig, 1566 verify_args.sig_size); 1567 if (r < 0) { 1568 ti->error = "Root hash verification failed"; 1569 goto bad; 1570 } 1571 1572 r = verity_init_sig(v, verify_args.sig, verify_args.sig_size); 1573 if (r < 0) { 1574 ti->error = "Cannot allocate root digest signature"; 1575 goto bad; 1576 } 1577 1578 v->hash_per_block_bits = 1579 __fls((1 << v->hash_dev_block_bits) / v->digest_size); 1580 1581 v->levels = 0; 1582 if (v->data_blocks) 1583 while (v->hash_per_block_bits * v->levels < 64 && 1584 (unsigned long long)(v->data_blocks - 1) >> 1585 (v->hash_per_block_bits * v->levels)) 1586 v->levels++; 1587 1588 if (v->levels > DM_VERITY_MAX_LEVELS) { 1589 ti->error = "Too many tree levels"; 1590 r = -E2BIG; 1591 goto bad; 1592 } 1593 1594 hash_position = v->hash_start; 1595 for (i = v->levels - 1; i >= 0; i--) { 1596 sector_t s; 1597 1598 v->hash_level_block[i] = hash_position; 1599 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) 1600 >> ((i + 1) * v->hash_per_block_bits); 1601 if (hash_position + s < hash_position) { 1602 ti->error = "Hash device offset overflow"; 1603 r = -E2BIG; 1604 goto bad; 1605 } 1606 hash_position += s; 1607 } 1608 v->hash_end = hash_position; 1609 1610 r = mempool_init_page_pool(&v->recheck_pool, 1, 0); 1611 if (unlikely(r)) { 1612 ti->error = "Cannot allocate mempool"; 1613 goto bad; 1614 } 1615 1616 v->io = dm_io_client_create(); 1617 if (IS_ERR(v->io)) { 1618 r = PTR_ERR(v->io); 1619 v->io = NULL; 1620 ti->error = "Cannot allocate dm io"; 1621 goto bad; 1622 } 1623 1624 v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1625 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), 1626 dm_bufio_alloc_callback, NULL, 1627 v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); 1628 if (IS_ERR(v->bufio)) { 1629 ti->error = "Cannot initialize dm-bufio"; 1630 r = PTR_ERR(v->bufio); 1631 v->bufio = NULL; 1632 goto bad; 1633 } 1634 1635 if (dm_bufio_get_device_size(v->bufio) < v->hash_end) { 1636 ti->error = "Hash device is too small"; 1637 r = -E2BIG; 1638 goto bad; 1639 } 1640 1641 /* 1642 * Using WQ_HIGHPRI improves throughput and completion latency by 1643 * reducing wait times when reading from a dm-verity device. 1644 * 1645 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI 1646 * allows verify_wq to preempt softirq since verification in softirq 1647 * will fall-back to using it for error handling (or if the bufio cache 1648 * doesn't have required hashes). 1649 */ 1650 v->verify_wq = alloc_workqueue("kverityd", 1651 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 1652 0); 1653 if (!v->verify_wq) { 1654 ti->error = "Cannot allocate workqueue"; 1655 r = -ENOMEM; 1656 goto bad; 1657 } 1658 1659 r = verity_fec_ctr(v); 1660 if (r) 1661 goto bad; 1662 1663 ti->per_io_data_size = roundup(ti->per_io_data_size, 1664 __alignof__(struct dm_verity_io)); 1665 1666 verity_verify_sig_opts_cleanup(&verify_args); 1667 1668 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); 1669 1670 return 0; 1671 1672 bad: 1673 1674 verity_verify_sig_opts_cleanup(&verify_args); 1675 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); 1676 verity_dtr(ti); 1677 1678 return r; 1679 } 1680 1681 /* 1682 * Get the verity mode (error behavior) of a verity target. 1683 * 1684 * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity 1685 * target. 1686 */ 1687 int dm_verity_get_mode(struct dm_target *ti) 1688 { 1689 struct dm_verity *v = ti->private; 1690 1691 if (!dm_is_verity_target(ti)) 1692 return -EINVAL; 1693 1694 return v->mode; 1695 } 1696 1697 /* 1698 * Get the root digest of a verity target. 1699 * 1700 * Returns a copy of the root digest, the caller is responsible for 1701 * freeing the memory of the digest. 1702 */ 1703 int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) 1704 { 1705 struct dm_verity *v = ti->private; 1706 1707 if (!dm_is_verity_target(ti)) 1708 return -EINVAL; 1709 1710 *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); 1711 if (*root_digest == NULL) 1712 return -ENOMEM; 1713 1714 *digest_size = v->digest_size; 1715 1716 return 0; 1717 } 1718 1719 #ifdef CONFIG_SECURITY 1720 1721 #ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG 1722 1723 static int verity_security_set_signature(struct block_device *bdev, 1724 struct dm_verity *v) 1725 { 1726 /* 1727 * if the dm-verity target is unsigned, v->root_digest_sig will 1728 * be NULL, and the hook call is still required to let LSMs mark 1729 * the device as unsigned. This information is crucial for LSMs to 1730 * block operations such as execution on unsigned files 1731 */ 1732 return security_bdev_setintegrity(bdev, 1733 LSM_INT_DMVERITY_SIG_VALID, 1734 v->root_digest_sig, 1735 v->sig_size); 1736 } 1737 1738 #else 1739 1740 static inline int verity_security_set_signature(struct block_device *bdev, 1741 struct dm_verity *v) 1742 { 1743 return 0; 1744 } 1745 1746 #endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */ 1747 1748 /* 1749 * Expose verity target's root hash and signature data to LSMs before resume. 1750 * 1751 * Returns 0 on success, or -ENOMEM if the system is out of memory. 1752 */ 1753 static int verity_preresume(struct dm_target *ti) 1754 { 1755 struct block_device *bdev; 1756 struct dm_verity_digest root_digest; 1757 struct dm_verity *v; 1758 int r; 1759 1760 v = ti->private; 1761 bdev = dm_disk(dm_table_get_md(ti->table))->part0; 1762 root_digest.digest = v->root_digest; 1763 root_digest.digest_len = v->digest_size; 1764 root_digest.alg = crypto_shash_alg_name(v->shash_tfm); 1765 1766 r = security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, &root_digest, 1767 sizeof(root_digest)); 1768 if (r) 1769 return r; 1770 1771 r = verity_security_set_signature(bdev, v); 1772 if (r) 1773 goto bad; 1774 1775 return 0; 1776 1777 bad: 1778 1779 security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, NULL, 0); 1780 1781 return r; 1782 } 1783 1784 #endif /* CONFIG_SECURITY */ 1785 1786 static struct target_type verity_target = { 1787 .name = "verity", 1788 /* Note: the LSMs depend on the singleton and immutable features */ 1789 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, 1790 .version = {1, 13, 0}, 1791 .module = THIS_MODULE, 1792 .ctr = verity_ctr, 1793 .dtr = verity_dtr, 1794 .map = verity_map, 1795 .postsuspend = verity_postsuspend, 1796 .status = verity_status, 1797 .prepare_ioctl = verity_prepare_ioctl, 1798 .iterate_devices = verity_iterate_devices, 1799 .io_hints = verity_io_hints, 1800 #ifdef CONFIG_SECURITY 1801 .preresume = verity_preresume, 1802 #endif /* CONFIG_SECURITY */ 1803 }; 1804 1805 static int __init dm_verity_init(void) 1806 { 1807 int r; 1808 1809 r = dm_verity_verify_sig_init(); 1810 if (r) 1811 return r; 1812 1813 r = dm_register_target(&verity_target); 1814 if (r) { 1815 dm_verity_verify_sig_exit(); 1816 return r; 1817 } 1818 1819 return 0; 1820 } 1821 module_init(dm_verity_init); 1822 1823 static void __exit dm_verity_exit(void) 1824 { 1825 dm_unregister_target(&verity_target); 1826 dm_verity_verify_sig_exit(); 1827 } 1828 module_exit(dm_verity_exit); 1829 1830 /* 1831 * Check whether a DM target is a verity target. 1832 */ 1833 bool dm_is_verity_target(struct dm_target *ti) 1834 { 1835 return ti->type == &verity_target; 1836 } 1837 1838 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); 1839 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); 1840 MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); 1841 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); 1842 MODULE_LICENSE("GPL"); 1843