1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. 4 * 5 * Author: Mikulas Patocka <mpatocka@redhat.com> 6 * 7 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors 8 * 9 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set 10 * default prefetch value. Data are read in "prefetch_cluster" chunks from the 11 * hash device. Setting this greatly improves performance when data and hash 12 * are on the same disk on different partitions on devices with poor random 13 * access behavior. 14 */ 15 16 #include "dm-verity.h" 17 #include "dm-verity-fec.h" 18 #include "dm-verity-verify-sig.h" 19 #include "dm-audit.h" 20 #include <linux/module.h> 21 #include <linux/reboot.h> 22 #include <linux/scatterlist.h> 23 #include <linux/string.h> 24 #include <linux/jump_label.h> 25 #include <linux/security.h> 26 27 #define DM_MSG_PREFIX "verity" 28 29 #define DM_VERITY_ENV_LENGTH 42 30 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" 31 32 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 33 34 #define DM_VERITY_MAX_CORRUPTED_ERRS 100 35 36 #define DM_VERITY_OPT_LOGGING "ignore_corruption" 37 #define DM_VERITY_OPT_RESTART "restart_on_corruption" 38 #define DM_VERITY_OPT_PANIC "panic_on_corruption" 39 #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" 40 #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" 41 #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" 42 43 #define DM_VERITY_OPTS_MAX (4 + DM_VERITY_OPTS_FEC + \ 44 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) 45 46 static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; 47 48 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); 49 50 static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); 51 52 /* Is at least one dm-verity instance using ahash_tfm instead of shash_tfm? */ 53 static DEFINE_STATIC_KEY_FALSE(ahash_enabled); 54 55 struct dm_verity_prefetch_work { 56 struct work_struct work; 57 struct dm_verity *v; 58 unsigned short ioprio; 59 sector_t block; 60 unsigned int n_blocks; 61 }; 62 63 /* 64 * Auxiliary structure appended to each dm-bufio buffer. If the value 65 * hash_verified is nonzero, hash of the block has been verified. 66 * 67 * The variable hash_verified is set to 0 when allocating the buffer, then 68 * it can be changed to 1 and it is never reset to 0 again. 69 * 70 * There is no lock around this value, a race condition can at worst cause 71 * that multiple processes verify the hash of the same buffer simultaneously 72 * and write 1 to hash_verified simultaneously. 73 * This condition is harmless, so we don't need locking. 74 */ 75 struct buffer_aux { 76 int hash_verified; 77 }; 78 79 /* 80 * Initialize struct buffer_aux for a freshly created buffer. 81 */ 82 static void dm_bufio_alloc_callback(struct dm_buffer *buf) 83 { 84 struct buffer_aux *aux = dm_bufio_get_aux_data(buf); 85 86 aux->hash_verified = 0; 87 } 88 89 /* 90 * Translate input sector number to the sector number on the target device. 91 */ 92 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) 93 { 94 return v->data_start + dm_target_offset(v->ti, bi_sector); 95 } 96 97 /* 98 * Return hash position of a specified block at a specified tree level 99 * (0 is the lowest level). 100 * The lowest "hash_per_block_bits"-bits of the result denote hash position 101 * inside a hash block. The remaining bits denote location of the hash block. 102 */ 103 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, 104 int level) 105 { 106 return block >> (level * v->hash_per_block_bits); 107 } 108 109 static int verity_ahash_update(struct dm_verity *v, struct ahash_request *req, 110 const u8 *data, size_t len, 111 struct crypto_wait *wait) 112 { 113 struct scatterlist sg; 114 115 if (likely(!is_vmalloc_addr(data))) { 116 sg_init_one(&sg, data, len); 117 ahash_request_set_crypt(req, &sg, NULL, len); 118 return crypto_wait_req(crypto_ahash_update(req), wait); 119 } 120 121 do { 122 int r; 123 size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data)); 124 125 flush_kernel_vmap_range((void *)data, this_step); 126 sg_init_table(&sg, 1); 127 sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data)); 128 ahash_request_set_crypt(req, &sg, NULL, this_step); 129 r = crypto_wait_req(crypto_ahash_update(req), wait); 130 if (unlikely(r)) 131 return r; 132 data += this_step; 133 len -= this_step; 134 } while (len); 135 136 return 0; 137 } 138 139 /* 140 * Wrapper for crypto_ahash_init, which handles verity salting. 141 */ 142 static int verity_ahash_init(struct dm_verity *v, struct ahash_request *req, 143 struct crypto_wait *wait, bool may_sleep) 144 { 145 int r; 146 147 ahash_request_set_tfm(req, v->ahash_tfm); 148 ahash_request_set_callback(req, 149 may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0, 150 crypto_req_done, (void *)wait); 151 crypto_init_wait(wait); 152 153 r = crypto_wait_req(crypto_ahash_init(req), wait); 154 155 if (unlikely(r < 0)) { 156 if (r != -ENOMEM) 157 DMERR("crypto_ahash_init failed: %d", r); 158 return r; 159 } 160 161 if (likely(v->salt_size && (v->version >= 1))) 162 r = verity_ahash_update(v, req, v->salt, v->salt_size, wait); 163 164 return r; 165 } 166 167 static int verity_ahash_final(struct dm_verity *v, struct ahash_request *req, 168 u8 *digest, struct crypto_wait *wait) 169 { 170 int r; 171 172 if (unlikely(v->salt_size && (!v->version))) { 173 r = verity_ahash_update(v, req, v->salt, v->salt_size, wait); 174 175 if (r < 0) { 176 DMERR("%s failed updating salt: %d", __func__, r); 177 goto out; 178 } 179 } 180 181 ahash_request_set_crypt(req, NULL, digest, 0); 182 r = crypto_wait_req(crypto_ahash_final(req), wait); 183 out: 184 return r; 185 } 186 187 int verity_hash(struct dm_verity *v, struct dm_verity_io *io, 188 const u8 *data, size_t len, u8 *digest, bool may_sleep) 189 { 190 int r; 191 192 if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) { 193 struct ahash_request *req = verity_io_hash_req(v, io); 194 struct crypto_wait wait; 195 196 r = verity_ahash_init(v, req, &wait, may_sleep) ?: 197 verity_ahash_update(v, req, data, len, &wait) ?: 198 verity_ahash_final(v, req, digest, &wait); 199 } else { 200 struct shash_desc *desc = verity_io_hash_req(v, io); 201 202 desc->tfm = v->shash_tfm; 203 r = crypto_shash_import(desc, v->initial_hashstate) ?: 204 crypto_shash_finup(desc, data, len, digest); 205 } 206 if (unlikely(r)) 207 DMERR("Error hashing block: %d", r); 208 return r; 209 } 210 211 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, 212 sector_t *hash_block, unsigned int *offset) 213 { 214 sector_t position = verity_position_at_level(v, block, level); 215 unsigned int idx; 216 217 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); 218 219 if (!offset) 220 return; 221 222 idx = position & ((1 << v->hash_per_block_bits) - 1); 223 if (!v->version) 224 *offset = idx * v->digest_size; 225 else 226 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); 227 } 228 229 /* 230 * Handle verification errors. 231 */ 232 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, 233 unsigned long long block) 234 { 235 char verity_env[DM_VERITY_ENV_LENGTH]; 236 char *envp[] = { verity_env, NULL }; 237 const char *type_str = ""; 238 struct mapped_device *md = dm_table_get_md(v->ti->table); 239 240 /* Corruption should be visible in device status in all modes */ 241 v->hash_failed = true; 242 243 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) 244 goto out; 245 246 v->corrupted_errs++; 247 248 switch (type) { 249 case DM_VERITY_BLOCK_TYPE_DATA: 250 type_str = "data"; 251 break; 252 case DM_VERITY_BLOCK_TYPE_METADATA: 253 type_str = "metadata"; 254 break; 255 default: 256 BUG(); 257 } 258 259 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, 260 type_str, block); 261 262 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { 263 DMERR("%s: reached maximum errors", v->data_dev->name); 264 dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); 265 } 266 267 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", 268 DM_VERITY_ENV_VAR_NAME, type, block); 269 270 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); 271 272 out: 273 if (v->mode == DM_VERITY_MODE_LOGGING) 274 return 0; 275 276 if (v->mode == DM_VERITY_MODE_RESTART) { 277 pr_emerg("dm-verity device corrupted\n"); 278 emergency_restart(); 279 } 280 281 if (v->mode == DM_VERITY_MODE_PANIC) 282 panic("dm-verity device corrupted"); 283 284 return 1; 285 } 286 287 /* 288 * Verify hash of a metadata block pertaining to the specified data block 289 * ("block" argument) at a specified level ("level" argument). 290 * 291 * On successful return, verity_io_want_digest(v, io) contains the hash value 292 * for a lower tree level or for the data block (if we're at the lowest level). 293 * 294 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. 295 * If "skip_unverified" is false, unverified buffer is hashed and verified 296 * against current value of verity_io_want_digest(v, io). 297 */ 298 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, 299 sector_t block, int level, bool skip_unverified, 300 u8 *want_digest) 301 { 302 struct dm_buffer *buf; 303 struct buffer_aux *aux; 304 u8 *data; 305 int r; 306 sector_t hash_block; 307 unsigned int offset; 308 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 309 310 verity_hash_at_level(v, block, level, &hash_block, &offset); 311 312 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 313 data = dm_bufio_get(v->bufio, hash_block, &buf); 314 if (data == NULL) { 315 /* 316 * In tasklet and the hash was not in the bufio cache. 317 * Return early and resume execution from a work-queue 318 * to read the hash from disk. 319 */ 320 return -EAGAIN; 321 } 322 } else { 323 data = dm_bufio_read_with_ioprio(v->bufio, hash_block, 324 &buf, bio_prio(bio)); 325 } 326 327 if (IS_ERR(data)) 328 return PTR_ERR(data); 329 330 aux = dm_bufio_get_aux_data(buf); 331 332 if (!aux->hash_verified) { 333 if (skip_unverified) { 334 r = 1; 335 goto release_ret_r; 336 } 337 338 r = verity_hash(v, io, data, 1 << v->hash_dev_block_bits, 339 verity_io_real_digest(v, io), !io->in_bh); 340 if (unlikely(r < 0)) 341 goto release_ret_r; 342 343 if (likely(memcmp(verity_io_real_digest(v, io), want_digest, 344 v->digest_size) == 0)) 345 aux->hash_verified = 1; 346 else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 347 /* 348 * Error handling code (FEC included) cannot be run in a 349 * tasklet since it may sleep, so fallback to work-queue. 350 */ 351 r = -EAGAIN; 352 goto release_ret_r; 353 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, 354 hash_block, data) == 0) 355 aux->hash_verified = 1; 356 else if (verity_handle_err(v, 357 DM_VERITY_BLOCK_TYPE_METADATA, 358 hash_block)) { 359 struct bio *bio = 360 dm_bio_from_per_bio_data(io, 361 v->ti->per_io_data_size); 362 dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio, 363 block, 0); 364 r = -EIO; 365 goto release_ret_r; 366 } 367 } 368 369 data += offset; 370 memcpy(want_digest, data, v->digest_size); 371 r = 0; 372 373 release_ret_r: 374 dm_bufio_release(buf); 375 return r; 376 } 377 378 /* 379 * Find a hash for a given block, write it to digest and verify the integrity 380 * of the hash tree if necessary. 381 */ 382 int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, 383 sector_t block, u8 *digest, bool *is_zero) 384 { 385 int r = 0, i; 386 387 if (likely(v->levels)) { 388 /* 389 * First, we try to get the requested hash for 390 * the current block. If the hash block itself is 391 * verified, zero is returned. If it isn't, this 392 * function returns 1 and we fall back to whole 393 * chain verification. 394 */ 395 r = verity_verify_level(v, io, block, 0, true, digest); 396 if (likely(r <= 0)) 397 goto out; 398 } 399 400 memcpy(digest, v->root_digest, v->digest_size); 401 402 for (i = v->levels - 1; i >= 0; i--) { 403 r = verity_verify_level(v, io, block, i, false, digest); 404 if (unlikely(r)) 405 goto out; 406 } 407 out: 408 if (!r && v->zero_digest) 409 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); 410 else 411 *is_zero = false; 412 413 return r; 414 } 415 416 static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, 417 sector_t cur_block, u8 *dest) 418 { 419 struct page *page; 420 void *buffer; 421 int r; 422 struct dm_io_request io_req; 423 struct dm_io_region io_loc; 424 425 page = mempool_alloc(&v->recheck_pool, GFP_NOIO); 426 buffer = page_to_virt(page); 427 428 io_req.bi_opf = REQ_OP_READ; 429 io_req.mem.type = DM_IO_KMEM; 430 io_req.mem.ptr.addr = buffer; 431 io_req.notify.fn = NULL; 432 io_req.client = v->io; 433 io_loc.bdev = v->data_dev->bdev; 434 io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); 435 io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); 436 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 437 if (unlikely(r)) 438 goto free_ret; 439 440 r = verity_hash(v, io, buffer, 1 << v->data_dev_block_bits, 441 verity_io_real_digest(v, io), true); 442 if (unlikely(r)) 443 goto free_ret; 444 445 if (memcmp(verity_io_real_digest(v, io), 446 verity_io_want_digest(v, io), v->digest_size)) { 447 r = -EIO; 448 goto free_ret; 449 } 450 451 memcpy(dest, buffer, 1 << v->data_dev_block_bits); 452 r = 0; 453 free_ret: 454 mempool_free(page, &v->recheck_pool); 455 456 return r; 457 } 458 459 static int verity_handle_data_hash_mismatch(struct dm_verity *v, 460 struct dm_verity_io *io, 461 struct bio *bio, sector_t blkno, 462 u8 *data) 463 { 464 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 465 /* 466 * Error handling code (FEC included) cannot be run in the 467 * BH workqueue, so fallback to a standard workqueue. 468 */ 469 return -EAGAIN; 470 } 471 if (verity_recheck(v, io, blkno, data) == 0) { 472 if (v->validated_blocks) 473 set_bit(blkno, v->validated_blocks); 474 return 0; 475 } 476 #if defined(CONFIG_DM_VERITY_FEC) 477 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, blkno, 478 data) == 0) 479 return 0; 480 #endif 481 if (bio->bi_status) 482 return -EIO; /* Error correction failed; Just return error */ 483 484 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, blkno)) { 485 dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", bio, blkno, 0); 486 return -EIO; 487 } 488 return 0; 489 } 490 491 /* 492 * Verify one "dm_verity_io" structure. 493 */ 494 static int verity_verify_io(struct dm_verity_io *io) 495 { 496 struct dm_verity *v = io->v; 497 const unsigned int block_size = 1 << v->data_dev_block_bits; 498 struct bvec_iter iter_copy; 499 struct bvec_iter *iter; 500 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 501 unsigned int b; 502 503 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { 504 /* 505 * Copy the iterator in case we need to restart 506 * verification in a work-queue. 507 */ 508 iter_copy = io->iter; 509 iter = &iter_copy; 510 } else 511 iter = &io->iter; 512 513 for (b = 0; b < io->n_blocks; 514 b++, bio_advance_iter(bio, iter, block_size)) { 515 int r; 516 sector_t cur_block = io->block + b; 517 bool is_zero; 518 struct bio_vec bv; 519 void *data; 520 521 if (v->validated_blocks && bio->bi_status == BLK_STS_OK && 522 likely(test_bit(cur_block, v->validated_blocks))) 523 continue; 524 525 r = verity_hash_for_block(v, io, cur_block, 526 verity_io_want_digest(v, io), 527 &is_zero); 528 if (unlikely(r < 0)) 529 return r; 530 531 bv = bio_iter_iovec(bio, *iter); 532 if (unlikely(bv.bv_len < block_size)) { 533 /* 534 * Data block spans pages. This should not happen, 535 * since dm-verity sets dma_alignment to the data block 536 * size minus 1, and dm-verity also doesn't allow the 537 * data block size to be greater than PAGE_SIZE. 538 */ 539 DMERR_LIMIT("unaligned io (data block spans pages)"); 540 return -EIO; 541 } 542 543 data = bvec_kmap_local(&bv); 544 545 if (is_zero) { 546 /* 547 * If we expect a zero block, don't validate, just 548 * return zeros. 549 */ 550 memset(data, 0, block_size); 551 kunmap_local(data); 552 continue; 553 } 554 555 r = verity_hash(v, io, data, block_size, 556 verity_io_real_digest(v, io), !io->in_bh); 557 if (unlikely(r < 0)) { 558 kunmap_local(data); 559 return r; 560 } 561 562 if (likely(memcmp(verity_io_real_digest(v, io), 563 verity_io_want_digest(v, io), v->digest_size) == 0)) { 564 if (v->validated_blocks) 565 set_bit(cur_block, v->validated_blocks); 566 kunmap_local(data); 567 continue; 568 } 569 r = verity_handle_data_hash_mismatch(v, io, bio, cur_block, 570 data); 571 kunmap_local(data); 572 if (unlikely(r)) 573 return r; 574 } 575 576 return 0; 577 } 578 579 /* 580 * Skip verity work in response to I/O error when system is shutting down. 581 */ 582 static inline bool verity_is_system_shutting_down(void) 583 { 584 return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF 585 || system_state == SYSTEM_RESTART; 586 } 587 588 /* 589 * End one "io" structure with a given error. 590 */ 591 static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) 592 { 593 struct dm_verity *v = io->v; 594 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); 595 596 bio->bi_end_io = io->orig_bi_end_io; 597 bio->bi_status = status; 598 599 if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) 600 verity_fec_finish_io(io); 601 602 if (unlikely(status != BLK_STS_OK) && 603 unlikely(!(bio->bi_opf & REQ_RAHEAD)) && 604 !verity_is_system_shutting_down()) { 605 if (v->mode == DM_VERITY_MODE_RESTART || 606 v->mode == DM_VERITY_MODE_PANIC) 607 DMERR_LIMIT("%s has error: %s", v->data_dev->name, 608 blk_status_to_str(status)); 609 610 if (v->mode == DM_VERITY_MODE_RESTART) { 611 pr_emerg("dm-verity device corrupted\n"); 612 emergency_restart(); 613 } 614 615 if (v->mode == DM_VERITY_MODE_PANIC) 616 panic("dm-verity device corrupted"); 617 } 618 619 bio_endio(bio); 620 } 621 622 static void verity_work(struct work_struct *w) 623 { 624 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); 625 626 io->in_bh = false; 627 628 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); 629 } 630 631 static void verity_bh_work(struct work_struct *w) 632 { 633 struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); 634 int err; 635 636 io->in_bh = true; 637 err = verity_verify_io(io); 638 if (err == -EAGAIN || err == -ENOMEM) { 639 /* fallback to retrying with work-queue */ 640 INIT_WORK(&io->work, verity_work); 641 queue_work(io->v->verify_wq, &io->work); 642 return; 643 } 644 645 verity_finish_io(io, errno_to_blk_status(err)); 646 } 647 648 static void verity_end_io(struct bio *bio) 649 { 650 struct dm_verity_io *io = bio->bi_private; 651 652 if (bio->bi_status && 653 (!verity_fec_is_enabled(io->v) || 654 verity_is_system_shutting_down() || 655 (bio->bi_opf & REQ_RAHEAD))) { 656 verity_finish_io(io, bio->bi_status); 657 return; 658 } 659 660 if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) { 661 INIT_WORK(&io->bh_work, verity_bh_work); 662 queue_work(system_bh_wq, &io->bh_work); 663 } else { 664 INIT_WORK(&io->work, verity_work); 665 queue_work(io->v->verify_wq, &io->work); 666 } 667 } 668 669 /* 670 * Prefetch buffers for the specified io. 671 * The root buffer is not prefetched, it is assumed that it will be cached 672 * all the time. 673 */ 674 static void verity_prefetch_io(struct work_struct *work) 675 { 676 struct dm_verity_prefetch_work *pw = 677 container_of(work, struct dm_verity_prefetch_work, work); 678 struct dm_verity *v = pw->v; 679 int i; 680 681 for (i = v->levels - 2; i >= 0; i--) { 682 sector_t hash_block_start; 683 sector_t hash_block_end; 684 685 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); 686 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); 687 688 if (!i) { 689 unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); 690 691 cluster >>= v->data_dev_block_bits; 692 if (unlikely(!cluster)) 693 goto no_prefetch_cluster; 694 695 if (unlikely(cluster & (cluster - 1))) 696 cluster = 1 << __fls(cluster); 697 698 hash_block_start &= ~(sector_t)(cluster - 1); 699 hash_block_end |= cluster - 1; 700 if (unlikely(hash_block_end >= v->hash_blocks)) 701 hash_block_end = v->hash_blocks - 1; 702 } 703 no_prefetch_cluster: 704 dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start, 705 hash_block_end - hash_block_start + 1, 706 pw->ioprio); 707 } 708 709 kfree(pw); 710 } 711 712 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, 713 unsigned short ioprio) 714 { 715 sector_t block = io->block; 716 unsigned int n_blocks = io->n_blocks; 717 struct dm_verity_prefetch_work *pw; 718 719 if (v->validated_blocks) { 720 while (n_blocks && test_bit(block, v->validated_blocks)) { 721 block++; 722 n_blocks--; 723 } 724 while (n_blocks && test_bit(block + n_blocks - 1, 725 v->validated_blocks)) 726 n_blocks--; 727 if (!n_blocks) 728 return; 729 } 730 731 pw = kmalloc(sizeof(struct dm_verity_prefetch_work), 732 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 733 734 if (!pw) 735 return; 736 737 INIT_WORK(&pw->work, verity_prefetch_io); 738 pw->v = v; 739 pw->block = block; 740 pw->n_blocks = n_blocks; 741 pw->ioprio = ioprio; 742 queue_work(v->verify_wq, &pw->work); 743 } 744 745 /* 746 * Bio map function. It allocates dm_verity_io structure and bio vector and 747 * fills them. Then it issues prefetches and the I/O. 748 */ 749 static int verity_map(struct dm_target *ti, struct bio *bio) 750 { 751 struct dm_verity *v = ti->private; 752 struct dm_verity_io *io; 753 754 bio_set_dev(bio, v->data_dev->bdev); 755 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); 756 757 if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & 758 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { 759 DMERR_LIMIT("unaligned io"); 760 return DM_MAPIO_KILL; 761 } 762 763 if (bio_end_sector(bio) >> 764 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { 765 DMERR_LIMIT("io out of range"); 766 return DM_MAPIO_KILL; 767 } 768 769 if (bio_data_dir(bio) == WRITE) 770 return DM_MAPIO_KILL; 771 772 io = dm_per_bio_data(bio, ti->per_io_data_size); 773 io->v = v; 774 io->orig_bi_end_io = bio->bi_end_io; 775 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); 776 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; 777 778 bio->bi_end_io = verity_end_io; 779 bio->bi_private = io; 780 io->iter = bio->bi_iter; 781 782 verity_fec_init_io(io); 783 784 verity_submit_prefetch(v, io, bio_prio(bio)); 785 786 submit_bio_noacct(bio); 787 788 return DM_MAPIO_SUBMITTED; 789 } 790 791 /* 792 * Status: V (valid) or C (corruption found) 793 */ 794 static void verity_status(struct dm_target *ti, status_type_t type, 795 unsigned int status_flags, char *result, unsigned int maxlen) 796 { 797 struct dm_verity *v = ti->private; 798 unsigned int args = 0; 799 unsigned int sz = 0; 800 unsigned int x; 801 802 switch (type) { 803 case STATUSTYPE_INFO: 804 DMEMIT("%c", v->hash_failed ? 'C' : 'V'); 805 break; 806 case STATUSTYPE_TABLE: 807 DMEMIT("%u %s %s %u %u %llu %llu %s ", 808 v->version, 809 v->data_dev->name, 810 v->hash_dev->name, 811 1 << v->data_dev_block_bits, 812 1 << v->hash_dev_block_bits, 813 (unsigned long long)v->data_blocks, 814 (unsigned long long)v->hash_start, 815 v->alg_name 816 ); 817 for (x = 0; x < v->digest_size; x++) 818 DMEMIT("%02x", v->root_digest[x]); 819 DMEMIT(" "); 820 if (!v->salt_size) 821 DMEMIT("-"); 822 else 823 for (x = 0; x < v->salt_size; x++) 824 DMEMIT("%02x", v->salt[x]); 825 if (v->mode != DM_VERITY_MODE_EIO) 826 args++; 827 if (verity_fec_is_enabled(v)) 828 args += DM_VERITY_OPTS_FEC; 829 if (v->zero_digest) 830 args++; 831 if (v->validated_blocks) 832 args++; 833 if (v->use_bh_wq) 834 args++; 835 if (v->signature_key_desc) 836 args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; 837 if (!args) 838 return; 839 DMEMIT(" %u", args); 840 if (v->mode != DM_VERITY_MODE_EIO) { 841 DMEMIT(" "); 842 switch (v->mode) { 843 case DM_VERITY_MODE_LOGGING: 844 DMEMIT(DM_VERITY_OPT_LOGGING); 845 break; 846 case DM_VERITY_MODE_RESTART: 847 DMEMIT(DM_VERITY_OPT_RESTART); 848 break; 849 case DM_VERITY_MODE_PANIC: 850 DMEMIT(DM_VERITY_OPT_PANIC); 851 break; 852 default: 853 BUG(); 854 } 855 } 856 if (v->zero_digest) 857 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); 858 if (v->validated_blocks) 859 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); 860 if (v->use_bh_wq) 861 DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); 862 sz = verity_fec_status_table(v, sz, result, maxlen); 863 if (v->signature_key_desc) 864 DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY 865 " %s", v->signature_key_desc); 866 break; 867 868 case STATUSTYPE_IMA: 869 DMEMIT_TARGET_NAME_VERSION(ti->type); 870 DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); 871 DMEMIT(",verity_version=%u", v->version); 872 DMEMIT(",data_device_name=%s", v->data_dev->name); 873 DMEMIT(",hash_device_name=%s", v->hash_dev->name); 874 DMEMIT(",verity_algorithm=%s", v->alg_name); 875 876 DMEMIT(",root_digest="); 877 for (x = 0; x < v->digest_size; x++) 878 DMEMIT("%02x", v->root_digest[x]); 879 880 DMEMIT(",salt="); 881 if (!v->salt_size) 882 DMEMIT("-"); 883 else 884 for (x = 0; x < v->salt_size; x++) 885 DMEMIT("%02x", v->salt[x]); 886 887 DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); 888 DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); 889 if (v->signature_key_desc) 890 DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); 891 892 if (v->mode != DM_VERITY_MODE_EIO) { 893 DMEMIT(",verity_mode="); 894 switch (v->mode) { 895 case DM_VERITY_MODE_LOGGING: 896 DMEMIT(DM_VERITY_OPT_LOGGING); 897 break; 898 case DM_VERITY_MODE_RESTART: 899 DMEMIT(DM_VERITY_OPT_RESTART); 900 break; 901 case DM_VERITY_MODE_PANIC: 902 DMEMIT(DM_VERITY_OPT_PANIC); 903 break; 904 default: 905 DMEMIT("invalid"); 906 } 907 } 908 DMEMIT(";"); 909 break; 910 } 911 } 912 913 static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) 914 { 915 struct dm_verity *v = ti->private; 916 917 *bdev = v->data_dev->bdev; 918 919 if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev)) 920 return 1; 921 return 0; 922 } 923 924 static int verity_iterate_devices(struct dm_target *ti, 925 iterate_devices_callout_fn fn, void *data) 926 { 927 struct dm_verity *v = ti->private; 928 929 return fn(ti, v->data_dev, v->data_start, ti->len, data); 930 } 931 932 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) 933 { 934 struct dm_verity *v = ti->private; 935 936 if (limits->logical_block_size < 1 << v->data_dev_block_bits) 937 limits->logical_block_size = 1 << v->data_dev_block_bits; 938 939 if (limits->physical_block_size < 1 << v->data_dev_block_bits) 940 limits->physical_block_size = 1 << v->data_dev_block_bits; 941 942 limits->io_min = limits->logical_block_size; 943 944 /* 945 * Similar to what dm-crypt does, opt dm-verity out of support for 946 * direct I/O that is aligned to less than the traditional direct I/O 947 * alignment requirement of logical_block_size. This prevents dm-verity 948 * data blocks from crossing pages, eliminating various edge cases. 949 */ 950 limits->dma_alignment = limits->logical_block_size - 1; 951 } 952 953 #ifdef CONFIG_SECURITY 954 955 static int verity_init_sig(struct dm_verity *v, const void *sig, 956 size_t sig_size) 957 { 958 v->sig_size = sig_size; 959 960 if (sig) { 961 v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL); 962 if (!v->root_digest_sig) 963 return -ENOMEM; 964 } 965 966 return 0; 967 } 968 969 static void verity_free_sig(struct dm_verity *v) 970 { 971 kfree(v->root_digest_sig); 972 } 973 974 #else 975 976 static inline int verity_init_sig(struct dm_verity *v, const void *sig, 977 size_t sig_size) 978 { 979 return 0; 980 } 981 982 static inline void verity_free_sig(struct dm_verity *v) 983 { 984 } 985 986 #endif /* CONFIG_SECURITY */ 987 988 static void verity_dtr(struct dm_target *ti) 989 { 990 struct dm_verity *v = ti->private; 991 992 if (v->verify_wq) 993 destroy_workqueue(v->verify_wq); 994 995 mempool_exit(&v->recheck_pool); 996 if (v->io) 997 dm_io_client_destroy(v->io); 998 999 if (v->bufio) 1000 dm_bufio_client_destroy(v->bufio); 1001 1002 kvfree(v->validated_blocks); 1003 kfree(v->salt); 1004 kfree(v->initial_hashstate); 1005 kfree(v->root_digest); 1006 kfree(v->zero_digest); 1007 verity_free_sig(v); 1008 1009 if (v->ahash_tfm) { 1010 static_branch_dec(&ahash_enabled); 1011 crypto_free_ahash(v->ahash_tfm); 1012 } else { 1013 crypto_free_shash(v->shash_tfm); 1014 } 1015 1016 kfree(v->alg_name); 1017 1018 if (v->hash_dev) 1019 dm_put_device(ti, v->hash_dev); 1020 1021 if (v->data_dev) 1022 dm_put_device(ti, v->data_dev); 1023 1024 verity_fec_dtr(v); 1025 1026 kfree(v->signature_key_desc); 1027 1028 if (v->use_bh_wq) 1029 static_branch_dec(&use_bh_wq_enabled); 1030 1031 kfree(v); 1032 1033 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); 1034 } 1035 1036 static int verity_alloc_most_once(struct dm_verity *v) 1037 { 1038 struct dm_target *ti = v->ti; 1039 1040 /* the bitset can only handle INT_MAX blocks */ 1041 if (v->data_blocks > INT_MAX) { 1042 ti->error = "device too large to use check_at_most_once"; 1043 return -E2BIG; 1044 } 1045 1046 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), 1047 sizeof(unsigned long), 1048 GFP_KERNEL); 1049 if (!v->validated_blocks) { 1050 ti->error = "failed to allocate bitset for check_at_most_once"; 1051 return -ENOMEM; 1052 } 1053 1054 return 0; 1055 } 1056 1057 static int verity_alloc_zero_digest(struct dm_verity *v) 1058 { 1059 int r = -ENOMEM; 1060 struct dm_verity_io *io; 1061 u8 *zero_data; 1062 1063 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); 1064 1065 if (!v->zero_digest) 1066 return r; 1067 1068 io = kmalloc(sizeof(*io) + v->hash_reqsize, GFP_KERNEL); 1069 1070 if (!io) 1071 return r; /* verity_dtr will free zero_digest */ 1072 1073 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); 1074 1075 if (!zero_data) 1076 goto out; 1077 1078 r = verity_hash(v, io, zero_data, 1 << v->data_dev_block_bits, 1079 v->zero_digest, true); 1080 1081 out: 1082 kfree(io); 1083 kfree(zero_data); 1084 1085 return r; 1086 } 1087 1088 static inline bool verity_is_verity_mode(const char *arg_name) 1089 { 1090 return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) || 1091 !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) || 1092 !strcasecmp(arg_name, DM_VERITY_OPT_PANIC)); 1093 } 1094 1095 static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) 1096 { 1097 if (v->mode) 1098 return -EINVAL; 1099 1100 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) 1101 v->mode = DM_VERITY_MODE_LOGGING; 1102 else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) 1103 v->mode = DM_VERITY_MODE_RESTART; 1104 else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC)) 1105 v->mode = DM_VERITY_MODE_PANIC; 1106 1107 return 0; 1108 } 1109 1110 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, 1111 struct dm_verity_sig_opts *verify_args, 1112 bool only_modifier_opts) 1113 { 1114 int r = 0; 1115 unsigned int argc; 1116 struct dm_target *ti = v->ti; 1117 const char *arg_name; 1118 1119 static const struct dm_arg _args[] = { 1120 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, 1121 }; 1122 1123 r = dm_read_arg_group(_args, as, &argc, &ti->error); 1124 if (r) 1125 return -EINVAL; 1126 1127 if (!argc) 1128 return 0; 1129 1130 do { 1131 arg_name = dm_shift_arg(as); 1132 argc--; 1133 1134 if (verity_is_verity_mode(arg_name)) { 1135 if (only_modifier_opts) 1136 continue; 1137 r = verity_parse_verity_mode(v, arg_name); 1138 if (r) { 1139 ti->error = "Conflicting error handling parameters"; 1140 return r; 1141 } 1142 continue; 1143 1144 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { 1145 if (only_modifier_opts) 1146 continue; 1147 r = verity_alloc_zero_digest(v); 1148 if (r) { 1149 ti->error = "Cannot allocate zero digest"; 1150 return r; 1151 } 1152 continue; 1153 1154 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { 1155 if (only_modifier_opts) 1156 continue; 1157 r = verity_alloc_most_once(v); 1158 if (r) 1159 return r; 1160 continue; 1161 1162 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { 1163 v->use_bh_wq = true; 1164 static_branch_inc(&use_bh_wq_enabled); 1165 continue; 1166 1167 } else if (verity_is_fec_opt_arg(arg_name)) { 1168 if (only_modifier_opts) 1169 continue; 1170 r = verity_fec_parse_opt_args(as, v, &argc, arg_name); 1171 if (r) 1172 return r; 1173 continue; 1174 1175 } else if (verity_verify_is_sig_opt_arg(arg_name)) { 1176 if (only_modifier_opts) 1177 continue; 1178 r = verity_verify_sig_parse_opt_args(as, v, 1179 verify_args, 1180 &argc, arg_name); 1181 if (r) 1182 return r; 1183 continue; 1184 1185 } else if (only_modifier_opts) { 1186 /* 1187 * Ignore unrecognized opt, could easily be an extra 1188 * argument to an option whose parsing was skipped. 1189 * Normal parsing (@only_modifier_opts=false) will 1190 * properly parse all options (and their extra args). 1191 */ 1192 continue; 1193 } 1194 1195 DMERR("Unrecognized verity feature request: %s", arg_name); 1196 ti->error = "Unrecognized verity feature request"; 1197 return -EINVAL; 1198 } while (argc && !r); 1199 1200 return r; 1201 } 1202 1203 static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name) 1204 { 1205 struct dm_target *ti = v->ti; 1206 struct crypto_ahash *ahash; 1207 struct crypto_shash *shash = NULL; 1208 const char *driver_name; 1209 1210 v->alg_name = kstrdup(alg_name, GFP_KERNEL); 1211 if (!v->alg_name) { 1212 ti->error = "Cannot allocate algorithm name"; 1213 return -ENOMEM; 1214 } 1215 1216 /* 1217 * Allocate the hash transformation object that this dm-verity instance 1218 * will use. The vast majority of dm-verity users use CPU-based 1219 * hashing, so when possible use the shash API to minimize the crypto 1220 * API overhead. If the ahash API resolves to a different driver 1221 * (likely an off-CPU hardware offload), use ahash instead. Also use 1222 * ahash if the obsolete dm-verity format with the appended salt is 1223 * being used, so that quirk only needs to be handled in one place. 1224 */ 1225 ahash = crypto_alloc_ahash(alg_name, 0, 1226 v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0); 1227 if (IS_ERR(ahash)) { 1228 ti->error = "Cannot initialize hash function"; 1229 return PTR_ERR(ahash); 1230 } 1231 driver_name = crypto_ahash_driver_name(ahash); 1232 if (v->version >= 1 /* salt prepended, not appended? */) { 1233 shash = crypto_alloc_shash(alg_name, 0, 0); 1234 if (!IS_ERR(shash) && 1235 strcmp(crypto_shash_driver_name(shash), driver_name) != 0) { 1236 /* 1237 * ahash gave a different driver than shash, so probably 1238 * this is a case of real hardware offload. Use ahash. 1239 */ 1240 crypto_free_shash(shash); 1241 shash = NULL; 1242 } 1243 } 1244 if (!IS_ERR_OR_NULL(shash)) { 1245 crypto_free_ahash(ahash); 1246 ahash = NULL; 1247 v->shash_tfm = shash; 1248 v->digest_size = crypto_shash_digestsize(shash); 1249 v->hash_reqsize = sizeof(struct shash_desc) + 1250 crypto_shash_descsize(shash); 1251 DMINFO("%s using shash \"%s\"", alg_name, driver_name); 1252 } else { 1253 v->ahash_tfm = ahash; 1254 static_branch_inc(&ahash_enabled); 1255 v->digest_size = crypto_ahash_digestsize(ahash); 1256 v->hash_reqsize = sizeof(struct ahash_request) + 1257 crypto_ahash_reqsize(ahash); 1258 DMINFO("%s using ahash \"%s\"", alg_name, driver_name); 1259 } 1260 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { 1261 ti->error = "Digest size too big"; 1262 return -EINVAL; 1263 } 1264 return 0; 1265 } 1266 1267 static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg) 1268 { 1269 struct dm_target *ti = v->ti; 1270 1271 if (strcmp(arg, "-") != 0) { 1272 v->salt_size = strlen(arg) / 2; 1273 v->salt = kmalloc(v->salt_size, GFP_KERNEL); 1274 if (!v->salt) { 1275 ti->error = "Cannot allocate salt"; 1276 return -ENOMEM; 1277 } 1278 if (strlen(arg) != v->salt_size * 2 || 1279 hex2bin(v->salt, arg, v->salt_size)) { 1280 ti->error = "Invalid salt"; 1281 return -EINVAL; 1282 } 1283 } 1284 if (v->shash_tfm) { 1285 SHASH_DESC_ON_STACK(desc, v->shash_tfm); 1286 int r; 1287 1288 /* 1289 * Compute the pre-salted hash state that can be passed to 1290 * crypto_shash_import() for each block later. 1291 */ 1292 v->initial_hashstate = kmalloc( 1293 crypto_shash_statesize(v->shash_tfm), GFP_KERNEL); 1294 if (!v->initial_hashstate) { 1295 ti->error = "Cannot allocate initial hash state"; 1296 return -ENOMEM; 1297 } 1298 desc->tfm = v->shash_tfm; 1299 r = crypto_shash_init(desc) ?: 1300 crypto_shash_update(desc, v->salt, v->salt_size) ?: 1301 crypto_shash_export(desc, v->initial_hashstate); 1302 if (r) { 1303 ti->error = "Cannot set up initial hash state"; 1304 return r; 1305 } 1306 } 1307 return 0; 1308 } 1309 1310 /* 1311 * Target parameters: 1312 * <version> The current format is version 1. 1313 * Vsn 0 is compatible with original Chromium OS releases. 1314 * <data device> 1315 * <hash device> 1316 * <data block size> 1317 * <hash block size> 1318 * <the number of data blocks> 1319 * <hash start block> 1320 * <algorithm> 1321 * <digest> 1322 * <salt> Hex string or "-" if no salt. 1323 */ 1324 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1325 { 1326 struct dm_verity *v; 1327 struct dm_verity_sig_opts verify_args = {0}; 1328 struct dm_arg_set as; 1329 unsigned int num; 1330 unsigned long long num_ll; 1331 int r; 1332 int i; 1333 sector_t hash_position; 1334 char dummy; 1335 char *root_hash_digest_to_validate; 1336 1337 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); 1338 if (!v) { 1339 ti->error = "Cannot allocate verity structure"; 1340 return -ENOMEM; 1341 } 1342 ti->private = v; 1343 v->ti = ti; 1344 1345 r = verity_fec_ctr_alloc(v); 1346 if (r) 1347 goto bad; 1348 1349 if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) { 1350 ti->error = "Device must be readonly"; 1351 r = -EINVAL; 1352 goto bad; 1353 } 1354 1355 if (argc < 10) { 1356 ti->error = "Not enough arguments"; 1357 r = -EINVAL; 1358 goto bad; 1359 } 1360 1361 /* Parse optional parameters that modify primary args */ 1362 if (argc > 10) { 1363 as.argc = argc - 10; 1364 as.argv = argv + 10; 1365 r = verity_parse_opt_args(&as, v, &verify_args, true); 1366 if (r < 0) 1367 goto bad; 1368 } 1369 1370 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || 1371 num > 1) { 1372 ti->error = "Invalid version"; 1373 r = -EINVAL; 1374 goto bad; 1375 } 1376 v->version = num; 1377 1378 r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev); 1379 if (r) { 1380 ti->error = "Data device lookup failed"; 1381 goto bad; 1382 } 1383 1384 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev); 1385 if (r) { 1386 ti->error = "Hash device lookup failed"; 1387 goto bad; 1388 } 1389 1390 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || 1391 !num || (num & (num - 1)) || 1392 num < bdev_logical_block_size(v->data_dev->bdev) || 1393 num > PAGE_SIZE) { 1394 ti->error = "Invalid data device block size"; 1395 r = -EINVAL; 1396 goto bad; 1397 } 1398 v->data_dev_block_bits = __ffs(num); 1399 1400 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || 1401 !num || (num & (num - 1)) || 1402 num < bdev_logical_block_size(v->hash_dev->bdev) || 1403 num > INT_MAX) { 1404 ti->error = "Invalid hash device block size"; 1405 r = -EINVAL; 1406 goto bad; 1407 } 1408 v->hash_dev_block_bits = __ffs(num); 1409 1410 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || 1411 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) 1412 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1413 ti->error = "Invalid data blocks"; 1414 r = -EINVAL; 1415 goto bad; 1416 } 1417 v->data_blocks = num_ll; 1418 1419 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { 1420 ti->error = "Data device is too small"; 1421 r = -EINVAL; 1422 goto bad; 1423 } 1424 1425 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || 1426 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) 1427 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { 1428 ti->error = "Invalid hash start"; 1429 r = -EINVAL; 1430 goto bad; 1431 } 1432 v->hash_start = num_ll; 1433 1434 r = verity_setup_hash_alg(v, argv[7]); 1435 if (r) 1436 goto bad; 1437 1438 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); 1439 if (!v->root_digest) { 1440 ti->error = "Cannot allocate root digest"; 1441 r = -ENOMEM; 1442 goto bad; 1443 } 1444 if (strlen(argv[8]) != v->digest_size * 2 || 1445 hex2bin(v->root_digest, argv[8], v->digest_size)) { 1446 ti->error = "Invalid root digest"; 1447 r = -EINVAL; 1448 goto bad; 1449 } 1450 root_hash_digest_to_validate = argv[8]; 1451 1452 r = verity_setup_salt_and_hashstate(v, argv[9]); 1453 if (r) 1454 goto bad; 1455 1456 argv += 10; 1457 argc -= 10; 1458 1459 /* Optional parameters */ 1460 if (argc) { 1461 as.argc = argc; 1462 as.argv = argv; 1463 r = verity_parse_opt_args(&as, v, &verify_args, false); 1464 if (r < 0) 1465 goto bad; 1466 } 1467 1468 /* Root hash signature is a optional parameter*/ 1469 r = verity_verify_root_hash(root_hash_digest_to_validate, 1470 strlen(root_hash_digest_to_validate), 1471 verify_args.sig, 1472 verify_args.sig_size); 1473 if (r < 0) { 1474 ti->error = "Root hash verification failed"; 1475 goto bad; 1476 } 1477 1478 r = verity_init_sig(v, verify_args.sig, verify_args.sig_size); 1479 if (r < 0) { 1480 ti->error = "Cannot allocate root digest signature"; 1481 goto bad; 1482 } 1483 1484 v->hash_per_block_bits = 1485 __fls((1 << v->hash_dev_block_bits) / v->digest_size); 1486 1487 v->levels = 0; 1488 if (v->data_blocks) 1489 while (v->hash_per_block_bits * v->levels < 64 && 1490 (unsigned long long)(v->data_blocks - 1) >> 1491 (v->hash_per_block_bits * v->levels)) 1492 v->levels++; 1493 1494 if (v->levels > DM_VERITY_MAX_LEVELS) { 1495 ti->error = "Too many tree levels"; 1496 r = -E2BIG; 1497 goto bad; 1498 } 1499 1500 hash_position = v->hash_start; 1501 for (i = v->levels - 1; i >= 0; i--) { 1502 sector_t s; 1503 1504 v->hash_level_block[i] = hash_position; 1505 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) 1506 >> ((i + 1) * v->hash_per_block_bits); 1507 if (hash_position + s < hash_position) { 1508 ti->error = "Hash device offset overflow"; 1509 r = -E2BIG; 1510 goto bad; 1511 } 1512 hash_position += s; 1513 } 1514 v->hash_blocks = hash_position; 1515 1516 r = mempool_init_page_pool(&v->recheck_pool, 1, 0); 1517 if (unlikely(r)) { 1518 ti->error = "Cannot allocate mempool"; 1519 goto bad; 1520 } 1521 1522 v->io = dm_io_client_create(); 1523 if (IS_ERR(v->io)) { 1524 r = PTR_ERR(v->io); 1525 v->io = NULL; 1526 ti->error = "Cannot allocate dm io"; 1527 goto bad; 1528 } 1529 1530 v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1531 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), 1532 dm_bufio_alloc_callback, NULL, 1533 v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); 1534 if (IS_ERR(v->bufio)) { 1535 ti->error = "Cannot initialize dm-bufio"; 1536 r = PTR_ERR(v->bufio); 1537 v->bufio = NULL; 1538 goto bad; 1539 } 1540 1541 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { 1542 ti->error = "Hash device is too small"; 1543 r = -E2BIG; 1544 goto bad; 1545 } 1546 1547 /* 1548 * Using WQ_HIGHPRI improves throughput and completion latency by 1549 * reducing wait times when reading from a dm-verity device. 1550 * 1551 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI 1552 * allows verify_wq to preempt softirq since verification in BH workqueue 1553 * will fall-back to using it for error handling (or if the bufio cache 1554 * doesn't have required hashes). 1555 */ 1556 v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1557 if (!v->verify_wq) { 1558 ti->error = "Cannot allocate workqueue"; 1559 r = -ENOMEM; 1560 goto bad; 1561 } 1562 1563 ti->per_io_data_size = sizeof(struct dm_verity_io) + v->hash_reqsize; 1564 1565 r = verity_fec_ctr(v); 1566 if (r) 1567 goto bad; 1568 1569 ti->per_io_data_size = roundup(ti->per_io_data_size, 1570 __alignof__(struct dm_verity_io)); 1571 1572 verity_verify_sig_opts_cleanup(&verify_args); 1573 1574 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); 1575 1576 return 0; 1577 1578 bad: 1579 1580 verity_verify_sig_opts_cleanup(&verify_args); 1581 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); 1582 verity_dtr(ti); 1583 1584 return r; 1585 } 1586 1587 /* 1588 * Get the verity mode (error behavior) of a verity target. 1589 * 1590 * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity 1591 * target. 1592 */ 1593 int dm_verity_get_mode(struct dm_target *ti) 1594 { 1595 struct dm_verity *v = ti->private; 1596 1597 if (!dm_is_verity_target(ti)) 1598 return -EINVAL; 1599 1600 return v->mode; 1601 } 1602 1603 /* 1604 * Get the root digest of a verity target. 1605 * 1606 * Returns a copy of the root digest, the caller is responsible for 1607 * freeing the memory of the digest. 1608 */ 1609 int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) 1610 { 1611 struct dm_verity *v = ti->private; 1612 1613 if (!dm_is_verity_target(ti)) 1614 return -EINVAL; 1615 1616 *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); 1617 if (*root_digest == NULL) 1618 return -ENOMEM; 1619 1620 *digest_size = v->digest_size; 1621 1622 return 0; 1623 } 1624 1625 #ifdef CONFIG_SECURITY 1626 1627 #ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG 1628 1629 static int verity_security_set_signature(struct block_device *bdev, 1630 struct dm_verity *v) 1631 { 1632 /* 1633 * if the dm-verity target is unsigned, v->root_digest_sig will 1634 * be NULL, and the hook call is still required to let LSMs mark 1635 * the device as unsigned. This information is crucial for LSMs to 1636 * block operations such as execution on unsigned files 1637 */ 1638 return security_bdev_setintegrity(bdev, 1639 LSM_INT_DMVERITY_SIG_VALID, 1640 v->root_digest_sig, 1641 v->sig_size); 1642 } 1643 1644 #else 1645 1646 static inline int verity_security_set_signature(struct block_device *bdev, 1647 struct dm_verity *v) 1648 { 1649 return 0; 1650 } 1651 1652 #endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */ 1653 1654 /* 1655 * Expose verity target's root hash and signature data to LSMs before resume. 1656 * 1657 * Returns 0 on success, or -ENOMEM if the system is out of memory. 1658 */ 1659 static int verity_preresume(struct dm_target *ti) 1660 { 1661 struct block_device *bdev; 1662 struct dm_verity_digest root_digest; 1663 struct dm_verity *v; 1664 int r; 1665 1666 v = ti->private; 1667 bdev = dm_disk(dm_table_get_md(ti->table))->part0; 1668 root_digest.digest = v->root_digest; 1669 root_digest.digest_len = v->digest_size; 1670 if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) 1671 root_digest.alg = crypto_ahash_alg_name(v->ahash_tfm); 1672 else 1673 root_digest.alg = crypto_shash_alg_name(v->shash_tfm); 1674 1675 r = security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, &root_digest, 1676 sizeof(root_digest)); 1677 if (r) 1678 return r; 1679 1680 r = verity_security_set_signature(bdev, v); 1681 if (r) 1682 goto bad; 1683 1684 return 0; 1685 1686 bad: 1687 1688 security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, NULL, 0); 1689 1690 return r; 1691 } 1692 1693 #endif /* CONFIG_SECURITY */ 1694 1695 static struct target_type verity_target = { 1696 .name = "verity", 1697 /* Note: the LSMs depend on the singleton and immutable features */ 1698 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, 1699 .version = {1, 10, 0}, 1700 .module = THIS_MODULE, 1701 .ctr = verity_ctr, 1702 .dtr = verity_dtr, 1703 .map = verity_map, 1704 .status = verity_status, 1705 .prepare_ioctl = verity_prepare_ioctl, 1706 .iterate_devices = verity_iterate_devices, 1707 .io_hints = verity_io_hints, 1708 #ifdef CONFIG_SECURITY 1709 .preresume = verity_preresume, 1710 #endif /* CONFIG_SECURITY */ 1711 }; 1712 module_dm(verity); 1713 1714 /* 1715 * Check whether a DM target is a verity target. 1716 */ 1717 bool dm_is_verity_target(struct dm_target *ti) 1718 { 1719 return ti->type == &verity_target; 1720 } 1721 1722 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); 1723 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); 1724 MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); 1725 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); 1726 MODULE_LICENSE("GPL"); 1727