1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 Red Hat, Inc.
4 *
5 * Author: Mikulas Patocka <mpatocka@redhat.com>
6 *
7 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
8 *
9 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
10 * default prefetch value. Data are read in "prefetch_cluster" chunks from the
11 * hash device. Setting this greatly improves performance when data and hash
12 * are on the same disk on different partitions on devices with poor random
13 * access behavior.
14 */
15
16 #include "dm-verity.h"
17 #include "dm-verity-fec.h"
18 #include "dm-verity-verify-sig.h"
19 #include "dm-audit.h"
20 #include <linux/module.h>
21 #include <linux/reboot.h>
22 #include <linux/string.h>
23 #include <linux/jump_label.h>
24 #include <linux/security.h>
25
26 #define DM_MSG_PREFIX "verity"
27
28 #define DM_VERITY_ENV_LENGTH 42
29 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
30
31 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
32 #define DM_VERITY_USE_BH_DEFAULT_BYTES 8192
33
34 #define DM_VERITY_MAX_CORRUPTED_ERRS 100
35
36 #define DM_VERITY_OPT_LOGGING "ignore_corruption"
37 #define DM_VERITY_OPT_RESTART "restart_on_corruption"
38 #define DM_VERITY_OPT_PANIC "panic_on_corruption"
39 #define DM_VERITY_OPT_ERROR_RESTART "restart_on_error"
40 #define DM_VERITY_OPT_ERROR_PANIC "panic_on_error"
41 #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
42 #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
43 #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet"
44
45 #define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \
46 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)
47
48 static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
49
50 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
51
52 static unsigned int dm_verity_use_bh_bytes[4] = {
53 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_NONE
54 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_RT
55 DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_BE
56 0 // IOPRIO_CLASS_IDLE
57 };
58
59 module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644);
60
61 static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
62
63 struct dm_verity_prefetch_work {
64 struct work_struct work;
65 struct dm_verity *v;
66 unsigned short ioprio;
67 sector_t block;
68 unsigned int n_blocks;
69 };
70
71 /*
72 * Auxiliary structure appended to each dm-bufio buffer. If the value
73 * hash_verified is nonzero, hash of the block has been verified.
74 *
75 * The variable hash_verified is set to 0 when allocating the buffer, then
76 * it can be changed to 1 and it is never reset to 0 again.
77 *
78 * There is no lock around this value, a race condition can at worst cause
79 * that multiple processes verify the hash of the same buffer simultaneously
80 * and write 1 to hash_verified simultaneously.
81 * This condition is harmless, so we don't need locking.
82 */
83 struct buffer_aux {
84 int hash_verified;
85 };
86
87 /*
88 * Initialize struct buffer_aux for a freshly created buffer.
89 */
dm_bufio_alloc_callback(struct dm_buffer * buf)90 static void dm_bufio_alloc_callback(struct dm_buffer *buf)
91 {
92 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
93
94 aux->hash_verified = 0;
95 }
96
97 /*
98 * Translate input sector number to the sector number on the target device.
99 */
verity_map_sector(struct dm_verity * v,sector_t bi_sector)100 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
101 {
102 return dm_target_offset(v->ti, bi_sector);
103 }
104
105 /*
106 * Return hash position of a specified block at a specified tree level
107 * (0 is the lowest level).
108 * The lowest "hash_per_block_bits"-bits of the result denote hash position
109 * inside a hash block. The remaining bits denote location of the hash block.
110 */
verity_position_at_level(struct dm_verity * v,sector_t block,int level)111 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
112 int level)
113 {
114 return block >> (level * v->hash_per_block_bits);
115 }
116
verity_hash(struct dm_verity * v,struct dm_verity_io * io,const u8 * data,size_t len,u8 * digest)117 int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
118 const u8 *data, size_t len, u8 *digest)
119 {
120 struct shash_desc *desc;
121 int r;
122
123 if (likely(v->use_sha256_lib)) {
124 struct sha256_ctx *ctx = &io->hash_ctx.sha256;
125
126 /*
127 * Fast path using SHA-256 library. This is enabled only for
128 * verity version 1, where the salt is at the beginning.
129 */
130 *ctx = *v->initial_hashstate.sha256;
131 sha256_update(ctx, data, len);
132 sha256_final(ctx, digest);
133 return 0;
134 }
135
136 desc = &io->hash_ctx.shash;
137 desc->tfm = v->shash_tfm;
138 if (unlikely(v->initial_hashstate.shash == NULL)) {
139 /* Version 0: salt at end */
140 r = crypto_shash_init(desc) ?:
141 crypto_shash_update(desc, data, len) ?:
142 crypto_shash_update(desc, v->salt, v->salt_size) ?:
143 crypto_shash_final(desc, digest);
144 } else {
145 /* Version 1: salt at beginning */
146 r = crypto_shash_import(desc, v->initial_hashstate.shash) ?:
147 crypto_shash_finup(desc, data, len, digest);
148 }
149 if (unlikely(r))
150 DMERR("Error hashing block: %d", r);
151 return r;
152 }
153
verity_hash_at_level(struct dm_verity * v,sector_t block,int level,sector_t * hash_block,unsigned int * offset)154 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
155 sector_t *hash_block, unsigned int *offset)
156 {
157 sector_t position = verity_position_at_level(v, block, level);
158 unsigned int idx;
159
160 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
161
162 if (!offset)
163 return;
164
165 idx = position & ((1 << v->hash_per_block_bits) - 1);
166 if (!v->version)
167 *offset = idx * v->digest_size;
168 else
169 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
170 }
171
172 /*
173 * Handle verification errors.
174 */
verity_handle_err(struct dm_verity * v,enum verity_block_type type,unsigned long long block)175 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
176 unsigned long long block)
177 {
178 char verity_env[DM_VERITY_ENV_LENGTH];
179 char *envp[] = { verity_env, NULL };
180 const char *type_str = "";
181 struct mapped_device *md = dm_table_get_md(v->ti->table);
182
183 /* Corruption should be visible in device status in all modes */
184 v->hash_failed = true;
185
186 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
187 goto out;
188
189 v->corrupted_errs++;
190
191 switch (type) {
192 case DM_VERITY_BLOCK_TYPE_DATA:
193 type_str = "data";
194 break;
195 case DM_VERITY_BLOCK_TYPE_METADATA:
196 type_str = "metadata";
197 break;
198 default:
199 BUG();
200 }
201
202 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,
203 type_str, block);
204
205 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) {
206 DMERR("%s: reached maximum errors", v->data_dev->name);
207 dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0);
208 }
209
210 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",
211 DM_VERITY_ENV_VAR_NAME, type, block);
212
213 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp);
214
215 out:
216 if (v->mode == DM_VERITY_MODE_LOGGING)
217 return 0;
218
219 if (v->mode == DM_VERITY_MODE_RESTART)
220 kernel_restart("dm-verity device corrupted");
221
222 if (v->mode == DM_VERITY_MODE_PANIC)
223 panic("dm-verity device corrupted");
224
225 return 1;
226 }
227
228 /*
229 * Verify hash of a metadata block pertaining to the specified data block
230 * ("block" argument) at a specified level ("level" argument).
231 *
232 * On successful return, want_digest contains the hash value for a lower tree
233 * level or for the data block (if we're at the lowest level).
234 *
235 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
236 * If "skip_unverified" is false, unverified buffer is hashed and verified
237 * against current value of want_digest.
238 */
verity_verify_level(struct dm_verity * v,struct dm_verity_io * io,sector_t block,int level,bool skip_unverified,u8 * want_digest)239 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
240 sector_t block, int level, bool skip_unverified,
241 u8 *want_digest)
242 {
243 struct dm_buffer *buf;
244 struct buffer_aux *aux;
245 u8 *data;
246 int r;
247 sector_t hash_block;
248 unsigned int offset;
249 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
250
251 verity_hash_at_level(v, block, level, &hash_block, &offset);
252
253 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
254 data = dm_bufio_get(v->bufio, hash_block, &buf);
255 if (IS_ERR_OR_NULL(data)) {
256 /*
257 * In tasklet and the hash was not in the bufio cache.
258 * Return early and resume execution from a work-queue
259 * to read the hash from disk.
260 */
261 return -EAGAIN;
262 }
263 } else {
264 data = dm_bufio_read_with_ioprio(v->bufio, hash_block,
265 &buf, bio->bi_ioprio);
266 }
267
268 if (IS_ERR(data)) {
269 if (skip_unverified)
270 return 1;
271 r = PTR_ERR(data);
272 data = dm_bufio_new(v->bufio, hash_block, &buf);
273 if (IS_ERR(data))
274 return r;
275 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA,
276 want_digest, hash_block, data) == 0) {
277 aux = dm_bufio_get_aux_data(buf);
278 aux->hash_verified = 1;
279 goto release_ok;
280 } else {
281 dm_bufio_release(buf);
282 dm_bufio_forget(v->bufio, hash_block);
283 return r;
284 }
285 }
286
287 aux = dm_bufio_get_aux_data(buf);
288
289 if (!aux->hash_verified) {
290 if (skip_unverified) {
291 r = 1;
292 goto release_ret_r;
293 }
294
295 r = verity_hash(v, io, data, 1 << v->hash_dev_block_bits,
296 io->tmp_digest);
297 if (unlikely(r < 0))
298 goto release_ret_r;
299
300 if (likely(memcmp(io->tmp_digest, want_digest,
301 v->digest_size) == 0))
302 aux->hash_verified = 1;
303 else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
304 /*
305 * Error handling code (FEC included) cannot be run in a
306 * tasklet since it may sleep, so fallback to work-queue.
307 */
308 r = -EAGAIN;
309 goto release_ret_r;
310 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA,
311 want_digest, hash_block, data) == 0)
312 aux->hash_verified = 1;
313 else if (verity_handle_err(v,
314 DM_VERITY_BLOCK_TYPE_METADATA,
315 hash_block)) {
316 struct bio *bio;
317 io->had_mismatch = true;
318 bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
319 dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio,
320 block, 0);
321 r = -EIO;
322 goto release_ret_r;
323 }
324 }
325
326 release_ok:
327 data += offset;
328 memcpy(want_digest, data, v->digest_size);
329 r = 0;
330
331 release_ret_r:
332 dm_bufio_release(buf);
333 return r;
334 }
335
336 /*
337 * Find a hash for a given block, write it to digest and verify the integrity
338 * of the hash tree if necessary.
339 */
verity_hash_for_block(struct dm_verity * v,struct dm_verity_io * io,sector_t block,u8 * digest,bool * is_zero)340 int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
341 sector_t block, u8 *digest, bool *is_zero)
342 {
343 int r = 0, i;
344
345 if (likely(v->levels)) {
346 /*
347 * First, we try to get the requested hash for
348 * the current block. If the hash block itself is
349 * verified, zero is returned. If it isn't, this
350 * function returns 1 and we fall back to whole
351 * chain verification.
352 */
353 r = verity_verify_level(v, io, block, 0, true, digest);
354 if (likely(r <= 0))
355 goto out;
356 }
357
358 memcpy(digest, v->root_digest, v->digest_size);
359
360 for (i = v->levels - 1; i >= 0; i--) {
361 r = verity_verify_level(v, io, block, i, false, digest);
362 if (unlikely(r))
363 goto out;
364 }
365 out:
366 if (!r && v->zero_digest)
367 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size);
368 else
369 *is_zero = false;
370
371 return r;
372 }
373
verity_recheck(struct dm_verity * v,struct dm_verity_io * io,const u8 * want_digest,sector_t cur_block,u8 * dest)374 static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
375 const u8 *want_digest, sector_t cur_block,
376 u8 *dest)
377 {
378 struct page *page;
379 void *buffer;
380 int r;
381 struct dm_io_request io_req;
382 struct dm_io_region io_loc;
383
384 page = mempool_alloc(&v->recheck_pool, GFP_NOIO);
385 buffer = page_to_virt(page);
386
387 io_req.bi_opf = REQ_OP_READ;
388 io_req.mem.type = DM_IO_KMEM;
389 io_req.mem.ptr.addr = buffer;
390 io_req.notify.fn = NULL;
391 io_req.client = v->io;
392 io_loc.bdev = v->data_dev->bdev;
393 io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
394 io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
395 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
396 if (unlikely(r))
397 goto free_ret;
398
399 r = verity_hash(v, io, buffer, 1 << v->data_dev_block_bits,
400 io->tmp_digest);
401 if (unlikely(r))
402 goto free_ret;
403
404 if (memcmp(io->tmp_digest, want_digest, v->digest_size)) {
405 r = -EIO;
406 goto free_ret;
407 }
408
409 memcpy(dest, buffer, 1 << v->data_dev_block_bits);
410 r = 0;
411 free_ret:
412 mempool_free(page, &v->recheck_pool);
413
414 return r;
415 }
416
verity_handle_data_hash_mismatch(struct dm_verity * v,struct dm_verity_io * io,struct bio * bio,struct pending_block * block)417 static int verity_handle_data_hash_mismatch(struct dm_verity *v,
418 struct dm_verity_io *io,
419 struct bio *bio,
420 struct pending_block *block)
421 {
422 const u8 *want_digest = block->want_digest;
423 sector_t blkno = block->blkno;
424 u8 *data = block->data;
425
426 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
427 /*
428 * Error handling code (FEC included) cannot be run in the
429 * BH workqueue, so fallback to a standard workqueue.
430 */
431 return -EAGAIN;
432 }
433 if (verity_recheck(v, io, want_digest, blkno, data) == 0) {
434 if (v->validated_blocks)
435 set_bit(blkno, v->validated_blocks);
436 return 0;
437 }
438 #if defined(CONFIG_DM_VERITY_FEC)
439 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, want_digest,
440 blkno, data) == 0)
441 return 0;
442 #endif
443 if (bio->bi_status)
444 return -EIO; /* Error correction failed; Just return error */
445
446 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, blkno)) {
447 io->had_mismatch = true;
448 dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", bio, blkno, 0);
449 return -EIO;
450 }
451 return 0;
452 }
453
verity_clear_pending_blocks(struct dm_verity_io * io)454 static void verity_clear_pending_blocks(struct dm_verity_io *io)
455 {
456 int i;
457
458 for (i = io->num_pending - 1; i >= 0; i--) {
459 kunmap_local(io->pending_blocks[i].data);
460 io->pending_blocks[i].data = NULL;
461 }
462 io->num_pending = 0;
463 }
464
verity_verify_pending_blocks(struct dm_verity * v,struct dm_verity_io * io,struct bio * bio)465 static int verity_verify_pending_blocks(struct dm_verity *v,
466 struct dm_verity_io *io,
467 struct bio *bio)
468 {
469 const unsigned int block_size = 1 << v->data_dev_block_bits;
470 int i, r;
471
472 if (io->num_pending == 2) {
473 /* num_pending == 2 implies that the algorithm is SHA-256 */
474 sha256_finup_2x(v->initial_hashstate.sha256,
475 io->pending_blocks[0].data,
476 io->pending_blocks[1].data, block_size,
477 io->pending_blocks[0].real_digest,
478 io->pending_blocks[1].real_digest);
479 } else {
480 for (i = 0; i < io->num_pending; i++) {
481 r = verity_hash(v, io, io->pending_blocks[i].data,
482 block_size,
483 io->pending_blocks[i].real_digest);
484 if (unlikely(r))
485 return r;
486 }
487 }
488
489 for (i = 0; i < io->num_pending; i++) {
490 struct pending_block *block = &io->pending_blocks[i];
491
492 if (likely(memcmp(block->real_digest, block->want_digest,
493 v->digest_size) == 0)) {
494 if (v->validated_blocks)
495 set_bit(block->blkno, v->validated_blocks);
496 } else {
497 r = verity_handle_data_hash_mismatch(v, io, bio, block);
498 if (unlikely(r))
499 return r;
500 }
501 }
502 verity_clear_pending_blocks(io);
503 return 0;
504 }
505
506 /*
507 * Verify one "dm_verity_io" structure.
508 */
verity_verify_io(struct dm_verity_io * io)509 static int verity_verify_io(struct dm_verity_io *io)
510 {
511 struct dm_verity *v = io->v;
512 const unsigned int block_size = 1 << v->data_dev_block_bits;
513 const int max_pending = v->use_sha256_finup_2x ? 2 : 1;
514 struct bvec_iter iter_copy;
515 struct bvec_iter *iter;
516 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
517 unsigned int b;
518 int r;
519
520 io->num_pending = 0;
521
522 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
523 /*
524 * Copy the iterator in case we need to restart
525 * verification in a work-queue.
526 */
527 iter_copy = io->iter;
528 iter = &iter_copy;
529 } else
530 iter = &io->iter;
531
532 for (b = 0; b < io->n_blocks;
533 b++, bio_advance_iter(bio, iter, block_size)) {
534 sector_t blkno = io->block + b;
535 struct pending_block *block;
536 bool is_zero;
537 struct bio_vec bv;
538 void *data;
539
540 if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&
541 likely(test_bit(blkno, v->validated_blocks)))
542 continue;
543
544 block = &io->pending_blocks[io->num_pending];
545
546 r = verity_hash_for_block(v, io, blkno, block->want_digest,
547 &is_zero);
548 if (unlikely(r < 0))
549 goto error;
550
551 bv = bio_iter_iovec(bio, *iter);
552 if (unlikely(bv.bv_len < block_size)) {
553 /*
554 * Data block spans pages. This should not happen,
555 * since dm-verity sets dma_alignment to the data block
556 * size minus 1, and dm-verity also doesn't allow the
557 * data block size to be greater than PAGE_SIZE.
558 */
559 DMERR_LIMIT("unaligned io (data block spans pages)");
560 r = -EIO;
561 goto error;
562 }
563
564 data = bvec_kmap_local(&bv);
565
566 if (is_zero) {
567 /*
568 * If we expect a zero block, don't validate, just
569 * return zeros.
570 */
571 memset(data, 0, block_size);
572 kunmap_local(data);
573 continue;
574 }
575 block->data = data;
576 block->blkno = blkno;
577 if (++io->num_pending == max_pending) {
578 r = verity_verify_pending_blocks(v, io, bio);
579 if (unlikely(r))
580 goto error;
581 }
582 }
583
584 if (io->num_pending) {
585 r = verity_verify_pending_blocks(v, io, bio);
586 if (unlikely(r))
587 goto error;
588 }
589
590 return 0;
591
592 error:
593 verity_clear_pending_blocks(io);
594 return r;
595 }
596
597 /*
598 * Skip verity work in response to I/O error when system is shutting down.
599 */
verity_is_system_shutting_down(void)600 static inline bool verity_is_system_shutting_down(void)
601 {
602 return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF
603 || system_state == SYSTEM_RESTART;
604 }
605
restart_io_error(struct work_struct * w)606 static void restart_io_error(struct work_struct *w)
607 {
608 kernel_restart("dm-verity device has I/O error");
609 }
610
611 /*
612 * End one "io" structure with a given error.
613 */
verity_finish_io(struct dm_verity_io * io,blk_status_t status)614 static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
615 {
616 struct dm_verity *v = io->v;
617 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
618
619 bio->bi_end_io = io->orig_bi_end_io;
620 bio->bi_status = status;
621
622 if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
623 verity_fec_finish_io(io);
624
625 if (unlikely(status != BLK_STS_OK) &&
626 unlikely(!(bio->bi_opf & REQ_RAHEAD)) &&
627 !io->had_mismatch &&
628 !verity_is_system_shutting_down()) {
629 if (v->error_mode == DM_VERITY_MODE_PANIC) {
630 panic("dm-verity device has I/O error");
631 }
632 if (v->error_mode == DM_VERITY_MODE_RESTART) {
633 static DECLARE_WORK(restart_work, restart_io_error);
634 queue_work(v->verify_wq, &restart_work);
635 /*
636 * We deliberately don't call bio_endio here, because
637 * the machine will be restarted anyway.
638 */
639 return;
640 }
641 }
642
643 bio_endio(bio);
644 }
645
verity_work(struct work_struct * w)646 static void verity_work(struct work_struct *w)
647 {
648 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
649
650 io->in_bh = false;
651
652 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
653 }
654
verity_bh_work(struct work_struct * w)655 static void verity_bh_work(struct work_struct *w)
656 {
657 struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
658 int err;
659
660 io->in_bh = true;
661 err = verity_verify_io(io);
662 if (err == -EAGAIN || err == -ENOMEM) {
663 /* fallback to retrying with work-queue */
664 INIT_WORK(&io->work, verity_work);
665 queue_work(io->v->verify_wq, &io->work);
666 return;
667 }
668
669 verity_finish_io(io, errno_to_blk_status(err));
670 }
671
verity_use_bh(unsigned int bytes,unsigned short ioprio)672 static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio)
673 {
674 return ioprio <= IOPRIO_CLASS_IDLE &&
675 bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]) &&
676 !need_resched();
677 }
678
verity_end_io(struct bio * bio)679 static void verity_end_io(struct bio *bio)
680 {
681 struct dm_verity_io *io = bio->bi_private;
682 unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio);
683 unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits;
684
685 if (bio->bi_status &&
686 (!verity_fec_is_enabled(io->v) ||
687 verity_is_system_shutting_down() ||
688 (bio->bi_opf & REQ_RAHEAD))) {
689 verity_finish_io(io, bio->bi_status);
690 return;
691 }
692
693 if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq &&
694 verity_use_bh(bytes, ioprio)) {
695 if (in_hardirq() || irqs_disabled()) {
696 INIT_WORK(&io->bh_work, verity_bh_work);
697 queue_work(system_bh_wq, &io->bh_work);
698 } else {
699 verity_bh_work(&io->bh_work);
700 }
701 } else {
702 INIT_WORK(&io->work, verity_work);
703 queue_work(io->v->verify_wq, &io->work);
704 }
705 }
706
707 /*
708 * Prefetch buffers for the specified io.
709 * The root buffer is not prefetched, it is assumed that it will be cached
710 * all the time.
711 */
verity_prefetch_io(struct work_struct * work)712 static void verity_prefetch_io(struct work_struct *work)
713 {
714 struct dm_verity_prefetch_work *pw =
715 container_of(work, struct dm_verity_prefetch_work, work);
716 struct dm_verity *v = pw->v;
717 int i;
718
719 for (i = v->levels - 2; i >= 0; i--) {
720 sector_t hash_block_start;
721 sector_t hash_block_end;
722
723 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);
724 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);
725
726 if (!i) {
727 unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster);
728
729 cluster >>= v->data_dev_block_bits;
730 if (unlikely(!cluster))
731 goto no_prefetch_cluster;
732
733 if (unlikely(cluster & (cluster - 1)))
734 cluster = 1 << __fls(cluster);
735
736 hash_block_start &= ~(sector_t)(cluster - 1);
737 hash_block_end |= cluster - 1;
738 if (unlikely(hash_block_end >= v->hash_blocks))
739 hash_block_end = v->hash_blocks - 1;
740 }
741 no_prefetch_cluster:
742 dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start,
743 hash_block_end - hash_block_start + 1,
744 pw->ioprio);
745 }
746
747 kfree(pw);
748 }
749
verity_submit_prefetch(struct dm_verity * v,struct dm_verity_io * io,unsigned short ioprio)750 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io,
751 unsigned short ioprio)
752 {
753 sector_t block = io->block;
754 unsigned int n_blocks = io->n_blocks;
755 struct dm_verity_prefetch_work *pw;
756
757 if (v->validated_blocks) {
758 while (n_blocks && test_bit(block, v->validated_blocks)) {
759 block++;
760 n_blocks--;
761 }
762 while (n_blocks && test_bit(block + n_blocks - 1,
763 v->validated_blocks))
764 n_blocks--;
765 if (!n_blocks)
766 return;
767 }
768
769 pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
770 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
771
772 if (!pw)
773 return;
774
775 INIT_WORK(&pw->work, verity_prefetch_io);
776 pw->v = v;
777 pw->block = block;
778 pw->n_blocks = n_blocks;
779 pw->ioprio = ioprio;
780 queue_work(v->verify_wq, &pw->work);
781 }
782
783 /*
784 * Bio map function. It allocates dm_verity_io structure and bio vector and
785 * fills them. Then it issues prefetches and the I/O.
786 */
verity_map(struct dm_target * ti,struct bio * bio)787 static int verity_map(struct dm_target *ti, struct bio *bio)
788 {
789 struct dm_verity *v = ti->private;
790 struct dm_verity_io *io;
791
792 bio_set_dev(bio, v->data_dev->bdev);
793 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);
794
795 if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &
796 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
797 DMERR_LIMIT("unaligned io");
798 return DM_MAPIO_KILL;
799 }
800
801 if (bio_end_sector(bio) >>
802 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
803 DMERR_LIMIT("io out of range");
804 return DM_MAPIO_KILL;
805 }
806
807 if (bio_data_dir(bio) == WRITE)
808 return DM_MAPIO_KILL;
809
810 io = dm_per_bio_data(bio, ti->per_io_data_size);
811 io->v = v;
812 io->orig_bi_end_io = bio->bi_end_io;
813 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
814 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
815 io->had_mismatch = false;
816
817 bio->bi_end_io = verity_end_io;
818 bio->bi_private = io;
819 io->iter = bio->bi_iter;
820
821 verity_fec_init_io(io);
822
823 verity_submit_prefetch(v, io, bio->bi_ioprio);
824
825 submit_bio_noacct(bio);
826
827 return DM_MAPIO_SUBMITTED;
828 }
829
verity_postsuspend(struct dm_target * ti)830 static void verity_postsuspend(struct dm_target *ti)
831 {
832 struct dm_verity *v = ti->private;
833 flush_workqueue(v->verify_wq);
834 dm_bufio_client_reset(v->bufio);
835 }
836
837 /*
838 * Status: V (valid) or C (corruption found)
839 */
verity_status(struct dm_target * ti,status_type_t type,unsigned int status_flags,char * result,unsigned int maxlen)840 static void verity_status(struct dm_target *ti, status_type_t type,
841 unsigned int status_flags, char *result, unsigned int maxlen)
842 {
843 struct dm_verity *v = ti->private;
844 unsigned int args = 0;
845 unsigned int sz = 0;
846 unsigned int x;
847
848 switch (type) {
849 case STATUSTYPE_INFO:
850 DMEMIT("%c", v->hash_failed ? 'C' : 'V');
851 if (verity_fec_is_enabled(v))
852 DMEMIT(" %lld", atomic64_read(&v->fec->corrected));
853 else
854 DMEMIT(" -");
855 break;
856 case STATUSTYPE_TABLE:
857 DMEMIT("%u %s %s %u %u %llu %llu %s ",
858 v->version,
859 v->data_dev->name,
860 v->hash_dev->name,
861 1 << v->data_dev_block_bits,
862 1 << v->hash_dev_block_bits,
863 (unsigned long long)v->data_blocks,
864 (unsigned long long)v->hash_start,
865 v->alg_name
866 );
867 for (x = 0; x < v->digest_size; x++)
868 DMEMIT("%02x", v->root_digest[x]);
869 DMEMIT(" ");
870 if (!v->salt_size)
871 DMEMIT("-");
872 else
873 for (x = 0; x < v->salt_size; x++)
874 DMEMIT("%02x", v->salt[x]);
875 if (v->mode != DM_VERITY_MODE_EIO)
876 args++;
877 if (v->error_mode != DM_VERITY_MODE_EIO)
878 args++;
879 if (verity_fec_is_enabled(v))
880 args += DM_VERITY_OPTS_FEC;
881 if (v->zero_digest)
882 args++;
883 if (v->validated_blocks)
884 args++;
885 if (v->use_bh_wq)
886 args++;
887 if (v->signature_key_desc)
888 args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
889 if (!args)
890 return;
891 DMEMIT(" %u", args);
892 if (v->mode != DM_VERITY_MODE_EIO) {
893 DMEMIT(" ");
894 switch (v->mode) {
895 case DM_VERITY_MODE_LOGGING:
896 DMEMIT(DM_VERITY_OPT_LOGGING);
897 break;
898 case DM_VERITY_MODE_RESTART:
899 DMEMIT(DM_VERITY_OPT_RESTART);
900 break;
901 case DM_VERITY_MODE_PANIC:
902 DMEMIT(DM_VERITY_OPT_PANIC);
903 break;
904 default:
905 BUG();
906 }
907 }
908 if (v->error_mode != DM_VERITY_MODE_EIO) {
909 DMEMIT(" ");
910 switch (v->error_mode) {
911 case DM_VERITY_MODE_RESTART:
912 DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
913 break;
914 case DM_VERITY_MODE_PANIC:
915 DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
916 break;
917 default:
918 BUG();
919 }
920 }
921 if (v->zero_digest)
922 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
923 if (v->validated_blocks)
924 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
925 if (v->use_bh_wq)
926 DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
927 sz = verity_fec_status_table(v, sz, result, maxlen);
928 if (v->signature_key_desc)
929 DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
930 " %s", v->signature_key_desc);
931 break;
932
933 case STATUSTYPE_IMA:
934 DMEMIT_TARGET_NAME_VERSION(ti->type);
935 DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V');
936 DMEMIT(",verity_version=%u", v->version);
937 DMEMIT(",data_device_name=%s", v->data_dev->name);
938 DMEMIT(",hash_device_name=%s", v->hash_dev->name);
939 DMEMIT(",verity_algorithm=%s", v->alg_name);
940
941 DMEMIT(",root_digest=");
942 for (x = 0; x < v->digest_size; x++)
943 DMEMIT("%02x", v->root_digest[x]);
944
945 DMEMIT(",salt=");
946 if (!v->salt_size)
947 DMEMIT("-");
948 else
949 for (x = 0; x < v->salt_size; x++)
950 DMEMIT("%02x", v->salt[x]);
951
952 DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n');
953 DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n');
954 if (v->signature_key_desc)
955 DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc);
956
957 if (v->mode != DM_VERITY_MODE_EIO) {
958 DMEMIT(",verity_mode=");
959 switch (v->mode) {
960 case DM_VERITY_MODE_LOGGING:
961 DMEMIT(DM_VERITY_OPT_LOGGING);
962 break;
963 case DM_VERITY_MODE_RESTART:
964 DMEMIT(DM_VERITY_OPT_RESTART);
965 break;
966 case DM_VERITY_MODE_PANIC:
967 DMEMIT(DM_VERITY_OPT_PANIC);
968 break;
969 default:
970 DMEMIT("invalid");
971 }
972 }
973 if (v->error_mode != DM_VERITY_MODE_EIO) {
974 DMEMIT(",verity_error_mode=");
975 switch (v->error_mode) {
976 case DM_VERITY_MODE_RESTART:
977 DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
978 break;
979 case DM_VERITY_MODE_PANIC:
980 DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
981 break;
982 default:
983 DMEMIT("invalid");
984 }
985 }
986 DMEMIT(";");
987 break;
988 }
989 }
990
verity_prepare_ioctl(struct dm_target * ti,struct block_device ** bdev,unsigned int cmd,unsigned long arg,bool * forward)991 static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev,
992 unsigned int cmd, unsigned long arg,
993 bool *forward)
994 {
995 struct dm_verity *v = ti->private;
996
997 *bdev = v->data_dev->bdev;
998
999 if (ti->len != bdev_nr_sectors(v->data_dev->bdev))
1000 return 1;
1001 return 0;
1002 }
1003
verity_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)1004 static int verity_iterate_devices(struct dm_target *ti,
1005 iterate_devices_callout_fn fn, void *data)
1006 {
1007 struct dm_verity *v = ti->private;
1008
1009 return fn(ti, v->data_dev, 0, ti->len, data);
1010 }
1011
verity_io_hints(struct dm_target * ti,struct queue_limits * limits)1012 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
1013 {
1014 struct dm_verity *v = ti->private;
1015
1016 if (limits->logical_block_size < 1 << v->data_dev_block_bits)
1017 limits->logical_block_size = 1 << v->data_dev_block_bits;
1018
1019 if (limits->physical_block_size < 1 << v->data_dev_block_bits)
1020 limits->physical_block_size = 1 << v->data_dev_block_bits;
1021
1022 limits->io_min = limits->logical_block_size;
1023
1024 /*
1025 * Similar to what dm-crypt does, opt dm-verity out of support for
1026 * direct I/O that is aligned to less than the traditional direct I/O
1027 * alignment requirement of logical_block_size. This prevents dm-verity
1028 * data blocks from crossing pages, eliminating various edge cases.
1029 */
1030 limits->dma_alignment = limits->logical_block_size - 1;
1031 }
1032
1033 #ifdef CONFIG_SECURITY
1034
verity_init_sig(struct dm_verity * v,const void * sig,size_t sig_size)1035 static int verity_init_sig(struct dm_verity *v, const void *sig,
1036 size_t sig_size)
1037 {
1038 v->sig_size = sig_size;
1039
1040 if (sig) {
1041 v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL);
1042 if (!v->root_digest_sig)
1043 return -ENOMEM;
1044 }
1045
1046 return 0;
1047 }
1048
verity_free_sig(struct dm_verity * v)1049 static void verity_free_sig(struct dm_verity *v)
1050 {
1051 kfree(v->root_digest_sig);
1052 }
1053
1054 #else
1055
verity_init_sig(struct dm_verity * v,const void * sig,size_t sig_size)1056 static inline int verity_init_sig(struct dm_verity *v, const void *sig,
1057 size_t sig_size)
1058 {
1059 return 0;
1060 }
1061
verity_free_sig(struct dm_verity * v)1062 static inline void verity_free_sig(struct dm_verity *v)
1063 {
1064 }
1065
1066 #endif /* CONFIG_SECURITY */
1067
verity_dtr(struct dm_target * ti)1068 static void verity_dtr(struct dm_target *ti)
1069 {
1070 struct dm_verity *v = ti->private;
1071
1072 if (v->verify_wq)
1073 destroy_workqueue(v->verify_wq);
1074
1075 mempool_exit(&v->recheck_pool);
1076 if (v->io)
1077 dm_io_client_destroy(v->io);
1078
1079 if (v->bufio)
1080 dm_bufio_client_destroy(v->bufio);
1081
1082 kvfree(v->validated_blocks);
1083 kfree(v->salt);
1084 kfree(v->initial_hashstate.shash);
1085 kfree(v->root_digest);
1086 kfree(v->zero_digest);
1087 verity_free_sig(v);
1088
1089 crypto_free_shash(v->shash_tfm);
1090
1091 kfree(v->alg_name);
1092
1093 if (v->hash_dev)
1094 dm_put_device(ti, v->hash_dev);
1095
1096 if (v->data_dev)
1097 dm_put_device(ti, v->data_dev);
1098
1099 verity_fec_dtr(v);
1100
1101 kfree(v->signature_key_desc);
1102
1103 if (v->use_bh_wq)
1104 static_branch_dec(&use_bh_wq_enabled);
1105
1106 kfree(v);
1107
1108 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
1109 }
1110
verity_alloc_most_once(struct dm_verity * v)1111 static int verity_alloc_most_once(struct dm_verity *v)
1112 {
1113 struct dm_target *ti = v->ti;
1114
1115 if (v->validated_blocks)
1116 return 0;
1117
1118 /* the bitset can only handle INT_MAX blocks */
1119 if (v->data_blocks > INT_MAX) {
1120 ti->error = "device too large to use check_at_most_once";
1121 return -E2BIG;
1122 }
1123
1124 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),
1125 sizeof(unsigned long),
1126 GFP_KERNEL);
1127 if (!v->validated_blocks) {
1128 ti->error = "failed to allocate bitset for check_at_most_once";
1129 return -ENOMEM;
1130 }
1131
1132 return 0;
1133 }
1134
verity_alloc_zero_digest(struct dm_verity * v)1135 static int verity_alloc_zero_digest(struct dm_verity *v)
1136 {
1137 int r = -ENOMEM;
1138 struct dm_verity_io *io;
1139 u8 *zero_data;
1140
1141 if (v->zero_digest)
1142 return 0;
1143
1144 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
1145
1146 if (!v->zero_digest)
1147 return r;
1148
1149 io = kmalloc(v->ti->per_io_data_size, GFP_KERNEL);
1150
1151 if (!io)
1152 return r; /* verity_dtr will free zero_digest */
1153
1154 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
1155
1156 if (!zero_data)
1157 goto out;
1158
1159 r = verity_hash(v, io, zero_data, 1 << v->data_dev_block_bits,
1160 v->zero_digest);
1161
1162 out:
1163 kfree(io);
1164 kfree(zero_data);
1165
1166 return r;
1167 }
1168
verity_is_verity_mode(const char * arg_name)1169 static inline bool verity_is_verity_mode(const char *arg_name)
1170 {
1171 return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) ||
1172 !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) ||
1173 !strcasecmp(arg_name, DM_VERITY_OPT_PANIC));
1174 }
1175
verity_parse_verity_mode(struct dm_verity * v,const char * arg_name)1176 static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name)
1177 {
1178 if (v->mode)
1179 return -EINVAL;
1180
1181 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING))
1182 v->mode = DM_VERITY_MODE_LOGGING;
1183 else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART))
1184 v->mode = DM_VERITY_MODE_RESTART;
1185 else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC))
1186 v->mode = DM_VERITY_MODE_PANIC;
1187
1188 return 0;
1189 }
1190
verity_is_verity_error_mode(const char * arg_name)1191 static inline bool verity_is_verity_error_mode(const char *arg_name)
1192 {
1193 return (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART) ||
1194 !strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC));
1195 }
1196
verity_parse_verity_error_mode(struct dm_verity * v,const char * arg_name)1197 static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name)
1198 {
1199 if (v->error_mode)
1200 return -EINVAL;
1201
1202 if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART))
1203 v->error_mode = DM_VERITY_MODE_RESTART;
1204 else if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC))
1205 v->error_mode = DM_VERITY_MODE_PANIC;
1206
1207 return 0;
1208 }
1209
verity_parse_opt_args(struct dm_arg_set * as,struct dm_verity * v,struct dm_verity_sig_opts * verify_args,bool only_modifier_opts)1210 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
1211 struct dm_verity_sig_opts *verify_args,
1212 bool only_modifier_opts)
1213 {
1214 int r = 0;
1215 unsigned int argc;
1216 struct dm_target *ti = v->ti;
1217 const char *arg_name;
1218
1219 static const struct dm_arg _args[] = {
1220 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
1221 };
1222
1223 r = dm_read_arg_group(_args, as, &argc, &ti->error);
1224 if (r)
1225 return -EINVAL;
1226
1227 if (!argc)
1228 return 0;
1229
1230 do {
1231 arg_name = dm_shift_arg(as);
1232 argc--;
1233
1234 if (verity_is_verity_mode(arg_name)) {
1235 if (only_modifier_opts)
1236 continue;
1237 r = verity_parse_verity_mode(v, arg_name);
1238 if (r) {
1239 ti->error = "Conflicting error handling parameters";
1240 return r;
1241 }
1242 continue;
1243
1244 } else if (verity_is_verity_error_mode(arg_name)) {
1245 if (only_modifier_opts)
1246 continue;
1247 r = verity_parse_verity_error_mode(v, arg_name);
1248 if (r) {
1249 ti->error = "Conflicting error handling parameters";
1250 return r;
1251 }
1252 continue;
1253
1254 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) {
1255 if (only_modifier_opts)
1256 continue;
1257 r = verity_alloc_zero_digest(v);
1258 if (r) {
1259 ti->error = "Cannot allocate zero digest";
1260 return r;
1261 }
1262 continue;
1263
1264 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) {
1265 if (only_modifier_opts)
1266 continue;
1267 r = verity_alloc_most_once(v);
1268 if (r)
1269 return r;
1270 continue;
1271
1272 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
1273 v->use_bh_wq = true;
1274 static_branch_inc(&use_bh_wq_enabled);
1275 continue;
1276
1277 } else if (verity_is_fec_opt_arg(arg_name)) {
1278 if (only_modifier_opts)
1279 continue;
1280 r = verity_fec_parse_opt_args(as, v, &argc, arg_name);
1281 if (r)
1282 return r;
1283 continue;
1284
1285 } else if (verity_verify_is_sig_opt_arg(arg_name)) {
1286 if (only_modifier_opts)
1287 continue;
1288 r = verity_verify_sig_parse_opt_args(as, v,
1289 verify_args,
1290 &argc, arg_name);
1291 if (r)
1292 return r;
1293 continue;
1294
1295 } else if (only_modifier_opts) {
1296 /*
1297 * Ignore unrecognized opt, could easily be an extra
1298 * argument to an option whose parsing was skipped.
1299 * Normal parsing (@only_modifier_opts=false) will
1300 * properly parse all options (and their extra args).
1301 */
1302 continue;
1303 }
1304
1305 DMERR("Unrecognized verity feature request: %s", arg_name);
1306 ti->error = "Unrecognized verity feature request";
1307 return -EINVAL;
1308 } while (argc && !r);
1309
1310 return r;
1311 }
1312
verity_setup_hash_alg(struct dm_verity * v,const char * alg_name)1313 static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
1314 {
1315 struct dm_target *ti = v->ti;
1316 struct crypto_shash *shash;
1317
1318 v->alg_name = kstrdup(alg_name, GFP_KERNEL);
1319 if (!v->alg_name) {
1320 ti->error = "Cannot allocate algorithm name";
1321 return -ENOMEM;
1322 }
1323
1324 shash = crypto_alloc_shash(alg_name, 0, 0);
1325 if (IS_ERR(shash)) {
1326 ti->error = "Cannot initialize hash function";
1327 return PTR_ERR(shash);
1328 }
1329 v->shash_tfm = shash;
1330 v->digest_size = crypto_shash_digestsize(shash);
1331 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
1332 ti->error = "Digest size too big";
1333 return -EINVAL;
1334 }
1335 if (likely(v->version && strcmp(alg_name, "sha256") == 0)) {
1336 /*
1337 * Fast path: use the library API for reduced overhead and
1338 * interleaved hashing support.
1339 */
1340 v->use_sha256_lib = true;
1341 if (sha256_finup_2x_is_optimized())
1342 v->use_sha256_finup_2x = true;
1343 ti->per_io_data_size =
1344 offsetofend(struct dm_verity_io, hash_ctx.sha256);
1345 } else {
1346 /* Fallback case: use the generic crypto API. */
1347 ti->per_io_data_size =
1348 offsetofend(struct dm_verity_io, hash_ctx.shash) +
1349 crypto_shash_descsize(shash);
1350 }
1351 return 0;
1352 }
1353
verity_setup_salt_and_hashstate(struct dm_verity * v,const char * arg)1354 static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)
1355 {
1356 struct dm_target *ti = v->ti;
1357
1358 if (strcmp(arg, "-") != 0) {
1359 v->salt_size = strlen(arg) / 2;
1360 v->salt = kmalloc(v->salt_size, GFP_KERNEL);
1361 if (!v->salt) {
1362 ti->error = "Cannot allocate salt";
1363 return -ENOMEM;
1364 }
1365 if (strlen(arg) != v->salt_size * 2 ||
1366 hex2bin(v->salt, arg, v->salt_size)) {
1367 ti->error = "Invalid salt";
1368 return -EINVAL;
1369 }
1370 }
1371 if (likely(v->use_sha256_lib)) {
1372 /* Implies version 1: salt at beginning */
1373 v->initial_hashstate.sha256 =
1374 kmalloc(sizeof(struct sha256_ctx), GFP_KERNEL);
1375 if (!v->initial_hashstate.sha256) {
1376 ti->error = "Cannot allocate initial hash state";
1377 return -ENOMEM;
1378 }
1379 sha256_init(v->initial_hashstate.sha256);
1380 sha256_update(v->initial_hashstate.sha256,
1381 v->salt, v->salt_size);
1382 } else if (v->version) { /* Version 1: salt at beginning */
1383 SHASH_DESC_ON_STACK(desc, v->shash_tfm);
1384 int r;
1385
1386 /*
1387 * Compute the pre-salted hash state that can be passed to
1388 * crypto_shash_import() for each block later.
1389 */
1390 v->initial_hashstate.shash = kmalloc(
1391 crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);
1392 if (!v->initial_hashstate.shash) {
1393 ti->error = "Cannot allocate initial hash state";
1394 return -ENOMEM;
1395 }
1396 desc->tfm = v->shash_tfm;
1397 r = crypto_shash_init(desc) ?:
1398 crypto_shash_update(desc, v->salt, v->salt_size) ?:
1399 crypto_shash_export(desc, v->initial_hashstate.shash);
1400 if (r) {
1401 ti->error = "Cannot set up initial hash state";
1402 return r;
1403 }
1404 }
1405 return 0;
1406 }
1407
1408 /*
1409 * Target parameters:
1410 * <version> The current format is version 1.
1411 * Vsn 0 is compatible with original Chromium OS releases.
1412 * <data device>
1413 * <hash device>
1414 * <data block size>
1415 * <hash block size>
1416 * <the number of data blocks>
1417 * <hash start block>
1418 * <algorithm>
1419 * <digest>
1420 * <salt> Hex string or "-" if no salt.
1421 */
verity_ctr(struct dm_target * ti,unsigned int argc,char ** argv)1422 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1423 {
1424 struct dm_verity *v;
1425 struct dm_verity_sig_opts verify_args = {0};
1426 struct dm_arg_set as;
1427 unsigned int num;
1428 unsigned long long num_ll;
1429 int r;
1430 int i;
1431 sector_t hash_position;
1432 char dummy;
1433 char *root_hash_digest_to_validate;
1434
1435 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
1436 if (!v) {
1437 ti->error = "Cannot allocate verity structure";
1438 return -ENOMEM;
1439 }
1440 ti->private = v;
1441 v->ti = ti;
1442
1443 r = verity_fec_ctr_alloc(v);
1444 if (r)
1445 goto bad;
1446
1447 if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) {
1448 ti->error = "Device must be readonly";
1449 r = -EINVAL;
1450 goto bad;
1451 }
1452
1453 if (argc < 10) {
1454 ti->error = "Not enough arguments";
1455 r = -EINVAL;
1456 goto bad;
1457 }
1458
1459 /* Parse optional parameters that modify primary args */
1460 if (argc > 10) {
1461 as.argc = argc - 10;
1462 as.argv = argv + 10;
1463 r = verity_parse_opt_args(&as, v, &verify_args, true);
1464 if (r < 0)
1465 goto bad;
1466 }
1467
1468 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
1469 num > 1) {
1470 ti->error = "Invalid version";
1471 r = -EINVAL;
1472 goto bad;
1473 }
1474 v->version = num;
1475
1476 r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev);
1477 if (r) {
1478 ti->error = "Data device lookup failed";
1479 goto bad;
1480 }
1481
1482 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev);
1483 if (r) {
1484 ti->error = "Hash device lookup failed";
1485 goto bad;
1486 }
1487
1488 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
1489 !num || (num & (num - 1)) ||
1490 num < bdev_logical_block_size(v->data_dev->bdev) ||
1491 num > PAGE_SIZE) {
1492 ti->error = "Invalid data device block size";
1493 r = -EINVAL;
1494 goto bad;
1495 }
1496 v->data_dev_block_bits = __ffs(num);
1497
1498 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
1499 !num || (num & (num - 1)) ||
1500 num < bdev_logical_block_size(v->hash_dev->bdev) ||
1501 num > INT_MAX) {
1502 ti->error = "Invalid hash device block size";
1503 r = -EINVAL;
1504 goto bad;
1505 }
1506 v->hash_dev_block_bits = __ffs(num);
1507
1508 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
1509 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
1510 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1511 ti->error = "Invalid data blocks";
1512 r = -EINVAL;
1513 goto bad;
1514 }
1515 v->data_blocks = num_ll;
1516
1517 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
1518 ti->error = "Data device is too small";
1519 r = -EINVAL;
1520 goto bad;
1521 }
1522
1523 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
1524 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
1525 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1526 ti->error = "Invalid hash start";
1527 r = -EINVAL;
1528 goto bad;
1529 }
1530 v->hash_start = num_ll;
1531
1532 r = verity_setup_hash_alg(v, argv[7]);
1533 if (r)
1534 goto bad;
1535
1536 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
1537 if (!v->root_digest) {
1538 ti->error = "Cannot allocate root digest";
1539 r = -ENOMEM;
1540 goto bad;
1541 }
1542 if (strlen(argv[8]) != v->digest_size * 2 ||
1543 hex2bin(v->root_digest, argv[8], v->digest_size)) {
1544 ti->error = "Invalid root digest";
1545 r = -EINVAL;
1546 goto bad;
1547 }
1548 root_hash_digest_to_validate = argv[8];
1549
1550 r = verity_setup_salt_and_hashstate(v, argv[9]);
1551 if (r)
1552 goto bad;
1553
1554 argv += 10;
1555 argc -= 10;
1556
1557 /* Optional parameters */
1558 if (argc) {
1559 as.argc = argc;
1560 as.argv = argv;
1561 r = verity_parse_opt_args(&as, v, &verify_args, false);
1562 if (r < 0)
1563 goto bad;
1564 }
1565
1566 /* Root hash signature is an optional parameter */
1567 r = verity_verify_root_hash(root_hash_digest_to_validate,
1568 strlen(root_hash_digest_to_validate),
1569 verify_args.sig,
1570 verify_args.sig_size);
1571 if (r < 0) {
1572 ti->error = "Root hash verification failed";
1573 goto bad;
1574 }
1575
1576 r = verity_init_sig(v, verify_args.sig, verify_args.sig_size);
1577 if (r < 0) {
1578 ti->error = "Cannot allocate root digest signature";
1579 goto bad;
1580 }
1581
1582 v->hash_per_block_bits =
1583 __fls((1 << v->hash_dev_block_bits) / v->digest_size);
1584
1585 v->levels = 0;
1586 if (v->data_blocks)
1587 while (v->hash_per_block_bits * v->levels < 64 &&
1588 (unsigned long long)(v->data_blocks - 1) >>
1589 (v->hash_per_block_bits * v->levels))
1590 v->levels++;
1591
1592 if (v->levels > DM_VERITY_MAX_LEVELS) {
1593 ti->error = "Too many tree levels";
1594 r = -E2BIG;
1595 goto bad;
1596 }
1597
1598 hash_position = v->hash_start;
1599 for (i = v->levels - 1; i >= 0; i--) {
1600 sector_t s;
1601
1602 v->hash_level_block[i] = hash_position;
1603 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
1604 >> ((i + 1) * v->hash_per_block_bits);
1605 if (hash_position + s < hash_position) {
1606 ti->error = "Hash device offset overflow";
1607 r = -E2BIG;
1608 goto bad;
1609 }
1610 hash_position += s;
1611 }
1612 v->hash_blocks = hash_position;
1613
1614 r = mempool_init_page_pool(&v->recheck_pool, 1, 0);
1615 if (unlikely(r)) {
1616 ti->error = "Cannot allocate mempool";
1617 goto bad;
1618 }
1619
1620 v->io = dm_io_client_create();
1621 if (IS_ERR(v->io)) {
1622 r = PTR_ERR(v->io);
1623 v->io = NULL;
1624 ti->error = "Cannot allocate dm io";
1625 goto bad;
1626 }
1627
1628 v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
1629 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
1630 dm_bufio_alloc_callback, NULL,
1631 v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
1632 if (IS_ERR(v->bufio)) {
1633 ti->error = "Cannot initialize dm-bufio";
1634 r = PTR_ERR(v->bufio);
1635 v->bufio = NULL;
1636 goto bad;
1637 }
1638
1639 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
1640 ti->error = "Hash device is too small";
1641 r = -E2BIG;
1642 goto bad;
1643 }
1644
1645 /*
1646 * Using WQ_HIGHPRI improves throughput and completion latency by
1647 * reducing wait times when reading from a dm-verity device.
1648 *
1649 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
1650 * allows verify_wq to preempt softirq since verification in BH workqueue
1651 * will fall-back to using it for error handling (or if the bufio cache
1652 * doesn't have required hashes).
1653 */
1654 v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1655 if (!v->verify_wq) {
1656 ti->error = "Cannot allocate workqueue";
1657 r = -ENOMEM;
1658 goto bad;
1659 }
1660
1661 r = verity_fec_ctr(v);
1662 if (r)
1663 goto bad;
1664
1665 ti->per_io_data_size = roundup(ti->per_io_data_size,
1666 __alignof__(struct dm_verity_io));
1667
1668 verity_verify_sig_opts_cleanup(&verify_args);
1669
1670 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
1671
1672 return 0;
1673
1674 bad:
1675
1676 verity_verify_sig_opts_cleanup(&verify_args);
1677 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
1678 verity_dtr(ti);
1679
1680 return r;
1681 }
1682
1683 /*
1684 * Get the verity mode (error behavior) of a verity target.
1685 *
1686 * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity
1687 * target.
1688 */
dm_verity_get_mode(struct dm_target * ti)1689 int dm_verity_get_mode(struct dm_target *ti)
1690 {
1691 struct dm_verity *v = ti->private;
1692
1693 if (!dm_is_verity_target(ti))
1694 return -EINVAL;
1695
1696 return v->mode;
1697 }
1698
1699 /*
1700 * Get the root digest of a verity target.
1701 *
1702 * Returns a copy of the root digest, the caller is responsible for
1703 * freeing the memory of the digest.
1704 */
dm_verity_get_root_digest(struct dm_target * ti,u8 ** root_digest,unsigned int * digest_size)1705 int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size)
1706 {
1707 struct dm_verity *v = ti->private;
1708
1709 if (!dm_is_verity_target(ti))
1710 return -EINVAL;
1711
1712 *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL);
1713 if (*root_digest == NULL)
1714 return -ENOMEM;
1715
1716 *digest_size = v->digest_size;
1717
1718 return 0;
1719 }
1720
1721 #ifdef CONFIG_SECURITY
1722
1723 #ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG
1724
verity_security_set_signature(struct block_device * bdev,struct dm_verity * v)1725 static int verity_security_set_signature(struct block_device *bdev,
1726 struct dm_verity *v)
1727 {
1728 /*
1729 * if the dm-verity target is unsigned, v->root_digest_sig will
1730 * be NULL, and the hook call is still required to let LSMs mark
1731 * the device as unsigned. This information is crucial for LSMs to
1732 * block operations such as execution on unsigned files
1733 */
1734 return security_bdev_setintegrity(bdev,
1735 LSM_INT_DMVERITY_SIG_VALID,
1736 v->root_digest_sig,
1737 v->sig_size);
1738 }
1739
1740 #else
1741
verity_security_set_signature(struct block_device * bdev,struct dm_verity * v)1742 static inline int verity_security_set_signature(struct block_device *bdev,
1743 struct dm_verity *v)
1744 {
1745 return 0;
1746 }
1747
1748 #endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */
1749
1750 /*
1751 * Expose verity target's root hash and signature data to LSMs before resume.
1752 *
1753 * Returns 0 on success, or -ENOMEM if the system is out of memory.
1754 */
verity_preresume(struct dm_target * ti)1755 static int verity_preresume(struct dm_target *ti)
1756 {
1757 struct block_device *bdev;
1758 struct dm_verity_digest root_digest;
1759 struct dm_verity *v;
1760 int r;
1761
1762 v = ti->private;
1763 bdev = dm_disk(dm_table_get_md(ti->table))->part0;
1764 root_digest.digest = v->root_digest;
1765 root_digest.digest_len = v->digest_size;
1766 root_digest.alg = crypto_shash_alg_name(v->shash_tfm);
1767
1768 r = security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, &root_digest,
1769 sizeof(root_digest));
1770 if (r)
1771 return r;
1772
1773 r = verity_security_set_signature(bdev, v);
1774 if (r)
1775 goto bad;
1776
1777 return 0;
1778
1779 bad:
1780
1781 security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, NULL, 0);
1782
1783 return r;
1784 }
1785
1786 #endif /* CONFIG_SECURITY */
1787
1788 static struct target_type verity_target = {
1789 .name = "verity",
1790 /* Note: the LSMs depend on the singleton and immutable features */
1791 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1792 .version = {1, 13, 0},
1793 .module = THIS_MODULE,
1794 .ctr = verity_ctr,
1795 .dtr = verity_dtr,
1796 .map = verity_map,
1797 .postsuspend = verity_postsuspend,
1798 .status = verity_status,
1799 .prepare_ioctl = verity_prepare_ioctl,
1800 .iterate_devices = verity_iterate_devices,
1801 .io_hints = verity_io_hints,
1802 #ifdef CONFIG_SECURITY
1803 .preresume = verity_preresume,
1804 #endif /* CONFIG_SECURITY */
1805 };
1806 module_dm(verity);
1807
1808 /*
1809 * Check whether a DM target is a verity target.
1810 */
dm_is_verity_target(struct dm_target * ti)1811 bool dm_is_verity_target(struct dm_target *ti)
1812 {
1813 return ti->type == &verity_target;
1814 }
1815
1816 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1817 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1818 MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1819 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
1820 MODULE_LICENSE("GPL");
1821