1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved. 4 * Copyright (C) 2016-2017 Milan Broz 5 * Copyright (C) 2016-2017 Mikulas Patocka 6 * 7 * This file is released under the GPL. 8 */ 9 10 #include "dm-bio-record.h" 11 12 #include <linux/compiler.h> 13 #include <linux/module.h> 14 #include <linux/device-mapper.h> 15 #include <linux/dm-io.h> 16 #include <linux/vmalloc.h> 17 #include <linux/sort.h> 18 #include <linux/rbtree.h> 19 #include <linux/delay.h> 20 #include <linux/random.h> 21 #include <linux/reboot.h> 22 #include <crypto/hash.h> 23 #include <crypto/skcipher.h> 24 #include <linux/async_tx.h> 25 #include <linux/dm-bufio.h> 26 27 #include "dm-audit.h" 28 29 #define DM_MSG_PREFIX "integrity" 30 31 #define DEFAULT_INTERLEAVE_SECTORS 32768 32 #define DEFAULT_JOURNAL_SIZE_FACTOR 7 33 #define DEFAULT_SECTORS_PER_BITMAP_BIT 32768 34 #define DEFAULT_BUFFER_SECTORS 128 35 #define DEFAULT_JOURNAL_WATERMARK 50 36 #define DEFAULT_SYNC_MSEC 10000 37 #define DEFAULT_MAX_JOURNAL_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 131072 : 8192) 38 #define MIN_LOG2_INTERLEAVE_SECTORS 3 39 #define MAX_LOG2_INTERLEAVE_SECTORS 31 40 #define METADATA_WORKQUEUE_MAX_ACTIVE 16 41 #define RECALC_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 32768 : 2048) 42 #define RECALC_WRITE_SUPER 16 43 #define BITMAP_BLOCK_SIZE 4096 /* don't change it */ 44 #define BITMAP_FLUSH_INTERVAL (10 * HZ) 45 #define DISCARD_FILLER 0xf6 46 #define SALT_SIZE 16 47 #define RECHECK_POOL_SIZE 256 48 49 /* 50 * Warning - DEBUG_PRINT prints security-sensitive data to the log, 51 * so it should not be enabled in the official kernel 52 */ 53 //#define DEBUG_PRINT 54 //#define INTERNAL_VERIFY 55 56 /* 57 * On disk structures 58 */ 59 60 #define SB_MAGIC "integrt" 61 #define SB_VERSION_1 1 62 #define SB_VERSION_2 2 63 #define SB_VERSION_3 3 64 #define SB_VERSION_4 4 65 #define SB_VERSION_5 5 66 #define SB_VERSION_6 6 67 #define SB_SECTORS 8 68 #define MAX_SECTORS_PER_BLOCK 8 69 70 struct superblock { 71 __u8 magic[8]; 72 __u8 version; 73 __u8 log2_interleave_sectors; 74 __le16 integrity_tag_size; 75 __le32 journal_sections; 76 __le64 provided_data_sectors; /* userspace uses this value */ 77 __le32 flags; 78 __u8 log2_sectors_per_block; 79 __u8 log2_blocks_per_bitmap_bit; 80 __u8 pad[2]; 81 __le64 recalc_sector; 82 __u8 pad2[8]; 83 __u8 salt[SALT_SIZE]; 84 }; 85 86 #define SB_FLAG_HAVE_JOURNAL_MAC 0x1 87 #define SB_FLAG_RECALCULATING 0x2 88 #define SB_FLAG_DIRTY_BITMAP 0x4 89 #define SB_FLAG_FIXED_PADDING 0x8 90 #define SB_FLAG_FIXED_HMAC 0x10 91 #define SB_FLAG_INLINE 0x20 92 93 #define JOURNAL_ENTRY_ROUNDUP 8 94 95 typedef __le64 commit_id_t; 96 #define JOURNAL_MAC_PER_SECTOR 8 97 98 struct journal_entry { 99 union { 100 struct { 101 __le32 sector_lo; 102 __le32 sector_hi; 103 } s; 104 __le64 sector; 105 } u; 106 commit_id_t last_bytes[]; 107 /* __u8 tag[0]; */ 108 }; 109 110 #define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block]) 111 112 #if BITS_PER_LONG == 64 113 #define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0) 114 #else 115 #define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0) 116 #endif 117 #define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) 118 #define journal_entry_is_unused(je) ((je)->u.s.sector_hi == cpu_to_le32(-1)) 119 #define journal_entry_set_unused(je) ((je)->u.s.sector_hi = cpu_to_le32(-1)) 120 #define journal_entry_is_inprogress(je) ((je)->u.s.sector_hi == cpu_to_le32(-2)) 121 #define journal_entry_set_inprogress(je) ((je)->u.s.sector_hi = cpu_to_le32(-2)) 122 123 #define JOURNAL_BLOCK_SECTORS 8 124 #define JOURNAL_SECTOR_DATA ((1 << SECTOR_SHIFT) - sizeof(commit_id_t)) 125 #define JOURNAL_MAC_SIZE (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS) 126 127 struct journal_sector { 128 struct_group(sectors, 129 __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR]; 130 __u8 mac[JOURNAL_MAC_PER_SECTOR]; 131 ); 132 commit_id_t commit_id; 133 }; 134 135 #define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK])) 136 137 #define METADATA_PADDING_SECTORS 8 138 139 #define N_COMMIT_IDS 4 140 141 static unsigned char prev_commit_seq(unsigned char seq) 142 { 143 return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS; 144 } 145 146 static unsigned char next_commit_seq(unsigned char seq) 147 { 148 return (seq + 1) % N_COMMIT_IDS; 149 } 150 151 /* 152 * In-memory structures 153 */ 154 155 struct journal_node { 156 struct rb_node node; 157 sector_t sector; 158 }; 159 160 struct alg_spec { 161 char *alg_string; 162 char *key_string; 163 __u8 *key; 164 unsigned int key_size; 165 }; 166 167 struct dm_integrity_c { 168 struct dm_dev *dev; 169 struct dm_dev *meta_dev; 170 unsigned int tag_size; 171 __s8 log2_tag_size; 172 unsigned int tuple_size; 173 sector_t start; 174 mempool_t journal_io_mempool; 175 struct dm_io_client *io; 176 struct dm_bufio_client *bufio; 177 struct workqueue_struct *metadata_wq; 178 struct superblock *sb; 179 unsigned int journal_pages; 180 unsigned int n_bitmap_blocks; 181 182 struct page_list *journal; 183 struct page_list *journal_io; 184 struct page_list *journal_xor; 185 struct page_list *recalc_bitmap; 186 struct page_list *may_write_bitmap; 187 struct bitmap_block_status *bbs; 188 unsigned int bitmap_flush_interval; 189 int synchronous_mode; 190 struct bio_list synchronous_bios; 191 struct delayed_work bitmap_flush_work; 192 193 struct crypto_skcipher *journal_crypt; 194 struct scatterlist **journal_scatterlist; 195 struct scatterlist **journal_io_scatterlist; 196 struct skcipher_request **sk_requests; 197 198 struct crypto_shash *journal_mac; 199 200 struct journal_node *journal_tree; 201 struct rb_root journal_tree_root; 202 203 sector_t provided_data_sectors; 204 205 unsigned short journal_entry_size; 206 unsigned char journal_entries_per_sector; 207 unsigned char journal_section_entries; 208 unsigned short journal_section_sectors; 209 unsigned int journal_sections; 210 unsigned int journal_entries; 211 sector_t data_device_sectors; 212 sector_t meta_device_sectors; 213 unsigned int initial_sectors; 214 unsigned int metadata_run; 215 __s8 log2_metadata_run; 216 __u8 log2_buffer_sectors; 217 __u8 sectors_per_block; 218 __u8 log2_blocks_per_bitmap_bit; 219 220 unsigned char mode; 221 222 int failed; 223 224 struct crypto_shash *internal_hash; 225 226 struct dm_target *ti; 227 228 /* these variables are locked with endio_wait.lock */ 229 struct rb_root in_progress; 230 struct list_head wait_list; 231 wait_queue_head_t endio_wait; 232 struct workqueue_struct *wait_wq; 233 struct workqueue_struct *offload_wq; 234 235 unsigned char commit_seq; 236 commit_id_t commit_ids[N_COMMIT_IDS]; 237 238 unsigned int committed_section; 239 unsigned int n_committed_sections; 240 241 unsigned int uncommitted_section; 242 unsigned int n_uncommitted_sections; 243 244 unsigned int free_section; 245 unsigned char free_section_entry; 246 unsigned int free_sectors; 247 248 unsigned int free_sectors_threshold; 249 250 struct workqueue_struct *commit_wq; 251 struct work_struct commit_work; 252 253 struct workqueue_struct *writer_wq; 254 struct work_struct writer_work; 255 256 struct workqueue_struct *recalc_wq; 257 struct work_struct recalc_work; 258 259 struct bio_list flush_bio_list; 260 261 unsigned long autocommit_jiffies; 262 struct timer_list autocommit_timer; 263 unsigned int autocommit_msec; 264 265 wait_queue_head_t copy_to_journal_wait; 266 267 struct completion crypto_backoff; 268 269 bool wrote_to_journal; 270 bool journal_uptodate; 271 bool just_formatted; 272 bool recalculate_flag; 273 bool reset_recalculate_flag; 274 bool discard; 275 bool fix_padding; 276 bool fix_hmac; 277 bool legacy_recalculate; 278 279 struct alg_spec internal_hash_alg; 280 struct alg_spec journal_crypt_alg; 281 struct alg_spec journal_mac_alg; 282 283 atomic64_t number_of_mismatches; 284 285 mempool_t recheck_pool; 286 struct bio_set recheck_bios; 287 struct bio_set recalc_bios; 288 289 struct notifier_block reboot_notifier; 290 }; 291 292 struct dm_integrity_range { 293 sector_t logical_sector; 294 sector_t n_sectors; 295 bool waiting; 296 union { 297 struct rb_node node; 298 struct { 299 struct task_struct *task; 300 struct list_head wait_entry; 301 }; 302 }; 303 }; 304 305 struct dm_integrity_io { 306 struct work_struct work; 307 308 struct dm_integrity_c *ic; 309 enum req_op op; 310 bool fua; 311 312 struct dm_integrity_range range; 313 314 sector_t metadata_block; 315 unsigned int metadata_offset; 316 317 atomic_t in_flight; 318 blk_status_t bi_status; 319 320 struct completion *completion; 321 322 struct dm_bio_details bio_details; 323 324 char *integrity_payload; 325 unsigned payload_len; 326 bool integrity_payload_from_mempool; 327 bool integrity_range_locked; 328 }; 329 330 struct journal_completion { 331 struct dm_integrity_c *ic; 332 atomic_t in_flight; 333 struct completion comp; 334 }; 335 336 struct journal_io { 337 struct dm_integrity_range range; 338 struct journal_completion *comp; 339 }; 340 341 struct bitmap_block_status { 342 struct work_struct work; 343 struct dm_integrity_c *ic; 344 unsigned int idx; 345 unsigned long *bitmap; 346 struct bio_list bio_queue; 347 spinlock_t bio_queue_lock; 348 349 }; 350 351 static struct kmem_cache *journal_io_cache; 352 353 #define JOURNAL_IO_MEMPOOL 32 354 355 #ifdef DEBUG_PRINT 356 #define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__) 357 #define DEBUG_bytes(bytes, len, msg, ...) printk(KERN_DEBUG msg "%s%*ph\n", ##__VA_ARGS__, \ 358 len ? ": " : "", len, bytes) 359 #else 360 #define DEBUG_print(x, ...) do { } while (0) 361 #define DEBUG_bytes(bytes, len, msg, ...) do { } while (0) 362 #endif 363 364 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); 365 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map); 366 static void integrity_bio_wait(struct work_struct *w); 367 static void dm_integrity_dtr(struct dm_target *ti); 368 369 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) 370 { 371 if (err == -EILSEQ) 372 atomic64_inc(&ic->number_of_mismatches); 373 if (!cmpxchg(&ic->failed, 0, err)) 374 DMERR("Error on %s: %d", msg, err); 375 } 376 377 static int dm_integrity_failed(struct dm_integrity_c *ic) 378 { 379 return READ_ONCE(ic->failed); 380 } 381 382 static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic) 383 { 384 if (ic->legacy_recalculate) 385 return false; 386 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ? 387 ic->internal_hash_alg.key || ic->journal_mac_alg.key : 388 ic->internal_hash_alg.key && !ic->journal_mac_alg.key) 389 return true; 390 return false; 391 } 392 393 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i, 394 unsigned int j, unsigned char seq) 395 { 396 /* 397 * Xor the number with section and sector, so that if a piece of 398 * journal is written at wrong place, it is detected. 399 */ 400 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j); 401 } 402 403 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, 404 sector_t *area, sector_t *offset) 405 { 406 if (!ic->meta_dev) { 407 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; 408 *area = data_sector >> log2_interleave_sectors; 409 *offset = (unsigned int)data_sector & ((1U << log2_interleave_sectors) - 1); 410 } else { 411 *area = 0; 412 *offset = data_sector; 413 } 414 } 415 416 #define sector_to_block(ic, n) \ 417 do { \ 418 BUG_ON((n) & (unsigned int)((ic)->sectors_per_block - 1)); \ 419 (n) >>= (ic)->sb->log2_sectors_per_block; \ 420 } while (0) 421 422 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, 423 sector_t offset, unsigned int *metadata_offset) 424 { 425 __u64 ms; 426 unsigned int mo; 427 428 ms = area << ic->sb->log2_interleave_sectors; 429 if (likely(ic->log2_metadata_run >= 0)) 430 ms += area << ic->log2_metadata_run; 431 else 432 ms += area * ic->metadata_run; 433 ms >>= ic->log2_buffer_sectors; 434 435 sector_to_block(ic, offset); 436 437 if (likely(ic->log2_tag_size >= 0)) { 438 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size); 439 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); 440 } else { 441 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors); 442 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); 443 } 444 *metadata_offset = mo; 445 return ms; 446 } 447 448 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset) 449 { 450 sector_t result; 451 452 if (ic->meta_dev) 453 return offset; 454 455 result = area << ic->sb->log2_interleave_sectors; 456 if (likely(ic->log2_metadata_run >= 0)) 457 result += (area + 1) << ic->log2_metadata_run; 458 else 459 result += (area + 1) * ic->metadata_run; 460 461 result += (sector_t)ic->initial_sectors + offset; 462 result += ic->start; 463 464 return result; 465 } 466 467 static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) 468 { 469 if (unlikely(*sec_ptr >= ic->journal_sections)) 470 *sec_ptr -= ic->journal_sections; 471 } 472 473 static void sb_set_version(struct dm_integrity_c *ic) 474 { 475 if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) 476 ic->sb->version = SB_VERSION_6; 477 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) 478 ic->sb->version = SB_VERSION_5; 479 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) 480 ic->sb->version = SB_VERSION_4; 481 else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) 482 ic->sb->version = SB_VERSION_3; 483 else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 484 ic->sb->version = SB_VERSION_2; 485 else 486 ic->sb->version = SB_VERSION_1; 487 } 488 489 static int sb_mac(struct dm_integrity_c *ic, bool wr) 490 { 491 SHASH_DESC_ON_STACK(desc, ic->journal_mac); 492 int r; 493 unsigned int mac_size = crypto_shash_digestsize(ic->journal_mac); 494 __u8 *sb = (__u8 *)ic->sb; 495 __u8 *mac = sb + (1 << SECTOR_SHIFT) - mac_size; 496 497 if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT || 498 mac_size > HASH_MAX_DIGESTSIZE) { 499 dm_integrity_io_error(ic, "digest is too long", -EINVAL); 500 return -EINVAL; 501 } 502 503 desc->tfm = ic->journal_mac; 504 505 if (likely(wr)) { 506 r = crypto_shash_digest(desc, sb, mac - sb, mac); 507 if (unlikely(r < 0)) { 508 dm_integrity_io_error(ic, "crypto_shash_digest", r); 509 return r; 510 } 511 } else { 512 __u8 actual_mac[HASH_MAX_DIGESTSIZE]; 513 514 r = crypto_shash_digest(desc, sb, mac - sb, actual_mac); 515 if (unlikely(r < 0)) { 516 dm_integrity_io_error(ic, "crypto_shash_digest", r); 517 return r; 518 } 519 if (memcmp(mac, actual_mac, mac_size)) { 520 dm_integrity_io_error(ic, "superblock mac", -EILSEQ); 521 dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0); 522 return -EILSEQ; 523 } 524 } 525 526 return 0; 527 } 528 529 static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) 530 { 531 struct dm_io_request io_req; 532 struct dm_io_region io_loc; 533 const enum req_op op = opf & REQ_OP_MASK; 534 int r; 535 536 io_req.bi_opf = opf; 537 io_req.mem.type = DM_IO_KMEM; 538 io_req.mem.ptr.addr = ic->sb; 539 io_req.notify.fn = NULL; 540 io_req.client = ic->io; 541 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; 542 io_loc.sector = ic->start; 543 io_loc.count = SB_SECTORS; 544 545 if (op == REQ_OP_WRITE) { 546 sb_set_version(ic); 547 if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { 548 r = sb_mac(ic, true); 549 if (unlikely(r)) 550 return r; 551 } 552 } 553 554 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 555 if (unlikely(r)) 556 return r; 557 558 if (op == REQ_OP_READ) { 559 if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { 560 r = sb_mac(ic, false); 561 if (unlikely(r)) 562 return r; 563 } 564 } 565 566 return 0; 567 } 568 569 #define BITMAP_OP_TEST_ALL_SET 0 570 #define BITMAP_OP_TEST_ALL_CLEAR 1 571 #define BITMAP_OP_SET 2 572 #define BITMAP_OP_CLEAR 3 573 574 static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap, 575 sector_t sector, sector_t n_sectors, int mode) 576 { 577 unsigned long bit, end_bit, this_end_bit, page, end_page; 578 unsigned long *data; 579 580 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { 581 DMCRIT("invalid bitmap access (%llx,%llx,%d,%d,%d)", 582 sector, 583 n_sectors, 584 ic->sb->log2_sectors_per_block, 585 ic->log2_blocks_per_bitmap_bit, 586 mode); 587 BUG(); 588 } 589 590 if (unlikely(!n_sectors)) 591 return true; 592 593 bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 594 end_bit = (sector + n_sectors - 1) >> 595 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 596 597 page = bit / (PAGE_SIZE * 8); 598 bit %= PAGE_SIZE * 8; 599 600 end_page = end_bit / (PAGE_SIZE * 8); 601 end_bit %= PAGE_SIZE * 8; 602 603 repeat: 604 if (page < end_page) 605 this_end_bit = PAGE_SIZE * 8 - 1; 606 else 607 this_end_bit = end_bit; 608 609 data = lowmem_page_address(bitmap[page].page); 610 611 if (mode == BITMAP_OP_TEST_ALL_SET) { 612 while (bit <= this_end_bit) { 613 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { 614 do { 615 if (data[bit / BITS_PER_LONG] != -1) 616 return false; 617 bit += BITS_PER_LONG; 618 } while (this_end_bit >= bit + BITS_PER_LONG - 1); 619 continue; 620 } 621 if (!test_bit(bit, data)) 622 return false; 623 bit++; 624 } 625 } else if (mode == BITMAP_OP_TEST_ALL_CLEAR) { 626 while (bit <= this_end_bit) { 627 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { 628 do { 629 if (data[bit / BITS_PER_LONG] != 0) 630 return false; 631 bit += BITS_PER_LONG; 632 } while (this_end_bit >= bit + BITS_PER_LONG - 1); 633 continue; 634 } 635 if (test_bit(bit, data)) 636 return false; 637 bit++; 638 } 639 } else if (mode == BITMAP_OP_SET) { 640 while (bit <= this_end_bit) { 641 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { 642 do { 643 data[bit / BITS_PER_LONG] = -1; 644 bit += BITS_PER_LONG; 645 } while (this_end_bit >= bit + BITS_PER_LONG - 1); 646 continue; 647 } 648 __set_bit(bit, data); 649 bit++; 650 } 651 } else if (mode == BITMAP_OP_CLEAR) { 652 if (!bit && this_end_bit == PAGE_SIZE * 8 - 1) 653 clear_page(data); 654 else { 655 while (bit <= this_end_bit) { 656 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { 657 do { 658 data[bit / BITS_PER_LONG] = 0; 659 bit += BITS_PER_LONG; 660 } while (this_end_bit >= bit + BITS_PER_LONG - 1); 661 continue; 662 } 663 __clear_bit(bit, data); 664 bit++; 665 } 666 } 667 } else { 668 BUG(); 669 } 670 671 if (unlikely(page < end_page)) { 672 bit = 0; 673 page++; 674 goto repeat; 675 } 676 677 return true; 678 } 679 680 static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src) 681 { 682 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); 683 unsigned int i; 684 685 for (i = 0; i < n_bitmap_pages; i++) { 686 unsigned long *dst_data = lowmem_page_address(dst[i].page); 687 unsigned long *src_data = lowmem_page_address(src[i].page); 688 689 copy_page(dst_data, src_data); 690 } 691 } 692 693 static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector) 694 { 695 unsigned int bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 696 unsigned int bitmap_block = bit / (BITMAP_BLOCK_SIZE * 8); 697 698 BUG_ON(bitmap_block >= ic->n_bitmap_blocks); 699 return &ic->bbs[bitmap_block]; 700 } 701 702 static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, 703 bool e, const char *function) 704 { 705 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY) 706 unsigned int limit = e ? ic->journal_section_entries : ic->journal_section_sectors; 707 708 if (unlikely(section >= ic->journal_sections) || 709 unlikely(offset >= limit)) { 710 DMCRIT("%s: invalid access at (%u,%u), limit (%u,%u)", 711 function, section, offset, ic->journal_sections, limit); 712 BUG(); 713 } 714 #endif 715 } 716 717 static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, 718 unsigned int *pl_index, unsigned int *pl_offset) 719 { 720 unsigned int sector; 721 722 access_journal_check(ic, section, offset, false, "page_list_location"); 723 724 sector = section * ic->journal_section_sectors + offset; 725 726 *pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 727 *pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 728 } 729 730 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, 731 unsigned int section, unsigned int offset, unsigned int *n_sectors) 732 { 733 unsigned int pl_index, pl_offset; 734 char *va; 735 736 page_list_location(ic, section, offset, &pl_index, &pl_offset); 737 738 if (n_sectors) 739 *n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT; 740 741 va = lowmem_page_address(pl[pl_index].page); 742 743 return (struct journal_sector *)(va + pl_offset); 744 } 745 746 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset) 747 { 748 return access_page_list(ic, ic->journal, section, offset, NULL); 749 } 750 751 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, unsigned int n) 752 { 753 unsigned int rel_sector, offset; 754 struct journal_sector *js; 755 756 access_journal_check(ic, section, n, true, "access_journal_entry"); 757 758 rel_sector = n % JOURNAL_BLOCK_SECTORS; 759 offset = n / JOURNAL_BLOCK_SECTORS; 760 761 js = access_journal(ic, section, rel_sector); 762 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size); 763 } 764 765 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, unsigned int n) 766 { 767 n <<= ic->sb->log2_sectors_per_block; 768 769 n += JOURNAL_BLOCK_SECTORS; 770 771 access_journal_check(ic, section, n, false, "access_journal_data"); 772 773 return access_journal(ic, section, n); 774 } 775 776 static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SIZE]) 777 { 778 SHASH_DESC_ON_STACK(desc, ic->journal_mac); 779 int r; 780 unsigned int j, size; 781 782 desc->tfm = ic->journal_mac; 783 784 r = crypto_shash_init(desc); 785 if (unlikely(r < 0)) { 786 dm_integrity_io_error(ic, "crypto_shash_init", r); 787 goto err; 788 } 789 790 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { 791 __le64 section_le; 792 793 r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE); 794 if (unlikely(r < 0)) { 795 dm_integrity_io_error(ic, "crypto_shash_update", r); 796 goto err; 797 } 798 799 section_le = cpu_to_le64(section); 800 r = crypto_shash_update(desc, (__u8 *)§ion_le, sizeof(section_le)); 801 if (unlikely(r < 0)) { 802 dm_integrity_io_error(ic, "crypto_shash_update", r); 803 goto err; 804 } 805 } 806 807 for (j = 0; j < ic->journal_section_entries; j++) { 808 struct journal_entry *je = access_journal_entry(ic, section, j); 809 810 r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof(je->u.sector)); 811 if (unlikely(r < 0)) { 812 dm_integrity_io_error(ic, "crypto_shash_update", r); 813 goto err; 814 } 815 } 816 817 size = crypto_shash_digestsize(ic->journal_mac); 818 819 if (likely(size <= JOURNAL_MAC_SIZE)) { 820 r = crypto_shash_final(desc, result); 821 if (unlikely(r < 0)) { 822 dm_integrity_io_error(ic, "crypto_shash_final", r); 823 goto err; 824 } 825 memset(result + size, 0, JOURNAL_MAC_SIZE - size); 826 } else { 827 __u8 digest[HASH_MAX_DIGESTSIZE]; 828 829 if (WARN_ON(size > sizeof(digest))) { 830 dm_integrity_io_error(ic, "digest_size", -EINVAL); 831 goto err; 832 } 833 r = crypto_shash_final(desc, digest); 834 if (unlikely(r < 0)) { 835 dm_integrity_io_error(ic, "crypto_shash_final", r); 836 goto err; 837 } 838 memcpy(result, digest, JOURNAL_MAC_SIZE); 839 } 840 841 return; 842 err: 843 memset(result, 0, JOURNAL_MAC_SIZE); 844 } 845 846 static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr) 847 { 848 __u8 result[JOURNAL_MAC_SIZE]; 849 unsigned int j; 850 851 if (!ic->journal_mac) 852 return; 853 854 section_mac(ic, section, result); 855 856 for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) { 857 struct journal_sector *js = access_journal(ic, section, j); 858 859 if (likely(wr)) 860 memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR); 861 else { 862 if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) { 863 dm_integrity_io_error(ic, "journal mac", -EILSEQ); 864 dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0); 865 } 866 } 867 } 868 } 869 870 static void complete_journal_op(void *context) 871 { 872 struct journal_completion *comp = context; 873 874 BUG_ON(!atomic_read(&comp->in_flight)); 875 if (likely(atomic_dec_and_test(&comp->in_flight))) 876 complete(&comp->comp); 877 } 878 879 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, 880 unsigned int n_sections, struct journal_completion *comp) 881 { 882 struct async_submit_ctl submit; 883 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT; 884 unsigned int pl_index, pl_offset, section_index; 885 struct page_list *source_pl, *target_pl; 886 887 if (likely(encrypt)) { 888 source_pl = ic->journal; 889 target_pl = ic->journal_io; 890 } else { 891 source_pl = ic->journal_io; 892 target_pl = ic->journal; 893 } 894 895 page_list_location(ic, section, 0, &pl_index, &pl_offset); 896 897 atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight); 898 899 init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL); 900 901 section_index = pl_index; 902 903 do { 904 size_t this_step; 905 struct page *src_pages[2]; 906 struct page *dst_page; 907 908 while (unlikely(pl_index == section_index)) { 909 unsigned int dummy; 910 911 if (likely(encrypt)) 912 rw_section_mac(ic, section, true); 913 section++; 914 n_sections--; 915 if (!n_sections) 916 break; 917 page_list_location(ic, section, 0, §ion_index, &dummy); 918 } 919 920 this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset); 921 dst_page = target_pl[pl_index].page; 922 src_pages[0] = source_pl[pl_index].page; 923 src_pages[1] = ic->journal_xor[pl_index].page; 924 925 async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit); 926 927 pl_index++; 928 pl_offset = 0; 929 n_bytes -= this_step; 930 } while (n_bytes); 931 932 BUG_ON(n_sections); 933 934 async_tx_issue_pending_all(); 935 } 936 937 static void complete_journal_encrypt(void *data, int err) 938 { 939 struct journal_completion *comp = data; 940 941 if (unlikely(err)) { 942 if (likely(err == -EINPROGRESS)) { 943 complete(&comp->ic->crypto_backoff); 944 return; 945 } 946 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err); 947 } 948 complete_journal_op(comp); 949 } 950 951 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp) 952 { 953 int r; 954 955 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 956 complete_journal_encrypt, comp); 957 if (likely(encrypt)) 958 r = crypto_skcipher_encrypt(req); 959 else 960 r = crypto_skcipher_decrypt(req); 961 if (likely(!r)) 962 return false; 963 if (likely(r == -EINPROGRESS)) 964 return true; 965 if (likely(r == -EBUSY)) { 966 wait_for_completion(&comp->ic->crypto_backoff); 967 reinit_completion(&comp->ic->crypto_backoff); 968 return true; 969 } 970 dm_integrity_io_error(comp->ic, "encrypt", r); 971 return false; 972 } 973 974 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, 975 unsigned int n_sections, struct journal_completion *comp) 976 { 977 struct scatterlist **source_sg; 978 struct scatterlist **target_sg; 979 980 atomic_add(2, &comp->in_flight); 981 982 if (likely(encrypt)) { 983 source_sg = ic->journal_scatterlist; 984 target_sg = ic->journal_io_scatterlist; 985 } else { 986 source_sg = ic->journal_io_scatterlist; 987 target_sg = ic->journal_scatterlist; 988 } 989 990 do { 991 struct skcipher_request *req; 992 unsigned int ivsize; 993 char *iv; 994 995 if (likely(encrypt)) 996 rw_section_mac(ic, section, true); 997 998 req = ic->sk_requests[section]; 999 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); 1000 iv = req->iv; 1001 1002 memcpy(iv, iv + ivsize, ivsize); 1003 1004 req->src = source_sg[section]; 1005 req->dst = target_sg[section]; 1006 1007 if (unlikely(do_crypt(encrypt, req, comp))) 1008 atomic_inc(&comp->in_flight); 1009 1010 section++; 1011 n_sections--; 1012 } while (n_sections); 1013 1014 atomic_dec(&comp->in_flight); 1015 complete_journal_op(comp); 1016 } 1017 1018 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, 1019 unsigned int n_sections, struct journal_completion *comp) 1020 { 1021 if (ic->journal_xor) 1022 return xor_journal(ic, encrypt, section, n_sections, comp); 1023 else 1024 return crypt_journal(ic, encrypt, section, n_sections, comp); 1025 } 1026 1027 static void complete_journal_io(unsigned long error, void *context) 1028 { 1029 struct journal_completion *comp = context; 1030 1031 if (unlikely(error != 0)) 1032 dm_integrity_io_error(comp->ic, "writing journal", -EIO); 1033 complete_journal_op(comp); 1034 } 1035 1036 static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, 1037 unsigned int sector, unsigned int n_sectors, 1038 struct journal_completion *comp) 1039 { 1040 struct dm_io_request io_req; 1041 struct dm_io_region io_loc; 1042 unsigned int pl_index, pl_offset; 1043 int r; 1044 1045 if (unlikely(dm_integrity_failed(ic))) { 1046 if (comp) 1047 complete_journal_io(-1UL, comp); 1048 return; 1049 } 1050 1051 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 1052 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 1053 1054 io_req.bi_opf = opf; 1055 io_req.mem.type = DM_IO_PAGE_LIST; 1056 if (ic->journal_io) 1057 io_req.mem.ptr.pl = &ic->journal_io[pl_index]; 1058 else 1059 io_req.mem.ptr.pl = &ic->journal[pl_index]; 1060 io_req.mem.offset = pl_offset; 1061 if (likely(comp != NULL)) { 1062 io_req.notify.fn = complete_journal_io; 1063 io_req.notify.context = comp; 1064 } else { 1065 io_req.notify.fn = NULL; 1066 } 1067 io_req.client = ic->io; 1068 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; 1069 io_loc.sector = ic->start + SB_SECTORS + sector; 1070 io_loc.count = n_sectors; 1071 1072 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 1073 if (unlikely(r)) { 1074 dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? 1075 "reading journal" : "writing journal", r); 1076 if (comp) { 1077 WARN_ONCE(1, "asynchronous dm_io failed: %d", r); 1078 complete_journal_io(-1UL, comp); 1079 } 1080 } 1081 } 1082 1083 static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf, 1084 unsigned int section, unsigned int n_sections, 1085 struct journal_completion *comp) 1086 { 1087 unsigned int sector, n_sectors; 1088 1089 sector = section * ic->journal_section_sectors; 1090 n_sectors = n_sections * ic->journal_section_sectors; 1091 1092 rw_journal_sectors(ic, opf, sector, n_sectors, comp); 1093 } 1094 1095 static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit_sections) 1096 { 1097 struct journal_completion io_comp; 1098 struct journal_completion crypt_comp_1; 1099 struct journal_completion crypt_comp_2; 1100 unsigned int i; 1101 1102 io_comp.ic = ic; 1103 init_completion(&io_comp.comp); 1104 1105 if (commit_start + commit_sections <= ic->journal_sections) { 1106 io_comp.in_flight = (atomic_t)ATOMIC_INIT(1); 1107 if (ic->journal_io) { 1108 crypt_comp_1.ic = ic; 1109 init_completion(&crypt_comp_1.comp); 1110 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 1111 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1); 1112 wait_for_completion_io(&crypt_comp_1.comp); 1113 } else { 1114 for (i = 0; i < commit_sections; i++) 1115 rw_section_mac(ic, commit_start + i, true); 1116 } 1117 rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start, 1118 commit_sections, &io_comp); 1119 } else { 1120 unsigned int to_end; 1121 1122 io_comp.in_flight = (atomic_t)ATOMIC_INIT(2); 1123 to_end = ic->journal_sections - commit_start; 1124 if (ic->journal_io) { 1125 crypt_comp_1.ic = ic; 1126 init_completion(&crypt_comp_1.comp); 1127 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 1128 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); 1129 if (try_wait_for_completion(&crypt_comp_1.comp)) { 1130 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 1131 commit_start, to_end, &io_comp); 1132 reinit_completion(&crypt_comp_1.comp); 1133 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 1134 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); 1135 wait_for_completion_io(&crypt_comp_1.comp); 1136 } else { 1137 crypt_comp_2.ic = ic; 1138 init_completion(&crypt_comp_2.comp); 1139 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0); 1140 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); 1141 wait_for_completion_io(&crypt_comp_1.comp); 1142 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); 1143 wait_for_completion_io(&crypt_comp_2.comp); 1144 } 1145 } else { 1146 for (i = 0; i < to_end; i++) 1147 rw_section_mac(ic, commit_start + i, true); 1148 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); 1149 for (i = 0; i < commit_sections - to_end; i++) 1150 rw_section_mac(ic, i, true); 1151 } 1152 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp); 1153 } 1154 1155 wait_for_completion_io(&io_comp.comp); 1156 } 1157 1158 static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, 1159 unsigned int n_sectors, sector_t target, io_notify_fn fn, void *data) 1160 { 1161 struct dm_io_request io_req; 1162 struct dm_io_region io_loc; 1163 int r; 1164 unsigned int sector, pl_index, pl_offset; 1165 1166 BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1)); 1167 1168 if (unlikely(dm_integrity_failed(ic))) { 1169 fn(-1UL, data); 1170 return; 1171 } 1172 1173 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset; 1174 1175 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 1176 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 1177 1178 io_req.bi_opf = REQ_OP_WRITE; 1179 io_req.mem.type = DM_IO_PAGE_LIST; 1180 io_req.mem.ptr.pl = &ic->journal[pl_index]; 1181 io_req.mem.offset = pl_offset; 1182 io_req.notify.fn = fn; 1183 io_req.notify.context = data; 1184 io_req.client = ic->io; 1185 io_loc.bdev = ic->dev->bdev; 1186 io_loc.sector = target; 1187 io_loc.count = n_sectors; 1188 1189 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 1190 if (unlikely(r)) { 1191 WARN_ONCE(1, "asynchronous dm_io failed: %d", r); 1192 fn(-1UL, data); 1193 } 1194 } 1195 1196 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2) 1197 { 1198 return range1->logical_sector < range2->logical_sector + range2->n_sectors && 1199 range1->logical_sector + range1->n_sectors > range2->logical_sector; 1200 } 1201 1202 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting) 1203 { 1204 struct rb_node **n = &ic->in_progress.rb_node; 1205 struct rb_node *parent; 1206 1207 BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned int)(ic->sectors_per_block - 1)); 1208 1209 if (likely(check_waiting)) { 1210 struct dm_integrity_range *range; 1211 1212 list_for_each_entry(range, &ic->wait_list, wait_entry) { 1213 if (unlikely(ranges_overlap(range, new_range))) 1214 return false; 1215 } 1216 } 1217 1218 parent = NULL; 1219 1220 while (*n) { 1221 struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node); 1222 1223 parent = *n; 1224 if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) 1225 n = &range->node.rb_left; 1226 else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) 1227 n = &range->node.rb_right; 1228 else 1229 return false; 1230 } 1231 1232 rb_link_node(&new_range->node, parent, n); 1233 rb_insert_color(&new_range->node, &ic->in_progress); 1234 1235 return true; 1236 } 1237 1238 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) 1239 { 1240 rb_erase(&range->node, &ic->in_progress); 1241 while (unlikely(!list_empty(&ic->wait_list))) { 1242 struct dm_integrity_range *last_range = 1243 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); 1244 struct task_struct *last_range_task; 1245 1246 last_range_task = last_range->task; 1247 list_del(&last_range->wait_entry); 1248 if (!add_new_range(ic, last_range, false)) { 1249 last_range->task = last_range_task; 1250 list_add(&last_range->wait_entry, &ic->wait_list); 1251 break; 1252 } 1253 last_range->waiting = false; 1254 wake_up_process(last_range_task); 1255 } 1256 } 1257 1258 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) 1259 { 1260 unsigned long flags; 1261 1262 spin_lock_irqsave(&ic->endio_wait.lock, flags); 1263 remove_range_unlocked(ic, range); 1264 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); 1265 } 1266 1267 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) 1268 { 1269 new_range->waiting = true; 1270 list_add_tail(&new_range->wait_entry, &ic->wait_list); 1271 new_range->task = current; 1272 do { 1273 __set_current_state(TASK_UNINTERRUPTIBLE); 1274 spin_unlock_irq(&ic->endio_wait.lock); 1275 io_schedule(); 1276 spin_lock_irq(&ic->endio_wait.lock); 1277 } while (unlikely(new_range->waiting)); 1278 } 1279 1280 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) 1281 { 1282 if (unlikely(!add_new_range(ic, new_range, true))) 1283 wait_and_add_new_range(ic, new_range); 1284 } 1285 1286 static void init_journal_node(struct journal_node *node) 1287 { 1288 RB_CLEAR_NODE(&node->node); 1289 node->sector = (sector_t)-1; 1290 } 1291 1292 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector) 1293 { 1294 struct rb_node **link; 1295 struct rb_node *parent; 1296 1297 node->sector = sector; 1298 BUG_ON(!RB_EMPTY_NODE(&node->node)); 1299 1300 link = &ic->journal_tree_root.rb_node; 1301 parent = NULL; 1302 1303 while (*link) { 1304 struct journal_node *j; 1305 1306 parent = *link; 1307 j = container_of(parent, struct journal_node, node); 1308 if (sector < j->sector) 1309 link = &j->node.rb_left; 1310 else 1311 link = &j->node.rb_right; 1312 } 1313 1314 rb_link_node(&node->node, parent, link); 1315 rb_insert_color(&node->node, &ic->journal_tree_root); 1316 } 1317 1318 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node) 1319 { 1320 BUG_ON(RB_EMPTY_NODE(&node->node)); 1321 rb_erase(&node->node, &ic->journal_tree_root); 1322 init_journal_node(node); 1323 } 1324 1325 #define NOT_FOUND (-1U) 1326 1327 static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) 1328 { 1329 struct rb_node *n = ic->journal_tree_root.rb_node; 1330 unsigned int found = NOT_FOUND; 1331 1332 *next_sector = (sector_t)-1; 1333 while (n) { 1334 struct journal_node *j = container_of(n, struct journal_node, node); 1335 1336 if (sector == j->sector) 1337 found = j - ic->journal_tree; 1338 1339 if (sector < j->sector) { 1340 *next_sector = j->sector; 1341 n = j->node.rb_left; 1342 } else 1343 n = j->node.rb_right; 1344 } 1345 1346 return found; 1347 } 1348 1349 static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector) 1350 { 1351 struct journal_node *node, *next_node; 1352 struct rb_node *next; 1353 1354 if (unlikely(pos >= ic->journal_entries)) 1355 return false; 1356 node = &ic->journal_tree[pos]; 1357 if (unlikely(RB_EMPTY_NODE(&node->node))) 1358 return false; 1359 if (unlikely(node->sector != sector)) 1360 return false; 1361 1362 next = rb_next(&node->node); 1363 if (unlikely(!next)) 1364 return true; 1365 1366 next_node = container_of(next, struct journal_node, node); 1367 return next_node->sector != sector; 1368 } 1369 1370 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node) 1371 { 1372 struct rb_node *next; 1373 struct journal_node *next_node; 1374 unsigned int next_section; 1375 1376 BUG_ON(RB_EMPTY_NODE(&node->node)); 1377 1378 next = rb_next(&node->node); 1379 if (unlikely(!next)) 1380 return false; 1381 1382 next_node = container_of(next, struct journal_node, node); 1383 1384 if (next_node->sector != node->sector) 1385 return false; 1386 1387 next_section = (unsigned int)(next_node - ic->journal_tree) / ic->journal_section_entries; 1388 if (next_section >= ic->committed_section && 1389 next_section < ic->committed_section + ic->n_committed_sections) 1390 return true; 1391 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections) 1392 return true; 1393 1394 return false; 1395 } 1396 1397 #define TAG_READ 0 1398 #define TAG_WRITE 1 1399 #define TAG_CMP 2 1400 1401 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block, 1402 unsigned int *metadata_offset, unsigned int total_size, int op) 1403 { 1404 #define MAY_BE_FILLER 1 1405 #define MAY_BE_HASH 2 1406 unsigned int hash_offset = 0; 1407 unsigned int may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); 1408 1409 do { 1410 unsigned char *data, *dp; 1411 struct dm_buffer *b; 1412 unsigned int to_copy; 1413 int r; 1414 1415 r = dm_integrity_failed(ic); 1416 if (unlikely(r)) 1417 return r; 1418 1419 data = dm_bufio_read(ic->bufio, *metadata_block, &b); 1420 if (IS_ERR(data)) 1421 return PTR_ERR(data); 1422 1423 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size); 1424 dp = data + *metadata_offset; 1425 if (op == TAG_READ) { 1426 memcpy(tag, dp, to_copy); 1427 } else if (op == TAG_WRITE) { 1428 if (memcmp(dp, tag, to_copy)) { 1429 memcpy(dp, tag, to_copy); 1430 dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy); 1431 } 1432 } else { 1433 /* e.g.: op == TAG_CMP */ 1434 1435 if (likely(is_power_of_2(ic->tag_size))) { 1436 if (unlikely(memcmp(dp, tag, to_copy))) 1437 if (unlikely(!ic->discard) || 1438 unlikely(memchr_inv(dp, DISCARD_FILLER, to_copy) != NULL)) { 1439 goto thorough_test; 1440 } 1441 } else { 1442 unsigned int i, ts; 1443 thorough_test: 1444 ts = total_size; 1445 1446 for (i = 0; i < to_copy; i++, ts--) { 1447 if (unlikely(dp[i] != tag[i])) 1448 may_be &= ~MAY_BE_HASH; 1449 if (likely(dp[i] != DISCARD_FILLER)) 1450 may_be &= ~MAY_BE_FILLER; 1451 hash_offset++; 1452 if (unlikely(hash_offset == ic->tag_size)) { 1453 if (unlikely(!may_be)) { 1454 dm_bufio_release(b); 1455 return ts; 1456 } 1457 hash_offset = 0; 1458 may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0); 1459 } 1460 } 1461 } 1462 } 1463 dm_bufio_release(b); 1464 1465 tag += to_copy; 1466 *metadata_offset += to_copy; 1467 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { 1468 (*metadata_block)++; 1469 *metadata_offset = 0; 1470 } 1471 1472 if (unlikely(!is_power_of_2(ic->tag_size))) 1473 hash_offset = (hash_offset + to_copy) % ic->tag_size; 1474 1475 total_size -= to_copy; 1476 } while (unlikely(total_size)); 1477 1478 return 0; 1479 #undef MAY_BE_FILLER 1480 #undef MAY_BE_HASH 1481 } 1482 1483 struct flush_request { 1484 struct dm_io_request io_req; 1485 struct dm_io_region io_reg; 1486 struct dm_integrity_c *ic; 1487 struct completion comp; 1488 }; 1489 1490 static void flush_notify(unsigned long error, void *fr_) 1491 { 1492 struct flush_request *fr = fr_; 1493 1494 if (unlikely(error != 0)) 1495 dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO); 1496 complete(&fr->comp); 1497 } 1498 1499 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data) 1500 { 1501 int r; 1502 struct flush_request fr; 1503 1504 if (!ic->meta_dev) 1505 flush_data = false; 1506 if (flush_data) { 1507 fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; 1508 fr.io_req.mem.type = DM_IO_KMEM; 1509 fr.io_req.mem.ptr.addr = NULL; 1510 fr.io_req.notify.fn = flush_notify; 1511 fr.io_req.notify.context = &fr; 1512 fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio); 1513 fr.io_reg.bdev = ic->dev->bdev; 1514 fr.io_reg.sector = 0; 1515 fr.io_reg.count = 0; 1516 fr.ic = ic; 1517 init_completion(&fr.comp); 1518 r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); 1519 BUG_ON(r); 1520 } 1521 1522 r = dm_bufio_write_dirty_buffers(ic->bufio); 1523 if (unlikely(r)) 1524 dm_integrity_io_error(ic, "writing tags", r); 1525 1526 if (flush_data) 1527 wait_for_completion(&fr.comp); 1528 } 1529 1530 static void sleep_on_endio_wait(struct dm_integrity_c *ic) 1531 { 1532 DECLARE_WAITQUEUE(wait, current); 1533 1534 __add_wait_queue(&ic->endio_wait, &wait); 1535 __set_current_state(TASK_UNINTERRUPTIBLE); 1536 spin_unlock_irq(&ic->endio_wait.lock); 1537 io_schedule(); 1538 spin_lock_irq(&ic->endio_wait.lock); 1539 __remove_wait_queue(&ic->endio_wait, &wait); 1540 } 1541 1542 static void autocommit_fn(struct timer_list *t) 1543 { 1544 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer); 1545 1546 if (likely(!dm_integrity_failed(ic))) 1547 queue_work(ic->commit_wq, &ic->commit_work); 1548 } 1549 1550 static void schedule_autocommit(struct dm_integrity_c *ic) 1551 { 1552 if (!timer_pending(&ic->autocommit_timer)) 1553 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies); 1554 } 1555 1556 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) 1557 { 1558 struct bio *bio; 1559 unsigned long flags; 1560 1561 spin_lock_irqsave(&ic->endio_wait.lock, flags); 1562 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1563 bio_list_add(&ic->flush_bio_list, bio); 1564 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); 1565 1566 queue_work(ic->commit_wq, &ic->commit_work); 1567 } 1568 1569 static void do_endio(struct dm_integrity_c *ic, struct bio *bio) 1570 { 1571 int r; 1572 1573 r = dm_integrity_failed(ic); 1574 if (unlikely(r) && !bio->bi_status) 1575 bio->bi_status = errno_to_blk_status(r); 1576 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { 1577 unsigned long flags; 1578 1579 spin_lock_irqsave(&ic->endio_wait.lock, flags); 1580 bio_list_add(&ic->synchronous_bios, bio); 1581 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); 1582 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); 1583 return; 1584 } 1585 bio_endio(bio); 1586 } 1587 1588 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) 1589 { 1590 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1591 1592 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) 1593 submit_flush_bio(ic, dio); 1594 else 1595 do_endio(ic, bio); 1596 } 1597 1598 static void dec_in_flight(struct dm_integrity_io *dio) 1599 { 1600 if (atomic_dec_and_test(&dio->in_flight)) { 1601 struct dm_integrity_c *ic = dio->ic; 1602 struct bio *bio; 1603 1604 remove_range(ic, &dio->range); 1605 1606 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) 1607 schedule_autocommit(ic); 1608 1609 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1610 if (unlikely(dio->bi_status) && !bio->bi_status) 1611 bio->bi_status = dio->bi_status; 1612 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { 1613 dio->range.logical_sector += dio->range.n_sectors; 1614 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); 1615 INIT_WORK(&dio->work, integrity_bio_wait); 1616 queue_work(ic->offload_wq, &dio->work); 1617 return; 1618 } 1619 do_endio_flush(ic, dio); 1620 } 1621 } 1622 1623 static void integrity_end_io(struct bio *bio) 1624 { 1625 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 1626 1627 dm_bio_restore(&dio->bio_details, bio); 1628 if (bio->bi_integrity) 1629 bio->bi_opf |= REQ_INTEGRITY; 1630 1631 if (dio->completion) 1632 complete(dio->completion); 1633 1634 dec_in_flight(dio); 1635 } 1636 1637 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, 1638 const char *data, char *result) 1639 { 1640 __le64 sector_le = cpu_to_le64(sector); 1641 SHASH_DESC_ON_STACK(req, ic->internal_hash); 1642 int r; 1643 unsigned int digest_size; 1644 1645 req->tfm = ic->internal_hash; 1646 1647 r = crypto_shash_init(req); 1648 if (unlikely(r < 0)) { 1649 dm_integrity_io_error(ic, "crypto_shash_init", r); 1650 goto failed; 1651 } 1652 1653 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { 1654 r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE); 1655 if (unlikely(r < 0)) { 1656 dm_integrity_io_error(ic, "crypto_shash_update", r); 1657 goto failed; 1658 } 1659 } 1660 1661 r = crypto_shash_update(req, (const __u8 *)§or_le, sizeof(sector_le)); 1662 if (unlikely(r < 0)) { 1663 dm_integrity_io_error(ic, "crypto_shash_update", r); 1664 goto failed; 1665 } 1666 1667 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT); 1668 if (unlikely(r < 0)) { 1669 dm_integrity_io_error(ic, "crypto_shash_update", r); 1670 goto failed; 1671 } 1672 1673 r = crypto_shash_final(req, result); 1674 if (unlikely(r < 0)) { 1675 dm_integrity_io_error(ic, "crypto_shash_final", r); 1676 goto failed; 1677 } 1678 1679 digest_size = crypto_shash_digestsize(ic->internal_hash); 1680 if (unlikely(digest_size < ic->tag_size)) 1681 memset(result + digest_size, 0, ic->tag_size - digest_size); 1682 1683 return; 1684 1685 failed: 1686 /* this shouldn't happen anyway, the hash functions have no reason to fail */ 1687 get_random_bytes(result, ic->tag_size); 1688 } 1689 1690 static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) 1691 { 1692 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1693 struct dm_integrity_c *ic = dio->ic; 1694 struct bvec_iter iter; 1695 struct bio_vec bv; 1696 sector_t sector, logical_sector, area, offset; 1697 struct page *page; 1698 1699 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); 1700 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, 1701 &dio->metadata_offset); 1702 sector = get_data_sector(ic, area, offset); 1703 logical_sector = dio->range.logical_sector; 1704 1705 page = mempool_alloc(&ic->recheck_pool, GFP_NOIO); 1706 1707 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { 1708 unsigned pos = 0; 1709 1710 do { 1711 sector_t alignment; 1712 char *mem; 1713 char *buffer = page_to_virt(page); 1714 int r; 1715 struct dm_io_request io_req; 1716 struct dm_io_region io_loc; 1717 io_req.bi_opf = REQ_OP_READ; 1718 io_req.mem.type = DM_IO_KMEM; 1719 io_req.mem.ptr.addr = buffer; 1720 io_req.notify.fn = NULL; 1721 io_req.client = ic->io; 1722 io_loc.bdev = ic->dev->bdev; 1723 io_loc.sector = sector; 1724 io_loc.count = ic->sectors_per_block; 1725 1726 /* Align the bio to logical block size */ 1727 alignment = dio->range.logical_sector | bio_sectors(bio) | (PAGE_SIZE >> SECTOR_SHIFT); 1728 alignment &= -alignment; 1729 io_loc.sector = round_down(io_loc.sector, alignment); 1730 io_loc.count += sector - io_loc.sector; 1731 buffer += (sector - io_loc.sector) << SECTOR_SHIFT; 1732 io_loc.count = round_up(io_loc.count, alignment); 1733 1734 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 1735 if (unlikely(r)) { 1736 dio->bi_status = errno_to_blk_status(r); 1737 goto free_ret; 1738 } 1739 1740 integrity_sector_checksum(ic, logical_sector, buffer, checksum); 1741 r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, 1742 &dio->metadata_offset, ic->tag_size, TAG_CMP); 1743 if (r) { 1744 if (r > 0) { 1745 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", 1746 bio->bi_bdev, logical_sector); 1747 atomic64_inc(&ic->number_of_mismatches); 1748 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum", 1749 bio, logical_sector, 0); 1750 r = -EILSEQ; 1751 } 1752 dio->bi_status = errno_to_blk_status(r); 1753 goto free_ret; 1754 } 1755 1756 mem = bvec_kmap_local(&bv); 1757 memcpy(mem + pos, buffer, ic->sectors_per_block << SECTOR_SHIFT); 1758 kunmap_local(mem); 1759 1760 pos += ic->sectors_per_block << SECTOR_SHIFT; 1761 sector += ic->sectors_per_block; 1762 logical_sector += ic->sectors_per_block; 1763 } while (pos < bv.bv_len); 1764 } 1765 free_ret: 1766 mempool_free(page, &ic->recheck_pool); 1767 } 1768 1769 static void integrity_metadata(struct work_struct *w) 1770 { 1771 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 1772 struct dm_integrity_c *ic = dio->ic; 1773 1774 int r; 1775 1776 if (ic->internal_hash) { 1777 struct bvec_iter iter; 1778 struct bio_vec bv; 1779 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash); 1780 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1781 char *checksums; 1782 unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; 1783 char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; 1784 sector_t sector; 1785 unsigned int sectors_to_process; 1786 1787 if (unlikely(ic->mode == 'R')) 1788 goto skip_io; 1789 1790 if (likely(dio->op != REQ_OP_DISCARD)) 1791 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space, 1792 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); 1793 else 1794 checksums = kmalloc(PAGE_SIZE, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); 1795 if (!checksums) { 1796 checksums = checksums_onstack; 1797 if (WARN_ON(extra_space && 1798 digest_size > sizeof(checksums_onstack))) { 1799 r = -EINVAL; 1800 goto error; 1801 } 1802 } 1803 1804 if (unlikely(dio->op == REQ_OP_DISCARD)) { 1805 unsigned int bi_size = dio->bio_details.bi_iter.bi_size; 1806 unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE; 1807 unsigned int max_blocks = max_size / ic->tag_size; 1808 1809 memset(checksums, DISCARD_FILLER, max_size); 1810 1811 while (bi_size) { 1812 unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); 1813 1814 this_step_blocks = min(this_step_blocks, max_blocks); 1815 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, 1816 this_step_blocks * ic->tag_size, TAG_WRITE); 1817 if (unlikely(r)) { 1818 if (likely(checksums != checksums_onstack)) 1819 kfree(checksums); 1820 goto error; 1821 } 1822 1823 bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); 1824 } 1825 1826 if (likely(checksums != checksums_onstack)) 1827 kfree(checksums); 1828 goto skip_io; 1829 } 1830 1831 sector = dio->range.logical_sector; 1832 sectors_to_process = dio->range.n_sectors; 1833 1834 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { 1835 struct bio_vec bv_copy = bv; 1836 unsigned int pos; 1837 char *mem, *checksums_ptr; 1838 1839 again: 1840 mem = bvec_kmap_local(&bv_copy); 1841 pos = 0; 1842 checksums_ptr = checksums; 1843 do { 1844 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); 1845 checksums_ptr += ic->tag_size; 1846 sectors_to_process -= ic->sectors_per_block; 1847 pos += ic->sectors_per_block << SECTOR_SHIFT; 1848 sector += ic->sectors_per_block; 1849 } while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack); 1850 kunmap_local(mem); 1851 1852 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, 1853 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE); 1854 if (unlikely(r)) { 1855 if (likely(checksums != checksums_onstack)) 1856 kfree(checksums); 1857 if (r > 0) { 1858 integrity_recheck(dio, checksums_onstack); 1859 goto skip_io; 1860 } 1861 goto error; 1862 } 1863 1864 if (!sectors_to_process) 1865 break; 1866 1867 if (unlikely(pos < bv_copy.bv_len)) { 1868 bv_copy.bv_offset += pos; 1869 bv_copy.bv_len -= pos; 1870 goto again; 1871 } 1872 } 1873 1874 if (likely(checksums != checksums_onstack)) 1875 kfree(checksums); 1876 } else { 1877 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity; 1878 1879 if (bip) { 1880 struct bio_vec biv; 1881 struct bvec_iter iter; 1882 unsigned int data_to_process = dio->range.n_sectors; 1883 1884 sector_to_block(ic, data_to_process); 1885 data_to_process *= ic->tag_size; 1886 1887 bip_for_each_vec(biv, bip, iter) { 1888 unsigned char *tag; 1889 unsigned int this_len; 1890 1891 BUG_ON(PageHighMem(biv.bv_page)); 1892 tag = bvec_virt(&biv); 1893 this_len = min(biv.bv_len, data_to_process); 1894 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, 1895 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE); 1896 if (unlikely(r)) 1897 goto error; 1898 data_to_process -= this_len; 1899 if (!data_to_process) 1900 break; 1901 } 1902 } 1903 } 1904 skip_io: 1905 dec_in_flight(dio); 1906 return; 1907 error: 1908 dio->bi_status = errno_to_blk_status(r); 1909 dec_in_flight(dio); 1910 } 1911 1912 static inline bool dm_integrity_check_limits(struct dm_integrity_c *ic, sector_t logical_sector, struct bio *bio) 1913 { 1914 if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { 1915 DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx", 1916 logical_sector, bio_sectors(bio), 1917 ic->provided_data_sectors); 1918 return false; 1919 } 1920 if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) { 1921 DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x", 1922 ic->sectors_per_block, 1923 logical_sector, bio_sectors(bio)); 1924 return false; 1925 } 1926 if (ic->sectors_per_block > 1 && likely(bio_op(bio) != REQ_OP_DISCARD)) { 1927 struct bvec_iter iter; 1928 struct bio_vec bv; 1929 1930 bio_for_each_segment(bv, bio, iter) { 1931 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { 1932 DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary", 1933 bv.bv_offset, bv.bv_len, ic->sectors_per_block); 1934 return false; 1935 } 1936 } 1937 } 1938 return true; 1939 } 1940 1941 static int dm_integrity_map(struct dm_target *ti, struct bio *bio) 1942 { 1943 struct dm_integrity_c *ic = ti->private; 1944 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 1945 struct bio_integrity_payload *bip; 1946 1947 sector_t area, offset; 1948 1949 dio->ic = ic; 1950 dio->bi_status = 0; 1951 dio->op = bio_op(bio); 1952 1953 if (ic->mode == 'I') { 1954 bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector); 1955 dio->integrity_payload = NULL; 1956 dio->integrity_payload_from_mempool = false; 1957 dio->integrity_range_locked = false; 1958 return dm_integrity_map_inline(dio, true); 1959 } 1960 1961 if (unlikely(dio->op == REQ_OP_DISCARD)) { 1962 if (ti->max_io_len) { 1963 sector_t sec = dm_target_offset(ti, bio->bi_iter.bi_sector); 1964 unsigned int log2_max_io_len = __fls(ti->max_io_len); 1965 sector_t start_boundary = sec >> log2_max_io_len; 1966 sector_t end_boundary = (sec + bio_sectors(bio) - 1) >> log2_max_io_len; 1967 1968 if (start_boundary < end_boundary) { 1969 sector_t len = ti->max_io_len - (sec & (ti->max_io_len - 1)); 1970 1971 dm_accept_partial_bio(bio, len); 1972 } 1973 } 1974 } 1975 1976 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1977 submit_flush_bio(ic, dio); 1978 return DM_MAPIO_SUBMITTED; 1979 } 1980 1981 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 1982 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA; 1983 if (unlikely(dio->fua)) { 1984 /* 1985 * Don't pass down the FUA flag because we have to flush 1986 * disk cache anyway. 1987 */ 1988 bio->bi_opf &= ~REQ_FUA; 1989 } 1990 if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio))) 1991 return DM_MAPIO_KILL; 1992 1993 bip = bio_integrity(bio); 1994 if (!ic->internal_hash) { 1995 if (bip) { 1996 unsigned int wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; 1997 1998 if (ic->log2_tag_size >= 0) 1999 wanted_tag_size <<= ic->log2_tag_size; 2000 else 2001 wanted_tag_size *= ic->tag_size; 2002 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) { 2003 DMERR("Invalid integrity data size %u, expected %u", 2004 bip->bip_iter.bi_size, wanted_tag_size); 2005 return DM_MAPIO_KILL; 2006 } 2007 } 2008 } else { 2009 if (unlikely(bip != NULL)) { 2010 DMERR("Unexpected integrity data when using internal hash"); 2011 return DM_MAPIO_KILL; 2012 } 2013 } 2014 2015 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) 2016 return DM_MAPIO_KILL; 2017 2018 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); 2019 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); 2020 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset); 2021 2022 dm_integrity_map_continue(dio, true); 2023 return DM_MAPIO_SUBMITTED; 2024 } 2025 2026 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, 2027 unsigned int journal_section, unsigned int journal_entry) 2028 { 2029 struct dm_integrity_c *ic = dio->ic; 2030 sector_t logical_sector; 2031 unsigned int n_sectors; 2032 2033 logical_sector = dio->range.logical_sector; 2034 n_sectors = dio->range.n_sectors; 2035 do { 2036 struct bio_vec bv = bio_iovec(bio); 2037 char *mem; 2038 2039 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors)) 2040 bv.bv_len = n_sectors << SECTOR_SHIFT; 2041 n_sectors -= bv.bv_len >> SECTOR_SHIFT; 2042 bio_advance_iter(bio, &bio->bi_iter, bv.bv_len); 2043 retry_kmap: 2044 mem = kmap_local_page(bv.bv_page); 2045 if (likely(dio->op == REQ_OP_WRITE)) 2046 flush_dcache_page(bv.bv_page); 2047 2048 do { 2049 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry); 2050 2051 if (unlikely(dio->op == REQ_OP_READ)) { 2052 struct journal_sector *js; 2053 char *mem_ptr; 2054 unsigned int s; 2055 2056 if (unlikely(journal_entry_is_inprogress(je))) { 2057 flush_dcache_page(bv.bv_page); 2058 kunmap_local(mem); 2059 2060 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); 2061 goto retry_kmap; 2062 } 2063 smp_rmb(); 2064 BUG_ON(journal_entry_get_sector(je) != logical_sector); 2065 js = access_journal_data(ic, journal_section, journal_entry); 2066 mem_ptr = mem + bv.bv_offset; 2067 s = 0; 2068 do { 2069 memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA); 2070 *(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s]; 2071 js++; 2072 mem_ptr += 1 << SECTOR_SHIFT; 2073 } while (++s < ic->sectors_per_block); 2074 #ifdef INTERNAL_VERIFY 2075 if (ic->internal_hash) { 2076 char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; 2077 2078 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); 2079 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { 2080 DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx", 2081 logical_sector); 2082 dm_audit_log_bio(DM_MSG_PREFIX, "journal-checksum", 2083 bio, logical_sector, 0); 2084 } 2085 } 2086 #endif 2087 } 2088 2089 if (!ic->internal_hash) { 2090 struct bio_integrity_payload *bip = bio_integrity(bio); 2091 unsigned int tag_todo = ic->tag_size; 2092 char *tag_ptr = journal_entry_tag(ic, je); 2093 2094 if (bip) { 2095 do { 2096 struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); 2097 unsigned int tag_now = min(biv.bv_len, tag_todo); 2098 char *tag_addr; 2099 2100 BUG_ON(PageHighMem(biv.bv_page)); 2101 tag_addr = bvec_virt(&biv); 2102 if (likely(dio->op == REQ_OP_WRITE)) 2103 memcpy(tag_ptr, tag_addr, tag_now); 2104 else 2105 memcpy(tag_addr, tag_ptr, tag_now); 2106 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now); 2107 tag_ptr += tag_now; 2108 tag_todo -= tag_now; 2109 } while (unlikely(tag_todo)); 2110 } else if (likely(dio->op == REQ_OP_WRITE)) 2111 memset(tag_ptr, 0, tag_todo); 2112 } 2113 2114 if (likely(dio->op == REQ_OP_WRITE)) { 2115 struct journal_sector *js; 2116 unsigned int s; 2117 2118 js = access_journal_data(ic, journal_section, journal_entry); 2119 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT); 2120 2121 s = 0; 2122 do { 2123 je->last_bytes[s] = js[s].commit_id; 2124 } while (++s < ic->sectors_per_block); 2125 2126 if (ic->internal_hash) { 2127 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash); 2128 2129 if (unlikely(digest_size > ic->tag_size)) { 2130 char checksums_onstack[HASH_MAX_DIGESTSIZE]; 2131 2132 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack); 2133 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); 2134 } else 2135 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je)); 2136 } 2137 2138 journal_entry_set_sector(je, logical_sector); 2139 } 2140 logical_sector += ic->sectors_per_block; 2141 2142 journal_entry++; 2143 if (unlikely(journal_entry == ic->journal_section_entries)) { 2144 journal_entry = 0; 2145 journal_section++; 2146 wraparound_section(ic, &journal_section); 2147 } 2148 2149 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT; 2150 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT); 2151 2152 if (unlikely(dio->op == REQ_OP_READ)) 2153 flush_dcache_page(bv.bv_page); 2154 kunmap_local(mem); 2155 } while (n_sectors); 2156 2157 if (likely(dio->op == REQ_OP_WRITE)) { 2158 smp_mb(); 2159 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) 2160 wake_up(&ic->copy_to_journal_wait); 2161 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) 2162 queue_work(ic->commit_wq, &ic->commit_work); 2163 else 2164 schedule_autocommit(ic); 2165 } else 2166 remove_range(ic, &dio->range); 2167 2168 if (unlikely(bio->bi_iter.bi_size)) { 2169 sector_t area, offset; 2170 2171 dio->range.logical_sector = logical_sector; 2172 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); 2173 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); 2174 return true; 2175 } 2176 2177 return false; 2178 } 2179 2180 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) 2181 { 2182 struct dm_integrity_c *ic = dio->ic; 2183 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2184 unsigned int journal_section, journal_entry; 2185 unsigned int journal_read_pos; 2186 sector_t recalc_sector; 2187 struct completion read_comp; 2188 bool discard_retried = false; 2189 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; 2190 2191 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') 2192 need_sync_io = true; 2193 2194 if (need_sync_io && from_map) { 2195 INIT_WORK(&dio->work, integrity_bio_wait); 2196 queue_work(ic->offload_wq, &dio->work); 2197 return; 2198 } 2199 2200 lock_retry: 2201 spin_lock_irq(&ic->endio_wait.lock); 2202 retry: 2203 if (unlikely(dm_integrity_failed(ic))) { 2204 spin_unlock_irq(&ic->endio_wait.lock); 2205 do_endio(ic, bio); 2206 return; 2207 } 2208 dio->range.n_sectors = bio_sectors(bio); 2209 journal_read_pos = NOT_FOUND; 2210 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { 2211 if (dio->op == REQ_OP_WRITE) { 2212 unsigned int next_entry, i, pos; 2213 unsigned int ws, we, range_sectors; 2214 2215 dio->range.n_sectors = min(dio->range.n_sectors, 2216 (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block); 2217 if (unlikely(!dio->range.n_sectors)) { 2218 if (from_map) 2219 goto offload_to_thread; 2220 sleep_on_endio_wait(ic); 2221 goto retry; 2222 } 2223 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; 2224 ic->free_sectors -= range_sectors; 2225 journal_section = ic->free_section; 2226 journal_entry = ic->free_section_entry; 2227 2228 next_entry = ic->free_section_entry + range_sectors; 2229 ic->free_section_entry = next_entry % ic->journal_section_entries; 2230 ic->free_section += next_entry / ic->journal_section_entries; 2231 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries; 2232 wraparound_section(ic, &ic->free_section); 2233 2234 pos = journal_section * ic->journal_section_entries + journal_entry; 2235 ws = journal_section; 2236 we = journal_entry; 2237 i = 0; 2238 do { 2239 struct journal_entry *je; 2240 2241 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); 2242 pos++; 2243 if (unlikely(pos >= ic->journal_entries)) 2244 pos = 0; 2245 2246 je = access_journal_entry(ic, ws, we); 2247 BUG_ON(!journal_entry_is_unused(je)); 2248 journal_entry_set_inprogress(je); 2249 we++; 2250 if (unlikely(we == ic->journal_section_entries)) { 2251 we = 0; 2252 ws++; 2253 wraparound_section(ic, &ws); 2254 } 2255 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); 2256 2257 spin_unlock_irq(&ic->endio_wait.lock); 2258 goto journal_read_write; 2259 } else { 2260 sector_t next_sector; 2261 2262 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); 2263 if (likely(journal_read_pos == NOT_FOUND)) { 2264 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) 2265 dio->range.n_sectors = next_sector - dio->range.logical_sector; 2266 } else { 2267 unsigned int i; 2268 unsigned int jp = journal_read_pos + 1; 2269 2270 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { 2271 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) 2272 break; 2273 } 2274 dio->range.n_sectors = i; 2275 } 2276 } 2277 } 2278 if (unlikely(!add_new_range(ic, &dio->range, true))) { 2279 /* 2280 * We must not sleep in the request routine because it could 2281 * stall bios on current->bio_list. 2282 * So, we offload the bio to a workqueue if we have to sleep. 2283 */ 2284 if (from_map) { 2285 offload_to_thread: 2286 spin_unlock_irq(&ic->endio_wait.lock); 2287 INIT_WORK(&dio->work, integrity_bio_wait); 2288 queue_work(ic->wait_wq, &dio->work); 2289 return; 2290 } 2291 if (journal_read_pos != NOT_FOUND) 2292 dio->range.n_sectors = ic->sectors_per_block; 2293 wait_and_add_new_range(ic, &dio->range); 2294 /* 2295 * wait_and_add_new_range drops the spinlock, so the journal 2296 * may have been changed arbitrarily. We need to recheck. 2297 * To simplify the code, we restrict I/O size to just one block. 2298 */ 2299 if (journal_read_pos != NOT_FOUND) { 2300 sector_t next_sector; 2301 unsigned int new_pos; 2302 2303 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); 2304 if (unlikely(new_pos != journal_read_pos)) { 2305 remove_range_unlocked(ic, &dio->range); 2306 goto retry; 2307 } 2308 } 2309 } 2310 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { 2311 sector_t next_sector; 2312 unsigned int new_pos; 2313 2314 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); 2315 if (unlikely(new_pos != NOT_FOUND) || 2316 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { 2317 remove_range_unlocked(ic, &dio->range); 2318 spin_unlock_irq(&ic->endio_wait.lock); 2319 queue_work(ic->commit_wq, &ic->commit_work); 2320 flush_workqueue(ic->commit_wq); 2321 queue_work(ic->writer_wq, &ic->writer_work); 2322 flush_workqueue(ic->writer_wq); 2323 discard_retried = true; 2324 goto lock_retry; 2325 } 2326 } 2327 recalc_sector = le64_to_cpu(ic->sb->recalc_sector); 2328 spin_unlock_irq(&ic->endio_wait.lock); 2329 2330 if (unlikely(journal_read_pos != NOT_FOUND)) { 2331 journal_section = journal_read_pos / ic->journal_section_entries; 2332 journal_entry = journal_read_pos % ic->journal_section_entries; 2333 goto journal_read_write; 2334 } 2335 2336 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { 2337 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, 2338 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { 2339 struct bitmap_block_status *bbs; 2340 2341 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); 2342 spin_lock(&bbs->bio_queue_lock); 2343 bio_list_add(&bbs->bio_queue, bio); 2344 spin_unlock(&bbs->bio_queue_lock); 2345 queue_work(ic->writer_wq, &bbs->work); 2346 return; 2347 } 2348 } 2349 2350 dio->in_flight = (atomic_t)ATOMIC_INIT(2); 2351 2352 if (need_sync_io) { 2353 init_completion(&read_comp); 2354 dio->completion = &read_comp; 2355 } else 2356 dio->completion = NULL; 2357 2358 dm_bio_record(&dio->bio_details, bio); 2359 bio_set_dev(bio, ic->dev->bdev); 2360 bio->bi_integrity = NULL; 2361 bio->bi_opf &= ~REQ_INTEGRITY; 2362 bio->bi_end_io = integrity_end_io; 2363 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; 2364 2365 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { 2366 integrity_metadata(&dio->work); 2367 dm_integrity_flush_buffers(ic, false); 2368 2369 dio->in_flight = (atomic_t)ATOMIC_INIT(1); 2370 dio->completion = NULL; 2371 2372 submit_bio_noacct(bio); 2373 2374 return; 2375 } 2376 2377 submit_bio_noacct(bio); 2378 2379 if (need_sync_io) { 2380 wait_for_completion_io(&read_comp); 2381 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 2382 dio->range.logical_sector + dio->range.n_sectors > recalc_sector) 2383 goto skip_check; 2384 if (ic->mode == 'B') { 2385 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, 2386 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) 2387 goto skip_check; 2388 } 2389 2390 if (likely(!bio->bi_status)) 2391 integrity_metadata(&dio->work); 2392 else 2393 skip_check: 2394 dec_in_flight(dio); 2395 } else { 2396 INIT_WORK(&dio->work, integrity_metadata); 2397 queue_work(ic->metadata_wq, &dio->work); 2398 } 2399 2400 return; 2401 2402 journal_read_write: 2403 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) 2404 goto lock_retry; 2405 2406 do_endio_flush(ic, dio); 2407 } 2408 2409 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) 2410 { 2411 struct dm_integrity_c *ic = dio->ic; 2412 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2413 struct bio_integrity_payload *bip; 2414 unsigned ret; 2415 sector_t recalc_sector; 2416 2417 if (unlikely(bio_integrity(bio))) { 2418 bio->bi_status = BLK_STS_NOTSUPP; 2419 bio_endio(bio); 2420 return DM_MAPIO_SUBMITTED; 2421 } 2422 2423 bio_set_dev(bio, ic->dev->bdev); 2424 if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) 2425 return DM_MAPIO_REMAPPED; 2426 2427 retry: 2428 if (!dio->integrity_payload) { 2429 unsigned digest_size, extra_size; 2430 dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block); 2431 digest_size = crypto_shash_digestsize(ic->internal_hash); 2432 extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; 2433 dio->payload_len += extra_size; 2434 dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 2435 if (unlikely(!dio->integrity_payload)) { 2436 const unsigned x_size = PAGE_SIZE << 1; 2437 if (dio->payload_len > x_size) { 2438 unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block; 2439 if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) { 2440 bio->bi_status = BLK_STS_NOTSUPP; 2441 bio_endio(bio); 2442 return DM_MAPIO_SUBMITTED; 2443 } 2444 dm_accept_partial_bio(bio, sectors); 2445 goto retry; 2446 } 2447 } 2448 } 2449 2450 dio->range.logical_sector = bio->bi_iter.bi_sector; 2451 dio->range.n_sectors = bio_sectors(bio); 2452 2453 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) 2454 goto skip_spinlock; 2455 #ifdef CONFIG_64BIT 2456 /* 2457 * On 64-bit CPUs we can optimize the lock away (so that it won't cause 2458 * cache line bouncing) and use acquire/release barriers instead. 2459 * 2460 * Paired with smp_store_release in integrity_recalc_inline. 2461 */ 2462 recalc_sector = le64_to_cpu(smp_load_acquire(&ic->sb->recalc_sector)); 2463 if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)) 2464 goto skip_spinlock; 2465 #endif 2466 spin_lock_irq(&ic->endio_wait.lock); 2467 recalc_sector = le64_to_cpu(ic->sb->recalc_sector); 2468 if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector) 2469 goto skip_unlock; 2470 if (unlikely(!add_new_range(ic, &dio->range, true))) { 2471 if (from_map) { 2472 spin_unlock_irq(&ic->endio_wait.lock); 2473 INIT_WORK(&dio->work, integrity_bio_wait); 2474 queue_work(ic->wait_wq, &dio->work); 2475 return DM_MAPIO_SUBMITTED; 2476 } 2477 wait_and_add_new_range(ic, &dio->range); 2478 } 2479 dio->integrity_range_locked = true; 2480 skip_unlock: 2481 spin_unlock_irq(&ic->endio_wait.lock); 2482 skip_spinlock: 2483 2484 if (unlikely(!dio->integrity_payload)) { 2485 dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO)); 2486 dio->integrity_payload_from_mempool = true; 2487 } 2488 2489 dio->bio_details.bi_iter = bio->bi_iter; 2490 2491 if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { 2492 return DM_MAPIO_KILL; 2493 } 2494 2495 bio->bi_iter.bi_sector += ic->start + SB_SECTORS; 2496 2497 bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 2498 if (IS_ERR(bip)) { 2499 bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); 2500 bio_endio(bio); 2501 return DM_MAPIO_SUBMITTED; 2502 } 2503 2504 if (dio->op == REQ_OP_WRITE) { 2505 unsigned pos = 0; 2506 while (dio->bio_details.bi_iter.bi_size) { 2507 struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); 2508 const char *mem = bvec_kmap_local(&bv); 2509 if (ic->tag_size < ic->tuple_size) 2510 memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size); 2511 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos); 2512 kunmap_local(mem); 2513 pos += ic->tuple_size; 2514 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); 2515 } 2516 } 2517 2518 ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload), 2519 dio->payload_len, offset_in_page(dio->integrity_payload)); 2520 if (unlikely(ret != dio->payload_len)) { 2521 bio->bi_status = BLK_STS_RESOURCE; 2522 bio_endio(bio); 2523 return DM_MAPIO_SUBMITTED; 2524 } 2525 2526 return DM_MAPIO_REMAPPED; 2527 } 2528 2529 static inline void dm_integrity_free_payload(struct dm_integrity_io *dio) 2530 { 2531 struct dm_integrity_c *ic = dio->ic; 2532 if (unlikely(dio->integrity_payload_from_mempool)) 2533 mempool_free(virt_to_page(dio->integrity_payload), &ic->recheck_pool); 2534 else 2535 kfree(dio->integrity_payload); 2536 dio->integrity_payload = NULL; 2537 dio->integrity_payload_from_mempool = false; 2538 } 2539 2540 static void dm_integrity_inline_recheck(struct work_struct *w) 2541 { 2542 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 2543 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2544 struct dm_integrity_c *ic = dio->ic; 2545 struct bio *outgoing_bio; 2546 void *outgoing_data; 2547 2548 dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO)); 2549 dio->integrity_payload_from_mempool = true; 2550 2551 outgoing_data = dio->integrity_payload + PAGE_SIZE; 2552 2553 while (dio->bio_details.bi_iter.bi_size) { 2554 char digest[HASH_MAX_DIGESTSIZE]; 2555 int r; 2556 struct bio_integrity_payload *bip; 2557 struct bio_vec bv; 2558 char *mem; 2559 2560 outgoing_bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recheck_bios); 2561 2562 r = bio_add_page(outgoing_bio, virt_to_page(outgoing_data), ic->sectors_per_block << SECTOR_SHIFT, 0); 2563 if (unlikely(r != (ic->sectors_per_block << SECTOR_SHIFT))) { 2564 bio_put(outgoing_bio); 2565 bio->bi_status = BLK_STS_RESOURCE; 2566 bio_endio(bio); 2567 return; 2568 } 2569 2570 bip = bio_integrity_alloc(outgoing_bio, GFP_NOIO, 1); 2571 if (IS_ERR(bip)) { 2572 bio_put(outgoing_bio); 2573 bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); 2574 bio_endio(bio); 2575 return; 2576 } 2577 2578 r = bio_integrity_add_page(outgoing_bio, virt_to_page(dio->integrity_payload), ic->tuple_size, 0); 2579 if (unlikely(r != ic->tuple_size)) { 2580 bio_put(outgoing_bio); 2581 bio->bi_status = BLK_STS_RESOURCE; 2582 bio_endio(bio); 2583 return; 2584 } 2585 2586 outgoing_bio->bi_iter.bi_sector = dio->bio_details.bi_iter.bi_sector + ic->start + SB_SECTORS; 2587 2588 r = submit_bio_wait(outgoing_bio); 2589 if (unlikely(r != 0)) { 2590 bio_put(outgoing_bio); 2591 bio->bi_status = errno_to_blk_status(r); 2592 bio_endio(bio); 2593 return; 2594 } 2595 bio_put(outgoing_bio); 2596 2597 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest); 2598 if (unlikely(memcmp(digest, dio->integrity_payload, min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) { 2599 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", 2600 ic->dev->bdev, dio->bio_details.bi_iter.bi_sector); 2601 atomic64_inc(&ic->number_of_mismatches); 2602 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum", 2603 bio, dio->bio_details.bi_iter.bi_sector, 0); 2604 2605 bio->bi_status = BLK_STS_PROTECTION; 2606 bio_endio(bio); 2607 return; 2608 } 2609 2610 bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); 2611 mem = bvec_kmap_local(&bv); 2612 memcpy(mem, outgoing_data, ic->sectors_per_block << SECTOR_SHIFT); 2613 kunmap_local(mem); 2614 2615 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); 2616 } 2617 2618 bio_endio(bio); 2619 } 2620 2621 static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) 2622 { 2623 struct dm_integrity_c *ic = ti->private; 2624 if (ic->mode == 'I') { 2625 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 2626 if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) { 2627 unsigned pos = 0; 2628 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 2629 unlikely(dio->integrity_range_locked)) 2630 goto skip_check; 2631 while (dio->bio_details.bi_iter.bi_size) { 2632 char digest[HASH_MAX_DIGESTSIZE]; 2633 struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); 2634 char *mem = bvec_kmap_local(&bv); 2635 //memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT); 2636 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, digest); 2637 if (unlikely(memcmp(digest, dio->integrity_payload + pos, 2638 min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) { 2639 kunmap_local(mem); 2640 dm_integrity_free_payload(dio); 2641 INIT_WORK(&dio->work, dm_integrity_inline_recheck); 2642 queue_work(ic->offload_wq, &dio->work); 2643 return DM_ENDIO_INCOMPLETE; 2644 } 2645 kunmap_local(mem); 2646 pos += ic->tuple_size; 2647 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); 2648 } 2649 } 2650 skip_check: 2651 dm_integrity_free_payload(dio); 2652 if (unlikely(dio->integrity_range_locked)) 2653 remove_range(ic, &dio->range); 2654 } 2655 return DM_ENDIO_DONE; 2656 } 2657 2658 static void integrity_bio_wait(struct work_struct *w) 2659 { 2660 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 2661 struct dm_integrity_c *ic = dio->ic; 2662 2663 if (ic->mode == 'I') { 2664 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 2665 int r = dm_integrity_map_inline(dio, false); 2666 switch (r) { 2667 case DM_MAPIO_KILL: 2668 bio->bi_status = BLK_STS_IOERR; 2669 fallthrough; 2670 case DM_MAPIO_REMAPPED: 2671 submit_bio_noacct(bio); 2672 fallthrough; 2673 case DM_MAPIO_SUBMITTED: 2674 return; 2675 default: 2676 BUG(); 2677 } 2678 } else { 2679 dm_integrity_map_continue(dio, false); 2680 } 2681 } 2682 2683 static void pad_uncommitted(struct dm_integrity_c *ic) 2684 { 2685 if (ic->free_section_entry) { 2686 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry; 2687 ic->free_section_entry = 0; 2688 ic->free_section++; 2689 wraparound_section(ic, &ic->free_section); 2690 ic->n_uncommitted_sections++; 2691 } 2692 if (WARN_ON(ic->journal_sections * ic->journal_section_entries != 2693 (ic->n_uncommitted_sections + ic->n_committed_sections) * 2694 ic->journal_section_entries + ic->free_sectors)) { 2695 DMCRIT("journal_sections %u, journal_section_entries %u, " 2696 "n_uncommitted_sections %u, n_committed_sections %u, " 2697 "journal_section_entries %u, free_sectors %u", 2698 ic->journal_sections, ic->journal_section_entries, 2699 ic->n_uncommitted_sections, ic->n_committed_sections, 2700 ic->journal_section_entries, ic->free_sectors); 2701 } 2702 } 2703 2704 static void integrity_commit(struct work_struct *w) 2705 { 2706 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work); 2707 unsigned int commit_start, commit_sections; 2708 unsigned int i, j, n; 2709 struct bio *flushes; 2710 2711 del_timer(&ic->autocommit_timer); 2712 2713 if (ic->mode == 'I') 2714 return; 2715 2716 spin_lock_irq(&ic->endio_wait.lock); 2717 flushes = bio_list_get(&ic->flush_bio_list); 2718 if (unlikely(ic->mode != 'J')) { 2719 spin_unlock_irq(&ic->endio_wait.lock); 2720 dm_integrity_flush_buffers(ic, true); 2721 goto release_flush_bios; 2722 } 2723 2724 pad_uncommitted(ic); 2725 commit_start = ic->uncommitted_section; 2726 commit_sections = ic->n_uncommitted_sections; 2727 spin_unlock_irq(&ic->endio_wait.lock); 2728 2729 if (!commit_sections) 2730 goto release_flush_bios; 2731 2732 ic->wrote_to_journal = true; 2733 2734 i = commit_start; 2735 for (n = 0; n < commit_sections; n++) { 2736 for (j = 0; j < ic->journal_section_entries; j++) { 2737 struct journal_entry *je; 2738 2739 je = access_journal_entry(ic, i, j); 2740 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); 2741 } 2742 for (j = 0; j < ic->journal_section_sectors; j++) { 2743 struct journal_sector *js; 2744 2745 js = access_journal(ic, i, j); 2746 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq); 2747 } 2748 i++; 2749 if (unlikely(i >= ic->journal_sections)) 2750 ic->commit_seq = next_commit_seq(ic->commit_seq); 2751 wraparound_section(ic, &i); 2752 } 2753 smp_rmb(); 2754 2755 write_journal(ic, commit_start, commit_sections); 2756 2757 spin_lock_irq(&ic->endio_wait.lock); 2758 ic->uncommitted_section += commit_sections; 2759 wraparound_section(ic, &ic->uncommitted_section); 2760 ic->n_uncommitted_sections -= commit_sections; 2761 ic->n_committed_sections += commit_sections; 2762 spin_unlock_irq(&ic->endio_wait.lock); 2763 2764 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) 2765 queue_work(ic->writer_wq, &ic->writer_work); 2766 2767 release_flush_bios: 2768 while (flushes) { 2769 struct bio *next = flushes->bi_next; 2770 2771 flushes->bi_next = NULL; 2772 do_endio(ic, flushes); 2773 flushes = next; 2774 } 2775 } 2776 2777 static void complete_copy_from_journal(unsigned long error, void *context) 2778 { 2779 struct journal_io *io = context; 2780 struct journal_completion *comp = io->comp; 2781 struct dm_integrity_c *ic = comp->ic; 2782 2783 remove_range(ic, &io->range); 2784 mempool_free(io, &ic->journal_io_mempool); 2785 if (unlikely(error != 0)) 2786 dm_integrity_io_error(ic, "copying from journal", -EIO); 2787 complete_journal_op(comp); 2788 } 2789 2790 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, 2791 struct journal_entry *je) 2792 { 2793 unsigned int s = 0; 2794 2795 do { 2796 js->commit_id = je->last_bytes[s]; 2797 js++; 2798 } while (++s < ic->sectors_per_block); 2799 } 2800 2801 static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start, 2802 unsigned int write_sections, bool from_replay) 2803 { 2804 unsigned int i, j, n; 2805 struct journal_completion comp; 2806 struct blk_plug plug; 2807 2808 blk_start_plug(&plug); 2809 2810 comp.ic = ic; 2811 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 2812 init_completion(&comp.comp); 2813 2814 i = write_start; 2815 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) { 2816 #ifndef INTERNAL_VERIFY 2817 if (unlikely(from_replay)) 2818 #endif 2819 rw_section_mac(ic, i, false); 2820 for (j = 0; j < ic->journal_section_entries; j++) { 2821 struct journal_entry *je = access_journal_entry(ic, i, j); 2822 sector_t sec, area, offset; 2823 unsigned int k, l, next_loop; 2824 sector_t metadata_block; 2825 unsigned int metadata_offset; 2826 struct journal_io *io; 2827 2828 if (journal_entry_is_unused(je)) 2829 continue; 2830 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay); 2831 sec = journal_entry_get_sector(je); 2832 if (unlikely(from_replay)) { 2833 if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) { 2834 dm_integrity_io_error(ic, "invalid sector in journal", -EIO); 2835 sec &= ~(sector_t)(ic->sectors_per_block - 1); 2836 } 2837 if (unlikely(sec >= ic->provided_data_sectors)) { 2838 journal_entry_set_unused(je); 2839 continue; 2840 } 2841 } 2842 get_area_and_offset(ic, sec, &area, &offset); 2843 restore_last_bytes(ic, access_journal_data(ic, i, j), je); 2844 for (k = j + 1; k < ic->journal_section_entries; k++) { 2845 struct journal_entry *je2 = access_journal_entry(ic, i, k); 2846 sector_t sec2, area2, offset2; 2847 2848 if (journal_entry_is_unused(je2)) 2849 break; 2850 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay); 2851 sec2 = journal_entry_get_sector(je2); 2852 if (unlikely(sec2 >= ic->provided_data_sectors)) 2853 break; 2854 get_area_and_offset(ic, sec2, &area2, &offset2); 2855 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block)) 2856 break; 2857 restore_last_bytes(ic, access_journal_data(ic, i, k), je2); 2858 } 2859 next_loop = k - 1; 2860 2861 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO); 2862 io->comp = ∁ 2863 io->range.logical_sector = sec; 2864 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; 2865 2866 spin_lock_irq(&ic->endio_wait.lock); 2867 add_new_range_and_wait(ic, &io->range); 2868 2869 if (likely(!from_replay)) { 2870 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; 2871 2872 /* don't write if there is newer committed sector */ 2873 while (j < k && find_newer_committed_node(ic, §ion_node[j])) { 2874 struct journal_entry *je2 = access_journal_entry(ic, i, j); 2875 2876 journal_entry_set_unused(je2); 2877 remove_journal_node(ic, §ion_node[j]); 2878 j++; 2879 sec += ic->sectors_per_block; 2880 offset += ic->sectors_per_block; 2881 } 2882 while (j < k && find_newer_committed_node(ic, §ion_node[k - 1])) { 2883 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1); 2884 2885 journal_entry_set_unused(je2); 2886 remove_journal_node(ic, §ion_node[k - 1]); 2887 k--; 2888 } 2889 if (j == k) { 2890 remove_range_unlocked(ic, &io->range); 2891 spin_unlock_irq(&ic->endio_wait.lock); 2892 mempool_free(io, &ic->journal_io_mempool); 2893 goto skip_io; 2894 } 2895 for (l = j; l < k; l++) 2896 remove_journal_node(ic, §ion_node[l]); 2897 } 2898 spin_unlock_irq(&ic->endio_wait.lock); 2899 2900 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); 2901 for (l = j; l < k; l++) { 2902 int r; 2903 struct journal_entry *je2 = access_journal_entry(ic, i, l); 2904 2905 if ( 2906 #ifndef INTERNAL_VERIFY 2907 unlikely(from_replay) && 2908 #endif 2909 ic->internal_hash) { 2910 char test_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; 2911 2912 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), 2913 (char *)access_journal_data(ic, i, l), test_tag); 2914 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { 2915 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); 2916 dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0); 2917 } 2918 } 2919 2920 journal_entry_set_unused(je2); 2921 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset, 2922 ic->tag_size, TAG_WRITE); 2923 if (unlikely(r)) 2924 dm_integrity_io_error(ic, "reading tags", r); 2925 } 2926 2927 atomic_inc(&comp.in_flight); 2928 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block, 2929 (k - j) << ic->sb->log2_sectors_per_block, 2930 get_data_sector(ic, area, offset), 2931 complete_copy_from_journal, io); 2932 skip_io: 2933 j = next_loop; 2934 } 2935 } 2936 2937 dm_bufio_write_dirty_buffers_async(ic->bufio); 2938 2939 blk_finish_plug(&plug); 2940 2941 complete_journal_op(&comp); 2942 wait_for_completion_io(&comp.comp); 2943 2944 dm_integrity_flush_buffers(ic, true); 2945 } 2946 2947 static void integrity_writer(struct work_struct *w) 2948 { 2949 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work); 2950 unsigned int write_start, write_sections; 2951 unsigned int prev_free_sectors; 2952 2953 spin_lock_irq(&ic->endio_wait.lock); 2954 write_start = ic->committed_section; 2955 write_sections = ic->n_committed_sections; 2956 spin_unlock_irq(&ic->endio_wait.lock); 2957 2958 if (!write_sections) 2959 return; 2960 2961 do_journal_write(ic, write_start, write_sections, false); 2962 2963 spin_lock_irq(&ic->endio_wait.lock); 2964 2965 ic->committed_section += write_sections; 2966 wraparound_section(ic, &ic->committed_section); 2967 ic->n_committed_sections -= write_sections; 2968 2969 prev_free_sectors = ic->free_sectors; 2970 ic->free_sectors += write_sections * ic->journal_section_entries; 2971 if (unlikely(!prev_free_sectors)) 2972 wake_up_locked(&ic->endio_wait); 2973 2974 spin_unlock_irq(&ic->endio_wait.lock); 2975 } 2976 2977 static void recalc_write_super(struct dm_integrity_c *ic) 2978 { 2979 int r; 2980 2981 dm_integrity_flush_buffers(ic, false); 2982 if (dm_integrity_failed(ic)) 2983 return; 2984 2985 r = sync_rw_sb(ic, REQ_OP_WRITE); 2986 if (unlikely(r)) 2987 dm_integrity_io_error(ic, "writing superblock", r); 2988 } 2989 2990 static void integrity_recalc(struct work_struct *w) 2991 { 2992 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); 2993 size_t recalc_tags_size; 2994 u8 *recalc_buffer = NULL; 2995 u8 *recalc_tags = NULL; 2996 struct dm_integrity_range range; 2997 struct dm_io_request io_req; 2998 struct dm_io_region io_loc; 2999 sector_t area, offset; 3000 sector_t metadata_block; 3001 unsigned int metadata_offset; 3002 sector_t logical_sector, n_sectors; 3003 __u8 *t; 3004 unsigned int i; 3005 int r; 3006 unsigned int super_counter = 0; 3007 unsigned recalc_sectors = RECALC_SECTORS; 3008 3009 retry: 3010 recalc_buffer = __vmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO); 3011 if (!recalc_buffer) { 3012 oom: 3013 recalc_sectors >>= 1; 3014 if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block) 3015 goto retry; 3016 DMCRIT("out of memory for recalculate buffer - recalculation disabled"); 3017 goto free_ret; 3018 } 3019 recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; 3020 if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size) 3021 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size; 3022 recalc_tags = kvmalloc(recalc_tags_size, GFP_NOIO); 3023 if (!recalc_tags) { 3024 vfree(recalc_buffer); 3025 recalc_buffer = NULL; 3026 goto oom; 3027 } 3028 3029 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector)); 3030 3031 spin_lock_irq(&ic->endio_wait.lock); 3032 3033 next_chunk: 3034 3035 if (unlikely(dm_post_suspending(ic->ti))) 3036 goto unlock_ret; 3037 3038 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); 3039 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { 3040 if (ic->mode == 'B') { 3041 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); 3042 DEBUG_print("queue_delayed_work: bitmap_flush_work\n"); 3043 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); 3044 } 3045 goto unlock_ret; 3046 } 3047 3048 get_area_and_offset(ic, range.logical_sector, &area, &offset); 3049 range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector); 3050 if (!ic->meta_dev) 3051 range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned int)offset); 3052 3053 add_new_range_and_wait(ic, &range); 3054 spin_unlock_irq(&ic->endio_wait.lock); 3055 logical_sector = range.logical_sector; 3056 n_sectors = range.n_sectors; 3057 3058 if (ic->mode == 'B') { 3059 if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) 3060 goto advance_and_next; 3061 3062 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, 3063 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { 3064 logical_sector += ic->sectors_per_block; 3065 n_sectors -= ic->sectors_per_block; 3066 cond_resched(); 3067 } 3068 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block, 3069 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { 3070 n_sectors -= ic->sectors_per_block; 3071 cond_resched(); 3072 } 3073 get_area_and_offset(ic, logical_sector, &area, &offset); 3074 } 3075 3076 DEBUG_print("recalculating: %llx, %llx\n", logical_sector, n_sectors); 3077 3078 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { 3079 recalc_write_super(ic); 3080 if (ic->mode == 'B') 3081 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); 3082 3083 super_counter = 0; 3084 } 3085 3086 if (unlikely(dm_integrity_failed(ic))) 3087 goto err; 3088 3089 io_req.bi_opf = REQ_OP_READ; 3090 io_req.mem.type = DM_IO_VMA; 3091 io_req.mem.ptr.addr = recalc_buffer; 3092 io_req.notify.fn = NULL; 3093 io_req.client = ic->io; 3094 io_loc.bdev = ic->dev->bdev; 3095 io_loc.sector = get_data_sector(ic, area, offset); 3096 io_loc.count = n_sectors; 3097 3098 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); 3099 if (unlikely(r)) { 3100 dm_integrity_io_error(ic, "reading data", r); 3101 goto err; 3102 } 3103 3104 t = recalc_tags; 3105 for (i = 0; i < n_sectors; i += ic->sectors_per_block) { 3106 integrity_sector_checksum(ic, logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t); 3107 t += ic->tag_size; 3108 } 3109 3110 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); 3111 3112 r = dm_integrity_rw_tag(ic, recalc_tags, &metadata_block, &metadata_offset, t - recalc_tags, TAG_WRITE); 3113 if (unlikely(r)) { 3114 dm_integrity_io_error(ic, "writing tags", r); 3115 goto err; 3116 } 3117 3118 if (ic->mode == 'B') { 3119 sector_t start, end; 3120 3121 start = (range.logical_sector >> 3122 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << 3123 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 3124 end = ((range.logical_sector + range.n_sectors) >> 3125 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << 3126 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 3127 block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR); 3128 } 3129 3130 advance_and_next: 3131 cond_resched(); 3132 3133 spin_lock_irq(&ic->endio_wait.lock); 3134 remove_range_unlocked(ic, &range); 3135 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); 3136 goto next_chunk; 3137 3138 err: 3139 remove_range(ic, &range); 3140 goto free_ret; 3141 3142 unlock_ret: 3143 spin_unlock_irq(&ic->endio_wait.lock); 3144 3145 recalc_write_super(ic); 3146 3147 free_ret: 3148 vfree(recalc_buffer); 3149 kvfree(recalc_tags); 3150 } 3151 3152 static void integrity_recalc_inline(struct work_struct *w) 3153 { 3154 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); 3155 size_t recalc_tags_size; 3156 u8 *recalc_buffer = NULL; 3157 u8 *recalc_tags = NULL; 3158 struct dm_integrity_range range; 3159 struct bio *bio; 3160 struct bio_integrity_payload *bip; 3161 __u8 *t; 3162 unsigned int i; 3163 int r; 3164 unsigned ret; 3165 unsigned int super_counter = 0; 3166 unsigned recalc_sectors = RECALC_SECTORS; 3167 3168 retry: 3169 recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN); 3170 if (!recalc_buffer) { 3171 oom: 3172 recalc_sectors >>= 1; 3173 if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block) 3174 goto retry; 3175 DMCRIT("out of memory for recalculate buffer - recalculation disabled"); 3176 goto free_ret; 3177 } 3178 3179 recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size; 3180 if (crypto_shash_digestsize(ic->internal_hash) > ic->tuple_size) 3181 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tuple_size; 3182 recalc_tags = kmalloc(recalc_tags_size, GFP_NOIO | __GFP_NOWARN); 3183 if (!recalc_tags) { 3184 kfree(recalc_buffer); 3185 recalc_buffer = NULL; 3186 goto oom; 3187 } 3188 3189 spin_lock_irq(&ic->endio_wait.lock); 3190 3191 next_chunk: 3192 if (unlikely(dm_post_suspending(ic->ti))) 3193 goto unlock_ret; 3194 3195 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); 3196 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) 3197 goto unlock_ret; 3198 range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector); 3199 3200 add_new_range_and_wait(ic, &range); 3201 spin_unlock_irq(&ic->endio_wait.lock); 3202 3203 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { 3204 recalc_write_super(ic); 3205 super_counter = 0; 3206 } 3207 3208 if (unlikely(dm_integrity_failed(ic))) 3209 goto err; 3210 3211 DEBUG_print("recalculating: %llx - %llx\n", range.logical_sector, range.n_sectors); 3212 3213 bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recalc_bios); 3214 bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; 3215 __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer)); 3216 r = submit_bio_wait(bio); 3217 bio_put(bio); 3218 if (unlikely(r)) { 3219 dm_integrity_io_error(ic, "reading data", r); 3220 goto err; 3221 } 3222 3223 t = recalc_tags; 3224 for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { 3225 memset(t, 0, ic->tuple_size); 3226 integrity_sector_checksum(ic, range.logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t); 3227 t += ic->tuple_size; 3228 } 3229 3230 bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_WRITE, GFP_NOIO, &ic->recalc_bios); 3231 bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; 3232 __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer)); 3233 3234 bip = bio_integrity_alloc(bio, GFP_NOIO, 1); 3235 if (unlikely(IS_ERR(bip))) { 3236 bio_put(bio); 3237 DMCRIT("out of memory for bio integrity payload - recalculation disabled"); 3238 goto err; 3239 } 3240 ret = bio_integrity_add_page(bio, virt_to_page(recalc_tags), t - recalc_tags, offset_in_page(recalc_tags)); 3241 if (unlikely(ret != t - recalc_tags)) { 3242 bio_put(bio); 3243 dm_integrity_io_error(ic, "attaching integrity tags", -ENOMEM); 3244 goto err; 3245 } 3246 3247 r = submit_bio_wait(bio); 3248 bio_put(bio); 3249 if (unlikely(r)) { 3250 dm_integrity_io_error(ic, "writing data", r); 3251 goto err; 3252 } 3253 3254 cond_resched(); 3255 spin_lock_irq(&ic->endio_wait.lock); 3256 remove_range_unlocked(ic, &range); 3257 #ifdef CONFIG_64BIT 3258 /* Paired with smp_load_acquire in dm_integrity_map_inline. */ 3259 smp_store_release(&ic->sb->recalc_sector, cpu_to_le64(range.logical_sector + range.n_sectors)); 3260 #else 3261 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); 3262 #endif 3263 goto next_chunk; 3264 3265 err: 3266 remove_range(ic, &range); 3267 goto free_ret; 3268 3269 unlock_ret: 3270 spin_unlock_irq(&ic->endio_wait.lock); 3271 3272 recalc_write_super(ic); 3273 3274 free_ret: 3275 kfree(recalc_buffer); 3276 kfree(recalc_tags); 3277 } 3278 3279 static void bitmap_block_work(struct work_struct *w) 3280 { 3281 struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work); 3282 struct dm_integrity_c *ic = bbs->ic; 3283 struct bio *bio; 3284 struct bio_list bio_queue; 3285 struct bio_list waiting; 3286 3287 bio_list_init(&waiting); 3288 3289 spin_lock(&bbs->bio_queue_lock); 3290 bio_queue = bbs->bio_queue; 3291 bio_list_init(&bbs->bio_queue); 3292 spin_unlock(&bbs->bio_queue_lock); 3293 3294 while ((bio = bio_list_pop(&bio_queue))) { 3295 struct dm_integrity_io *dio; 3296 3297 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 3298 3299 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, 3300 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { 3301 remove_range(ic, &dio->range); 3302 INIT_WORK(&dio->work, integrity_bio_wait); 3303 queue_work(ic->offload_wq, &dio->work); 3304 } else { 3305 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, 3306 dio->range.n_sectors, BITMAP_OP_SET); 3307 bio_list_add(&waiting, bio); 3308 } 3309 } 3310 3311 if (bio_list_empty(&waiting)) 3312 return; 3313 3314 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 3315 bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), 3316 BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL); 3317 3318 while ((bio = bio_list_pop(&waiting))) { 3319 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 3320 3321 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, 3322 dio->range.n_sectors, BITMAP_OP_SET); 3323 3324 remove_range(ic, &dio->range); 3325 INIT_WORK(&dio->work, integrity_bio_wait); 3326 queue_work(ic->offload_wq, &dio->work); 3327 } 3328 3329 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); 3330 } 3331 3332 static void bitmap_flush_work(struct work_struct *work) 3333 { 3334 struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work); 3335 struct dm_integrity_range range; 3336 unsigned long limit; 3337 struct bio *bio; 3338 3339 dm_integrity_flush_buffers(ic, false); 3340 3341 range.logical_sector = 0; 3342 range.n_sectors = ic->provided_data_sectors; 3343 3344 spin_lock_irq(&ic->endio_wait.lock); 3345 add_new_range_and_wait(ic, &range); 3346 spin_unlock_irq(&ic->endio_wait.lock); 3347 3348 dm_integrity_flush_buffers(ic, true); 3349 3350 limit = ic->provided_data_sectors; 3351 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { 3352 limit = le64_to_cpu(ic->sb->recalc_sector) 3353 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit) 3354 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); 3355 } 3356 /*DEBUG_print("zeroing journal\n");*/ 3357 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR); 3358 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR); 3359 3360 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, 3361 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3362 3363 spin_lock_irq(&ic->endio_wait.lock); 3364 remove_range_unlocked(ic, &range); 3365 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { 3366 bio_endio(bio); 3367 spin_unlock_irq(&ic->endio_wait.lock); 3368 spin_lock_irq(&ic->endio_wait.lock); 3369 } 3370 spin_unlock_irq(&ic->endio_wait.lock); 3371 } 3372 3373 3374 static void init_journal(struct dm_integrity_c *ic, unsigned int start_section, 3375 unsigned int n_sections, unsigned char commit_seq) 3376 { 3377 unsigned int i, j, n; 3378 3379 if (!n_sections) 3380 return; 3381 3382 for (n = 0; n < n_sections; n++) { 3383 i = start_section + n; 3384 wraparound_section(ic, &i); 3385 for (j = 0; j < ic->journal_section_sectors; j++) { 3386 struct journal_sector *js = access_journal(ic, i, j); 3387 3388 BUILD_BUG_ON(sizeof(js->sectors) != JOURNAL_SECTOR_DATA); 3389 memset(&js->sectors, 0, sizeof(js->sectors)); 3390 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq); 3391 } 3392 for (j = 0; j < ic->journal_section_entries; j++) { 3393 struct journal_entry *je = access_journal_entry(ic, i, j); 3394 3395 journal_entry_set_unused(je); 3396 } 3397 } 3398 3399 write_journal(ic, start_section, n_sections); 3400 } 3401 3402 static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t id) 3403 { 3404 unsigned char k; 3405 3406 for (k = 0; k < N_COMMIT_IDS; k++) { 3407 if (dm_integrity_commit_id(ic, i, j, k) == id) 3408 return k; 3409 } 3410 dm_integrity_io_error(ic, "journal commit id", -EIO); 3411 return -EIO; 3412 } 3413 3414 static void replay_journal(struct dm_integrity_c *ic) 3415 { 3416 unsigned int i, j; 3417 bool used_commit_ids[N_COMMIT_IDS]; 3418 unsigned int max_commit_id_sections[N_COMMIT_IDS]; 3419 unsigned int write_start, write_sections; 3420 unsigned int continue_section; 3421 bool journal_empty; 3422 unsigned char unused, last_used, want_commit_seq; 3423 3424 if (ic->mode == 'R') 3425 return; 3426 3427 if (ic->journal_uptodate) 3428 return; 3429 3430 last_used = 0; 3431 write_start = 0; 3432 3433 if (!ic->just_formatted) { 3434 DEBUG_print("reading journal\n"); 3435 rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL); 3436 if (ic->journal_io) 3437 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal"); 3438 if (ic->journal_io) { 3439 struct journal_completion crypt_comp; 3440 3441 crypt_comp.ic = ic; 3442 init_completion(&crypt_comp.comp); 3443 crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0); 3444 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp); 3445 wait_for_completion(&crypt_comp.comp); 3446 } 3447 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal"); 3448 } 3449 3450 if (dm_integrity_failed(ic)) 3451 goto clear_journal; 3452 3453 journal_empty = true; 3454 memset(used_commit_ids, 0, sizeof(used_commit_ids)); 3455 memset(max_commit_id_sections, 0, sizeof(max_commit_id_sections)); 3456 for (i = 0; i < ic->journal_sections; i++) { 3457 for (j = 0; j < ic->journal_section_sectors; j++) { 3458 int k; 3459 struct journal_sector *js = access_journal(ic, i, j); 3460 3461 k = find_commit_seq(ic, i, j, js->commit_id); 3462 if (k < 0) 3463 goto clear_journal; 3464 used_commit_ids[k] = true; 3465 max_commit_id_sections[k] = i; 3466 } 3467 if (journal_empty) { 3468 for (j = 0; j < ic->journal_section_entries; j++) { 3469 struct journal_entry *je = access_journal_entry(ic, i, j); 3470 3471 if (!journal_entry_is_unused(je)) { 3472 journal_empty = false; 3473 break; 3474 } 3475 } 3476 } 3477 } 3478 3479 if (!used_commit_ids[N_COMMIT_IDS - 1]) { 3480 unused = N_COMMIT_IDS - 1; 3481 while (unused && !used_commit_ids[unused - 1]) 3482 unused--; 3483 } else { 3484 for (unused = 0; unused < N_COMMIT_IDS; unused++) 3485 if (!used_commit_ids[unused]) 3486 break; 3487 if (unused == N_COMMIT_IDS) { 3488 dm_integrity_io_error(ic, "journal commit ids", -EIO); 3489 goto clear_journal; 3490 } 3491 } 3492 DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n", 3493 unused, used_commit_ids[0], used_commit_ids[1], 3494 used_commit_ids[2], used_commit_ids[3]); 3495 3496 last_used = prev_commit_seq(unused); 3497 want_commit_seq = prev_commit_seq(last_used); 3498 3499 if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)]) 3500 journal_empty = true; 3501 3502 write_start = max_commit_id_sections[last_used] + 1; 3503 if (unlikely(write_start >= ic->journal_sections)) 3504 want_commit_seq = next_commit_seq(want_commit_seq); 3505 wraparound_section(ic, &write_start); 3506 3507 i = write_start; 3508 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) { 3509 for (j = 0; j < ic->journal_section_sectors; j++) { 3510 struct journal_sector *js = access_journal(ic, i, j); 3511 3512 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) { 3513 /* 3514 * This could be caused by crash during writing. 3515 * We won't replay the inconsistent part of the 3516 * journal. 3517 */ 3518 DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n", 3519 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq); 3520 goto brk; 3521 } 3522 } 3523 i++; 3524 if (unlikely(i >= ic->journal_sections)) 3525 want_commit_seq = next_commit_seq(want_commit_seq); 3526 wraparound_section(ic, &i); 3527 } 3528 brk: 3529 3530 if (!journal_empty) { 3531 DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n", 3532 write_sections, write_start, want_commit_seq); 3533 do_journal_write(ic, write_start, write_sections, true); 3534 } 3535 3536 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) { 3537 continue_section = write_start; 3538 ic->commit_seq = want_commit_seq; 3539 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq); 3540 } else { 3541 unsigned int s; 3542 unsigned char erase_seq; 3543 3544 clear_journal: 3545 DEBUG_print("clearing journal\n"); 3546 3547 erase_seq = prev_commit_seq(prev_commit_seq(last_used)); 3548 s = write_start; 3549 init_journal(ic, s, 1, erase_seq); 3550 s++; 3551 wraparound_section(ic, &s); 3552 if (ic->journal_sections >= 2) { 3553 init_journal(ic, s, ic->journal_sections - 2, erase_seq); 3554 s += ic->journal_sections - 2; 3555 wraparound_section(ic, &s); 3556 init_journal(ic, s, 1, erase_seq); 3557 } 3558 3559 continue_section = 0; 3560 ic->commit_seq = next_commit_seq(erase_seq); 3561 } 3562 3563 ic->committed_section = continue_section; 3564 ic->n_committed_sections = 0; 3565 3566 ic->uncommitted_section = continue_section; 3567 ic->n_uncommitted_sections = 0; 3568 3569 ic->free_section = continue_section; 3570 ic->free_section_entry = 0; 3571 ic->free_sectors = ic->journal_entries; 3572 3573 ic->journal_tree_root = RB_ROOT; 3574 for (i = 0; i < ic->journal_entries; i++) 3575 init_journal_node(&ic->journal_tree[i]); 3576 } 3577 3578 static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic) 3579 { 3580 DEBUG_print("%s\n", __func__); 3581 3582 if (ic->mode == 'B') { 3583 ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1; 3584 ic->synchronous_mode = 1; 3585 3586 cancel_delayed_work_sync(&ic->bitmap_flush_work); 3587 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); 3588 flush_workqueue(ic->commit_wq); 3589 } 3590 } 3591 3592 static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x) 3593 { 3594 struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier); 3595 3596 DEBUG_print("%s\n", __func__); 3597 3598 dm_integrity_enter_synchronous_mode(ic); 3599 3600 return NOTIFY_DONE; 3601 } 3602 3603 static void dm_integrity_postsuspend(struct dm_target *ti) 3604 { 3605 struct dm_integrity_c *ic = ti->private; 3606 int r; 3607 3608 WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier)); 3609 3610 del_timer_sync(&ic->autocommit_timer); 3611 3612 if (ic->recalc_wq) 3613 drain_workqueue(ic->recalc_wq); 3614 3615 if (ic->mode == 'B') 3616 cancel_delayed_work_sync(&ic->bitmap_flush_work); 3617 3618 queue_work(ic->commit_wq, &ic->commit_work); 3619 drain_workqueue(ic->commit_wq); 3620 3621 if (ic->mode == 'J') { 3622 queue_work(ic->writer_wq, &ic->writer_work); 3623 drain_workqueue(ic->writer_wq); 3624 dm_integrity_flush_buffers(ic, true); 3625 if (ic->wrote_to_journal) { 3626 init_journal(ic, ic->free_section, 3627 ic->journal_sections - ic->free_section, ic->commit_seq); 3628 if (ic->free_section) { 3629 init_journal(ic, 0, ic->free_section, 3630 next_commit_seq(ic->commit_seq)); 3631 } 3632 } 3633 } 3634 3635 if (ic->mode == 'B') { 3636 dm_integrity_flush_buffers(ic, true); 3637 #if 1 3638 /* set to 0 to test bitmap replay code */ 3639 init_journal(ic, 0, ic->journal_sections, 0); 3640 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); 3641 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 3642 if (unlikely(r)) 3643 dm_integrity_io_error(ic, "writing superblock", r); 3644 #endif 3645 } 3646 3647 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); 3648 3649 ic->journal_uptodate = true; 3650 } 3651 3652 static void dm_integrity_resume(struct dm_target *ti) 3653 { 3654 struct dm_integrity_c *ic = ti->private; 3655 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); 3656 int r; 3657 3658 DEBUG_print("resume\n"); 3659 3660 ic->wrote_to_journal = false; 3661 3662 if (ic->provided_data_sectors != old_provided_data_sectors) { 3663 if (ic->provided_data_sectors > old_provided_data_sectors && 3664 ic->mode == 'B' && 3665 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { 3666 rw_journal_sectors(ic, REQ_OP_READ, 0, 3667 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3668 block_bitmap_op(ic, ic->journal, old_provided_data_sectors, 3669 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET); 3670 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, 3671 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3672 } 3673 3674 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); 3675 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 3676 if (unlikely(r)) 3677 dm_integrity_io_error(ic, "writing superblock", r); 3678 } 3679 3680 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) { 3681 DEBUG_print("resume dirty_bitmap\n"); 3682 rw_journal_sectors(ic, REQ_OP_READ, 0, 3683 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3684 if (ic->mode == 'B') { 3685 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && 3686 !ic->reset_recalculate_flag) { 3687 block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal); 3688 block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal); 3689 if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, 3690 BITMAP_OP_TEST_ALL_CLEAR)) { 3691 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 3692 ic->sb->recalc_sector = cpu_to_le64(0); 3693 } 3694 } else { 3695 DEBUG_print("non-matching blocks_per_bitmap_bit: %u, %u\n", 3696 ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit); 3697 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; 3698 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); 3699 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); 3700 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET); 3701 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, 3702 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3703 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 3704 ic->sb->recalc_sector = cpu_to_le64(0); 3705 } 3706 } else { 3707 if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && 3708 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) || 3709 ic->reset_recalculate_flag) { 3710 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 3711 ic->sb->recalc_sector = cpu_to_le64(0); 3712 } 3713 init_journal(ic, 0, ic->journal_sections, 0); 3714 replay_journal(ic); 3715 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); 3716 } 3717 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 3718 if (unlikely(r)) 3719 dm_integrity_io_error(ic, "writing superblock", r); 3720 } else { 3721 replay_journal(ic); 3722 if (ic->reset_recalculate_flag) { 3723 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 3724 ic->sb->recalc_sector = cpu_to_le64(0); 3725 } 3726 if (ic->mode == 'B') { 3727 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP); 3728 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; 3729 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 3730 if (unlikely(r)) 3731 dm_integrity_io_error(ic, "writing superblock", r); 3732 3733 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); 3734 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); 3735 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); 3736 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 3737 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) { 3738 block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector), 3739 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); 3740 block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector), 3741 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); 3742 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector), 3743 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); 3744 } 3745 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, 3746 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); 3747 } 3748 } 3749 3750 DEBUG_print("testing recalc: %x\n", ic->sb->flags); 3751 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { 3752 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); 3753 3754 DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors); 3755 if (recalc_pos < ic->provided_data_sectors) { 3756 queue_work(ic->recalc_wq, &ic->recalc_work); 3757 } else if (recalc_pos > ic->provided_data_sectors) { 3758 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors); 3759 recalc_write_super(ic); 3760 } 3761 } 3762 3763 ic->reboot_notifier.notifier_call = dm_integrity_reboot; 3764 ic->reboot_notifier.next = NULL; 3765 ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */ 3766 WARN_ON(register_reboot_notifier(&ic->reboot_notifier)); 3767 3768 #if 0 3769 /* set to 1 to stress test synchronous mode */ 3770 dm_integrity_enter_synchronous_mode(ic); 3771 #endif 3772 } 3773 3774 static void dm_integrity_status(struct dm_target *ti, status_type_t type, 3775 unsigned int status_flags, char *result, unsigned int maxlen) 3776 { 3777 struct dm_integrity_c *ic = ti->private; 3778 unsigned int arg_count; 3779 size_t sz = 0; 3780 3781 switch (type) { 3782 case STATUSTYPE_INFO: 3783 DMEMIT("%llu %llu", 3784 (unsigned long long)atomic64_read(&ic->number_of_mismatches), 3785 ic->provided_data_sectors); 3786 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 3787 DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector)); 3788 else 3789 DMEMIT(" -"); 3790 break; 3791 3792 case STATUSTYPE_TABLE: { 3793 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; 3794 3795 watermark_percentage += ic->journal_entries / 2; 3796 do_div(watermark_percentage, ic->journal_entries); 3797 arg_count = 3; 3798 arg_count += !!ic->meta_dev; 3799 arg_count += ic->sectors_per_block != 1; 3800 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); 3801 arg_count += ic->reset_recalculate_flag; 3802 arg_count += ic->discard; 3803 arg_count += ic->mode == 'J'; 3804 arg_count += ic->mode == 'J'; 3805 arg_count += ic->mode == 'B'; 3806 arg_count += ic->mode == 'B'; 3807 arg_count += !!ic->internal_hash_alg.alg_string; 3808 arg_count += !!ic->journal_crypt_alg.alg_string; 3809 arg_count += !!ic->journal_mac_alg.alg_string; 3810 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0; 3811 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0; 3812 arg_count += ic->legacy_recalculate; 3813 DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start, 3814 ic->tag_size, ic->mode, arg_count); 3815 if (ic->meta_dev) 3816 DMEMIT(" meta_device:%s", ic->meta_dev->name); 3817 if (ic->sectors_per_block != 1) 3818 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); 3819 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 3820 DMEMIT(" recalculate"); 3821 if (ic->reset_recalculate_flag) 3822 DMEMIT(" reset_recalculate"); 3823 if (ic->discard) 3824 DMEMIT(" allow_discards"); 3825 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); 3826 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); 3827 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); 3828 if (ic->mode == 'J') { 3829 DMEMIT(" journal_watermark:%u", (unsigned int)watermark_percentage); 3830 DMEMIT(" commit_time:%u", ic->autocommit_msec); 3831 } 3832 if (ic->mode == 'B') { 3833 DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit); 3834 DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval)); 3835 } 3836 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) 3837 DMEMIT(" fix_padding"); 3838 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) 3839 DMEMIT(" fix_hmac"); 3840 if (ic->legacy_recalculate) 3841 DMEMIT(" legacy_recalculate"); 3842 3843 #define EMIT_ALG(a, n) \ 3844 do { \ 3845 if (ic->a.alg_string) { \ 3846 DMEMIT(" %s:%s", n, ic->a.alg_string); \ 3847 if (ic->a.key_string) \ 3848 DMEMIT(":%s", ic->a.key_string);\ 3849 } \ 3850 } while (0) 3851 EMIT_ALG(internal_hash_alg, "internal_hash"); 3852 EMIT_ALG(journal_crypt_alg, "journal_crypt"); 3853 EMIT_ALG(journal_mac_alg, "journal_mac"); 3854 break; 3855 } 3856 case STATUSTYPE_IMA: 3857 DMEMIT_TARGET_NAME_VERSION(ti->type); 3858 DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c", 3859 ic->dev->name, ic->start, ic->tag_size, ic->mode); 3860 3861 if (ic->meta_dev) 3862 DMEMIT(",meta_device=%s", ic->meta_dev->name); 3863 if (ic->sectors_per_block != 1) 3864 DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT); 3865 3866 DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? 3867 'y' : 'n'); 3868 DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); 3869 DMEMIT(",fix_padding=%c", 3870 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); 3871 DMEMIT(",fix_hmac=%c", 3872 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n'); 3873 DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n'); 3874 3875 DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS); 3876 DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors); 3877 DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors); 3878 DMEMIT(";"); 3879 break; 3880 } 3881 } 3882 3883 static int dm_integrity_iterate_devices(struct dm_target *ti, 3884 iterate_devices_callout_fn fn, void *data) 3885 { 3886 struct dm_integrity_c *ic = ti->private; 3887 3888 if (!ic->meta_dev) 3889 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); 3890 else 3891 return fn(ti, ic->dev, 0, ti->len, data); 3892 } 3893 3894 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits) 3895 { 3896 struct dm_integrity_c *ic = ti->private; 3897 3898 if (ic->sectors_per_block > 1) { 3899 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT; 3900 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT; 3901 limits->io_min = ic->sectors_per_block << SECTOR_SHIFT; 3902 limits->dma_alignment = limits->logical_block_size - 1; 3903 limits->discard_granularity = ic->sectors_per_block << SECTOR_SHIFT; 3904 } 3905 3906 if (!ic->internal_hash) { 3907 struct blk_integrity *bi = &limits->integrity; 3908 3909 memset(bi, 0, sizeof(*bi)); 3910 bi->tuple_size = ic->tag_size; 3911 bi->tag_size = bi->tuple_size; 3912 bi->interval_exp = 3913 ic->sb->log2_sectors_per_block + SECTOR_SHIFT; 3914 } 3915 3916 limits->max_integrity_segments = USHRT_MAX; 3917 } 3918 3919 static void calculate_journal_section_size(struct dm_integrity_c *ic) 3920 { 3921 unsigned int sector_space = JOURNAL_SECTOR_DATA; 3922 3923 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections); 3924 ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size, 3925 JOURNAL_ENTRY_ROUNDUP); 3926 3927 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) 3928 sector_space -= JOURNAL_MAC_PER_SECTOR; 3929 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size; 3930 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS; 3931 ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS; 3932 ic->journal_entries = ic->journal_section_entries * ic->journal_sections; 3933 } 3934 3935 static int calculate_device_limits(struct dm_integrity_c *ic) 3936 { 3937 __u64 initial_sectors; 3938 3939 calculate_journal_section_size(ic); 3940 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections; 3941 if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX) 3942 return -EINVAL; 3943 ic->initial_sectors = initial_sectors; 3944 3945 if (ic->mode == 'I') { 3946 if (ic->initial_sectors + ic->provided_data_sectors > ic->meta_device_sectors) 3947 return -EINVAL; 3948 } else if (!ic->meta_dev) { 3949 sector_t last_sector, last_area, last_offset; 3950 3951 /* we have to maintain excessive padding for compatibility with existing volumes */ 3952 __u64 metadata_run_padding = 3953 ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ? 3954 (__u64)(METADATA_PADDING_SECTORS << SECTOR_SHIFT) : 3955 (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS); 3956 3957 ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), 3958 metadata_run_padding) >> SECTOR_SHIFT; 3959 if (!(ic->metadata_run & (ic->metadata_run - 1))) 3960 ic->log2_metadata_run = __ffs(ic->metadata_run); 3961 else 3962 ic->log2_metadata_run = -1; 3963 3964 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); 3965 last_sector = get_data_sector(ic, last_area, last_offset); 3966 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) 3967 return -EINVAL; 3968 } else { 3969 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; 3970 3971 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) 3972 >> (ic->log2_buffer_sectors + SECTOR_SHIFT); 3973 meta_size <<= ic->log2_buffer_sectors; 3974 if (ic->initial_sectors + meta_size < ic->initial_sectors || 3975 ic->initial_sectors + meta_size > ic->meta_device_sectors) 3976 return -EINVAL; 3977 ic->metadata_run = 1; 3978 ic->log2_metadata_run = 0; 3979 } 3980 3981 return 0; 3982 } 3983 3984 static void get_provided_data_sectors(struct dm_integrity_c *ic) 3985 { 3986 if (!ic->meta_dev) { 3987 int test_bit; 3988 3989 ic->provided_data_sectors = 0; 3990 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { 3991 __u64 prev_data_sectors = ic->provided_data_sectors; 3992 3993 ic->provided_data_sectors |= (sector_t)1 << test_bit; 3994 if (calculate_device_limits(ic)) 3995 ic->provided_data_sectors = prev_data_sectors; 3996 } 3997 } else { 3998 ic->provided_data_sectors = ic->data_device_sectors; 3999 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); 4000 } 4001 } 4002 4003 static int initialize_superblock(struct dm_integrity_c *ic, 4004 unsigned int journal_sectors, unsigned int interleave_sectors) 4005 { 4006 unsigned int journal_sections; 4007 int test_bit; 4008 4009 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); 4010 memcpy(ic->sb->magic, SB_MAGIC, 8); 4011 if (ic->mode == 'I') 4012 ic->sb->flags |= cpu_to_le32(SB_FLAG_INLINE); 4013 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); 4014 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); 4015 if (ic->journal_mac_alg.alg_string) 4016 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC); 4017 4018 calculate_journal_section_size(ic); 4019 journal_sections = journal_sectors / ic->journal_section_sectors; 4020 if (!journal_sections) 4021 journal_sections = 1; 4022 if (ic->mode == 'I') 4023 journal_sections = 0; 4024 4025 if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) { 4026 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC); 4027 get_random_bytes(ic->sb->salt, SALT_SIZE); 4028 } 4029 4030 if (!ic->meta_dev) { 4031 if (ic->fix_padding) 4032 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); 4033 ic->sb->journal_sections = cpu_to_le32(journal_sections); 4034 if (!interleave_sectors) 4035 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; 4036 ic->sb->log2_interleave_sectors = __fls(interleave_sectors); 4037 ic->sb->log2_interleave_sectors = max_t(__u8, MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); 4038 ic->sb->log2_interleave_sectors = min_t(__u8, MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); 4039 4040 get_provided_data_sectors(ic); 4041 if (!ic->provided_data_sectors) 4042 return -EINVAL; 4043 } else { 4044 ic->sb->log2_interleave_sectors = 0; 4045 4046 get_provided_data_sectors(ic); 4047 if (!ic->provided_data_sectors) 4048 return -EINVAL; 4049 4050 try_smaller_buffer: 4051 ic->sb->journal_sections = cpu_to_le32(0); 4052 for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) { 4053 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); 4054 __u32 test_journal_sections = prev_journal_sections | (1U << test_bit); 4055 4056 if (test_journal_sections > journal_sections) 4057 continue; 4058 ic->sb->journal_sections = cpu_to_le32(test_journal_sections); 4059 if (calculate_device_limits(ic)) 4060 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections); 4061 4062 } 4063 if (!le32_to_cpu(ic->sb->journal_sections)) { 4064 if (ic->log2_buffer_sectors > 3) { 4065 ic->log2_buffer_sectors--; 4066 goto try_smaller_buffer; 4067 } 4068 return -EINVAL; 4069 } 4070 } 4071 4072 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); 4073 4074 sb_set_version(ic); 4075 4076 return 0; 4077 } 4078 4079 static void dm_integrity_free_page_list(struct page_list *pl) 4080 { 4081 unsigned int i; 4082 4083 if (!pl) 4084 return; 4085 for (i = 0; pl[i].page; i++) 4086 __free_page(pl[i].page); 4087 kvfree(pl); 4088 } 4089 4090 static struct page_list *dm_integrity_alloc_page_list(unsigned int n_pages) 4091 { 4092 struct page_list *pl; 4093 unsigned int i; 4094 4095 pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO); 4096 if (!pl) 4097 return NULL; 4098 4099 for (i = 0; i < n_pages; i++) { 4100 pl[i].page = alloc_page(GFP_KERNEL); 4101 if (!pl[i].page) { 4102 dm_integrity_free_page_list(pl); 4103 return NULL; 4104 } 4105 if (i) 4106 pl[i - 1].next = &pl[i]; 4107 } 4108 pl[i].page = NULL; 4109 pl[i].next = NULL; 4110 4111 return pl; 4112 } 4113 4114 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl) 4115 { 4116 unsigned int i; 4117 4118 for (i = 0; i < ic->journal_sections; i++) 4119 kvfree(sl[i]); 4120 kvfree(sl); 4121 } 4122 4123 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, 4124 struct page_list *pl) 4125 { 4126 struct scatterlist **sl; 4127 unsigned int i; 4128 4129 sl = kvmalloc_array(ic->journal_sections, 4130 sizeof(struct scatterlist *), 4131 GFP_KERNEL | __GFP_ZERO); 4132 if (!sl) 4133 return NULL; 4134 4135 for (i = 0; i < ic->journal_sections; i++) { 4136 struct scatterlist *s; 4137 unsigned int start_index, start_offset; 4138 unsigned int end_index, end_offset; 4139 unsigned int n_pages; 4140 unsigned int idx; 4141 4142 page_list_location(ic, i, 0, &start_index, &start_offset); 4143 page_list_location(ic, i, ic->journal_section_sectors - 1, 4144 &end_index, &end_offset); 4145 4146 n_pages = (end_index - start_index + 1); 4147 4148 s = kvmalloc_array(n_pages, sizeof(struct scatterlist), 4149 GFP_KERNEL); 4150 if (!s) { 4151 dm_integrity_free_journal_scatterlist(ic, sl); 4152 return NULL; 4153 } 4154 4155 sg_init_table(s, n_pages); 4156 for (idx = start_index; idx <= end_index; idx++) { 4157 char *va = lowmem_page_address(pl[idx].page); 4158 unsigned int start = 0, end = PAGE_SIZE; 4159 4160 if (idx == start_index) 4161 start = start_offset; 4162 if (idx == end_index) 4163 end = end_offset + (1 << SECTOR_SHIFT); 4164 sg_set_buf(&s[idx - start_index], va + start, end - start); 4165 } 4166 4167 sl[i] = s; 4168 } 4169 4170 return sl; 4171 } 4172 4173 static void free_alg(struct alg_spec *a) 4174 { 4175 kfree_sensitive(a->alg_string); 4176 kfree_sensitive(a->key); 4177 memset(a, 0, sizeof(*a)); 4178 } 4179 4180 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval) 4181 { 4182 char *k; 4183 4184 free_alg(a); 4185 4186 a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL); 4187 if (!a->alg_string) 4188 goto nomem; 4189 4190 k = strchr(a->alg_string, ':'); 4191 if (k) { 4192 *k = 0; 4193 a->key_string = k + 1; 4194 if (strlen(a->key_string) & 1) 4195 goto inval; 4196 4197 a->key_size = strlen(a->key_string) / 2; 4198 a->key = kmalloc(a->key_size, GFP_KERNEL); 4199 if (!a->key) 4200 goto nomem; 4201 if (hex2bin(a->key, a->key_string, a->key_size)) 4202 goto inval; 4203 } 4204 4205 return 0; 4206 inval: 4207 *error = error_inval; 4208 return -EINVAL; 4209 nomem: 4210 *error = "Out of memory for an argument"; 4211 return -ENOMEM; 4212 } 4213 4214 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, 4215 char *error_alg, char *error_key) 4216 { 4217 int r; 4218 4219 if (a->alg_string) { 4220 *hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 4221 if (IS_ERR(*hash)) { 4222 *error = error_alg; 4223 r = PTR_ERR(*hash); 4224 *hash = NULL; 4225 return r; 4226 } 4227 4228 if (a->key) { 4229 r = crypto_shash_setkey(*hash, a->key, a->key_size); 4230 if (r) { 4231 *error = error_key; 4232 return r; 4233 } 4234 } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) { 4235 *error = error_key; 4236 return -ENOKEY; 4237 } 4238 } 4239 4240 return 0; 4241 } 4242 4243 static int create_journal(struct dm_integrity_c *ic, char **error) 4244 { 4245 int r = 0; 4246 unsigned int i; 4247 __u64 journal_pages, journal_desc_size, journal_tree_size; 4248 unsigned char *crypt_data = NULL, *crypt_iv = NULL; 4249 struct skcipher_request *req = NULL; 4250 4251 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL); 4252 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL); 4253 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL); 4254 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL); 4255 4256 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors, 4257 PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT); 4258 journal_desc_size = journal_pages * sizeof(struct page_list); 4259 if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) { 4260 *error = "Journal doesn't fit into memory"; 4261 r = -ENOMEM; 4262 goto bad; 4263 } 4264 ic->journal_pages = journal_pages; 4265 4266 ic->journal = dm_integrity_alloc_page_list(ic->journal_pages); 4267 if (!ic->journal) { 4268 *error = "Could not allocate memory for journal"; 4269 r = -ENOMEM; 4270 goto bad; 4271 } 4272 if (ic->journal_crypt_alg.alg_string) { 4273 unsigned int ivsize, blocksize; 4274 struct journal_completion comp; 4275 4276 comp.ic = ic; 4277 ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); 4278 if (IS_ERR(ic->journal_crypt)) { 4279 *error = "Invalid journal cipher"; 4280 r = PTR_ERR(ic->journal_crypt); 4281 ic->journal_crypt = NULL; 4282 goto bad; 4283 } 4284 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); 4285 blocksize = crypto_skcipher_blocksize(ic->journal_crypt); 4286 4287 if (ic->journal_crypt_alg.key) { 4288 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key, 4289 ic->journal_crypt_alg.key_size); 4290 if (r) { 4291 *error = "Error setting encryption key"; 4292 goto bad; 4293 } 4294 } 4295 DEBUG_print("cipher %s, block size %u iv size %u\n", 4296 ic->journal_crypt_alg.alg_string, blocksize, ivsize); 4297 4298 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages); 4299 if (!ic->journal_io) { 4300 *error = "Could not allocate memory for journal io"; 4301 r = -ENOMEM; 4302 goto bad; 4303 } 4304 4305 if (blocksize == 1) { 4306 struct scatterlist *sg; 4307 4308 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 4309 if (!req) { 4310 *error = "Could not allocate crypt request"; 4311 r = -ENOMEM; 4312 goto bad; 4313 } 4314 4315 crypt_iv = kzalloc(ivsize, GFP_KERNEL); 4316 if (!crypt_iv) { 4317 *error = "Could not allocate iv"; 4318 r = -ENOMEM; 4319 goto bad; 4320 } 4321 4322 ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages); 4323 if (!ic->journal_xor) { 4324 *error = "Could not allocate memory for journal xor"; 4325 r = -ENOMEM; 4326 goto bad; 4327 } 4328 4329 sg = kvmalloc_array(ic->journal_pages + 1, 4330 sizeof(struct scatterlist), 4331 GFP_KERNEL); 4332 if (!sg) { 4333 *error = "Unable to allocate sg list"; 4334 r = -ENOMEM; 4335 goto bad; 4336 } 4337 sg_init_table(sg, ic->journal_pages + 1); 4338 for (i = 0; i < ic->journal_pages; i++) { 4339 char *va = lowmem_page_address(ic->journal_xor[i].page); 4340 4341 clear_page(va); 4342 sg_set_buf(&sg[i], va, PAGE_SIZE); 4343 } 4344 sg_set_buf(&sg[i], &ic->commit_ids, sizeof(ic->commit_ids)); 4345 4346 skcipher_request_set_crypt(req, sg, sg, 4347 PAGE_SIZE * ic->journal_pages + sizeof(ic->commit_ids), crypt_iv); 4348 init_completion(&comp.comp); 4349 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 4350 if (do_crypt(true, req, &comp)) 4351 wait_for_completion(&comp.comp); 4352 kvfree(sg); 4353 r = dm_integrity_failed(ic); 4354 if (r) { 4355 *error = "Unable to encrypt journal"; 4356 goto bad; 4357 } 4358 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data"); 4359 4360 crypto_free_skcipher(ic->journal_crypt); 4361 ic->journal_crypt = NULL; 4362 } else { 4363 unsigned int crypt_len = roundup(ivsize, blocksize); 4364 4365 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 4366 if (!req) { 4367 *error = "Could not allocate crypt request"; 4368 r = -ENOMEM; 4369 goto bad; 4370 } 4371 4372 crypt_iv = kmalloc(ivsize, GFP_KERNEL); 4373 if (!crypt_iv) { 4374 *error = "Could not allocate iv"; 4375 r = -ENOMEM; 4376 goto bad; 4377 } 4378 4379 crypt_data = kmalloc(crypt_len, GFP_KERNEL); 4380 if (!crypt_data) { 4381 *error = "Unable to allocate crypt data"; 4382 r = -ENOMEM; 4383 goto bad; 4384 } 4385 4386 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal); 4387 if (!ic->journal_scatterlist) { 4388 *error = "Unable to allocate sg list"; 4389 r = -ENOMEM; 4390 goto bad; 4391 } 4392 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io); 4393 if (!ic->journal_io_scatterlist) { 4394 *error = "Unable to allocate sg list"; 4395 r = -ENOMEM; 4396 goto bad; 4397 } 4398 ic->sk_requests = kvmalloc_array(ic->journal_sections, 4399 sizeof(struct skcipher_request *), 4400 GFP_KERNEL | __GFP_ZERO); 4401 if (!ic->sk_requests) { 4402 *error = "Unable to allocate sk requests"; 4403 r = -ENOMEM; 4404 goto bad; 4405 } 4406 for (i = 0; i < ic->journal_sections; i++) { 4407 struct scatterlist sg; 4408 struct skcipher_request *section_req; 4409 __le32 section_le = cpu_to_le32(i); 4410 4411 memset(crypt_iv, 0x00, ivsize); 4412 memset(crypt_data, 0x00, crypt_len); 4413 memcpy(crypt_data, §ion_le, min_t(size_t, crypt_len, sizeof(section_le))); 4414 4415 sg_init_one(&sg, crypt_data, crypt_len); 4416 skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv); 4417 init_completion(&comp.comp); 4418 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 4419 if (do_crypt(true, req, &comp)) 4420 wait_for_completion(&comp.comp); 4421 4422 r = dm_integrity_failed(ic); 4423 if (r) { 4424 *error = "Unable to generate iv"; 4425 goto bad; 4426 } 4427 4428 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 4429 if (!section_req) { 4430 *error = "Unable to allocate crypt request"; 4431 r = -ENOMEM; 4432 goto bad; 4433 } 4434 section_req->iv = kmalloc_array(ivsize, 2, 4435 GFP_KERNEL); 4436 if (!section_req->iv) { 4437 skcipher_request_free(section_req); 4438 *error = "Unable to allocate iv"; 4439 r = -ENOMEM; 4440 goto bad; 4441 } 4442 memcpy(section_req->iv + ivsize, crypt_data, ivsize); 4443 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT; 4444 ic->sk_requests[i] = section_req; 4445 DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i); 4446 } 4447 } 4448 } 4449 4450 for (i = 0; i < N_COMMIT_IDS; i++) { 4451 unsigned int j; 4452 4453 retest_commit_id: 4454 for (j = 0; j < i; j++) { 4455 if (ic->commit_ids[j] == ic->commit_ids[i]) { 4456 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1); 4457 goto retest_commit_id; 4458 } 4459 } 4460 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]); 4461 } 4462 4463 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node); 4464 if (journal_tree_size > ULONG_MAX) { 4465 *error = "Journal doesn't fit into memory"; 4466 r = -ENOMEM; 4467 goto bad; 4468 } 4469 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL); 4470 if (!ic->journal_tree) { 4471 *error = "Could not allocate memory for journal tree"; 4472 r = -ENOMEM; 4473 } 4474 bad: 4475 kfree(crypt_data); 4476 kfree(crypt_iv); 4477 skcipher_request_free(req); 4478 4479 return r; 4480 } 4481 4482 /* 4483 * Construct a integrity mapping 4484 * 4485 * Arguments: 4486 * device 4487 * offset from the start of the device 4488 * tag size 4489 * D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode 4490 * number of optional arguments 4491 * optional arguments: 4492 * journal_sectors 4493 * interleave_sectors 4494 * buffer_sectors 4495 * journal_watermark 4496 * commit_time 4497 * meta_device 4498 * block_size 4499 * sectors_per_bit 4500 * bitmap_flush_interval 4501 * internal_hash 4502 * journal_crypt 4503 * journal_mac 4504 * recalculate 4505 */ 4506 static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv) 4507 { 4508 struct dm_integrity_c *ic; 4509 char dummy; 4510 int r; 4511 unsigned int extra_args; 4512 struct dm_arg_set as; 4513 static const struct dm_arg _args[] = { 4514 {0, 18, "Invalid number of feature args"}, 4515 }; 4516 unsigned int journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; 4517 bool should_write_sb; 4518 __u64 threshold; 4519 unsigned long long start; 4520 __s8 log2_sectors_per_bitmap_bit = -1; 4521 __s8 log2_blocks_per_bitmap_bit; 4522 __u64 bits_in_journal; 4523 __u64 n_bitmap_bits; 4524 4525 #define DIRECT_ARGUMENTS 4 4526 4527 if (argc <= DIRECT_ARGUMENTS) { 4528 ti->error = "Invalid argument count"; 4529 return -EINVAL; 4530 } 4531 4532 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL); 4533 if (!ic) { 4534 ti->error = "Cannot allocate integrity context"; 4535 return -ENOMEM; 4536 } 4537 ti->private = ic; 4538 ti->per_io_data_size = sizeof(struct dm_integrity_io); 4539 ic->ti = ti; 4540 4541 ic->in_progress = RB_ROOT; 4542 INIT_LIST_HEAD(&ic->wait_list); 4543 init_waitqueue_head(&ic->endio_wait); 4544 bio_list_init(&ic->flush_bio_list); 4545 init_waitqueue_head(&ic->copy_to_journal_wait); 4546 init_completion(&ic->crypto_backoff); 4547 atomic64_set(&ic->number_of_mismatches, 0); 4548 ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL; 4549 4550 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); 4551 if (r) { 4552 ti->error = "Device lookup failed"; 4553 goto bad; 4554 } 4555 4556 if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) { 4557 ti->error = "Invalid starting offset"; 4558 r = -EINVAL; 4559 goto bad; 4560 } 4561 ic->start = start; 4562 4563 if (strcmp(argv[2], "-")) { 4564 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) { 4565 ti->error = "Invalid tag size"; 4566 r = -EINVAL; 4567 goto bad; 4568 } 4569 } 4570 4571 if (!strcmp(argv[3], "J") || !strcmp(argv[3], "B") || 4572 !strcmp(argv[3], "D") || !strcmp(argv[3], "R") || 4573 !strcmp(argv[3], "I")) { 4574 ic->mode = argv[3][0]; 4575 } else { 4576 ti->error = "Invalid mode (expecting J, B, D, R, I)"; 4577 r = -EINVAL; 4578 goto bad; 4579 } 4580 4581 journal_sectors = 0; 4582 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; 4583 buffer_sectors = DEFAULT_BUFFER_SECTORS; 4584 journal_watermark = DEFAULT_JOURNAL_WATERMARK; 4585 sync_msec = DEFAULT_SYNC_MSEC; 4586 ic->sectors_per_block = 1; 4587 4588 as.argc = argc - DIRECT_ARGUMENTS; 4589 as.argv = argv + DIRECT_ARGUMENTS; 4590 r = dm_read_arg_group(_args, &as, &extra_args, &ti->error); 4591 if (r) 4592 goto bad; 4593 4594 while (extra_args--) { 4595 const char *opt_string; 4596 unsigned int val; 4597 unsigned long long llval; 4598 4599 opt_string = dm_shift_arg(&as); 4600 if (!opt_string) { 4601 r = -EINVAL; 4602 ti->error = "Not enough feature arguments"; 4603 goto bad; 4604 } 4605 if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1) 4606 journal_sectors = val ? val : 1; 4607 else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1) 4608 interleave_sectors = val; 4609 else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1) 4610 buffer_sectors = val; 4611 else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100) 4612 journal_watermark = val; 4613 else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1) 4614 sync_msec = val; 4615 else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) { 4616 if (ic->meta_dev) { 4617 dm_put_device(ti, ic->meta_dev); 4618 ic->meta_dev = NULL; 4619 } 4620 r = dm_get_device(ti, strchr(opt_string, ':') + 1, 4621 dm_table_get_mode(ti->table), &ic->meta_dev); 4622 if (r) { 4623 ti->error = "Device lookup failed"; 4624 goto bad; 4625 } 4626 } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) { 4627 if (val < 1 << SECTOR_SHIFT || 4628 val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT || 4629 (val & (val - 1))) { 4630 r = -EINVAL; 4631 ti->error = "Invalid block_size argument"; 4632 goto bad; 4633 } 4634 ic->sectors_per_block = val >> SECTOR_SHIFT; 4635 } else if (sscanf(opt_string, "sectors_per_bit:%llu%c", &llval, &dummy) == 1) { 4636 log2_sectors_per_bitmap_bit = !llval ? 0 : __ilog2_u64(llval); 4637 } else if (sscanf(opt_string, "bitmap_flush_interval:%u%c", &val, &dummy) == 1) { 4638 if ((uint64_t)val >= (uint64_t)UINT_MAX * 1000 / HZ) { 4639 r = -EINVAL; 4640 ti->error = "Invalid bitmap_flush_interval argument"; 4641 goto bad; 4642 } 4643 ic->bitmap_flush_interval = msecs_to_jiffies(val); 4644 } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) { 4645 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error, 4646 "Invalid internal_hash argument"); 4647 if (r) 4648 goto bad; 4649 } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) { 4650 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error, 4651 "Invalid journal_crypt argument"); 4652 if (r) 4653 goto bad; 4654 } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) { 4655 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, 4656 "Invalid journal_mac argument"); 4657 if (r) 4658 goto bad; 4659 } else if (!strcmp(opt_string, "recalculate")) { 4660 ic->recalculate_flag = true; 4661 } else if (!strcmp(opt_string, "reset_recalculate")) { 4662 ic->recalculate_flag = true; 4663 ic->reset_recalculate_flag = true; 4664 } else if (!strcmp(opt_string, "allow_discards")) { 4665 ic->discard = true; 4666 } else if (!strcmp(opt_string, "fix_padding")) { 4667 ic->fix_padding = true; 4668 } else if (!strcmp(opt_string, "fix_hmac")) { 4669 ic->fix_hmac = true; 4670 } else if (!strcmp(opt_string, "legacy_recalculate")) { 4671 ic->legacy_recalculate = true; 4672 } else { 4673 r = -EINVAL; 4674 ti->error = "Invalid argument"; 4675 goto bad; 4676 } 4677 } 4678 4679 ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev); 4680 if (!ic->meta_dev) 4681 ic->meta_device_sectors = ic->data_device_sectors; 4682 else 4683 ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev); 4684 4685 if (!journal_sectors) { 4686 journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, 4687 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); 4688 } 4689 4690 if (!buffer_sectors) 4691 buffer_sectors = 1; 4692 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); 4693 4694 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, 4695 "Invalid internal hash", "Error setting internal hash key"); 4696 if (r) 4697 goto bad; 4698 4699 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, 4700 "Invalid journal mac", "Error setting journal mac key"); 4701 if (r) 4702 goto bad; 4703 4704 if (!ic->tag_size) { 4705 if (!ic->internal_hash) { 4706 ti->error = "Unknown tag size"; 4707 r = -EINVAL; 4708 goto bad; 4709 } 4710 ic->tag_size = crypto_shash_digestsize(ic->internal_hash); 4711 } 4712 if (ic->tag_size > MAX_TAG_SIZE) { 4713 ti->error = "Too big tag size"; 4714 r = -EINVAL; 4715 goto bad; 4716 } 4717 if (!(ic->tag_size & (ic->tag_size - 1))) 4718 ic->log2_tag_size = __ffs(ic->tag_size); 4719 else 4720 ic->log2_tag_size = -1; 4721 4722 if (ic->mode == 'I') { 4723 struct blk_integrity *bi; 4724 if (ic->meta_dev) { 4725 r = -EINVAL; 4726 ti->error = "Metadata device not supported in inline mode"; 4727 goto bad; 4728 } 4729 if (!ic->internal_hash_alg.alg_string) { 4730 r = -EINVAL; 4731 ti->error = "Internal hash not set in inline mode"; 4732 goto bad; 4733 } 4734 if (ic->journal_crypt_alg.alg_string || ic->journal_mac_alg.alg_string) { 4735 r = -EINVAL; 4736 ti->error = "Journal crypt not supported in inline mode"; 4737 goto bad; 4738 } 4739 if (ic->discard) { 4740 r = -EINVAL; 4741 ti->error = "Discards not supported in inline mode"; 4742 goto bad; 4743 } 4744 bi = blk_get_integrity(ic->dev->bdev->bd_disk); 4745 if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) { 4746 r = -EINVAL; 4747 ti->error = "Integrity profile not supported"; 4748 goto bad; 4749 } 4750 /*printk("tag_size: %u, tuple_size: %u\n", bi->tag_size, bi->tuple_size);*/ 4751 if (bi->tuple_size < ic->tag_size) { 4752 r = -EINVAL; 4753 ti->error = "The integrity profile is smaller than tag size"; 4754 goto bad; 4755 } 4756 if ((unsigned long)bi->tuple_size > PAGE_SIZE / 2) { 4757 r = -EINVAL; 4758 ti->error = "Too big tuple size"; 4759 goto bad; 4760 } 4761 ic->tuple_size = bi->tuple_size; 4762 if (1 << bi->interval_exp != ic->sectors_per_block << SECTOR_SHIFT) { 4763 r = -EINVAL; 4764 ti->error = "Integrity profile sector size mismatch"; 4765 goto bad; 4766 } 4767 } 4768 4769 if (ic->mode == 'B' && !ic->internal_hash) { 4770 r = -EINVAL; 4771 ti->error = "Bitmap mode can be only used with internal hash"; 4772 goto bad; 4773 } 4774 4775 if (ic->discard && !ic->internal_hash) { 4776 r = -EINVAL; 4777 ti->error = "Discard can be only used with internal hash"; 4778 goto bad; 4779 } 4780 4781 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); 4782 ic->autocommit_msec = sync_msec; 4783 timer_setup(&ic->autocommit_timer, autocommit_fn, 0); 4784 4785 ic->io = dm_io_client_create(); 4786 if (IS_ERR(ic->io)) { 4787 r = PTR_ERR(ic->io); 4788 ic->io = NULL; 4789 ti->error = "Cannot allocate dm io"; 4790 goto bad; 4791 } 4792 4793 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache); 4794 if (r) { 4795 ti->error = "Cannot allocate mempool"; 4796 goto bad; 4797 } 4798 4799 r = mempool_init_page_pool(&ic->recheck_pool, 1, ic->mode == 'I' ? 1 : 0); 4800 if (r) { 4801 ti->error = "Cannot allocate mempool"; 4802 goto bad; 4803 } 4804 4805 if (ic->mode == 'I') { 4806 r = bioset_init(&ic->recheck_bios, RECHECK_POOL_SIZE, 0, BIOSET_NEED_BVECS); 4807 if (r) { 4808 ti->error = "Cannot allocate bio set"; 4809 goto bad; 4810 } 4811 r = bioset_integrity_create(&ic->recheck_bios, RECHECK_POOL_SIZE); 4812 if (r) { 4813 ti->error = "Cannot allocate bio integrity set"; 4814 r = -ENOMEM; 4815 goto bad; 4816 } 4817 r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS); 4818 if (r) { 4819 ti->error = "Cannot allocate bio set"; 4820 goto bad; 4821 } 4822 r = bioset_integrity_create(&ic->recalc_bios, 1); 4823 if (r) { 4824 ti->error = "Cannot allocate bio integrity set"; 4825 r = -ENOMEM; 4826 goto bad; 4827 } 4828 } 4829 4830 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", 4831 WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE); 4832 if (!ic->metadata_wq) { 4833 ti->error = "Cannot allocate workqueue"; 4834 r = -ENOMEM; 4835 goto bad; 4836 } 4837 4838 /* 4839 * If this workqueue weren't ordered, it would cause bio reordering 4840 * and reduced performance. 4841 */ 4842 ic->wait_wq = alloc_ordered_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM); 4843 if (!ic->wait_wq) { 4844 ti->error = "Cannot allocate workqueue"; 4845 r = -ENOMEM; 4846 goto bad; 4847 } 4848 4849 ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM, 4850 METADATA_WORKQUEUE_MAX_ACTIVE); 4851 if (!ic->offload_wq) { 4852 ti->error = "Cannot allocate workqueue"; 4853 r = -ENOMEM; 4854 goto bad; 4855 } 4856 4857 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); 4858 if (!ic->commit_wq) { 4859 ti->error = "Cannot allocate workqueue"; 4860 r = -ENOMEM; 4861 goto bad; 4862 } 4863 INIT_WORK(&ic->commit_work, integrity_commit); 4864 4865 if (ic->mode == 'J' || ic->mode == 'B') { 4866 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); 4867 if (!ic->writer_wq) { 4868 ti->error = "Cannot allocate workqueue"; 4869 r = -ENOMEM; 4870 goto bad; 4871 } 4872 INIT_WORK(&ic->writer_work, integrity_writer); 4873 } 4874 4875 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL); 4876 if (!ic->sb) { 4877 r = -ENOMEM; 4878 ti->error = "Cannot allocate superblock area"; 4879 goto bad; 4880 } 4881 4882 r = sync_rw_sb(ic, REQ_OP_READ); 4883 if (r) { 4884 ti->error = "Error reading superblock"; 4885 goto bad; 4886 } 4887 should_write_sb = false; 4888 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) { 4889 if (ic->mode != 'R') { 4890 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) { 4891 r = -EINVAL; 4892 ti->error = "The device is not initialized"; 4893 goto bad; 4894 } 4895 } 4896 4897 r = initialize_superblock(ic, journal_sectors, interleave_sectors); 4898 if (r) { 4899 ti->error = "Could not initialize superblock"; 4900 goto bad; 4901 } 4902 if (ic->mode != 'R') 4903 should_write_sb = true; 4904 } 4905 4906 if (!ic->sb->version || ic->sb->version > SB_VERSION_6) { 4907 r = -EINVAL; 4908 ti->error = "Unknown version"; 4909 goto bad; 4910 } 4911 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) != (ic->mode == 'I')) { 4912 r = -EINVAL; 4913 ti->error = "Inline flag mismatch"; 4914 goto bad; 4915 } 4916 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) { 4917 r = -EINVAL; 4918 ti->error = "Tag size doesn't match the information in superblock"; 4919 goto bad; 4920 } 4921 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) { 4922 r = -EINVAL; 4923 ti->error = "Block size doesn't match the information in superblock"; 4924 goto bad; 4925 } 4926 if (ic->mode != 'I') { 4927 if (!le32_to_cpu(ic->sb->journal_sections)) { 4928 r = -EINVAL; 4929 ti->error = "Corrupted superblock, journal_sections is 0"; 4930 goto bad; 4931 } 4932 } else { 4933 if (le32_to_cpu(ic->sb->journal_sections)) { 4934 r = -EINVAL; 4935 ti->error = "Corrupted superblock, journal_sections is not 0"; 4936 goto bad; 4937 } 4938 } 4939 /* make sure that ti->max_io_len doesn't overflow */ 4940 if (!ic->meta_dev) { 4941 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || 4942 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { 4943 r = -EINVAL; 4944 ti->error = "Invalid interleave_sectors in the superblock"; 4945 goto bad; 4946 } 4947 } else { 4948 if (ic->sb->log2_interleave_sectors) { 4949 r = -EINVAL; 4950 ti->error = "Invalid interleave_sectors in the superblock"; 4951 goto bad; 4952 } 4953 } 4954 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { 4955 r = -EINVAL; 4956 ti->error = "Journal mac mismatch"; 4957 goto bad; 4958 } 4959 4960 get_provided_data_sectors(ic); 4961 if (!ic->provided_data_sectors) { 4962 r = -EINVAL; 4963 ti->error = "The device is too small"; 4964 goto bad; 4965 } 4966 4967 try_smaller_buffer: 4968 r = calculate_device_limits(ic); 4969 if (r) { 4970 if (ic->meta_dev) { 4971 if (ic->log2_buffer_sectors > 3) { 4972 ic->log2_buffer_sectors--; 4973 goto try_smaller_buffer; 4974 } 4975 } 4976 ti->error = "The device is too small"; 4977 goto bad; 4978 } 4979 4980 if (log2_sectors_per_bitmap_bit < 0) 4981 log2_sectors_per_bitmap_bit = __fls(DEFAULT_SECTORS_PER_BITMAP_BIT); 4982 if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block) 4983 log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block; 4984 4985 bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3); 4986 if (bits_in_journal > UINT_MAX) 4987 bits_in_journal = UINT_MAX; 4988 if (bits_in_journal) 4989 while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit) - 1) >> log2_sectors_per_bitmap_bit) 4990 log2_sectors_per_bitmap_bit++; 4991 4992 log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block; 4993 ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; 4994 if (should_write_sb) 4995 ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; 4996 4997 n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) 4998 + (((sector_t)1 << log2_blocks_per_bitmap_bit) - 1)) >> log2_blocks_per_bitmap_bit; 4999 ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8); 5000 5001 if (!ic->meta_dev) 5002 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); 5003 5004 if (ti->len > ic->provided_data_sectors) { 5005 r = -EINVAL; 5006 ti->error = "Not enough provided sectors for requested mapping size"; 5007 goto bad; 5008 } 5009 5010 threshold = (__u64)ic->journal_entries * (100 - journal_watermark); 5011 threshold += 50; 5012 do_div(threshold, 100); 5013 ic->free_sectors_threshold = threshold; 5014 5015 DEBUG_print("initialized:\n"); 5016 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size)); 5017 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size); 5018 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector); 5019 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries); 5020 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors); 5021 DEBUG_print(" journal_sections %u\n", (unsigned int)le32_to_cpu(ic->sb->journal_sections)); 5022 DEBUG_print(" journal_entries %u\n", ic->journal_entries); 5023 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); 5024 DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev)); 5025 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); 5026 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); 5027 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); 5028 DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data_sectors); 5029 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); 5030 DEBUG_print(" bits_in_journal %llu\n", bits_in_journal); 5031 5032 if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { 5033 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 5034 ic->sb->recalc_sector = cpu_to_le64(0); 5035 } 5036 5037 if (ic->internal_hash) { 5038 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); 5039 if (!ic->recalc_wq) { 5040 ti->error = "Cannot allocate workqueue"; 5041 r = -ENOMEM; 5042 goto bad; 5043 } 5044 INIT_WORK(&ic->recalc_work, ic->mode == 'I' ? integrity_recalc_inline : integrity_recalc); 5045 } else { 5046 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { 5047 ti->error = "Recalculate can only be specified with internal_hash"; 5048 r = -EINVAL; 5049 goto bad; 5050 } 5051 } 5052 5053 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 5054 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors && 5055 dm_integrity_disable_recalculate(ic)) { 5056 ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\""; 5057 r = -EOPNOTSUPP; 5058 goto bad; 5059 } 5060 5061 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, 5062 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0); 5063 if (IS_ERR(ic->bufio)) { 5064 r = PTR_ERR(ic->bufio); 5065 ti->error = "Cannot initialize dm-bufio"; 5066 ic->bufio = NULL; 5067 goto bad; 5068 } 5069 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); 5070 5071 if (ic->mode != 'R' && ic->mode != 'I') { 5072 r = create_journal(ic, &ti->error); 5073 if (r) 5074 goto bad; 5075 5076 } 5077 5078 if (ic->mode == 'B') { 5079 unsigned int i; 5080 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); 5081 5082 ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); 5083 if (!ic->recalc_bitmap) { 5084 r = -ENOMEM; 5085 goto bad; 5086 } 5087 ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); 5088 if (!ic->may_write_bitmap) { 5089 r = -ENOMEM; 5090 goto bad; 5091 } 5092 ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL); 5093 if (!ic->bbs) { 5094 r = -ENOMEM; 5095 goto bad; 5096 } 5097 INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work); 5098 for (i = 0; i < ic->n_bitmap_blocks; i++) { 5099 struct bitmap_block_status *bbs = &ic->bbs[i]; 5100 unsigned int sector, pl_index, pl_offset; 5101 5102 INIT_WORK(&bbs->work, bitmap_block_work); 5103 bbs->ic = ic; 5104 bbs->idx = i; 5105 bio_list_init(&bbs->bio_queue); 5106 spin_lock_init(&bbs->bio_queue_lock); 5107 5108 sector = i * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT); 5109 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 5110 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 5111 5112 bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset; 5113 } 5114 } 5115 5116 if (should_write_sb) { 5117 init_journal(ic, 0, ic->journal_sections, 0); 5118 r = dm_integrity_failed(ic); 5119 if (unlikely(r)) { 5120 ti->error = "Error initializing journal"; 5121 goto bad; 5122 } 5123 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); 5124 if (r) { 5125 ti->error = "Error initializing superblock"; 5126 goto bad; 5127 } 5128 ic->just_formatted = true; 5129 } 5130 5131 if (!ic->meta_dev && ic->mode != 'I') { 5132 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); 5133 if (r) 5134 goto bad; 5135 } 5136 if (ic->mode == 'B') { 5137 unsigned int max_io_len; 5138 5139 max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_SIZE * 8); 5140 if (!max_io_len) 5141 max_io_len = 1U << 31; 5142 DEBUG_print("max_io_len: old %u, new %u\n", ti->max_io_len, max_io_len); 5143 if (!ti->max_io_len || ti->max_io_len > max_io_len) { 5144 r = dm_set_target_max_io_len(ti, max_io_len); 5145 if (r) 5146 goto bad; 5147 } 5148 } 5149 5150 ti->num_flush_bios = 1; 5151 ti->flush_supported = true; 5152 if (ic->discard) 5153 ti->num_discard_bios = 1; 5154 5155 if (ic->mode == 'I') 5156 ti->mempool_needs_integrity = true; 5157 5158 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); 5159 return 0; 5160 5161 bad: 5162 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); 5163 dm_integrity_dtr(ti); 5164 return r; 5165 } 5166 5167 static void dm_integrity_dtr(struct dm_target *ti) 5168 { 5169 struct dm_integrity_c *ic = ti->private; 5170 5171 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); 5172 BUG_ON(!list_empty(&ic->wait_list)); 5173 5174 if (ic->mode == 'B') 5175 cancel_delayed_work_sync(&ic->bitmap_flush_work); 5176 if (ic->metadata_wq) 5177 destroy_workqueue(ic->metadata_wq); 5178 if (ic->wait_wq) 5179 destroy_workqueue(ic->wait_wq); 5180 if (ic->offload_wq) 5181 destroy_workqueue(ic->offload_wq); 5182 if (ic->commit_wq) 5183 destroy_workqueue(ic->commit_wq); 5184 if (ic->writer_wq) 5185 destroy_workqueue(ic->writer_wq); 5186 if (ic->recalc_wq) 5187 destroy_workqueue(ic->recalc_wq); 5188 kvfree(ic->bbs); 5189 if (ic->bufio) 5190 dm_bufio_client_destroy(ic->bufio); 5191 bioset_exit(&ic->recalc_bios); 5192 bioset_exit(&ic->recheck_bios); 5193 mempool_exit(&ic->recheck_pool); 5194 mempool_exit(&ic->journal_io_mempool); 5195 if (ic->io) 5196 dm_io_client_destroy(ic->io); 5197 if (ic->dev) 5198 dm_put_device(ti, ic->dev); 5199 if (ic->meta_dev) 5200 dm_put_device(ti, ic->meta_dev); 5201 dm_integrity_free_page_list(ic->journal); 5202 dm_integrity_free_page_list(ic->journal_io); 5203 dm_integrity_free_page_list(ic->journal_xor); 5204 dm_integrity_free_page_list(ic->recalc_bitmap); 5205 dm_integrity_free_page_list(ic->may_write_bitmap); 5206 if (ic->journal_scatterlist) 5207 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist); 5208 if (ic->journal_io_scatterlist) 5209 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist); 5210 if (ic->sk_requests) { 5211 unsigned int i; 5212 5213 for (i = 0; i < ic->journal_sections; i++) { 5214 struct skcipher_request *req; 5215 5216 req = ic->sk_requests[i]; 5217 if (req) { 5218 kfree_sensitive(req->iv); 5219 skcipher_request_free(req); 5220 } 5221 } 5222 kvfree(ic->sk_requests); 5223 } 5224 kvfree(ic->journal_tree); 5225 if (ic->sb) 5226 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT); 5227 5228 if (ic->internal_hash) 5229 crypto_free_shash(ic->internal_hash); 5230 free_alg(&ic->internal_hash_alg); 5231 5232 if (ic->journal_crypt) 5233 crypto_free_skcipher(ic->journal_crypt); 5234 free_alg(&ic->journal_crypt_alg); 5235 5236 if (ic->journal_mac) 5237 crypto_free_shash(ic->journal_mac); 5238 free_alg(&ic->journal_mac_alg); 5239 5240 kfree(ic); 5241 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); 5242 } 5243 5244 static struct target_type integrity_target = { 5245 .name = "integrity", 5246 .version = {1, 13, 0}, 5247 .module = THIS_MODULE, 5248 .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, 5249 .ctr = dm_integrity_ctr, 5250 .dtr = dm_integrity_dtr, 5251 .map = dm_integrity_map, 5252 .end_io = dm_integrity_end_io, 5253 .postsuspend = dm_integrity_postsuspend, 5254 .resume = dm_integrity_resume, 5255 .status = dm_integrity_status, 5256 .iterate_devices = dm_integrity_iterate_devices, 5257 .io_hints = dm_integrity_io_hints, 5258 }; 5259 5260 static int __init dm_integrity_init(void) 5261 { 5262 int r; 5263 5264 journal_io_cache = kmem_cache_create("integrity_journal_io", 5265 sizeof(struct journal_io), 0, 0, NULL); 5266 if (!journal_io_cache) { 5267 DMERR("can't allocate journal io cache"); 5268 return -ENOMEM; 5269 } 5270 5271 r = dm_register_target(&integrity_target); 5272 if (r < 0) { 5273 kmem_cache_destroy(journal_io_cache); 5274 return r; 5275 } 5276 5277 return 0; 5278 } 5279 5280 static void __exit dm_integrity_exit(void) 5281 { 5282 dm_unregister_target(&integrity_target); 5283 kmem_cache_destroy(journal_io_cache); 5284 } 5285 5286 module_init(dm_integrity_init); 5287 module_exit(dm_integrity_exit); 5288 5289 MODULE_AUTHOR("Milan Broz"); 5290 MODULE_AUTHOR("Mikulas Patocka"); 5291 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension"); 5292 MODULE_LICENSE("GPL"); 5293