1 /* 2 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved. 3 * Copyright (C) 2016-2017 Milan Broz 4 * Copyright (C) 2016-2017 Mikulas Patocka 5 * 6 * This file is released under the GPL. 7 */ 8 9 #include <linux/compiler.h> 10 #include <linux/module.h> 11 #include <linux/device-mapper.h> 12 #include <linux/dm-io.h> 13 #include <linux/vmalloc.h> 14 #include <linux/sort.h> 15 #include <linux/rbtree.h> 16 #include <linux/delay.h> 17 #include <linux/random.h> 18 #include <crypto/hash.h> 19 #include <crypto/skcipher.h> 20 #include <linux/async_tx.h> 21 #include <linux/dm-bufio.h> 22 23 #define DM_MSG_PREFIX "integrity" 24 25 #define DEFAULT_INTERLEAVE_SECTORS 32768 26 #define DEFAULT_JOURNAL_SIZE_FACTOR 7 27 #define DEFAULT_BUFFER_SECTORS 128 28 #define DEFAULT_JOURNAL_WATERMARK 50 29 #define DEFAULT_SYNC_MSEC 10000 30 #define DEFAULT_MAX_JOURNAL_SECTORS 131072 31 #define MIN_LOG2_INTERLEAVE_SECTORS 3 32 #define MAX_LOG2_INTERLEAVE_SECTORS 31 33 #define METADATA_WORKQUEUE_MAX_ACTIVE 16 34 #define RECALC_SECTORS 8192 35 #define RECALC_WRITE_SUPER 16 36 37 /* 38 * Warning - DEBUG_PRINT prints security-sensitive data to the log, 39 * so it should not be enabled in the official kernel 40 */ 41 //#define DEBUG_PRINT 42 //#define INTERNAL_VERIFY 43 44 /* 45 * On disk structures 46 */ 47 48 #define SB_MAGIC "integrt" 49 #define SB_VERSION_1 1 50 #define SB_VERSION_2 2 51 #define SB_SECTORS 8 52 #define MAX_SECTORS_PER_BLOCK 8 53 54 struct superblock { 55 __u8 magic[8]; 56 __u8 version; 57 __u8 log2_interleave_sectors; 58 __u16 integrity_tag_size; 59 __u32 journal_sections; 60 __u64 provided_data_sectors; /* userspace uses this value */ 61 __u32 flags; 62 __u8 log2_sectors_per_block; 63 __u8 pad[3]; 64 __u64 recalc_sector; 65 }; 66 67 #define SB_FLAG_HAVE_JOURNAL_MAC 0x1 68 #define SB_FLAG_RECALCULATING 0x2 69 70 #define JOURNAL_ENTRY_ROUNDUP 8 71 72 typedef __u64 commit_id_t; 73 #define JOURNAL_MAC_PER_SECTOR 8 74 75 struct journal_entry { 76 union { 77 struct { 78 __u32 sector_lo; 79 __u32 sector_hi; 80 } s; 81 __u64 sector; 82 } u; 83 commit_id_t last_bytes[0]; 84 /* __u8 tag[0]; */ 85 }; 86 87 #define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block]) 88 89 #if BITS_PER_LONG == 64 90 #define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0) 91 #define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) 92 #elif defined(CONFIG_LBDAF) 93 #define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0) 94 #define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) 95 #else 96 #define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32(0)); } while (0) 97 #define journal_entry_get_sector(je) le32_to_cpu((je)->u.s.sector_lo) 98 #endif 99 #define journal_entry_is_unused(je) ((je)->u.s.sector_hi == cpu_to_le32(-1)) 100 #define journal_entry_set_unused(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0) 101 #define journal_entry_is_inprogress(je) ((je)->u.s.sector_hi == cpu_to_le32(-2)) 102 #define journal_entry_set_inprogress(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0) 103 104 #define JOURNAL_BLOCK_SECTORS 8 105 #define JOURNAL_SECTOR_DATA ((1 << SECTOR_SHIFT) - sizeof(commit_id_t)) 106 #define JOURNAL_MAC_SIZE (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS) 107 108 struct journal_sector { 109 __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR]; 110 __u8 mac[JOURNAL_MAC_PER_SECTOR]; 111 commit_id_t commit_id; 112 }; 113 114 #define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK])) 115 116 #define METADATA_PADDING_SECTORS 8 117 118 #define N_COMMIT_IDS 4 119 120 static unsigned char prev_commit_seq(unsigned char seq) 121 { 122 return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS; 123 } 124 125 static unsigned char next_commit_seq(unsigned char seq) 126 { 127 return (seq + 1) % N_COMMIT_IDS; 128 } 129 130 /* 131 * In-memory structures 132 */ 133 134 struct journal_node { 135 struct rb_node node; 136 sector_t sector; 137 }; 138 139 struct alg_spec { 140 char *alg_string; 141 char *key_string; 142 __u8 *key; 143 unsigned key_size; 144 }; 145 146 struct dm_integrity_c { 147 struct dm_dev *dev; 148 struct dm_dev *meta_dev; 149 unsigned tag_size; 150 __s8 log2_tag_size; 151 sector_t start; 152 mempool_t journal_io_mempool; 153 struct dm_io_client *io; 154 struct dm_bufio_client *bufio; 155 struct workqueue_struct *metadata_wq; 156 struct superblock *sb; 157 unsigned journal_pages; 158 struct page_list *journal; 159 struct page_list *journal_io; 160 struct page_list *journal_xor; 161 162 struct crypto_skcipher *journal_crypt; 163 struct scatterlist **journal_scatterlist; 164 struct scatterlist **journal_io_scatterlist; 165 struct skcipher_request **sk_requests; 166 167 struct crypto_shash *journal_mac; 168 169 struct journal_node *journal_tree; 170 struct rb_root journal_tree_root; 171 172 sector_t provided_data_sectors; 173 174 unsigned short journal_entry_size; 175 unsigned char journal_entries_per_sector; 176 unsigned char journal_section_entries; 177 unsigned short journal_section_sectors; 178 unsigned journal_sections; 179 unsigned journal_entries; 180 sector_t data_device_sectors; 181 sector_t meta_device_sectors; 182 unsigned initial_sectors; 183 unsigned metadata_run; 184 __s8 log2_metadata_run; 185 __u8 log2_buffer_sectors; 186 __u8 sectors_per_block; 187 188 unsigned char mode; 189 int suspending; 190 191 int failed; 192 193 struct crypto_shash *internal_hash; 194 195 /* these variables are locked with endio_wait.lock */ 196 struct rb_root in_progress; 197 struct list_head wait_list; 198 wait_queue_head_t endio_wait; 199 struct workqueue_struct *wait_wq; 200 201 unsigned char commit_seq; 202 commit_id_t commit_ids[N_COMMIT_IDS]; 203 204 unsigned committed_section; 205 unsigned n_committed_sections; 206 207 unsigned uncommitted_section; 208 unsigned n_uncommitted_sections; 209 210 unsigned free_section; 211 unsigned char free_section_entry; 212 unsigned free_sectors; 213 214 unsigned free_sectors_threshold; 215 216 struct workqueue_struct *commit_wq; 217 struct work_struct commit_work; 218 219 struct workqueue_struct *writer_wq; 220 struct work_struct writer_work; 221 222 struct workqueue_struct *recalc_wq; 223 struct work_struct recalc_work; 224 u8 *recalc_buffer; 225 u8 *recalc_tags; 226 227 struct bio_list flush_bio_list; 228 229 unsigned long autocommit_jiffies; 230 struct timer_list autocommit_timer; 231 unsigned autocommit_msec; 232 233 wait_queue_head_t copy_to_journal_wait; 234 235 struct completion crypto_backoff; 236 237 bool journal_uptodate; 238 bool just_formatted; 239 240 struct alg_spec internal_hash_alg; 241 struct alg_spec journal_crypt_alg; 242 struct alg_spec journal_mac_alg; 243 244 atomic64_t number_of_mismatches; 245 }; 246 247 struct dm_integrity_range { 248 sector_t logical_sector; 249 unsigned n_sectors; 250 bool waiting; 251 union { 252 struct rb_node node; 253 struct { 254 struct task_struct *task; 255 struct list_head wait_entry; 256 }; 257 }; 258 }; 259 260 struct dm_integrity_io { 261 struct work_struct work; 262 263 struct dm_integrity_c *ic; 264 bool write; 265 bool fua; 266 267 struct dm_integrity_range range; 268 269 sector_t metadata_block; 270 unsigned metadata_offset; 271 272 atomic_t in_flight; 273 blk_status_t bi_status; 274 275 struct completion *completion; 276 277 struct gendisk *orig_bi_disk; 278 u8 orig_bi_partno; 279 bio_end_io_t *orig_bi_end_io; 280 struct bio_integrity_payload *orig_bi_integrity; 281 struct bvec_iter orig_bi_iter; 282 }; 283 284 struct journal_completion { 285 struct dm_integrity_c *ic; 286 atomic_t in_flight; 287 struct completion comp; 288 }; 289 290 struct journal_io { 291 struct dm_integrity_range range; 292 struct journal_completion *comp; 293 }; 294 295 static struct kmem_cache *journal_io_cache; 296 297 #define JOURNAL_IO_MEMPOOL 32 298 299 #ifdef DEBUG_PRINT 300 #define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__) 301 static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...) 302 { 303 va_list args; 304 va_start(args, msg); 305 vprintk(msg, args); 306 va_end(args); 307 if (len) 308 pr_cont(":"); 309 while (len) { 310 pr_cont(" %02x", *bytes); 311 bytes++; 312 len--; 313 } 314 pr_cont("\n"); 315 } 316 #define DEBUG_bytes(bytes, len, msg, ...) __DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__) 317 #else 318 #define DEBUG_print(x, ...) do { } while (0) 319 #define DEBUG_bytes(bytes, len, msg, ...) do { } while (0) 320 #endif 321 322 /* 323 * DM Integrity profile, protection is performed layer above (dm-crypt) 324 */ 325 static const struct blk_integrity_profile dm_integrity_profile = { 326 .name = "DM-DIF-EXT-TAG", 327 .generate_fn = NULL, 328 .verify_fn = NULL, 329 }; 330 331 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); 332 static void integrity_bio_wait(struct work_struct *w); 333 static void dm_integrity_dtr(struct dm_target *ti); 334 335 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) 336 { 337 if (err == -EILSEQ) 338 atomic64_inc(&ic->number_of_mismatches); 339 if (!cmpxchg(&ic->failed, 0, err)) 340 DMERR("Error on %s: %d", msg, err); 341 } 342 343 static int dm_integrity_failed(struct dm_integrity_c *ic) 344 { 345 return READ_ONCE(ic->failed); 346 } 347 348 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i, 349 unsigned j, unsigned char seq) 350 { 351 /* 352 * Xor the number with section and sector, so that if a piece of 353 * journal is written at wrong place, it is detected. 354 */ 355 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j); 356 } 357 358 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, 359 sector_t *area, sector_t *offset) 360 { 361 if (!ic->meta_dev) { 362 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; 363 *area = data_sector >> log2_interleave_sectors; 364 *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1); 365 } else { 366 *area = 0; 367 *offset = data_sector; 368 } 369 } 370 371 #define sector_to_block(ic, n) \ 372 do { \ 373 BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1)); \ 374 (n) >>= (ic)->sb->log2_sectors_per_block; \ 375 } while (0) 376 377 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, 378 sector_t offset, unsigned *metadata_offset) 379 { 380 __u64 ms; 381 unsigned mo; 382 383 ms = area << ic->sb->log2_interleave_sectors; 384 if (likely(ic->log2_metadata_run >= 0)) 385 ms += area << ic->log2_metadata_run; 386 else 387 ms += area * ic->metadata_run; 388 ms >>= ic->log2_buffer_sectors; 389 390 sector_to_block(ic, offset); 391 392 if (likely(ic->log2_tag_size >= 0)) { 393 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size); 394 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); 395 } else { 396 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors); 397 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1); 398 } 399 *metadata_offset = mo; 400 return ms; 401 } 402 403 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset) 404 { 405 sector_t result; 406 407 if (ic->meta_dev) 408 return offset; 409 410 result = area << ic->sb->log2_interleave_sectors; 411 if (likely(ic->log2_metadata_run >= 0)) 412 result += (area + 1) << ic->log2_metadata_run; 413 else 414 result += (area + 1) * ic->metadata_run; 415 416 result += (sector_t)ic->initial_sectors + offset; 417 result += ic->start; 418 419 return result; 420 } 421 422 static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr) 423 { 424 if (unlikely(*sec_ptr >= ic->journal_sections)) 425 *sec_ptr -= ic->journal_sections; 426 } 427 428 static void sb_set_version(struct dm_integrity_c *ic) 429 { 430 if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 431 ic->sb->version = SB_VERSION_2; 432 else 433 ic->sb->version = SB_VERSION_1; 434 } 435 436 static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) 437 { 438 struct dm_io_request io_req; 439 struct dm_io_region io_loc; 440 441 io_req.bi_op = op; 442 io_req.bi_op_flags = op_flags; 443 io_req.mem.type = DM_IO_KMEM; 444 io_req.mem.ptr.addr = ic->sb; 445 io_req.notify.fn = NULL; 446 io_req.client = ic->io; 447 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; 448 io_loc.sector = ic->start; 449 io_loc.count = SB_SECTORS; 450 451 return dm_io(&io_req, 1, &io_loc, NULL); 452 } 453 454 static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset, 455 bool e, const char *function) 456 { 457 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY) 458 unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors; 459 460 if (unlikely(section >= ic->journal_sections) || 461 unlikely(offset >= limit)) { 462 printk(KERN_CRIT "%s: invalid access at (%u,%u), limit (%u,%u)\n", 463 function, section, offset, ic->journal_sections, limit); 464 BUG(); 465 } 466 #endif 467 } 468 469 static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset, 470 unsigned *pl_index, unsigned *pl_offset) 471 { 472 unsigned sector; 473 474 access_journal_check(ic, section, offset, false, "page_list_location"); 475 476 sector = section * ic->journal_section_sectors + offset; 477 478 *pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 479 *pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 480 } 481 482 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, 483 unsigned section, unsigned offset, unsigned *n_sectors) 484 { 485 unsigned pl_index, pl_offset; 486 char *va; 487 488 page_list_location(ic, section, offset, &pl_index, &pl_offset); 489 490 if (n_sectors) 491 *n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT; 492 493 va = lowmem_page_address(pl[pl_index].page); 494 495 return (struct journal_sector *)(va + pl_offset); 496 } 497 498 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset) 499 { 500 return access_page_list(ic, ic->journal, section, offset, NULL); 501 } 502 503 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n) 504 { 505 unsigned rel_sector, offset; 506 struct journal_sector *js; 507 508 access_journal_check(ic, section, n, true, "access_journal_entry"); 509 510 rel_sector = n % JOURNAL_BLOCK_SECTORS; 511 offset = n / JOURNAL_BLOCK_SECTORS; 512 513 js = access_journal(ic, section, rel_sector); 514 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size); 515 } 516 517 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n) 518 { 519 n <<= ic->sb->log2_sectors_per_block; 520 521 n += JOURNAL_BLOCK_SECTORS; 522 523 access_journal_check(ic, section, n, false, "access_journal_data"); 524 525 return access_journal(ic, section, n); 526 } 527 528 static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE]) 529 { 530 SHASH_DESC_ON_STACK(desc, ic->journal_mac); 531 int r; 532 unsigned j, size; 533 534 desc->tfm = ic->journal_mac; 535 desc->flags = 0; 536 537 r = crypto_shash_init(desc); 538 if (unlikely(r)) { 539 dm_integrity_io_error(ic, "crypto_shash_init", r); 540 goto err; 541 } 542 543 for (j = 0; j < ic->journal_section_entries; j++) { 544 struct journal_entry *je = access_journal_entry(ic, section, j); 545 r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector); 546 if (unlikely(r)) { 547 dm_integrity_io_error(ic, "crypto_shash_update", r); 548 goto err; 549 } 550 } 551 552 size = crypto_shash_digestsize(ic->journal_mac); 553 554 if (likely(size <= JOURNAL_MAC_SIZE)) { 555 r = crypto_shash_final(desc, result); 556 if (unlikely(r)) { 557 dm_integrity_io_error(ic, "crypto_shash_final", r); 558 goto err; 559 } 560 memset(result + size, 0, JOURNAL_MAC_SIZE - size); 561 } else { 562 __u8 digest[HASH_MAX_DIGESTSIZE]; 563 564 if (WARN_ON(size > sizeof(digest))) { 565 dm_integrity_io_error(ic, "digest_size", -EINVAL); 566 goto err; 567 } 568 r = crypto_shash_final(desc, digest); 569 if (unlikely(r)) { 570 dm_integrity_io_error(ic, "crypto_shash_final", r); 571 goto err; 572 } 573 memcpy(result, digest, JOURNAL_MAC_SIZE); 574 } 575 576 return; 577 err: 578 memset(result, 0, JOURNAL_MAC_SIZE); 579 } 580 581 static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr) 582 { 583 __u8 result[JOURNAL_MAC_SIZE]; 584 unsigned j; 585 586 if (!ic->journal_mac) 587 return; 588 589 section_mac(ic, section, result); 590 591 for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) { 592 struct journal_sector *js = access_journal(ic, section, j); 593 594 if (likely(wr)) 595 memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR); 596 else { 597 if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) 598 dm_integrity_io_error(ic, "journal mac", -EILSEQ); 599 } 600 } 601 } 602 603 static void complete_journal_op(void *context) 604 { 605 struct journal_completion *comp = context; 606 BUG_ON(!atomic_read(&comp->in_flight)); 607 if (likely(atomic_dec_and_test(&comp->in_flight))) 608 complete(&comp->comp); 609 } 610 611 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, 612 unsigned n_sections, struct journal_completion *comp) 613 { 614 struct async_submit_ctl submit; 615 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT; 616 unsigned pl_index, pl_offset, section_index; 617 struct page_list *source_pl, *target_pl; 618 619 if (likely(encrypt)) { 620 source_pl = ic->journal; 621 target_pl = ic->journal_io; 622 } else { 623 source_pl = ic->journal_io; 624 target_pl = ic->journal; 625 } 626 627 page_list_location(ic, section, 0, &pl_index, &pl_offset); 628 629 atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight); 630 631 init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL); 632 633 section_index = pl_index; 634 635 do { 636 size_t this_step; 637 struct page *src_pages[2]; 638 struct page *dst_page; 639 640 while (unlikely(pl_index == section_index)) { 641 unsigned dummy; 642 if (likely(encrypt)) 643 rw_section_mac(ic, section, true); 644 section++; 645 n_sections--; 646 if (!n_sections) 647 break; 648 page_list_location(ic, section, 0, §ion_index, &dummy); 649 } 650 651 this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset); 652 dst_page = target_pl[pl_index].page; 653 src_pages[0] = source_pl[pl_index].page; 654 src_pages[1] = ic->journal_xor[pl_index].page; 655 656 async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit); 657 658 pl_index++; 659 pl_offset = 0; 660 n_bytes -= this_step; 661 } while (n_bytes); 662 663 BUG_ON(n_sections); 664 665 async_tx_issue_pending_all(); 666 } 667 668 static void complete_journal_encrypt(struct crypto_async_request *req, int err) 669 { 670 struct journal_completion *comp = req->data; 671 if (unlikely(err)) { 672 if (likely(err == -EINPROGRESS)) { 673 complete(&comp->ic->crypto_backoff); 674 return; 675 } 676 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err); 677 } 678 complete_journal_op(comp); 679 } 680 681 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp) 682 { 683 int r; 684 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 685 complete_journal_encrypt, comp); 686 if (likely(encrypt)) 687 r = crypto_skcipher_encrypt(req); 688 else 689 r = crypto_skcipher_decrypt(req); 690 if (likely(!r)) 691 return false; 692 if (likely(r == -EINPROGRESS)) 693 return true; 694 if (likely(r == -EBUSY)) { 695 wait_for_completion(&comp->ic->crypto_backoff); 696 reinit_completion(&comp->ic->crypto_backoff); 697 return true; 698 } 699 dm_integrity_io_error(comp->ic, "encrypt", r); 700 return false; 701 } 702 703 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, 704 unsigned n_sections, struct journal_completion *comp) 705 { 706 struct scatterlist **source_sg; 707 struct scatterlist **target_sg; 708 709 atomic_add(2, &comp->in_flight); 710 711 if (likely(encrypt)) { 712 source_sg = ic->journal_scatterlist; 713 target_sg = ic->journal_io_scatterlist; 714 } else { 715 source_sg = ic->journal_io_scatterlist; 716 target_sg = ic->journal_scatterlist; 717 } 718 719 do { 720 struct skcipher_request *req; 721 unsigned ivsize; 722 char *iv; 723 724 if (likely(encrypt)) 725 rw_section_mac(ic, section, true); 726 727 req = ic->sk_requests[section]; 728 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); 729 iv = req->iv; 730 731 memcpy(iv, iv + ivsize, ivsize); 732 733 req->src = source_sg[section]; 734 req->dst = target_sg[section]; 735 736 if (unlikely(do_crypt(encrypt, req, comp))) 737 atomic_inc(&comp->in_flight); 738 739 section++; 740 n_sections--; 741 } while (n_sections); 742 743 atomic_dec(&comp->in_flight); 744 complete_journal_op(comp); 745 } 746 747 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, 748 unsigned n_sections, struct journal_completion *comp) 749 { 750 if (ic->journal_xor) 751 return xor_journal(ic, encrypt, section, n_sections, comp); 752 else 753 return crypt_journal(ic, encrypt, section, n_sections, comp); 754 } 755 756 static void complete_journal_io(unsigned long error, void *context) 757 { 758 struct journal_completion *comp = context; 759 if (unlikely(error != 0)) 760 dm_integrity_io_error(comp->ic, "writing journal", -EIO); 761 complete_journal_op(comp); 762 } 763 764 static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section, 765 unsigned n_sections, struct journal_completion *comp) 766 { 767 struct dm_io_request io_req; 768 struct dm_io_region io_loc; 769 unsigned sector, n_sectors, pl_index, pl_offset; 770 int r; 771 772 if (unlikely(dm_integrity_failed(ic))) { 773 if (comp) 774 complete_journal_io(-1UL, comp); 775 return; 776 } 777 778 sector = section * ic->journal_section_sectors; 779 n_sectors = n_sections * ic->journal_section_sectors; 780 781 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 782 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 783 784 io_req.bi_op = op; 785 io_req.bi_op_flags = op_flags; 786 io_req.mem.type = DM_IO_PAGE_LIST; 787 if (ic->journal_io) 788 io_req.mem.ptr.pl = &ic->journal_io[pl_index]; 789 else 790 io_req.mem.ptr.pl = &ic->journal[pl_index]; 791 io_req.mem.offset = pl_offset; 792 if (likely(comp != NULL)) { 793 io_req.notify.fn = complete_journal_io; 794 io_req.notify.context = comp; 795 } else { 796 io_req.notify.fn = NULL; 797 } 798 io_req.client = ic->io; 799 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; 800 io_loc.sector = ic->start + SB_SECTORS + sector; 801 io_loc.count = n_sectors; 802 803 r = dm_io(&io_req, 1, &io_loc, NULL); 804 if (unlikely(r)) { 805 dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r); 806 if (comp) { 807 WARN_ONCE(1, "asynchronous dm_io failed: %d", r); 808 complete_journal_io(-1UL, comp); 809 } 810 } 811 } 812 813 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections) 814 { 815 struct journal_completion io_comp; 816 struct journal_completion crypt_comp_1; 817 struct journal_completion crypt_comp_2; 818 unsigned i; 819 820 io_comp.ic = ic; 821 init_completion(&io_comp.comp); 822 823 if (commit_start + commit_sections <= ic->journal_sections) { 824 io_comp.in_flight = (atomic_t)ATOMIC_INIT(1); 825 if (ic->journal_io) { 826 crypt_comp_1.ic = ic; 827 init_completion(&crypt_comp_1.comp); 828 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 829 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1); 830 wait_for_completion_io(&crypt_comp_1.comp); 831 } else { 832 for (i = 0; i < commit_sections; i++) 833 rw_section_mac(ic, commit_start + i, true); 834 } 835 rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start, 836 commit_sections, &io_comp); 837 } else { 838 unsigned to_end; 839 io_comp.in_flight = (atomic_t)ATOMIC_INIT(2); 840 to_end = ic->journal_sections - commit_start; 841 if (ic->journal_io) { 842 crypt_comp_1.ic = ic; 843 init_completion(&crypt_comp_1.comp); 844 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 845 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); 846 if (try_wait_for_completion(&crypt_comp_1.comp)) { 847 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); 848 reinit_completion(&crypt_comp_1.comp); 849 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); 850 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); 851 wait_for_completion_io(&crypt_comp_1.comp); 852 } else { 853 crypt_comp_2.ic = ic; 854 init_completion(&crypt_comp_2.comp); 855 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0); 856 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); 857 wait_for_completion_io(&crypt_comp_1.comp); 858 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); 859 wait_for_completion_io(&crypt_comp_2.comp); 860 } 861 } else { 862 for (i = 0; i < to_end; i++) 863 rw_section_mac(ic, commit_start + i, true); 864 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); 865 for (i = 0; i < commit_sections - to_end; i++) 866 rw_section_mac(ic, i, true); 867 } 868 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp); 869 } 870 871 wait_for_completion_io(&io_comp.comp); 872 } 873 874 static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset, 875 unsigned n_sectors, sector_t target, io_notify_fn fn, void *data) 876 { 877 struct dm_io_request io_req; 878 struct dm_io_region io_loc; 879 int r; 880 unsigned sector, pl_index, pl_offset; 881 882 BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1)); 883 884 if (unlikely(dm_integrity_failed(ic))) { 885 fn(-1UL, data); 886 return; 887 } 888 889 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset; 890 891 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); 892 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); 893 894 io_req.bi_op = REQ_OP_WRITE; 895 io_req.bi_op_flags = 0; 896 io_req.mem.type = DM_IO_PAGE_LIST; 897 io_req.mem.ptr.pl = &ic->journal[pl_index]; 898 io_req.mem.offset = pl_offset; 899 io_req.notify.fn = fn; 900 io_req.notify.context = data; 901 io_req.client = ic->io; 902 io_loc.bdev = ic->dev->bdev; 903 io_loc.sector = target; 904 io_loc.count = n_sectors; 905 906 r = dm_io(&io_req, 1, &io_loc, NULL); 907 if (unlikely(r)) { 908 WARN_ONCE(1, "asynchronous dm_io failed: %d", r); 909 fn(-1UL, data); 910 } 911 } 912 913 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2) 914 { 915 return range1->logical_sector < range2->logical_sector + range2->n_sectors && 916 range1->logical_sector + range1->n_sectors > range2->logical_sector; 917 } 918 919 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting) 920 { 921 struct rb_node **n = &ic->in_progress.rb_node; 922 struct rb_node *parent; 923 924 BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1)); 925 926 if (likely(check_waiting)) { 927 struct dm_integrity_range *range; 928 list_for_each_entry(range, &ic->wait_list, wait_entry) { 929 if (unlikely(ranges_overlap(range, new_range))) 930 return false; 931 } 932 } 933 934 parent = NULL; 935 936 while (*n) { 937 struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node); 938 939 parent = *n; 940 if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) { 941 n = &range->node.rb_left; 942 } else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) { 943 n = &range->node.rb_right; 944 } else { 945 return false; 946 } 947 } 948 949 rb_link_node(&new_range->node, parent, n); 950 rb_insert_color(&new_range->node, &ic->in_progress); 951 952 return true; 953 } 954 955 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) 956 { 957 rb_erase(&range->node, &ic->in_progress); 958 while (unlikely(!list_empty(&ic->wait_list))) { 959 struct dm_integrity_range *last_range = 960 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); 961 struct task_struct *last_range_task; 962 last_range_task = last_range->task; 963 list_del(&last_range->wait_entry); 964 if (!add_new_range(ic, last_range, false)) { 965 last_range->task = last_range_task; 966 list_add(&last_range->wait_entry, &ic->wait_list); 967 break; 968 } 969 last_range->waiting = false; 970 wake_up_process(last_range_task); 971 } 972 } 973 974 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) 975 { 976 unsigned long flags; 977 978 spin_lock_irqsave(&ic->endio_wait.lock, flags); 979 remove_range_unlocked(ic, range); 980 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); 981 } 982 983 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) 984 { 985 new_range->waiting = true; 986 list_add_tail(&new_range->wait_entry, &ic->wait_list); 987 new_range->task = current; 988 do { 989 __set_current_state(TASK_UNINTERRUPTIBLE); 990 spin_unlock_irq(&ic->endio_wait.lock); 991 io_schedule(); 992 spin_lock_irq(&ic->endio_wait.lock); 993 } while (unlikely(new_range->waiting)); 994 } 995 996 static void init_journal_node(struct journal_node *node) 997 { 998 RB_CLEAR_NODE(&node->node); 999 node->sector = (sector_t)-1; 1000 } 1001 1002 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector) 1003 { 1004 struct rb_node **link; 1005 struct rb_node *parent; 1006 1007 node->sector = sector; 1008 BUG_ON(!RB_EMPTY_NODE(&node->node)); 1009 1010 link = &ic->journal_tree_root.rb_node; 1011 parent = NULL; 1012 1013 while (*link) { 1014 struct journal_node *j; 1015 parent = *link; 1016 j = container_of(parent, struct journal_node, node); 1017 if (sector < j->sector) 1018 link = &j->node.rb_left; 1019 else 1020 link = &j->node.rb_right; 1021 } 1022 1023 rb_link_node(&node->node, parent, link); 1024 rb_insert_color(&node->node, &ic->journal_tree_root); 1025 } 1026 1027 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node) 1028 { 1029 BUG_ON(RB_EMPTY_NODE(&node->node)); 1030 rb_erase(&node->node, &ic->journal_tree_root); 1031 init_journal_node(node); 1032 } 1033 1034 #define NOT_FOUND (-1U) 1035 1036 static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) 1037 { 1038 struct rb_node *n = ic->journal_tree_root.rb_node; 1039 unsigned found = NOT_FOUND; 1040 *next_sector = (sector_t)-1; 1041 while (n) { 1042 struct journal_node *j = container_of(n, struct journal_node, node); 1043 if (sector == j->sector) { 1044 found = j - ic->journal_tree; 1045 } 1046 if (sector < j->sector) { 1047 *next_sector = j->sector; 1048 n = j->node.rb_left; 1049 } else { 1050 n = j->node.rb_right; 1051 } 1052 } 1053 1054 return found; 1055 } 1056 1057 static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector) 1058 { 1059 struct journal_node *node, *next_node; 1060 struct rb_node *next; 1061 1062 if (unlikely(pos >= ic->journal_entries)) 1063 return false; 1064 node = &ic->journal_tree[pos]; 1065 if (unlikely(RB_EMPTY_NODE(&node->node))) 1066 return false; 1067 if (unlikely(node->sector != sector)) 1068 return false; 1069 1070 next = rb_next(&node->node); 1071 if (unlikely(!next)) 1072 return true; 1073 1074 next_node = container_of(next, struct journal_node, node); 1075 return next_node->sector != sector; 1076 } 1077 1078 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node) 1079 { 1080 struct rb_node *next; 1081 struct journal_node *next_node; 1082 unsigned next_section; 1083 1084 BUG_ON(RB_EMPTY_NODE(&node->node)); 1085 1086 next = rb_next(&node->node); 1087 if (unlikely(!next)) 1088 return false; 1089 1090 next_node = container_of(next, struct journal_node, node); 1091 1092 if (next_node->sector != node->sector) 1093 return false; 1094 1095 next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries; 1096 if (next_section >= ic->committed_section && 1097 next_section < ic->committed_section + ic->n_committed_sections) 1098 return true; 1099 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections) 1100 return true; 1101 1102 return false; 1103 } 1104 1105 #define TAG_READ 0 1106 #define TAG_WRITE 1 1107 #define TAG_CMP 2 1108 1109 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block, 1110 unsigned *metadata_offset, unsigned total_size, int op) 1111 { 1112 do { 1113 unsigned char *data, *dp; 1114 struct dm_buffer *b; 1115 unsigned to_copy; 1116 int r; 1117 1118 r = dm_integrity_failed(ic); 1119 if (unlikely(r)) 1120 return r; 1121 1122 data = dm_bufio_read(ic->bufio, *metadata_block, &b); 1123 if (IS_ERR(data)) 1124 return PTR_ERR(data); 1125 1126 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size); 1127 dp = data + *metadata_offset; 1128 if (op == TAG_READ) { 1129 memcpy(tag, dp, to_copy); 1130 } else if (op == TAG_WRITE) { 1131 memcpy(dp, tag, to_copy); 1132 dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy); 1133 } else { 1134 /* e.g.: op == TAG_CMP */ 1135 if (unlikely(memcmp(dp, tag, to_copy))) { 1136 unsigned i; 1137 1138 for (i = 0; i < to_copy; i++) { 1139 if (dp[i] != tag[i]) 1140 break; 1141 total_size--; 1142 } 1143 dm_bufio_release(b); 1144 return total_size; 1145 } 1146 } 1147 dm_bufio_release(b); 1148 1149 tag += to_copy; 1150 *metadata_offset += to_copy; 1151 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { 1152 (*metadata_block)++; 1153 *metadata_offset = 0; 1154 } 1155 total_size -= to_copy; 1156 } while (unlikely(total_size)); 1157 1158 return 0; 1159 } 1160 1161 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic) 1162 { 1163 int r; 1164 r = dm_bufio_write_dirty_buffers(ic->bufio); 1165 if (unlikely(r)) 1166 dm_integrity_io_error(ic, "writing tags", r); 1167 } 1168 1169 static void sleep_on_endio_wait(struct dm_integrity_c *ic) 1170 { 1171 DECLARE_WAITQUEUE(wait, current); 1172 __add_wait_queue(&ic->endio_wait, &wait); 1173 __set_current_state(TASK_UNINTERRUPTIBLE); 1174 spin_unlock_irq(&ic->endio_wait.lock); 1175 io_schedule(); 1176 spin_lock_irq(&ic->endio_wait.lock); 1177 __remove_wait_queue(&ic->endio_wait, &wait); 1178 } 1179 1180 static void autocommit_fn(struct timer_list *t) 1181 { 1182 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer); 1183 1184 if (likely(!dm_integrity_failed(ic))) 1185 queue_work(ic->commit_wq, &ic->commit_work); 1186 } 1187 1188 static void schedule_autocommit(struct dm_integrity_c *ic) 1189 { 1190 if (!timer_pending(&ic->autocommit_timer)) 1191 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies); 1192 } 1193 1194 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) 1195 { 1196 struct bio *bio; 1197 unsigned long flags; 1198 1199 spin_lock_irqsave(&ic->endio_wait.lock, flags); 1200 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1201 bio_list_add(&ic->flush_bio_list, bio); 1202 spin_unlock_irqrestore(&ic->endio_wait.lock, flags); 1203 1204 queue_work(ic->commit_wq, &ic->commit_work); 1205 } 1206 1207 static void do_endio(struct dm_integrity_c *ic, struct bio *bio) 1208 { 1209 int r = dm_integrity_failed(ic); 1210 if (unlikely(r) && !bio->bi_status) 1211 bio->bi_status = errno_to_blk_status(r); 1212 bio_endio(bio); 1213 } 1214 1215 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) 1216 { 1217 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1218 1219 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) 1220 submit_flush_bio(ic, dio); 1221 else 1222 do_endio(ic, bio); 1223 } 1224 1225 static void dec_in_flight(struct dm_integrity_io *dio) 1226 { 1227 if (atomic_dec_and_test(&dio->in_flight)) { 1228 struct dm_integrity_c *ic = dio->ic; 1229 struct bio *bio; 1230 1231 remove_range(ic, &dio->range); 1232 1233 if (unlikely(dio->write)) 1234 schedule_autocommit(ic); 1235 1236 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1237 1238 if (unlikely(dio->bi_status) && !bio->bi_status) 1239 bio->bi_status = dio->bi_status; 1240 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { 1241 dio->range.logical_sector += dio->range.n_sectors; 1242 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); 1243 INIT_WORK(&dio->work, integrity_bio_wait); 1244 queue_work(ic->wait_wq, &dio->work); 1245 return; 1246 } 1247 do_endio_flush(ic, dio); 1248 } 1249 } 1250 1251 static void integrity_end_io(struct bio *bio) 1252 { 1253 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 1254 1255 bio->bi_iter = dio->orig_bi_iter; 1256 bio->bi_disk = dio->orig_bi_disk; 1257 bio->bi_partno = dio->orig_bi_partno; 1258 if (dio->orig_bi_integrity) { 1259 bio->bi_integrity = dio->orig_bi_integrity; 1260 bio->bi_opf |= REQ_INTEGRITY; 1261 } 1262 bio->bi_end_io = dio->orig_bi_end_io; 1263 1264 if (dio->completion) 1265 complete(dio->completion); 1266 1267 dec_in_flight(dio); 1268 } 1269 1270 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, 1271 const char *data, char *result) 1272 { 1273 __u64 sector_le = cpu_to_le64(sector); 1274 SHASH_DESC_ON_STACK(req, ic->internal_hash); 1275 int r; 1276 unsigned digest_size; 1277 1278 req->tfm = ic->internal_hash; 1279 req->flags = 0; 1280 1281 r = crypto_shash_init(req); 1282 if (unlikely(r < 0)) { 1283 dm_integrity_io_error(ic, "crypto_shash_init", r); 1284 goto failed; 1285 } 1286 1287 r = crypto_shash_update(req, (const __u8 *)§or_le, sizeof sector_le); 1288 if (unlikely(r < 0)) { 1289 dm_integrity_io_error(ic, "crypto_shash_update", r); 1290 goto failed; 1291 } 1292 1293 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT); 1294 if (unlikely(r < 0)) { 1295 dm_integrity_io_error(ic, "crypto_shash_update", r); 1296 goto failed; 1297 } 1298 1299 r = crypto_shash_final(req, result); 1300 if (unlikely(r < 0)) { 1301 dm_integrity_io_error(ic, "crypto_shash_final", r); 1302 goto failed; 1303 } 1304 1305 digest_size = crypto_shash_digestsize(ic->internal_hash); 1306 if (unlikely(digest_size < ic->tag_size)) 1307 memset(result + digest_size, 0, ic->tag_size - digest_size); 1308 1309 return; 1310 1311 failed: 1312 /* this shouldn't happen anyway, the hash functions have no reason to fail */ 1313 get_random_bytes(result, ic->tag_size); 1314 } 1315 1316 static void integrity_metadata(struct work_struct *w) 1317 { 1318 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 1319 struct dm_integrity_c *ic = dio->ic; 1320 1321 int r; 1322 1323 if (ic->internal_hash) { 1324 struct bvec_iter iter; 1325 struct bio_vec bv; 1326 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); 1327 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1328 char *checksums; 1329 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; 1330 char checksums_onstack[HASH_MAX_DIGESTSIZE]; 1331 unsigned sectors_to_process = dio->range.n_sectors; 1332 sector_t sector = dio->range.logical_sector; 1333 1334 if (unlikely(ic->mode == 'R')) 1335 goto skip_io; 1336 1337 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space, 1338 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); 1339 if (!checksums) { 1340 checksums = checksums_onstack; 1341 if (WARN_ON(extra_space && 1342 digest_size > sizeof(checksums_onstack))) { 1343 r = -EINVAL; 1344 goto error; 1345 } 1346 } 1347 1348 __bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) { 1349 unsigned pos; 1350 char *mem, *checksums_ptr; 1351 1352 again: 1353 mem = (char *)kmap_atomic(bv.bv_page) + bv.bv_offset; 1354 pos = 0; 1355 checksums_ptr = checksums; 1356 do { 1357 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); 1358 checksums_ptr += ic->tag_size; 1359 sectors_to_process -= ic->sectors_per_block; 1360 pos += ic->sectors_per_block << SECTOR_SHIFT; 1361 sector += ic->sectors_per_block; 1362 } while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack); 1363 kunmap_atomic(mem); 1364 1365 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, 1366 checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE); 1367 if (unlikely(r)) { 1368 if (r > 0) { 1369 DMERR_LIMIT("Checksum failed at sector 0x%llx", 1370 (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size))); 1371 r = -EILSEQ; 1372 atomic64_inc(&ic->number_of_mismatches); 1373 } 1374 if (likely(checksums != checksums_onstack)) 1375 kfree(checksums); 1376 goto error; 1377 } 1378 1379 if (!sectors_to_process) 1380 break; 1381 1382 if (unlikely(pos < bv.bv_len)) { 1383 bv.bv_offset += pos; 1384 bv.bv_len -= pos; 1385 goto again; 1386 } 1387 } 1388 1389 if (likely(checksums != checksums_onstack)) 1390 kfree(checksums); 1391 } else { 1392 struct bio_integrity_payload *bip = dio->orig_bi_integrity; 1393 1394 if (bip) { 1395 struct bio_vec biv; 1396 struct bvec_iter iter; 1397 unsigned data_to_process = dio->range.n_sectors; 1398 sector_to_block(ic, data_to_process); 1399 data_to_process *= ic->tag_size; 1400 1401 bip_for_each_vec(biv, bip, iter) { 1402 unsigned char *tag; 1403 unsigned this_len; 1404 1405 BUG_ON(PageHighMem(biv.bv_page)); 1406 tag = lowmem_page_address(biv.bv_page) + biv.bv_offset; 1407 this_len = min(biv.bv_len, data_to_process); 1408 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, 1409 this_len, !dio->write ? TAG_READ : TAG_WRITE); 1410 if (unlikely(r)) 1411 goto error; 1412 data_to_process -= this_len; 1413 if (!data_to_process) 1414 break; 1415 } 1416 } 1417 } 1418 skip_io: 1419 dec_in_flight(dio); 1420 return; 1421 error: 1422 dio->bi_status = errno_to_blk_status(r); 1423 dec_in_flight(dio); 1424 } 1425 1426 static int dm_integrity_map(struct dm_target *ti, struct bio *bio) 1427 { 1428 struct dm_integrity_c *ic = ti->private; 1429 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); 1430 struct bio_integrity_payload *bip; 1431 1432 sector_t area, offset; 1433 1434 dio->ic = ic; 1435 dio->bi_status = 0; 1436 1437 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1438 submit_flush_bio(ic, dio); 1439 return DM_MAPIO_SUBMITTED; 1440 } 1441 1442 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 1443 dio->write = bio_op(bio) == REQ_OP_WRITE; 1444 dio->fua = dio->write && bio->bi_opf & REQ_FUA; 1445 if (unlikely(dio->fua)) { 1446 /* 1447 * Don't pass down the FUA flag because we have to flush 1448 * disk cache anyway. 1449 */ 1450 bio->bi_opf &= ~REQ_FUA; 1451 } 1452 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { 1453 DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx", 1454 (unsigned long long)dio->range.logical_sector, bio_sectors(bio), 1455 (unsigned long long)ic->provided_data_sectors); 1456 return DM_MAPIO_KILL; 1457 } 1458 if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) { 1459 DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x", 1460 ic->sectors_per_block, 1461 (unsigned long long)dio->range.logical_sector, bio_sectors(bio)); 1462 return DM_MAPIO_KILL; 1463 } 1464 1465 if (ic->sectors_per_block > 1) { 1466 struct bvec_iter iter; 1467 struct bio_vec bv; 1468 bio_for_each_segment(bv, bio, iter) { 1469 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { 1470 DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary", 1471 bv.bv_offset, bv.bv_len, ic->sectors_per_block); 1472 return DM_MAPIO_KILL; 1473 } 1474 } 1475 } 1476 1477 bip = bio_integrity(bio); 1478 if (!ic->internal_hash) { 1479 if (bip) { 1480 unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; 1481 if (ic->log2_tag_size >= 0) 1482 wanted_tag_size <<= ic->log2_tag_size; 1483 else 1484 wanted_tag_size *= ic->tag_size; 1485 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) { 1486 DMERR("Invalid integrity data size %u, expected %u", bip->bip_iter.bi_size, wanted_tag_size); 1487 return DM_MAPIO_KILL; 1488 } 1489 } 1490 } else { 1491 if (unlikely(bip != NULL)) { 1492 DMERR("Unexpected integrity data when using internal hash"); 1493 return DM_MAPIO_KILL; 1494 } 1495 } 1496 1497 if (unlikely(ic->mode == 'R') && unlikely(dio->write)) 1498 return DM_MAPIO_KILL; 1499 1500 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); 1501 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); 1502 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset); 1503 1504 dm_integrity_map_continue(dio, true); 1505 return DM_MAPIO_SUBMITTED; 1506 } 1507 1508 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, 1509 unsigned journal_section, unsigned journal_entry) 1510 { 1511 struct dm_integrity_c *ic = dio->ic; 1512 sector_t logical_sector; 1513 unsigned n_sectors; 1514 1515 logical_sector = dio->range.logical_sector; 1516 n_sectors = dio->range.n_sectors; 1517 do { 1518 struct bio_vec bv = bio_iovec(bio); 1519 char *mem; 1520 1521 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors)) 1522 bv.bv_len = n_sectors << SECTOR_SHIFT; 1523 n_sectors -= bv.bv_len >> SECTOR_SHIFT; 1524 bio_advance_iter(bio, &bio->bi_iter, bv.bv_len); 1525 retry_kmap: 1526 mem = kmap_atomic(bv.bv_page); 1527 if (likely(dio->write)) 1528 flush_dcache_page(bv.bv_page); 1529 1530 do { 1531 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry); 1532 1533 if (unlikely(!dio->write)) { 1534 struct journal_sector *js; 1535 char *mem_ptr; 1536 unsigned s; 1537 1538 if (unlikely(journal_entry_is_inprogress(je))) { 1539 flush_dcache_page(bv.bv_page); 1540 kunmap_atomic(mem); 1541 1542 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); 1543 goto retry_kmap; 1544 } 1545 smp_rmb(); 1546 BUG_ON(journal_entry_get_sector(je) != logical_sector); 1547 js = access_journal_data(ic, journal_section, journal_entry); 1548 mem_ptr = mem + bv.bv_offset; 1549 s = 0; 1550 do { 1551 memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA); 1552 *(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s]; 1553 js++; 1554 mem_ptr += 1 << SECTOR_SHIFT; 1555 } while (++s < ic->sectors_per_block); 1556 #ifdef INTERNAL_VERIFY 1557 if (ic->internal_hash) { 1558 char checksums_onstack[max(HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; 1559 1560 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); 1561 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { 1562 DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx", 1563 (unsigned long long)logical_sector); 1564 } 1565 } 1566 #endif 1567 } 1568 1569 if (!ic->internal_hash) { 1570 struct bio_integrity_payload *bip = bio_integrity(bio); 1571 unsigned tag_todo = ic->tag_size; 1572 char *tag_ptr = journal_entry_tag(ic, je); 1573 1574 if (bip) do { 1575 struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); 1576 unsigned tag_now = min(biv.bv_len, tag_todo); 1577 char *tag_addr; 1578 BUG_ON(PageHighMem(biv.bv_page)); 1579 tag_addr = lowmem_page_address(biv.bv_page) + biv.bv_offset; 1580 if (likely(dio->write)) 1581 memcpy(tag_ptr, tag_addr, tag_now); 1582 else 1583 memcpy(tag_addr, tag_ptr, tag_now); 1584 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now); 1585 tag_ptr += tag_now; 1586 tag_todo -= tag_now; 1587 } while (unlikely(tag_todo)); else { 1588 if (likely(dio->write)) 1589 memset(tag_ptr, 0, tag_todo); 1590 } 1591 } 1592 1593 if (likely(dio->write)) { 1594 struct journal_sector *js; 1595 unsigned s; 1596 1597 js = access_journal_data(ic, journal_section, journal_entry); 1598 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT); 1599 1600 s = 0; 1601 do { 1602 je->last_bytes[s] = js[s].commit_id; 1603 } while (++s < ic->sectors_per_block); 1604 1605 if (ic->internal_hash) { 1606 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); 1607 if (unlikely(digest_size > ic->tag_size)) { 1608 char checksums_onstack[HASH_MAX_DIGESTSIZE]; 1609 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack); 1610 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); 1611 } else 1612 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je)); 1613 } 1614 1615 journal_entry_set_sector(je, logical_sector); 1616 } 1617 logical_sector += ic->sectors_per_block; 1618 1619 journal_entry++; 1620 if (unlikely(journal_entry == ic->journal_section_entries)) { 1621 journal_entry = 0; 1622 journal_section++; 1623 wraparound_section(ic, &journal_section); 1624 } 1625 1626 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT; 1627 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT); 1628 1629 if (unlikely(!dio->write)) 1630 flush_dcache_page(bv.bv_page); 1631 kunmap_atomic(mem); 1632 } while (n_sectors); 1633 1634 if (likely(dio->write)) { 1635 smp_mb(); 1636 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) 1637 wake_up(&ic->copy_to_journal_wait); 1638 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) { 1639 queue_work(ic->commit_wq, &ic->commit_work); 1640 } else { 1641 schedule_autocommit(ic); 1642 } 1643 } else { 1644 remove_range(ic, &dio->range); 1645 } 1646 1647 if (unlikely(bio->bi_iter.bi_size)) { 1648 sector_t area, offset; 1649 1650 dio->range.logical_sector = logical_sector; 1651 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); 1652 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); 1653 return true; 1654 } 1655 1656 return false; 1657 } 1658 1659 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) 1660 { 1661 struct dm_integrity_c *ic = dio->ic; 1662 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); 1663 unsigned journal_section, journal_entry; 1664 unsigned journal_read_pos; 1665 struct completion read_comp; 1666 bool need_sync_io = ic->internal_hash && !dio->write; 1667 1668 if (need_sync_io && from_map) { 1669 INIT_WORK(&dio->work, integrity_bio_wait); 1670 queue_work(ic->metadata_wq, &dio->work); 1671 return; 1672 } 1673 1674 lock_retry: 1675 spin_lock_irq(&ic->endio_wait.lock); 1676 retry: 1677 if (unlikely(dm_integrity_failed(ic))) { 1678 spin_unlock_irq(&ic->endio_wait.lock); 1679 do_endio(ic, bio); 1680 return; 1681 } 1682 dio->range.n_sectors = bio_sectors(bio); 1683 journal_read_pos = NOT_FOUND; 1684 if (likely(ic->mode == 'J')) { 1685 if (dio->write) { 1686 unsigned next_entry, i, pos; 1687 unsigned ws, we, range_sectors; 1688 1689 dio->range.n_sectors = min(dio->range.n_sectors, 1690 ic->free_sectors << ic->sb->log2_sectors_per_block); 1691 if (unlikely(!dio->range.n_sectors)) { 1692 if (from_map) 1693 goto offload_to_thread; 1694 sleep_on_endio_wait(ic); 1695 goto retry; 1696 } 1697 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; 1698 ic->free_sectors -= range_sectors; 1699 journal_section = ic->free_section; 1700 journal_entry = ic->free_section_entry; 1701 1702 next_entry = ic->free_section_entry + range_sectors; 1703 ic->free_section_entry = next_entry % ic->journal_section_entries; 1704 ic->free_section += next_entry / ic->journal_section_entries; 1705 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries; 1706 wraparound_section(ic, &ic->free_section); 1707 1708 pos = journal_section * ic->journal_section_entries + journal_entry; 1709 ws = journal_section; 1710 we = journal_entry; 1711 i = 0; 1712 do { 1713 struct journal_entry *je; 1714 1715 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); 1716 pos++; 1717 if (unlikely(pos >= ic->journal_entries)) 1718 pos = 0; 1719 1720 je = access_journal_entry(ic, ws, we); 1721 BUG_ON(!journal_entry_is_unused(je)); 1722 journal_entry_set_inprogress(je); 1723 we++; 1724 if (unlikely(we == ic->journal_section_entries)) { 1725 we = 0; 1726 ws++; 1727 wraparound_section(ic, &ws); 1728 } 1729 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); 1730 1731 spin_unlock_irq(&ic->endio_wait.lock); 1732 goto journal_read_write; 1733 } else { 1734 sector_t next_sector; 1735 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); 1736 if (likely(journal_read_pos == NOT_FOUND)) { 1737 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) 1738 dio->range.n_sectors = next_sector - dio->range.logical_sector; 1739 } else { 1740 unsigned i; 1741 unsigned jp = journal_read_pos + 1; 1742 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { 1743 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) 1744 break; 1745 } 1746 dio->range.n_sectors = i; 1747 } 1748 } 1749 } 1750 if (unlikely(!add_new_range(ic, &dio->range, true))) { 1751 /* 1752 * We must not sleep in the request routine because it could 1753 * stall bios on current->bio_list. 1754 * So, we offload the bio to a workqueue if we have to sleep. 1755 */ 1756 if (from_map) { 1757 offload_to_thread: 1758 spin_unlock_irq(&ic->endio_wait.lock); 1759 INIT_WORK(&dio->work, integrity_bio_wait); 1760 queue_work(ic->wait_wq, &dio->work); 1761 return; 1762 } 1763 wait_and_add_new_range(ic, &dio->range); 1764 } 1765 spin_unlock_irq(&ic->endio_wait.lock); 1766 1767 if (unlikely(journal_read_pos != NOT_FOUND)) { 1768 journal_section = journal_read_pos / ic->journal_section_entries; 1769 journal_entry = journal_read_pos % ic->journal_section_entries; 1770 goto journal_read_write; 1771 } 1772 1773 dio->in_flight = (atomic_t)ATOMIC_INIT(2); 1774 1775 if (need_sync_io) { 1776 init_completion(&read_comp); 1777 dio->completion = &read_comp; 1778 } else 1779 dio->completion = NULL; 1780 1781 dio->orig_bi_iter = bio->bi_iter; 1782 1783 dio->orig_bi_disk = bio->bi_disk; 1784 dio->orig_bi_partno = bio->bi_partno; 1785 bio_set_dev(bio, ic->dev->bdev); 1786 1787 dio->orig_bi_integrity = bio_integrity(bio); 1788 bio->bi_integrity = NULL; 1789 bio->bi_opf &= ~REQ_INTEGRITY; 1790 1791 dio->orig_bi_end_io = bio->bi_end_io; 1792 bio->bi_end_io = integrity_end_io; 1793 1794 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; 1795 generic_make_request(bio); 1796 1797 if (need_sync_io) { 1798 wait_for_completion_io(&read_comp); 1799 if (unlikely(ic->recalc_wq != NULL) && 1800 ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && 1801 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) 1802 goto skip_check; 1803 if (likely(!bio->bi_status)) 1804 integrity_metadata(&dio->work); 1805 else 1806 skip_check: 1807 dec_in_flight(dio); 1808 1809 } else { 1810 INIT_WORK(&dio->work, integrity_metadata); 1811 queue_work(ic->metadata_wq, &dio->work); 1812 } 1813 1814 return; 1815 1816 journal_read_write: 1817 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) 1818 goto lock_retry; 1819 1820 do_endio_flush(ic, dio); 1821 } 1822 1823 1824 static void integrity_bio_wait(struct work_struct *w) 1825 { 1826 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); 1827 1828 dm_integrity_map_continue(dio, false); 1829 } 1830 1831 static void pad_uncommitted(struct dm_integrity_c *ic) 1832 { 1833 if (ic->free_section_entry) { 1834 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry; 1835 ic->free_section_entry = 0; 1836 ic->free_section++; 1837 wraparound_section(ic, &ic->free_section); 1838 ic->n_uncommitted_sections++; 1839 } 1840 WARN_ON(ic->journal_sections * ic->journal_section_entries != 1841 (ic->n_uncommitted_sections + ic->n_committed_sections) * ic->journal_section_entries + ic->free_sectors); 1842 } 1843 1844 static void integrity_commit(struct work_struct *w) 1845 { 1846 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work); 1847 unsigned commit_start, commit_sections; 1848 unsigned i, j, n; 1849 struct bio *flushes; 1850 1851 del_timer(&ic->autocommit_timer); 1852 1853 spin_lock_irq(&ic->endio_wait.lock); 1854 flushes = bio_list_get(&ic->flush_bio_list); 1855 if (unlikely(ic->mode != 'J')) { 1856 spin_unlock_irq(&ic->endio_wait.lock); 1857 dm_integrity_flush_buffers(ic); 1858 goto release_flush_bios; 1859 } 1860 1861 pad_uncommitted(ic); 1862 commit_start = ic->uncommitted_section; 1863 commit_sections = ic->n_uncommitted_sections; 1864 spin_unlock_irq(&ic->endio_wait.lock); 1865 1866 if (!commit_sections) 1867 goto release_flush_bios; 1868 1869 i = commit_start; 1870 for (n = 0; n < commit_sections; n++) { 1871 for (j = 0; j < ic->journal_section_entries; j++) { 1872 struct journal_entry *je; 1873 je = access_journal_entry(ic, i, j); 1874 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); 1875 } 1876 for (j = 0; j < ic->journal_section_sectors; j++) { 1877 struct journal_sector *js; 1878 js = access_journal(ic, i, j); 1879 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq); 1880 } 1881 i++; 1882 if (unlikely(i >= ic->journal_sections)) 1883 ic->commit_seq = next_commit_seq(ic->commit_seq); 1884 wraparound_section(ic, &i); 1885 } 1886 smp_rmb(); 1887 1888 write_journal(ic, commit_start, commit_sections); 1889 1890 spin_lock_irq(&ic->endio_wait.lock); 1891 ic->uncommitted_section += commit_sections; 1892 wraparound_section(ic, &ic->uncommitted_section); 1893 ic->n_uncommitted_sections -= commit_sections; 1894 ic->n_committed_sections += commit_sections; 1895 spin_unlock_irq(&ic->endio_wait.lock); 1896 1897 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) 1898 queue_work(ic->writer_wq, &ic->writer_work); 1899 1900 release_flush_bios: 1901 while (flushes) { 1902 struct bio *next = flushes->bi_next; 1903 flushes->bi_next = NULL; 1904 do_endio(ic, flushes); 1905 flushes = next; 1906 } 1907 } 1908 1909 static void complete_copy_from_journal(unsigned long error, void *context) 1910 { 1911 struct journal_io *io = context; 1912 struct journal_completion *comp = io->comp; 1913 struct dm_integrity_c *ic = comp->ic; 1914 remove_range(ic, &io->range); 1915 mempool_free(io, &ic->journal_io_mempool); 1916 if (unlikely(error != 0)) 1917 dm_integrity_io_error(ic, "copying from journal", -EIO); 1918 complete_journal_op(comp); 1919 } 1920 1921 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, 1922 struct journal_entry *je) 1923 { 1924 unsigned s = 0; 1925 do { 1926 js->commit_id = je->last_bytes[s]; 1927 js++; 1928 } while (++s < ic->sectors_per_block); 1929 } 1930 1931 static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, 1932 unsigned write_sections, bool from_replay) 1933 { 1934 unsigned i, j, n; 1935 struct journal_completion comp; 1936 struct blk_plug plug; 1937 1938 blk_start_plug(&plug); 1939 1940 comp.ic = ic; 1941 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 1942 init_completion(&comp.comp); 1943 1944 i = write_start; 1945 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) { 1946 #ifndef INTERNAL_VERIFY 1947 if (unlikely(from_replay)) 1948 #endif 1949 rw_section_mac(ic, i, false); 1950 for (j = 0; j < ic->journal_section_entries; j++) { 1951 struct journal_entry *je = access_journal_entry(ic, i, j); 1952 sector_t sec, area, offset; 1953 unsigned k, l, next_loop; 1954 sector_t metadata_block; 1955 unsigned metadata_offset; 1956 struct journal_io *io; 1957 1958 if (journal_entry_is_unused(je)) 1959 continue; 1960 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay); 1961 sec = journal_entry_get_sector(je); 1962 if (unlikely(from_replay)) { 1963 if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) { 1964 dm_integrity_io_error(ic, "invalid sector in journal", -EIO); 1965 sec &= ~(sector_t)(ic->sectors_per_block - 1); 1966 } 1967 } 1968 get_area_and_offset(ic, sec, &area, &offset); 1969 restore_last_bytes(ic, access_journal_data(ic, i, j), je); 1970 for (k = j + 1; k < ic->journal_section_entries; k++) { 1971 struct journal_entry *je2 = access_journal_entry(ic, i, k); 1972 sector_t sec2, area2, offset2; 1973 if (journal_entry_is_unused(je2)) 1974 break; 1975 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay); 1976 sec2 = journal_entry_get_sector(je2); 1977 get_area_and_offset(ic, sec2, &area2, &offset2); 1978 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block)) 1979 break; 1980 restore_last_bytes(ic, access_journal_data(ic, i, k), je2); 1981 } 1982 next_loop = k - 1; 1983 1984 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO); 1985 io->comp = ∁ 1986 io->range.logical_sector = sec; 1987 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; 1988 1989 spin_lock_irq(&ic->endio_wait.lock); 1990 if (unlikely(!add_new_range(ic, &io->range, true))) 1991 wait_and_add_new_range(ic, &io->range); 1992 1993 if (likely(!from_replay)) { 1994 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; 1995 1996 /* don't write if there is newer committed sector */ 1997 while (j < k && find_newer_committed_node(ic, §ion_node[j])) { 1998 struct journal_entry *je2 = access_journal_entry(ic, i, j); 1999 2000 journal_entry_set_unused(je2); 2001 remove_journal_node(ic, §ion_node[j]); 2002 j++; 2003 sec += ic->sectors_per_block; 2004 offset += ic->sectors_per_block; 2005 } 2006 while (j < k && find_newer_committed_node(ic, §ion_node[k - 1])) { 2007 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1); 2008 2009 journal_entry_set_unused(je2); 2010 remove_journal_node(ic, §ion_node[k - 1]); 2011 k--; 2012 } 2013 if (j == k) { 2014 remove_range_unlocked(ic, &io->range); 2015 spin_unlock_irq(&ic->endio_wait.lock); 2016 mempool_free(io, &ic->journal_io_mempool); 2017 goto skip_io; 2018 } 2019 for (l = j; l < k; l++) { 2020 remove_journal_node(ic, §ion_node[l]); 2021 } 2022 } 2023 spin_unlock_irq(&ic->endio_wait.lock); 2024 2025 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); 2026 for (l = j; l < k; l++) { 2027 int r; 2028 struct journal_entry *je2 = access_journal_entry(ic, i, l); 2029 2030 if ( 2031 #ifndef INTERNAL_VERIFY 2032 unlikely(from_replay) && 2033 #endif 2034 ic->internal_hash) { 2035 char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; 2036 2037 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), 2038 (char *)access_journal_data(ic, i, l), test_tag); 2039 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) 2040 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); 2041 } 2042 2043 journal_entry_set_unused(je2); 2044 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset, 2045 ic->tag_size, TAG_WRITE); 2046 if (unlikely(r)) { 2047 dm_integrity_io_error(ic, "reading tags", r); 2048 } 2049 } 2050 2051 atomic_inc(&comp.in_flight); 2052 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block, 2053 (k - j) << ic->sb->log2_sectors_per_block, 2054 get_data_sector(ic, area, offset), 2055 complete_copy_from_journal, io); 2056 skip_io: 2057 j = next_loop; 2058 } 2059 } 2060 2061 dm_bufio_write_dirty_buffers_async(ic->bufio); 2062 2063 blk_finish_plug(&plug); 2064 2065 complete_journal_op(&comp); 2066 wait_for_completion_io(&comp.comp); 2067 2068 dm_integrity_flush_buffers(ic); 2069 } 2070 2071 static void integrity_writer(struct work_struct *w) 2072 { 2073 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work); 2074 unsigned write_start, write_sections; 2075 2076 unsigned prev_free_sectors; 2077 2078 /* the following test is not needed, but it tests the replay code */ 2079 if (READ_ONCE(ic->suspending) && !ic->meta_dev) 2080 return; 2081 2082 spin_lock_irq(&ic->endio_wait.lock); 2083 write_start = ic->committed_section; 2084 write_sections = ic->n_committed_sections; 2085 spin_unlock_irq(&ic->endio_wait.lock); 2086 2087 if (!write_sections) 2088 return; 2089 2090 do_journal_write(ic, write_start, write_sections, false); 2091 2092 spin_lock_irq(&ic->endio_wait.lock); 2093 2094 ic->committed_section += write_sections; 2095 wraparound_section(ic, &ic->committed_section); 2096 ic->n_committed_sections -= write_sections; 2097 2098 prev_free_sectors = ic->free_sectors; 2099 ic->free_sectors += write_sections * ic->journal_section_entries; 2100 if (unlikely(!prev_free_sectors)) 2101 wake_up_locked(&ic->endio_wait); 2102 2103 spin_unlock_irq(&ic->endio_wait.lock); 2104 } 2105 2106 static void recalc_write_super(struct dm_integrity_c *ic) 2107 { 2108 int r; 2109 2110 dm_integrity_flush_buffers(ic); 2111 if (dm_integrity_failed(ic)) 2112 return; 2113 2114 sb_set_version(ic); 2115 r = sync_rw_sb(ic, REQ_OP_WRITE, 0); 2116 if (unlikely(r)) 2117 dm_integrity_io_error(ic, "writing superblock", r); 2118 } 2119 2120 static void integrity_recalc(struct work_struct *w) 2121 { 2122 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); 2123 struct dm_integrity_range range; 2124 struct dm_io_request io_req; 2125 struct dm_io_region io_loc; 2126 sector_t area, offset; 2127 sector_t metadata_block; 2128 unsigned metadata_offset; 2129 __u8 *t; 2130 unsigned i; 2131 int r; 2132 unsigned super_counter = 0; 2133 2134 spin_lock_irq(&ic->endio_wait.lock); 2135 2136 next_chunk: 2137 2138 if (unlikely(READ_ONCE(ic->suspending))) 2139 goto unlock_ret; 2140 2141 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); 2142 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) 2143 goto unlock_ret; 2144 2145 get_area_and_offset(ic, range.logical_sector, &area, &offset); 2146 range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector); 2147 if (!ic->meta_dev) 2148 range.n_sectors = min(range.n_sectors, (1U << ic->sb->log2_interleave_sectors) - (unsigned)offset); 2149 2150 if (unlikely(!add_new_range(ic, &range, true))) 2151 wait_and_add_new_range(ic, &range); 2152 2153 spin_unlock_irq(&ic->endio_wait.lock); 2154 2155 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { 2156 recalc_write_super(ic); 2157 super_counter = 0; 2158 } 2159 2160 if (unlikely(dm_integrity_failed(ic))) 2161 goto err; 2162 2163 io_req.bi_op = REQ_OP_READ; 2164 io_req.bi_op_flags = 0; 2165 io_req.mem.type = DM_IO_VMA; 2166 io_req.mem.ptr.addr = ic->recalc_buffer; 2167 io_req.notify.fn = NULL; 2168 io_req.client = ic->io; 2169 io_loc.bdev = ic->dev->bdev; 2170 io_loc.sector = get_data_sector(ic, area, offset); 2171 io_loc.count = range.n_sectors; 2172 2173 r = dm_io(&io_req, 1, &io_loc, NULL); 2174 if (unlikely(r)) { 2175 dm_integrity_io_error(ic, "reading data", r); 2176 goto err; 2177 } 2178 2179 t = ic->recalc_tags; 2180 for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { 2181 integrity_sector_checksum(ic, range.logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t); 2182 t += ic->tag_size; 2183 } 2184 2185 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); 2186 2187 r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE); 2188 if (unlikely(r)) { 2189 dm_integrity_io_error(ic, "writing tags", r); 2190 goto err; 2191 } 2192 2193 spin_lock_irq(&ic->endio_wait.lock); 2194 remove_range_unlocked(ic, &range); 2195 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); 2196 goto next_chunk; 2197 2198 err: 2199 remove_range(ic, &range); 2200 return; 2201 2202 unlock_ret: 2203 spin_unlock_irq(&ic->endio_wait.lock); 2204 2205 recalc_write_super(ic); 2206 } 2207 2208 static void init_journal(struct dm_integrity_c *ic, unsigned start_section, 2209 unsigned n_sections, unsigned char commit_seq) 2210 { 2211 unsigned i, j, n; 2212 2213 if (!n_sections) 2214 return; 2215 2216 for (n = 0; n < n_sections; n++) { 2217 i = start_section + n; 2218 wraparound_section(ic, &i); 2219 for (j = 0; j < ic->journal_section_sectors; j++) { 2220 struct journal_sector *js = access_journal(ic, i, j); 2221 memset(&js->entries, 0, JOURNAL_SECTOR_DATA); 2222 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq); 2223 } 2224 for (j = 0; j < ic->journal_section_entries; j++) { 2225 struct journal_entry *je = access_journal_entry(ic, i, j); 2226 journal_entry_set_unused(je); 2227 } 2228 } 2229 2230 write_journal(ic, start_section, n_sections); 2231 } 2232 2233 static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id) 2234 { 2235 unsigned char k; 2236 for (k = 0; k < N_COMMIT_IDS; k++) { 2237 if (dm_integrity_commit_id(ic, i, j, k) == id) 2238 return k; 2239 } 2240 dm_integrity_io_error(ic, "journal commit id", -EIO); 2241 return -EIO; 2242 } 2243 2244 static void replay_journal(struct dm_integrity_c *ic) 2245 { 2246 unsigned i, j; 2247 bool used_commit_ids[N_COMMIT_IDS]; 2248 unsigned max_commit_id_sections[N_COMMIT_IDS]; 2249 unsigned write_start, write_sections; 2250 unsigned continue_section; 2251 bool journal_empty; 2252 unsigned char unused, last_used, want_commit_seq; 2253 2254 if (ic->mode == 'R') 2255 return; 2256 2257 if (ic->journal_uptodate) 2258 return; 2259 2260 last_used = 0; 2261 write_start = 0; 2262 2263 if (!ic->just_formatted) { 2264 DEBUG_print("reading journal\n"); 2265 rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL); 2266 if (ic->journal_io) 2267 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal"); 2268 if (ic->journal_io) { 2269 struct journal_completion crypt_comp; 2270 crypt_comp.ic = ic; 2271 init_completion(&crypt_comp.comp); 2272 crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0); 2273 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp); 2274 wait_for_completion(&crypt_comp.comp); 2275 } 2276 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal"); 2277 } 2278 2279 if (dm_integrity_failed(ic)) 2280 goto clear_journal; 2281 2282 journal_empty = true; 2283 memset(used_commit_ids, 0, sizeof used_commit_ids); 2284 memset(max_commit_id_sections, 0, sizeof max_commit_id_sections); 2285 for (i = 0; i < ic->journal_sections; i++) { 2286 for (j = 0; j < ic->journal_section_sectors; j++) { 2287 int k; 2288 struct journal_sector *js = access_journal(ic, i, j); 2289 k = find_commit_seq(ic, i, j, js->commit_id); 2290 if (k < 0) 2291 goto clear_journal; 2292 used_commit_ids[k] = true; 2293 max_commit_id_sections[k] = i; 2294 } 2295 if (journal_empty) { 2296 for (j = 0; j < ic->journal_section_entries; j++) { 2297 struct journal_entry *je = access_journal_entry(ic, i, j); 2298 if (!journal_entry_is_unused(je)) { 2299 journal_empty = false; 2300 break; 2301 } 2302 } 2303 } 2304 } 2305 2306 if (!used_commit_ids[N_COMMIT_IDS - 1]) { 2307 unused = N_COMMIT_IDS - 1; 2308 while (unused && !used_commit_ids[unused - 1]) 2309 unused--; 2310 } else { 2311 for (unused = 0; unused < N_COMMIT_IDS; unused++) 2312 if (!used_commit_ids[unused]) 2313 break; 2314 if (unused == N_COMMIT_IDS) { 2315 dm_integrity_io_error(ic, "journal commit ids", -EIO); 2316 goto clear_journal; 2317 } 2318 } 2319 DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n", 2320 unused, used_commit_ids[0], used_commit_ids[1], 2321 used_commit_ids[2], used_commit_ids[3]); 2322 2323 last_used = prev_commit_seq(unused); 2324 want_commit_seq = prev_commit_seq(last_used); 2325 2326 if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)]) 2327 journal_empty = true; 2328 2329 write_start = max_commit_id_sections[last_used] + 1; 2330 if (unlikely(write_start >= ic->journal_sections)) 2331 want_commit_seq = next_commit_seq(want_commit_seq); 2332 wraparound_section(ic, &write_start); 2333 2334 i = write_start; 2335 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) { 2336 for (j = 0; j < ic->journal_section_sectors; j++) { 2337 struct journal_sector *js = access_journal(ic, i, j); 2338 2339 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) { 2340 /* 2341 * This could be caused by crash during writing. 2342 * We won't replay the inconsistent part of the 2343 * journal. 2344 */ 2345 DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n", 2346 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq); 2347 goto brk; 2348 } 2349 } 2350 i++; 2351 if (unlikely(i >= ic->journal_sections)) 2352 want_commit_seq = next_commit_seq(want_commit_seq); 2353 wraparound_section(ic, &i); 2354 } 2355 brk: 2356 2357 if (!journal_empty) { 2358 DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n", 2359 write_sections, write_start, want_commit_seq); 2360 do_journal_write(ic, write_start, write_sections, true); 2361 } 2362 2363 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) { 2364 continue_section = write_start; 2365 ic->commit_seq = want_commit_seq; 2366 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq); 2367 } else { 2368 unsigned s; 2369 unsigned char erase_seq; 2370 clear_journal: 2371 DEBUG_print("clearing journal\n"); 2372 2373 erase_seq = prev_commit_seq(prev_commit_seq(last_used)); 2374 s = write_start; 2375 init_journal(ic, s, 1, erase_seq); 2376 s++; 2377 wraparound_section(ic, &s); 2378 if (ic->journal_sections >= 2) { 2379 init_journal(ic, s, ic->journal_sections - 2, erase_seq); 2380 s += ic->journal_sections - 2; 2381 wraparound_section(ic, &s); 2382 init_journal(ic, s, 1, erase_seq); 2383 } 2384 2385 continue_section = 0; 2386 ic->commit_seq = next_commit_seq(erase_seq); 2387 } 2388 2389 ic->committed_section = continue_section; 2390 ic->n_committed_sections = 0; 2391 2392 ic->uncommitted_section = continue_section; 2393 ic->n_uncommitted_sections = 0; 2394 2395 ic->free_section = continue_section; 2396 ic->free_section_entry = 0; 2397 ic->free_sectors = ic->journal_entries; 2398 2399 ic->journal_tree_root = RB_ROOT; 2400 for (i = 0; i < ic->journal_entries; i++) 2401 init_journal_node(&ic->journal_tree[i]); 2402 } 2403 2404 static void dm_integrity_postsuspend(struct dm_target *ti) 2405 { 2406 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; 2407 2408 del_timer_sync(&ic->autocommit_timer); 2409 2410 WRITE_ONCE(ic->suspending, 1); 2411 2412 if (ic->recalc_wq) 2413 drain_workqueue(ic->recalc_wq); 2414 2415 queue_work(ic->commit_wq, &ic->commit_work); 2416 drain_workqueue(ic->commit_wq); 2417 2418 if (ic->mode == 'J') { 2419 if (ic->meta_dev) 2420 queue_work(ic->writer_wq, &ic->writer_work); 2421 drain_workqueue(ic->writer_wq); 2422 dm_integrity_flush_buffers(ic); 2423 } 2424 2425 WRITE_ONCE(ic->suspending, 0); 2426 2427 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); 2428 2429 ic->journal_uptodate = true; 2430 } 2431 2432 static void dm_integrity_resume(struct dm_target *ti) 2433 { 2434 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; 2435 2436 replay_journal(ic); 2437 2438 if (ic->recalc_wq && ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { 2439 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); 2440 if (recalc_pos < ic->provided_data_sectors) { 2441 queue_work(ic->recalc_wq, &ic->recalc_work); 2442 } else if (recalc_pos > ic->provided_data_sectors) { 2443 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors); 2444 recalc_write_super(ic); 2445 } 2446 } 2447 } 2448 2449 static void dm_integrity_status(struct dm_target *ti, status_type_t type, 2450 unsigned status_flags, char *result, unsigned maxlen) 2451 { 2452 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; 2453 unsigned arg_count; 2454 size_t sz = 0; 2455 2456 switch (type) { 2457 case STATUSTYPE_INFO: 2458 DMEMIT("%llu %llu", 2459 (unsigned long long)atomic64_read(&ic->number_of_mismatches), 2460 (unsigned long long)ic->provided_data_sectors); 2461 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 2462 DMEMIT(" %llu", (unsigned long long)le64_to_cpu(ic->sb->recalc_sector)); 2463 else 2464 DMEMIT(" -"); 2465 break; 2466 2467 case STATUSTYPE_TABLE: { 2468 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; 2469 watermark_percentage += ic->journal_entries / 2; 2470 do_div(watermark_percentage, ic->journal_entries); 2471 arg_count = 5; 2472 arg_count += !!ic->meta_dev; 2473 arg_count += ic->sectors_per_block != 1; 2474 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); 2475 arg_count += !!ic->internal_hash_alg.alg_string; 2476 arg_count += !!ic->journal_crypt_alg.alg_string; 2477 arg_count += !!ic->journal_mac_alg.alg_string; 2478 DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start, 2479 ic->tag_size, ic->mode, arg_count); 2480 if (ic->meta_dev) 2481 DMEMIT(" meta_device:%s", ic->meta_dev->name); 2482 if (ic->sectors_per_block != 1) 2483 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); 2484 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) 2485 DMEMIT(" recalculate"); 2486 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); 2487 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); 2488 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); 2489 DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage); 2490 DMEMIT(" commit_time:%u", ic->autocommit_msec); 2491 2492 #define EMIT_ALG(a, n) \ 2493 do { \ 2494 if (ic->a.alg_string) { \ 2495 DMEMIT(" %s:%s", n, ic->a.alg_string); \ 2496 if (ic->a.key_string) \ 2497 DMEMIT(":%s", ic->a.key_string);\ 2498 } \ 2499 } while (0) 2500 EMIT_ALG(internal_hash_alg, "internal_hash"); 2501 EMIT_ALG(journal_crypt_alg, "journal_crypt"); 2502 EMIT_ALG(journal_mac_alg, "journal_mac"); 2503 break; 2504 } 2505 } 2506 } 2507 2508 static int dm_integrity_iterate_devices(struct dm_target *ti, 2509 iterate_devices_callout_fn fn, void *data) 2510 { 2511 struct dm_integrity_c *ic = ti->private; 2512 2513 if (!ic->meta_dev) 2514 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); 2515 else 2516 return fn(ti, ic->dev, 0, ti->len, data); 2517 } 2518 2519 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits) 2520 { 2521 struct dm_integrity_c *ic = ti->private; 2522 2523 if (ic->sectors_per_block > 1) { 2524 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT; 2525 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT; 2526 blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); 2527 } 2528 } 2529 2530 static void calculate_journal_section_size(struct dm_integrity_c *ic) 2531 { 2532 unsigned sector_space = JOURNAL_SECTOR_DATA; 2533 2534 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections); 2535 ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size, 2536 JOURNAL_ENTRY_ROUNDUP); 2537 2538 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) 2539 sector_space -= JOURNAL_MAC_PER_SECTOR; 2540 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size; 2541 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS; 2542 ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS; 2543 ic->journal_entries = ic->journal_section_entries * ic->journal_sections; 2544 } 2545 2546 static int calculate_device_limits(struct dm_integrity_c *ic) 2547 { 2548 __u64 initial_sectors; 2549 2550 calculate_journal_section_size(ic); 2551 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections; 2552 if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX) 2553 return -EINVAL; 2554 ic->initial_sectors = initial_sectors; 2555 2556 if (!ic->meta_dev) { 2557 sector_t last_sector, last_area, last_offset; 2558 2559 ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), 2560 (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT; 2561 if (!(ic->metadata_run & (ic->metadata_run - 1))) 2562 ic->log2_metadata_run = __ffs(ic->metadata_run); 2563 else 2564 ic->log2_metadata_run = -1; 2565 2566 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); 2567 last_sector = get_data_sector(ic, last_area, last_offset); 2568 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) 2569 return -EINVAL; 2570 } else { 2571 __u64 meta_size = ic->provided_data_sectors * ic->tag_size; 2572 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) 2573 >> (ic->log2_buffer_sectors + SECTOR_SHIFT); 2574 meta_size <<= ic->log2_buffer_sectors; 2575 if (ic->initial_sectors + meta_size < ic->initial_sectors || 2576 ic->initial_sectors + meta_size > ic->meta_device_sectors) 2577 return -EINVAL; 2578 ic->metadata_run = 1; 2579 ic->log2_metadata_run = 0; 2580 } 2581 2582 return 0; 2583 } 2584 2585 static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors) 2586 { 2587 unsigned journal_sections; 2588 int test_bit; 2589 2590 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); 2591 memcpy(ic->sb->magic, SB_MAGIC, 8); 2592 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); 2593 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); 2594 if (ic->journal_mac_alg.alg_string) 2595 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC); 2596 2597 calculate_journal_section_size(ic); 2598 journal_sections = journal_sectors / ic->journal_section_sectors; 2599 if (!journal_sections) 2600 journal_sections = 1; 2601 2602 if (!ic->meta_dev) { 2603 ic->sb->journal_sections = cpu_to_le32(journal_sections); 2604 if (!interleave_sectors) 2605 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; 2606 ic->sb->log2_interleave_sectors = __fls(interleave_sectors); 2607 ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); 2608 ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); 2609 2610 ic->provided_data_sectors = 0; 2611 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { 2612 __u64 prev_data_sectors = ic->provided_data_sectors; 2613 2614 ic->provided_data_sectors |= (sector_t)1 << test_bit; 2615 if (calculate_device_limits(ic)) 2616 ic->provided_data_sectors = prev_data_sectors; 2617 } 2618 if (!ic->provided_data_sectors) 2619 return -EINVAL; 2620 } else { 2621 ic->sb->log2_interleave_sectors = 0; 2622 ic->provided_data_sectors = ic->data_device_sectors; 2623 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); 2624 2625 try_smaller_buffer: 2626 ic->sb->journal_sections = cpu_to_le32(0); 2627 for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) { 2628 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); 2629 __u32 test_journal_sections = prev_journal_sections | (1U << test_bit); 2630 if (test_journal_sections > journal_sections) 2631 continue; 2632 ic->sb->journal_sections = cpu_to_le32(test_journal_sections); 2633 if (calculate_device_limits(ic)) 2634 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections); 2635 2636 } 2637 if (!le32_to_cpu(ic->sb->journal_sections)) { 2638 if (ic->log2_buffer_sectors > 3) { 2639 ic->log2_buffer_sectors--; 2640 goto try_smaller_buffer; 2641 } 2642 return -EINVAL; 2643 } 2644 } 2645 2646 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); 2647 2648 sb_set_version(ic); 2649 2650 return 0; 2651 } 2652 2653 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) 2654 { 2655 struct gendisk *disk = dm_disk(dm_table_get_md(ti->table)); 2656 struct blk_integrity bi; 2657 2658 memset(&bi, 0, sizeof(bi)); 2659 bi.profile = &dm_integrity_profile; 2660 bi.tuple_size = ic->tag_size; 2661 bi.tag_size = bi.tuple_size; 2662 bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; 2663 2664 blk_integrity_register(disk, &bi); 2665 blk_queue_max_integrity_segments(disk->queue, UINT_MAX); 2666 } 2667 2668 static void dm_integrity_free_page_list(struct dm_integrity_c *ic, struct page_list *pl) 2669 { 2670 unsigned i; 2671 2672 if (!pl) 2673 return; 2674 for (i = 0; i < ic->journal_pages; i++) 2675 if (pl[i].page) 2676 __free_page(pl[i].page); 2677 kvfree(pl); 2678 } 2679 2680 static struct page_list *dm_integrity_alloc_page_list(struct dm_integrity_c *ic) 2681 { 2682 size_t page_list_desc_size = ic->journal_pages * sizeof(struct page_list); 2683 struct page_list *pl; 2684 unsigned i; 2685 2686 pl = kvmalloc(page_list_desc_size, GFP_KERNEL | __GFP_ZERO); 2687 if (!pl) 2688 return NULL; 2689 2690 for (i = 0; i < ic->journal_pages; i++) { 2691 pl[i].page = alloc_page(GFP_KERNEL); 2692 if (!pl[i].page) { 2693 dm_integrity_free_page_list(ic, pl); 2694 return NULL; 2695 } 2696 if (i) 2697 pl[i - 1].next = &pl[i]; 2698 } 2699 2700 return pl; 2701 } 2702 2703 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl) 2704 { 2705 unsigned i; 2706 for (i = 0; i < ic->journal_sections; i++) 2707 kvfree(sl[i]); 2708 kvfree(sl); 2709 } 2710 2711 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl) 2712 { 2713 struct scatterlist **sl; 2714 unsigned i; 2715 2716 sl = kvmalloc_array(ic->journal_sections, 2717 sizeof(struct scatterlist *), 2718 GFP_KERNEL | __GFP_ZERO); 2719 if (!sl) 2720 return NULL; 2721 2722 for (i = 0; i < ic->journal_sections; i++) { 2723 struct scatterlist *s; 2724 unsigned start_index, start_offset; 2725 unsigned end_index, end_offset; 2726 unsigned n_pages; 2727 unsigned idx; 2728 2729 page_list_location(ic, i, 0, &start_index, &start_offset); 2730 page_list_location(ic, i, ic->journal_section_sectors - 1, &end_index, &end_offset); 2731 2732 n_pages = (end_index - start_index + 1); 2733 2734 s = kvmalloc_array(n_pages, sizeof(struct scatterlist), 2735 GFP_KERNEL); 2736 if (!s) { 2737 dm_integrity_free_journal_scatterlist(ic, sl); 2738 return NULL; 2739 } 2740 2741 sg_init_table(s, n_pages); 2742 for (idx = start_index; idx <= end_index; idx++) { 2743 char *va = lowmem_page_address(pl[idx].page); 2744 unsigned start = 0, end = PAGE_SIZE; 2745 if (idx == start_index) 2746 start = start_offset; 2747 if (idx == end_index) 2748 end = end_offset + (1 << SECTOR_SHIFT); 2749 sg_set_buf(&s[idx - start_index], va + start, end - start); 2750 } 2751 2752 sl[i] = s; 2753 } 2754 2755 return sl; 2756 } 2757 2758 static void free_alg(struct alg_spec *a) 2759 { 2760 kzfree(a->alg_string); 2761 kzfree(a->key); 2762 memset(a, 0, sizeof *a); 2763 } 2764 2765 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval) 2766 { 2767 char *k; 2768 2769 free_alg(a); 2770 2771 a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL); 2772 if (!a->alg_string) 2773 goto nomem; 2774 2775 k = strchr(a->alg_string, ':'); 2776 if (k) { 2777 *k = 0; 2778 a->key_string = k + 1; 2779 if (strlen(a->key_string) & 1) 2780 goto inval; 2781 2782 a->key_size = strlen(a->key_string) / 2; 2783 a->key = kmalloc(a->key_size, GFP_KERNEL); 2784 if (!a->key) 2785 goto nomem; 2786 if (hex2bin(a->key, a->key_string, a->key_size)) 2787 goto inval; 2788 } 2789 2790 return 0; 2791 inval: 2792 *error = error_inval; 2793 return -EINVAL; 2794 nomem: 2795 *error = "Out of memory for an argument"; 2796 return -ENOMEM; 2797 } 2798 2799 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, 2800 char *error_alg, char *error_key) 2801 { 2802 int r; 2803 2804 if (a->alg_string) { 2805 *hash = crypto_alloc_shash(a->alg_string, 0, 0); 2806 if (IS_ERR(*hash)) { 2807 *error = error_alg; 2808 r = PTR_ERR(*hash); 2809 *hash = NULL; 2810 return r; 2811 } 2812 2813 if (a->key) { 2814 r = crypto_shash_setkey(*hash, a->key, a->key_size); 2815 if (r) { 2816 *error = error_key; 2817 return r; 2818 } 2819 } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) { 2820 *error = error_key; 2821 return -ENOKEY; 2822 } 2823 } 2824 2825 return 0; 2826 } 2827 2828 static int create_journal(struct dm_integrity_c *ic, char **error) 2829 { 2830 int r = 0; 2831 unsigned i; 2832 __u64 journal_pages, journal_desc_size, journal_tree_size; 2833 unsigned char *crypt_data = NULL, *crypt_iv = NULL; 2834 struct skcipher_request *req = NULL; 2835 2836 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL); 2837 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL); 2838 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL); 2839 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL); 2840 2841 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors, 2842 PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT); 2843 journal_desc_size = journal_pages * sizeof(struct page_list); 2844 if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) { 2845 *error = "Journal doesn't fit into memory"; 2846 r = -ENOMEM; 2847 goto bad; 2848 } 2849 ic->journal_pages = journal_pages; 2850 2851 ic->journal = dm_integrity_alloc_page_list(ic); 2852 if (!ic->journal) { 2853 *error = "Could not allocate memory for journal"; 2854 r = -ENOMEM; 2855 goto bad; 2856 } 2857 if (ic->journal_crypt_alg.alg_string) { 2858 unsigned ivsize, blocksize; 2859 struct journal_completion comp; 2860 2861 comp.ic = ic; 2862 ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, 0); 2863 if (IS_ERR(ic->journal_crypt)) { 2864 *error = "Invalid journal cipher"; 2865 r = PTR_ERR(ic->journal_crypt); 2866 ic->journal_crypt = NULL; 2867 goto bad; 2868 } 2869 ivsize = crypto_skcipher_ivsize(ic->journal_crypt); 2870 blocksize = crypto_skcipher_blocksize(ic->journal_crypt); 2871 2872 if (ic->journal_crypt_alg.key) { 2873 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key, 2874 ic->journal_crypt_alg.key_size); 2875 if (r) { 2876 *error = "Error setting encryption key"; 2877 goto bad; 2878 } 2879 } 2880 DEBUG_print("cipher %s, block size %u iv size %u\n", 2881 ic->journal_crypt_alg.alg_string, blocksize, ivsize); 2882 2883 ic->journal_io = dm_integrity_alloc_page_list(ic); 2884 if (!ic->journal_io) { 2885 *error = "Could not allocate memory for journal io"; 2886 r = -ENOMEM; 2887 goto bad; 2888 } 2889 2890 if (blocksize == 1) { 2891 struct scatterlist *sg; 2892 2893 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 2894 if (!req) { 2895 *error = "Could not allocate crypt request"; 2896 r = -ENOMEM; 2897 goto bad; 2898 } 2899 2900 crypt_iv = kmalloc(ivsize, GFP_KERNEL); 2901 if (!crypt_iv) { 2902 *error = "Could not allocate iv"; 2903 r = -ENOMEM; 2904 goto bad; 2905 } 2906 2907 ic->journal_xor = dm_integrity_alloc_page_list(ic); 2908 if (!ic->journal_xor) { 2909 *error = "Could not allocate memory for journal xor"; 2910 r = -ENOMEM; 2911 goto bad; 2912 } 2913 2914 sg = kvmalloc_array(ic->journal_pages + 1, 2915 sizeof(struct scatterlist), 2916 GFP_KERNEL); 2917 if (!sg) { 2918 *error = "Unable to allocate sg list"; 2919 r = -ENOMEM; 2920 goto bad; 2921 } 2922 sg_init_table(sg, ic->journal_pages + 1); 2923 for (i = 0; i < ic->journal_pages; i++) { 2924 char *va = lowmem_page_address(ic->journal_xor[i].page); 2925 clear_page(va); 2926 sg_set_buf(&sg[i], va, PAGE_SIZE); 2927 } 2928 sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids); 2929 memset(crypt_iv, 0x00, ivsize); 2930 2931 skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv); 2932 init_completion(&comp.comp); 2933 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 2934 if (do_crypt(true, req, &comp)) 2935 wait_for_completion(&comp.comp); 2936 kvfree(sg); 2937 r = dm_integrity_failed(ic); 2938 if (r) { 2939 *error = "Unable to encrypt journal"; 2940 goto bad; 2941 } 2942 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data"); 2943 2944 crypto_free_skcipher(ic->journal_crypt); 2945 ic->journal_crypt = NULL; 2946 } else { 2947 unsigned crypt_len = roundup(ivsize, blocksize); 2948 2949 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 2950 if (!req) { 2951 *error = "Could not allocate crypt request"; 2952 r = -ENOMEM; 2953 goto bad; 2954 } 2955 2956 crypt_iv = kmalloc(ivsize, GFP_KERNEL); 2957 if (!crypt_iv) { 2958 *error = "Could not allocate iv"; 2959 r = -ENOMEM; 2960 goto bad; 2961 } 2962 2963 crypt_data = kmalloc(crypt_len, GFP_KERNEL); 2964 if (!crypt_data) { 2965 *error = "Unable to allocate crypt data"; 2966 r = -ENOMEM; 2967 goto bad; 2968 } 2969 2970 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal); 2971 if (!ic->journal_scatterlist) { 2972 *error = "Unable to allocate sg list"; 2973 r = -ENOMEM; 2974 goto bad; 2975 } 2976 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io); 2977 if (!ic->journal_io_scatterlist) { 2978 *error = "Unable to allocate sg list"; 2979 r = -ENOMEM; 2980 goto bad; 2981 } 2982 ic->sk_requests = kvmalloc_array(ic->journal_sections, 2983 sizeof(struct skcipher_request *), 2984 GFP_KERNEL | __GFP_ZERO); 2985 if (!ic->sk_requests) { 2986 *error = "Unable to allocate sk requests"; 2987 r = -ENOMEM; 2988 goto bad; 2989 } 2990 for (i = 0; i < ic->journal_sections; i++) { 2991 struct scatterlist sg; 2992 struct skcipher_request *section_req; 2993 __u32 section_le = cpu_to_le32(i); 2994 2995 memset(crypt_iv, 0x00, ivsize); 2996 memset(crypt_data, 0x00, crypt_len); 2997 memcpy(crypt_data, §ion_le, min((size_t)crypt_len, sizeof(section_le))); 2998 2999 sg_init_one(&sg, crypt_data, crypt_len); 3000 skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv); 3001 init_completion(&comp.comp); 3002 comp.in_flight = (atomic_t)ATOMIC_INIT(1); 3003 if (do_crypt(true, req, &comp)) 3004 wait_for_completion(&comp.comp); 3005 3006 r = dm_integrity_failed(ic); 3007 if (r) { 3008 *error = "Unable to generate iv"; 3009 goto bad; 3010 } 3011 3012 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); 3013 if (!section_req) { 3014 *error = "Unable to allocate crypt request"; 3015 r = -ENOMEM; 3016 goto bad; 3017 } 3018 section_req->iv = kmalloc_array(ivsize, 2, 3019 GFP_KERNEL); 3020 if (!section_req->iv) { 3021 skcipher_request_free(section_req); 3022 *error = "Unable to allocate iv"; 3023 r = -ENOMEM; 3024 goto bad; 3025 } 3026 memcpy(section_req->iv + ivsize, crypt_data, ivsize); 3027 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT; 3028 ic->sk_requests[i] = section_req; 3029 DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i); 3030 } 3031 } 3032 } 3033 3034 for (i = 0; i < N_COMMIT_IDS; i++) { 3035 unsigned j; 3036 retest_commit_id: 3037 for (j = 0; j < i; j++) { 3038 if (ic->commit_ids[j] == ic->commit_ids[i]) { 3039 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1); 3040 goto retest_commit_id; 3041 } 3042 } 3043 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]); 3044 } 3045 3046 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node); 3047 if (journal_tree_size > ULONG_MAX) { 3048 *error = "Journal doesn't fit into memory"; 3049 r = -ENOMEM; 3050 goto bad; 3051 } 3052 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL); 3053 if (!ic->journal_tree) { 3054 *error = "Could not allocate memory for journal tree"; 3055 r = -ENOMEM; 3056 } 3057 bad: 3058 kfree(crypt_data); 3059 kfree(crypt_iv); 3060 skcipher_request_free(req); 3061 3062 return r; 3063 } 3064 3065 /* 3066 * Construct a integrity mapping 3067 * 3068 * Arguments: 3069 * device 3070 * offset from the start of the device 3071 * tag size 3072 * D - direct writes, J - journal writes, R - recovery mode 3073 * number of optional arguments 3074 * optional arguments: 3075 * journal_sectors 3076 * interleave_sectors 3077 * buffer_sectors 3078 * journal_watermark 3079 * commit_time 3080 * internal_hash 3081 * journal_crypt 3082 * journal_mac 3083 * block_size 3084 */ 3085 static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) 3086 { 3087 struct dm_integrity_c *ic; 3088 char dummy; 3089 int r; 3090 unsigned extra_args; 3091 struct dm_arg_set as; 3092 static const struct dm_arg _args[] = { 3093 {0, 9, "Invalid number of feature args"}, 3094 }; 3095 unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; 3096 bool recalculate; 3097 bool should_write_sb; 3098 __u64 threshold; 3099 unsigned long long start; 3100 3101 #define DIRECT_ARGUMENTS 4 3102 3103 if (argc <= DIRECT_ARGUMENTS) { 3104 ti->error = "Invalid argument count"; 3105 return -EINVAL; 3106 } 3107 3108 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL); 3109 if (!ic) { 3110 ti->error = "Cannot allocate integrity context"; 3111 return -ENOMEM; 3112 } 3113 ti->private = ic; 3114 ti->per_io_data_size = sizeof(struct dm_integrity_io); 3115 3116 ic->in_progress = RB_ROOT; 3117 INIT_LIST_HEAD(&ic->wait_list); 3118 init_waitqueue_head(&ic->endio_wait); 3119 bio_list_init(&ic->flush_bio_list); 3120 init_waitqueue_head(&ic->copy_to_journal_wait); 3121 init_completion(&ic->crypto_backoff); 3122 atomic64_set(&ic->number_of_mismatches, 0); 3123 3124 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); 3125 if (r) { 3126 ti->error = "Device lookup failed"; 3127 goto bad; 3128 } 3129 3130 if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) { 3131 ti->error = "Invalid starting offset"; 3132 r = -EINVAL; 3133 goto bad; 3134 } 3135 ic->start = start; 3136 3137 if (strcmp(argv[2], "-")) { 3138 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) { 3139 ti->error = "Invalid tag size"; 3140 r = -EINVAL; 3141 goto bad; 3142 } 3143 } 3144 3145 if (!strcmp(argv[3], "J") || !strcmp(argv[3], "D") || !strcmp(argv[3], "R")) 3146 ic->mode = argv[3][0]; 3147 else { 3148 ti->error = "Invalid mode (expecting J, D, R)"; 3149 r = -EINVAL; 3150 goto bad; 3151 } 3152 3153 journal_sectors = 0; 3154 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; 3155 buffer_sectors = DEFAULT_BUFFER_SECTORS; 3156 journal_watermark = DEFAULT_JOURNAL_WATERMARK; 3157 sync_msec = DEFAULT_SYNC_MSEC; 3158 recalculate = false; 3159 ic->sectors_per_block = 1; 3160 3161 as.argc = argc - DIRECT_ARGUMENTS; 3162 as.argv = argv + DIRECT_ARGUMENTS; 3163 r = dm_read_arg_group(_args, &as, &extra_args, &ti->error); 3164 if (r) 3165 goto bad; 3166 3167 while (extra_args--) { 3168 const char *opt_string; 3169 unsigned val; 3170 opt_string = dm_shift_arg(&as); 3171 if (!opt_string) { 3172 r = -EINVAL; 3173 ti->error = "Not enough feature arguments"; 3174 goto bad; 3175 } 3176 if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1) 3177 journal_sectors = val ? val : 1; 3178 else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1) 3179 interleave_sectors = val; 3180 else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1) 3181 buffer_sectors = val; 3182 else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100) 3183 journal_watermark = val; 3184 else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1) 3185 sync_msec = val; 3186 else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) { 3187 if (ic->meta_dev) { 3188 dm_put_device(ti, ic->meta_dev); 3189 ic->meta_dev = NULL; 3190 } 3191 r = dm_get_device(ti, strchr(opt_string, ':') + 1, dm_table_get_mode(ti->table), &ic->meta_dev); 3192 if (r) { 3193 ti->error = "Device lookup failed"; 3194 goto bad; 3195 } 3196 } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) { 3197 if (val < 1 << SECTOR_SHIFT || 3198 val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT || 3199 (val & (val -1))) { 3200 r = -EINVAL; 3201 ti->error = "Invalid block_size argument"; 3202 goto bad; 3203 } 3204 ic->sectors_per_block = val >> SECTOR_SHIFT; 3205 } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) { 3206 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error, 3207 "Invalid internal_hash argument"); 3208 if (r) 3209 goto bad; 3210 } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) { 3211 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error, 3212 "Invalid journal_crypt argument"); 3213 if (r) 3214 goto bad; 3215 } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) { 3216 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, 3217 "Invalid journal_mac argument"); 3218 if (r) 3219 goto bad; 3220 } else if (!strcmp(opt_string, "recalculate")) { 3221 recalculate = true; 3222 } else { 3223 r = -EINVAL; 3224 ti->error = "Invalid argument"; 3225 goto bad; 3226 } 3227 } 3228 3229 ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; 3230 if (!ic->meta_dev) 3231 ic->meta_device_sectors = ic->data_device_sectors; 3232 else 3233 ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT; 3234 3235 if (!journal_sectors) { 3236 journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, 3237 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); 3238 } 3239 3240 if (!buffer_sectors) 3241 buffer_sectors = 1; 3242 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); 3243 3244 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, 3245 "Invalid internal hash", "Error setting internal hash key"); 3246 if (r) 3247 goto bad; 3248 3249 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, 3250 "Invalid journal mac", "Error setting journal mac key"); 3251 if (r) 3252 goto bad; 3253 3254 if (!ic->tag_size) { 3255 if (!ic->internal_hash) { 3256 ti->error = "Unknown tag size"; 3257 r = -EINVAL; 3258 goto bad; 3259 } 3260 ic->tag_size = crypto_shash_digestsize(ic->internal_hash); 3261 } 3262 if (ic->tag_size > MAX_TAG_SIZE) { 3263 ti->error = "Too big tag size"; 3264 r = -EINVAL; 3265 goto bad; 3266 } 3267 if (!(ic->tag_size & (ic->tag_size - 1))) 3268 ic->log2_tag_size = __ffs(ic->tag_size); 3269 else 3270 ic->log2_tag_size = -1; 3271 3272 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); 3273 ic->autocommit_msec = sync_msec; 3274 timer_setup(&ic->autocommit_timer, autocommit_fn, 0); 3275 3276 ic->io = dm_io_client_create(); 3277 if (IS_ERR(ic->io)) { 3278 r = PTR_ERR(ic->io); 3279 ic->io = NULL; 3280 ti->error = "Cannot allocate dm io"; 3281 goto bad; 3282 } 3283 3284 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache); 3285 if (r) { 3286 ti->error = "Cannot allocate mempool"; 3287 goto bad; 3288 } 3289 3290 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", 3291 WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE); 3292 if (!ic->metadata_wq) { 3293 ti->error = "Cannot allocate workqueue"; 3294 r = -ENOMEM; 3295 goto bad; 3296 } 3297 3298 /* 3299 * If this workqueue were percpu, it would cause bio reordering 3300 * and reduced performance. 3301 */ 3302 ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); 3303 if (!ic->wait_wq) { 3304 ti->error = "Cannot allocate workqueue"; 3305 r = -ENOMEM; 3306 goto bad; 3307 } 3308 3309 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); 3310 if (!ic->commit_wq) { 3311 ti->error = "Cannot allocate workqueue"; 3312 r = -ENOMEM; 3313 goto bad; 3314 } 3315 INIT_WORK(&ic->commit_work, integrity_commit); 3316 3317 if (ic->mode == 'J') { 3318 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); 3319 if (!ic->writer_wq) { 3320 ti->error = "Cannot allocate workqueue"; 3321 r = -ENOMEM; 3322 goto bad; 3323 } 3324 INIT_WORK(&ic->writer_work, integrity_writer); 3325 } 3326 3327 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL); 3328 if (!ic->sb) { 3329 r = -ENOMEM; 3330 ti->error = "Cannot allocate superblock area"; 3331 goto bad; 3332 } 3333 3334 r = sync_rw_sb(ic, REQ_OP_READ, 0); 3335 if (r) { 3336 ti->error = "Error reading superblock"; 3337 goto bad; 3338 } 3339 should_write_sb = false; 3340 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) { 3341 if (ic->mode != 'R') { 3342 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) { 3343 r = -EINVAL; 3344 ti->error = "The device is not initialized"; 3345 goto bad; 3346 } 3347 } 3348 3349 r = initialize_superblock(ic, journal_sectors, interleave_sectors); 3350 if (r) { 3351 ti->error = "Could not initialize superblock"; 3352 goto bad; 3353 } 3354 if (ic->mode != 'R') 3355 should_write_sb = true; 3356 } 3357 3358 if (!ic->sb->version || ic->sb->version > SB_VERSION_2) { 3359 r = -EINVAL; 3360 ti->error = "Unknown version"; 3361 goto bad; 3362 } 3363 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) { 3364 r = -EINVAL; 3365 ti->error = "Tag size doesn't match the information in superblock"; 3366 goto bad; 3367 } 3368 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) { 3369 r = -EINVAL; 3370 ti->error = "Block size doesn't match the information in superblock"; 3371 goto bad; 3372 } 3373 if (!le32_to_cpu(ic->sb->journal_sections)) { 3374 r = -EINVAL; 3375 ti->error = "Corrupted superblock, journal_sections is 0"; 3376 goto bad; 3377 } 3378 /* make sure that ti->max_io_len doesn't overflow */ 3379 if (!ic->meta_dev) { 3380 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || 3381 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { 3382 r = -EINVAL; 3383 ti->error = "Invalid interleave_sectors in the superblock"; 3384 goto bad; 3385 } 3386 } else { 3387 if (ic->sb->log2_interleave_sectors) { 3388 r = -EINVAL; 3389 ti->error = "Invalid interleave_sectors in the superblock"; 3390 goto bad; 3391 } 3392 } 3393 ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); 3394 if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) { 3395 /* test for overflow */ 3396 r = -EINVAL; 3397 ti->error = "The superblock has 64-bit device size, but the kernel was compiled with 32-bit sectors"; 3398 goto bad; 3399 } 3400 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { 3401 r = -EINVAL; 3402 ti->error = "Journal mac mismatch"; 3403 goto bad; 3404 } 3405 3406 try_smaller_buffer: 3407 r = calculate_device_limits(ic); 3408 if (r) { 3409 if (ic->meta_dev) { 3410 if (ic->log2_buffer_sectors > 3) { 3411 ic->log2_buffer_sectors--; 3412 goto try_smaller_buffer; 3413 } 3414 } 3415 ti->error = "The device is too small"; 3416 goto bad; 3417 } 3418 if (!ic->meta_dev) 3419 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); 3420 3421 if (ti->len > ic->provided_data_sectors) { 3422 r = -EINVAL; 3423 ti->error = "Not enough provided sectors for requested mapping size"; 3424 goto bad; 3425 } 3426 3427 3428 threshold = (__u64)ic->journal_entries * (100 - journal_watermark); 3429 threshold += 50; 3430 do_div(threshold, 100); 3431 ic->free_sectors_threshold = threshold; 3432 3433 DEBUG_print("initialized:\n"); 3434 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size)); 3435 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size); 3436 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector); 3437 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries); 3438 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors); 3439 DEBUG_print(" journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections)); 3440 DEBUG_print(" journal_entries %u\n", ic->journal_entries); 3441 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); 3442 DEBUG_print(" device_sectors 0x%llx\n", (unsigned long long)ic->device_sectors); 3443 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); 3444 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); 3445 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); 3446 DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", (unsigned long long)ic->provided_data_sectors, 3447 (unsigned long long)ic->provided_data_sectors); 3448 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); 3449 3450 if (recalculate && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { 3451 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); 3452 ic->sb->recalc_sector = cpu_to_le64(0); 3453 } 3454 3455 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { 3456 if (!ic->internal_hash) { 3457 r = -EINVAL; 3458 ti->error = "Recalculate is only valid with internal hash"; 3459 goto bad; 3460 } 3461 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); 3462 if (!ic->recalc_wq ) { 3463 ti->error = "Cannot allocate workqueue"; 3464 r = -ENOMEM; 3465 goto bad; 3466 } 3467 INIT_WORK(&ic->recalc_work, integrity_recalc); 3468 ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT); 3469 if (!ic->recalc_buffer) { 3470 ti->error = "Cannot allocate buffer for recalculating"; 3471 r = -ENOMEM; 3472 goto bad; 3473 } 3474 ic->recalc_tags = kvmalloc_array(RECALC_SECTORS >> ic->sb->log2_sectors_per_block, 3475 ic->tag_size, GFP_KERNEL); 3476 if (!ic->recalc_tags) { 3477 ti->error = "Cannot allocate tags for recalculating"; 3478 r = -ENOMEM; 3479 goto bad; 3480 } 3481 } 3482 3483 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, 3484 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL); 3485 if (IS_ERR(ic->bufio)) { 3486 r = PTR_ERR(ic->bufio); 3487 ti->error = "Cannot initialize dm-bufio"; 3488 ic->bufio = NULL; 3489 goto bad; 3490 } 3491 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); 3492 3493 if (ic->mode != 'R') { 3494 r = create_journal(ic, &ti->error); 3495 if (r) 3496 goto bad; 3497 } 3498 3499 if (should_write_sb) { 3500 int r; 3501 3502 init_journal(ic, 0, ic->journal_sections, 0); 3503 r = dm_integrity_failed(ic); 3504 if (unlikely(r)) { 3505 ti->error = "Error initializing journal"; 3506 goto bad; 3507 } 3508 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); 3509 if (r) { 3510 ti->error = "Error initializing superblock"; 3511 goto bad; 3512 } 3513 ic->just_formatted = true; 3514 } 3515 3516 if (!ic->meta_dev) { 3517 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); 3518 if (r) 3519 goto bad; 3520 } 3521 3522 if (!ic->internal_hash) 3523 dm_integrity_set(ti, ic); 3524 3525 ti->num_flush_bios = 1; 3526 ti->flush_supported = true; 3527 3528 return 0; 3529 bad: 3530 dm_integrity_dtr(ti); 3531 return r; 3532 } 3533 3534 static void dm_integrity_dtr(struct dm_target *ti) 3535 { 3536 struct dm_integrity_c *ic = ti->private; 3537 3538 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); 3539 BUG_ON(!list_empty(&ic->wait_list)); 3540 3541 if (ic->metadata_wq) 3542 destroy_workqueue(ic->metadata_wq); 3543 if (ic->wait_wq) 3544 destroy_workqueue(ic->wait_wq); 3545 if (ic->commit_wq) 3546 destroy_workqueue(ic->commit_wq); 3547 if (ic->writer_wq) 3548 destroy_workqueue(ic->writer_wq); 3549 if (ic->recalc_wq) 3550 destroy_workqueue(ic->recalc_wq); 3551 if (ic->recalc_buffer) 3552 vfree(ic->recalc_buffer); 3553 if (ic->recalc_tags) 3554 kvfree(ic->recalc_tags); 3555 if (ic->bufio) 3556 dm_bufio_client_destroy(ic->bufio); 3557 mempool_exit(&ic->journal_io_mempool); 3558 if (ic->io) 3559 dm_io_client_destroy(ic->io); 3560 if (ic->dev) 3561 dm_put_device(ti, ic->dev); 3562 if (ic->meta_dev) 3563 dm_put_device(ti, ic->meta_dev); 3564 dm_integrity_free_page_list(ic, ic->journal); 3565 dm_integrity_free_page_list(ic, ic->journal_io); 3566 dm_integrity_free_page_list(ic, ic->journal_xor); 3567 if (ic->journal_scatterlist) 3568 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist); 3569 if (ic->journal_io_scatterlist) 3570 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist); 3571 if (ic->sk_requests) { 3572 unsigned i; 3573 3574 for (i = 0; i < ic->journal_sections; i++) { 3575 struct skcipher_request *req = ic->sk_requests[i]; 3576 if (req) { 3577 kzfree(req->iv); 3578 skcipher_request_free(req); 3579 } 3580 } 3581 kvfree(ic->sk_requests); 3582 } 3583 kvfree(ic->journal_tree); 3584 if (ic->sb) 3585 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT); 3586 3587 if (ic->internal_hash) 3588 crypto_free_shash(ic->internal_hash); 3589 free_alg(&ic->internal_hash_alg); 3590 3591 if (ic->journal_crypt) 3592 crypto_free_skcipher(ic->journal_crypt); 3593 free_alg(&ic->journal_crypt_alg); 3594 3595 if (ic->journal_mac) 3596 crypto_free_shash(ic->journal_mac); 3597 free_alg(&ic->journal_mac_alg); 3598 3599 kfree(ic); 3600 } 3601 3602 static struct target_type integrity_target = { 3603 .name = "integrity", 3604 .version = {1, 2, 0}, 3605 .module = THIS_MODULE, 3606 .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, 3607 .ctr = dm_integrity_ctr, 3608 .dtr = dm_integrity_dtr, 3609 .map = dm_integrity_map, 3610 .postsuspend = dm_integrity_postsuspend, 3611 .resume = dm_integrity_resume, 3612 .status = dm_integrity_status, 3613 .iterate_devices = dm_integrity_iterate_devices, 3614 .io_hints = dm_integrity_io_hints, 3615 }; 3616 3617 static int __init dm_integrity_init(void) 3618 { 3619 int r; 3620 3621 journal_io_cache = kmem_cache_create("integrity_journal_io", 3622 sizeof(struct journal_io), 0, 0, NULL); 3623 if (!journal_io_cache) { 3624 DMERR("can't allocate journal io cache"); 3625 return -ENOMEM; 3626 } 3627 3628 r = dm_register_target(&integrity_target); 3629 3630 if (r < 0) 3631 DMERR("register failed %d", r); 3632 3633 return r; 3634 } 3635 3636 static void __exit dm_integrity_exit(void) 3637 { 3638 dm_unregister_target(&integrity_target); 3639 kmem_cache_destroy(journal_io_cache); 3640 } 3641 3642 module_init(dm_integrity_init); 3643 module_exit(dm_integrity_exit); 3644 3645 MODULE_AUTHOR("Milan Broz"); 3646 MODULE_AUTHOR("Mikulas Patocka"); 3647 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension"); 3648 MODULE_LICENSE("GPL"); 3649