1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Red Hat. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-bio-prison-v2.h" 10 #include "dm-bio-record.h" 11 #include "dm-cache-metadata.h" 12 #include "dm-io-tracker.h" 13 #include "dm-cache-background-tracker.h" 14 15 #include <linux/dm-io.h> 16 #include <linux/dm-kcopyd.h> 17 #include <linux/jiffies.h> 18 #include <linux/init.h> 19 #include <linux/kstrtox.h> 20 #include <linux/math64.h> 21 #include <linux/mempool.h> 22 #include <linux/module.h> 23 #include <linux/rwsem.h> 24 #include <linux/slab.h> 25 #include <linux/vmalloc.h> 26 27 #define DM_MSG_PREFIX "cache" 28 29 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle, 30 "A percentage of time allocated for copying to and/or from cache"); 31 32 /*----------------------------------------------------------------*/ 33 34 /* 35 * Glossary: 36 * 37 * oblock: index of an origin block 38 * cblock: index of a cache block 39 * promotion: movement of a block from origin to cache 40 * demotion: movement of a block from cache to origin 41 * migration: movement of a block between the origin and cache device, 42 * either direction 43 */ 44 45 /*----------------------------------------------------------------*/ 46 47 /* 48 * Represents a chunk of future work. 'input' allows continuations to pass 49 * values between themselves, typically error values. 50 */ 51 struct continuation { 52 struct work_struct ws; 53 blk_status_t input; 54 }; 55 56 static inline void init_continuation(struct continuation *k, 57 void (*fn)(struct work_struct *)) 58 { 59 INIT_WORK(&k->ws, fn); 60 k->input = 0; 61 } 62 63 static inline void queue_continuation(struct workqueue_struct *wq, 64 struct continuation *k) 65 { 66 queue_work(wq, &k->ws); 67 } 68 69 /*----------------------------------------------------------------*/ 70 71 /* 72 * The batcher collects together pieces of work that need a particular 73 * operation to occur before they can proceed (typically a commit). 74 */ 75 struct batcher { 76 /* 77 * The operation that everyone is waiting for. 78 */ 79 blk_status_t (*commit_op)(void *context); 80 void *commit_context; 81 82 /* 83 * This is how bios should be issued once the commit op is complete 84 * (accounted_request). 85 */ 86 void (*issue_op)(struct bio *bio, void *context); 87 void *issue_context; 88 89 /* 90 * Queued work gets put on here after commit. 91 */ 92 struct workqueue_struct *wq; 93 94 spinlock_t lock; 95 struct list_head work_items; 96 struct bio_list bios; 97 struct work_struct commit_work; 98 99 bool commit_scheduled; 100 }; 101 102 static void __commit(struct work_struct *_ws) 103 { 104 struct batcher *b = container_of(_ws, struct batcher, commit_work); 105 blk_status_t r; 106 struct list_head work_items; 107 struct work_struct *ws, *tmp; 108 struct continuation *k; 109 struct bio *bio; 110 struct bio_list bios; 111 112 INIT_LIST_HEAD(&work_items); 113 bio_list_init(&bios); 114 115 /* 116 * We have to grab these before the commit_op to avoid a race 117 * condition. 118 */ 119 spin_lock_irq(&b->lock); 120 list_splice_init(&b->work_items, &work_items); 121 bio_list_merge_init(&bios, &b->bios); 122 b->commit_scheduled = false; 123 spin_unlock_irq(&b->lock); 124 125 r = b->commit_op(b->commit_context); 126 127 list_for_each_entry_safe(ws, tmp, &work_items, entry) { 128 k = container_of(ws, struct continuation, ws); 129 k->input = r; 130 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */ 131 queue_work(b->wq, ws); 132 } 133 134 while ((bio = bio_list_pop(&bios))) { 135 if (r) { 136 bio->bi_status = r; 137 bio_endio(bio); 138 } else 139 b->issue_op(bio, b->issue_context); 140 } 141 } 142 143 static void batcher_init(struct batcher *b, 144 blk_status_t (*commit_op)(void *), 145 void *commit_context, 146 void (*issue_op)(struct bio *bio, void *), 147 void *issue_context, 148 struct workqueue_struct *wq) 149 { 150 b->commit_op = commit_op; 151 b->commit_context = commit_context; 152 b->issue_op = issue_op; 153 b->issue_context = issue_context; 154 b->wq = wq; 155 156 spin_lock_init(&b->lock); 157 INIT_LIST_HEAD(&b->work_items); 158 bio_list_init(&b->bios); 159 INIT_WORK(&b->commit_work, __commit); 160 b->commit_scheduled = false; 161 } 162 163 static void async_commit(struct batcher *b) 164 { 165 queue_work(b->wq, &b->commit_work); 166 } 167 168 static void continue_after_commit(struct batcher *b, struct continuation *k) 169 { 170 bool commit_scheduled; 171 172 spin_lock_irq(&b->lock); 173 commit_scheduled = b->commit_scheduled; 174 list_add_tail(&k->ws.entry, &b->work_items); 175 spin_unlock_irq(&b->lock); 176 177 if (commit_scheduled) 178 async_commit(b); 179 } 180 181 /* 182 * Bios are errored if commit failed. 183 */ 184 static void issue_after_commit(struct batcher *b, struct bio *bio) 185 { 186 bool commit_scheduled; 187 188 spin_lock_irq(&b->lock); 189 commit_scheduled = b->commit_scheduled; 190 bio_list_add(&b->bios, bio); 191 spin_unlock_irq(&b->lock); 192 193 if (commit_scheduled) 194 async_commit(b); 195 } 196 197 /* 198 * Call this if some urgent work is waiting for the commit to complete. 199 */ 200 static void schedule_commit(struct batcher *b) 201 { 202 bool immediate; 203 204 spin_lock_irq(&b->lock); 205 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios); 206 b->commit_scheduled = true; 207 spin_unlock_irq(&b->lock); 208 209 if (immediate) 210 async_commit(b); 211 } 212 213 /* 214 * There are a couple of places where we let a bio run, but want to do some 215 * work before calling its endio function. We do this by temporarily 216 * changing the endio fn. 217 */ 218 struct dm_hook_info { 219 bio_end_io_t *bi_end_io; 220 }; 221 222 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio, 223 bio_end_io_t *bi_end_io, void *bi_private) 224 { 225 h->bi_end_io = bio->bi_end_io; 226 227 bio->bi_end_io = bi_end_io; 228 bio->bi_private = bi_private; 229 } 230 231 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) 232 { 233 bio->bi_end_io = h->bi_end_io; 234 } 235 236 /*----------------------------------------------------------------*/ 237 238 #define MIGRATION_POOL_SIZE 128 239 #define COMMIT_PERIOD HZ 240 #define MIGRATION_COUNT_WINDOW 10 241 242 /* 243 * The block size of the device holding cache data must be 244 * between 32KB and 1GB. 245 */ 246 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT) 247 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 248 249 enum cache_metadata_mode { 250 CM_WRITE, /* metadata may be changed */ 251 CM_READ_ONLY, /* metadata may not be changed */ 252 CM_FAIL 253 }; 254 255 enum cache_io_mode { 256 /* 257 * Data is written to cached blocks only. These blocks are marked 258 * dirty. If you lose the cache device you will lose data. 259 * Potential performance increase for both reads and writes. 260 */ 261 CM_IO_WRITEBACK, 262 263 /* 264 * Data is written to both cache and origin. Blocks are never 265 * dirty. Potential performance benfit for reads only. 266 */ 267 CM_IO_WRITETHROUGH, 268 269 /* 270 * A degraded mode useful for various cache coherency situations 271 * (eg, rolling back snapshots). Reads and writes always go to the 272 * origin. If a write goes to a cached oblock, then the cache 273 * block is invalidated. 274 */ 275 CM_IO_PASSTHROUGH 276 }; 277 278 struct cache_features { 279 enum cache_metadata_mode mode; 280 enum cache_io_mode io_mode; 281 unsigned int metadata_version; 282 bool discard_passdown:1; 283 }; 284 285 struct cache_stats { 286 atomic_t read_hit; 287 atomic_t read_miss; 288 atomic_t write_hit; 289 atomic_t write_miss; 290 atomic_t demotion; 291 atomic_t promotion; 292 atomic_t writeback; 293 atomic_t copies_avoided; 294 atomic_t cache_cell_clash; 295 atomic_t commit_count; 296 atomic_t discard_count; 297 }; 298 299 struct cache { 300 struct dm_target *ti; 301 spinlock_t lock; 302 303 /* 304 * Fields for converting from sectors to blocks. 305 */ 306 int sectors_per_block_shift; 307 sector_t sectors_per_block; 308 309 struct dm_cache_metadata *cmd; 310 311 /* 312 * Metadata is written to this device. 313 */ 314 struct dm_dev *metadata_dev; 315 316 /* 317 * The slower of the two data devices. Typically a spindle. 318 */ 319 struct dm_dev *origin_dev; 320 321 /* 322 * The faster of the two data devices. Typically an SSD. 323 */ 324 struct dm_dev *cache_dev; 325 326 /* 327 * Size of the origin device in _complete_ blocks and native sectors. 328 */ 329 dm_oblock_t origin_blocks; 330 sector_t origin_sectors; 331 332 /* 333 * Size of the cache device in blocks. 334 */ 335 dm_cblock_t cache_size; 336 337 /* 338 * Invalidation fields. 339 */ 340 spinlock_t invalidation_lock; 341 struct list_head invalidation_requests; 342 343 sector_t migration_threshold; 344 345 /* 346 * The number of in flight migrations that are performing 347 * background io. eg, promotion, writeback. 348 */ 349 atomic_t nr_io_migrations; 350 351 struct bio_list deferred_bios; 352 353 struct rw_semaphore quiesce_lock; 354 355 /* 356 * origin_blocks entries, discarded if set. 357 */ 358 dm_dblock_t discard_nr_blocks; 359 unsigned long *discard_bitset; 360 uint32_t discard_block_size; /* a power of 2 times sectors per block */ 361 362 /* 363 * Rather than reconstructing the table line for the status we just 364 * save it and regurgitate. 365 */ 366 unsigned int nr_ctr_args; 367 const char **ctr_args; 368 369 struct dm_kcopyd_client *copier; 370 struct work_struct deferred_bio_worker; 371 struct work_struct migration_worker; 372 struct workqueue_struct *wq; 373 struct delayed_work waker; 374 struct dm_bio_prison_v2 *prison; 375 376 /* 377 * cache_size entries, dirty if set 378 */ 379 unsigned long *dirty_bitset; 380 atomic_t nr_dirty; 381 382 unsigned int policy_nr_args; 383 struct dm_cache_policy *policy; 384 385 /* 386 * Cache features such as write-through. 387 */ 388 struct cache_features features; 389 390 struct cache_stats stats; 391 392 bool need_tick_bio:1; 393 bool sized:1; 394 bool invalidate:1; 395 bool commit_requested:1; 396 bool loaded_mappings:1; 397 bool loaded_discards:1; 398 399 /* background work management */ 400 bool background_work_allowed; 401 unsigned background_work_nr; 402 spinlock_t background_work_lock; 403 wait_queue_head_t background_work_wait; 404 405 struct batcher committer; 406 struct work_struct commit_ws; 407 408 struct dm_io_tracker tracker; 409 410 mempool_t migration_pool; 411 412 struct bio_set bs; 413 414 /* 415 * Cache_size entries. Set bits indicate blocks mapped beyond the 416 * target length, which are marked for invalidation. 417 */ 418 unsigned long *invalid_bitset; 419 }; 420 421 struct per_bio_data { 422 bool tick:1; 423 unsigned int req_nr:2; 424 struct dm_bio_prison_cell_v2 *cell; 425 struct dm_hook_info hook_info; 426 sector_t len; 427 }; 428 429 struct dm_cache_migration { 430 struct continuation k; 431 struct cache *cache; 432 433 struct policy_work *op; 434 struct bio *overwrite_bio; 435 struct dm_bio_prison_cell_v2 *cell; 436 437 dm_cblock_t invalidate_cblock; 438 dm_oblock_t invalidate_oblock; 439 }; 440 441 /*----------------------------------------------------------------*/ 442 443 static bool writethrough_mode(struct cache *cache) 444 { 445 return cache->features.io_mode == CM_IO_WRITETHROUGH; 446 } 447 448 static bool writeback_mode(struct cache *cache) 449 { 450 return cache->features.io_mode == CM_IO_WRITEBACK; 451 } 452 453 static inline bool passthrough_mode(struct cache *cache) 454 { 455 return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH); 456 } 457 458 /*----------------------------------------------------------------*/ 459 460 static void wake_deferred_bio_worker(struct cache *cache) 461 { 462 queue_work(cache->wq, &cache->deferred_bio_worker); 463 } 464 465 static void wake_migration_worker(struct cache *cache) 466 { 467 if (passthrough_mode(cache)) 468 return; 469 470 queue_work(cache->wq, &cache->migration_worker); 471 } 472 473 /*----------------------------------------------------------------*/ 474 475 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache) 476 { 477 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO); 478 } 479 480 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell) 481 { 482 dm_bio_prison_free_cell_v2(cache->prison, cell); 483 } 484 485 static struct dm_cache_migration *alloc_migration(struct cache *cache) 486 { 487 struct dm_cache_migration *mg; 488 489 mg = mempool_alloc(&cache->migration_pool, GFP_NOIO); 490 491 memset(mg, 0, sizeof(*mg)); 492 493 mg->cache = cache; 494 495 return mg; 496 } 497 498 static void free_migration(struct dm_cache_migration *mg) 499 { 500 mempool_free(mg, &mg->cache->migration_pool); 501 } 502 503 /*----------------------------------------------------------------*/ 504 505 static inline dm_oblock_t oblock_succ(dm_oblock_t b) 506 { 507 return to_oblock(from_oblock(b) + 1ull); 508 } 509 510 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key) 511 { 512 key->virtual = 0; 513 key->dev = 0; 514 key->block_begin = from_oblock(begin); 515 key->block_end = from_oblock(end); 516 } 517 518 /* 519 * We have two lock levels. Level 0, which is used to prevent WRITEs, and 520 * level 1 which prevents *both* READs and WRITEs. 521 */ 522 #define WRITE_LOCK_LEVEL 0 523 #define READ_WRITE_LOCK_LEVEL 1 524 525 static unsigned int lock_level(struct bio *bio) 526 { 527 return bio_data_dir(bio) == WRITE ? 528 WRITE_LOCK_LEVEL : 529 READ_WRITE_LOCK_LEVEL; 530 } 531 532 /* 533 *-------------------------------------------------------------- 534 * Per bio data 535 *-------------------------------------------------------------- 536 */ 537 538 static struct per_bio_data *get_per_bio_data(struct bio *bio) 539 { 540 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); 541 542 BUG_ON(!pb); 543 return pb; 544 } 545 546 static struct per_bio_data *init_per_bio_data(struct bio *bio) 547 { 548 struct per_bio_data *pb = get_per_bio_data(bio); 549 550 pb->tick = false; 551 pb->req_nr = dm_bio_get_target_bio_nr(bio); 552 pb->cell = NULL; 553 pb->len = 0; 554 555 return pb; 556 } 557 558 /*----------------------------------------------------------------*/ 559 560 static void defer_bio(struct cache *cache, struct bio *bio) 561 { 562 spin_lock_irq(&cache->lock); 563 bio_list_add(&cache->deferred_bios, bio); 564 spin_unlock_irq(&cache->lock); 565 566 wake_deferred_bio_worker(cache); 567 } 568 569 static void defer_bios(struct cache *cache, struct bio_list *bios) 570 { 571 spin_lock_irq(&cache->lock); 572 bio_list_merge_init(&cache->deferred_bios, bios); 573 spin_unlock_irq(&cache->lock); 574 575 wake_deferred_bio_worker(cache); 576 } 577 578 /*----------------------------------------------------------------*/ 579 580 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio) 581 { 582 bool r; 583 struct per_bio_data *pb; 584 struct dm_cell_key_v2 key; 585 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL); 586 struct dm_bio_prison_cell_v2 *cell_prealloc, *cell; 587 588 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */ 589 590 build_key(oblock, end, &key); 591 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell); 592 if (!r) { 593 /* 594 * Failed to get the lock. 595 */ 596 free_prison_cell(cache, cell_prealloc); 597 return r; 598 } 599 600 if (cell != cell_prealloc) 601 free_prison_cell(cache, cell_prealloc); 602 603 pb = get_per_bio_data(bio); 604 pb->cell = cell; 605 606 return r; 607 } 608 609 /*----------------------------------------------------------------*/ 610 611 static bool is_dirty(struct cache *cache, dm_cblock_t b) 612 { 613 return test_bit(from_cblock(b), cache->dirty_bitset); 614 } 615 616 static void set_dirty(struct cache *cache, dm_cblock_t cblock) 617 { 618 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) { 619 atomic_inc(&cache->nr_dirty); 620 policy_set_dirty(cache->policy, cblock); 621 } 622 } 623 624 /* 625 * These two are called when setting after migrations to force the policy 626 * and dirty bitset to be in sync. 627 */ 628 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock) 629 { 630 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) 631 atomic_inc(&cache->nr_dirty); 632 policy_set_dirty(cache->policy, cblock); 633 } 634 635 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock) 636 { 637 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) { 638 if (atomic_dec_return(&cache->nr_dirty) == 0) 639 dm_table_event(cache->ti->table); 640 } 641 642 policy_clear_dirty(cache->policy, cblock); 643 } 644 645 /*----------------------------------------------------------------*/ 646 647 static bool block_size_is_power_of_two(struct cache *cache) 648 { 649 return cache->sectors_per_block_shift >= 0; 650 } 651 652 static dm_block_t block_div(dm_block_t b, uint32_t n) 653 { 654 do_div(b, n); 655 656 return b; 657 } 658 659 static dm_block_t oblocks_per_dblock(struct cache *cache) 660 { 661 dm_block_t oblocks = cache->discard_block_size; 662 663 if (block_size_is_power_of_two(cache)) 664 oblocks >>= cache->sectors_per_block_shift; 665 else 666 oblocks = block_div(oblocks, cache->sectors_per_block); 667 668 return oblocks; 669 } 670 671 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock) 672 { 673 return to_dblock(block_div(from_oblock(oblock), 674 oblocks_per_dblock(cache))); 675 } 676 677 static void set_discard(struct cache *cache, dm_dblock_t b) 678 { 679 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks)); 680 atomic_inc(&cache->stats.discard_count); 681 682 spin_lock_irq(&cache->lock); 683 set_bit(from_dblock(b), cache->discard_bitset); 684 spin_unlock_irq(&cache->lock); 685 } 686 687 static void clear_discard(struct cache *cache, dm_dblock_t b) 688 { 689 spin_lock_irq(&cache->lock); 690 clear_bit(from_dblock(b), cache->discard_bitset); 691 spin_unlock_irq(&cache->lock); 692 } 693 694 static bool is_discarded(struct cache *cache, dm_dblock_t b) 695 { 696 int r; 697 698 spin_lock_irq(&cache->lock); 699 r = test_bit(from_dblock(b), cache->discard_bitset); 700 spin_unlock_irq(&cache->lock); 701 702 return r; 703 } 704 705 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) 706 { 707 int r; 708 709 spin_lock_irq(&cache->lock); 710 r = test_bit(from_dblock(oblock_to_dblock(cache, b)), 711 cache->discard_bitset); 712 spin_unlock_irq(&cache->lock); 713 714 return r; 715 } 716 717 /* 718 * ------------------------------------------------------------- 719 * Remapping 720 *-------------------------------------------------------------- 721 */ 722 static void remap_to_origin(struct cache *cache, struct bio *bio) 723 { 724 bio_set_dev(bio, cache->origin_dev->bdev); 725 } 726 727 static void remap_to_cache(struct cache *cache, struct bio *bio, 728 dm_cblock_t cblock) 729 { 730 sector_t bi_sector = bio->bi_iter.bi_sector; 731 sector_t block = from_cblock(cblock); 732 733 bio_set_dev(bio, cache->cache_dev->bdev); 734 if (!block_size_is_power_of_two(cache)) 735 bio->bi_iter.bi_sector = 736 (block * cache->sectors_per_block) + 737 sector_div(bi_sector, cache->sectors_per_block); 738 else 739 bio->bi_iter.bi_sector = 740 (block << cache->sectors_per_block_shift) | 741 (bi_sector & (cache->sectors_per_block - 1)); 742 } 743 744 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) 745 { 746 struct per_bio_data *pb; 747 748 spin_lock_irq(&cache->lock); 749 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) && 750 bio_op(bio) != REQ_OP_DISCARD) { 751 pb = get_per_bio_data(bio); 752 pb->tick = true; 753 cache->need_tick_bio = false; 754 } 755 spin_unlock_irq(&cache->lock); 756 } 757 758 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, 759 dm_oblock_t oblock) 760 { 761 // FIXME: check_if_tick_bio_needed() is called way too much through this interface 762 check_if_tick_bio_needed(cache, bio); 763 remap_to_origin(cache, bio); 764 if (bio_data_dir(bio) == WRITE) 765 clear_discard(cache, oblock_to_dblock(cache, oblock)); 766 } 767 768 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, 769 dm_oblock_t oblock, dm_cblock_t cblock) 770 { 771 check_if_tick_bio_needed(cache, bio); 772 remap_to_cache(cache, bio, cblock); 773 if (bio_data_dir(bio) == WRITE) { 774 set_dirty(cache, cblock); 775 clear_discard(cache, oblock_to_dblock(cache, oblock)); 776 } 777 } 778 779 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) 780 { 781 sector_t block_nr = bio->bi_iter.bi_sector; 782 783 if (!block_size_is_power_of_two(cache)) 784 (void) sector_div(block_nr, cache->sectors_per_block); 785 else 786 block_nr >>= cache->sectors_per_block_shift; 787 788 return to_oblock(block_nr); 789 } 790 791 static bool accountable_bio(struct cache *cache, struct bio *bio) 792 { 793 return bio_op(bio) != REQ_OP_DISCARD; 794 } 795 796 static void accounted_begin(struct cache *cache, struct bio *bio) 797 { 798 struct per_bio_data *pb; 799 800 if (accountable_bio(cache, bio)) { 801 pb = get_per_bio_data(bio); 802 pb->len = bio_sectors(bio); 803 dm_iot_io_begin(&cache->tracker, pb->len); 804 } 805 } 806 807 static void accounted_complete(struct cache *cache, struct bio *bio) 808 { 809 struct per_bio_data *pb = get_per_bio_data(bio); 810 811 dm_iot_io_end(&cache->tracker, pb->len); 812 } 813 814 static void accounted_request(struct cache *cache, struct bio *bio) 815 { 816 accounted_begin(cache, bio); 817 dm_submit_bio_remap(bio, NULL); 818 } 819 820 static void issue_op(struct bio *bio, void *context) 821 { 822 struct cache *cache = context; 823 824 accounted_request(cache, bio); 825 } 826 827 /* 828 * When running in writethrough mode we need to send writes to clean blocks 829 * to both the cache and origin devices. Clone the bio and send them in parallel. 830 */ 831 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio, 832 dm_oblock_t oblock, dm_cblock_t cblock) 833 { 834 struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio, 835 GFP_NOIO, &cache->bs); 836 837 BUG_ON(!origin_bio); 838 839 bio_chain(origin_bio, bio); 840 841 if (bio_data_dir(origin_bio) == WRITE) 842 clear_discard(cache, oblock_to_dblock(cache, oblock)); 843 submit_bio(origin_bio); 844 845 remap_to_cache(cache, bio, cblock); 846 } 847 848 /* 849 *-------------------------------------------------------------- 850 * Failure modes 851 *-------------------------------------------------------------- 852 */ 853 static enum cache_metadata_mode get_cache_mode(struct cache *cache) 854 { 855 return cache->features.mode; 856 } 857 858 static const char *cache_device_name(struct cache *cache) 859 { 860 return dm_table_device_name(cache->ti->table); 861 } 862 863 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode) 864 { 865 static const char *descs[] = { 866 "write", 867 "read-only", 868 "fail" 869 }; 870 871 dm_table_event(cache->ti->table); 872 DMINFO("%s: switching cache to %s mode", 873 cache_device_name(cache), descs[(int)mode]); 874 } 875 876 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode) 877 { 878 bool needs_check; 879 enum cache_metadata_mode old_mode = get_cache_mode(cache); 880 881 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) { 882 DMERR("%s: unable to read needs_check flag, setting failure mode.", 883 cache_device_name(cache)); 884 new_mode = CM_FAIL; 885 } 886 887 if (new_mode == CM_WRITE && needs_check) { 888 DMERR("%s: unable to switch cache to write mode until repaired.", 889 cache_device_name(cache)); 890 if (old_mode != new_mode) 891 new_mode = old_mode; 892 else 893 new_mode = CM_READ_ONLY; 894 } 895 896 /* Never move out of fail mode */ 897 if (old_mode == CM_FAIL) 898 new_mode = CM_FAIL; 899 900 switch (new_mode) { 901 case CM_FAIL: 902 case CM_READ_ONLY: 903 dm_cache_metadata_set_read_only(cache->cmd); 904 break; 905 906 case CM_WRITE: 907 dm_cache_metadata_set_read_write(cache->cmd); 908 break; 909 } 910 911 cache->features.mode = new_mode; 912 913 if (new_mode != old_mode) 914 notify_mode_switch(cache, new_mode); 915 } 916 917 static void abort_transaction(struct cache *cache) 918 { 919 const char *dev_name = cache_device_name(cache); 920 921 if (get_cache_mode(cache) >= CM_READ_ONLY) 922 return; 923 924 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 925 if (dm_cache_metadata_abort(cache->cmd)) { 926 DMERR("%s: failed to abort metadata transaction", dev_name); 927 set_cache_mode(cache, CM_FAIL); 928 } 929 930 if (dm_cache_metadata_set_needs_check(cache->cmd)) { 931 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 932 set_cache_mode(cache, CM_FAIL); 933 } 934 } 935 936 static void metadata_operation_failed(struct cache *cache, const char *op, int r) 937 { 938 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 939 cache_device_name(cache), op, r); 940 abort_transaction(cache); 941 set_cache_mode(cache, CM_READ_ONLY); 942 } 943 944 /*----------------------------------------------------------------*/ 945 946 static void load_stats(struct cache *cache) 947 { 948 struct dm_cache_statistics stats; 949 950 dm_cache_metadata_get_stats(cache->cmd, &stats); 951 atomic_set(&cache->stats.read_hit, stats.read_hits); 952 atomic_set(&cache->stats.read_miss, stats.read_misses); 953 atomic_set(&cache->stats.write_hit, stats.write_hits); 954 atomic_set(&cache->stats.write_miss, stats.write_misses); 955 } 956 957 static void save_stats(struct cache *cache) 958 { 959 struct dm_cache_statistics stats; 960 961 if (get_cache_mode(cache) >= CM_READ_ONLY) 962 return; 963 964 stats.read_hits = atomic_read(&cache->stats.read_hit); 965 stats.read_misses = atomic_read(&cache->stats.read_miss); 966 stats.write_hits = atomic_read(&cache->stats.write_hit); 967 stats.write_misses = atomic_read(&cache->stats.write_miss); 968 969 dm_cache_metadata_set_stats(cache->cmd, &stats); 970 } 971 972 static void update_stats(struct cache_stats *stats, enum policy_operation op) 973 { 974 switch (op) { 975 case POLICY_PROMOTE: 976 atomic_inc(&stats->promotion); 977 break; 978 979 case POLICY_DEMOTE: 980 atomic_inc(&stats->demotion); 981 break; 982 983 case POLICY_WRITEBACK: 984 atomic_inc(&stats->writeback); 985 break; 986 } 987 } 988 989 /* 990 *--------------------------------------------------------------------- 991 * Migration processing 992 * 993 * Migration covers moving data from the origin device to the cache, or 994 * vice versa. 995 *--------------------------------------------------------------------- 996 */ 997 static void inc_io_migrations(struct cache *cache) 998 { 999 atomic_inc(&cache->nr_io_migrations); 1000 } 1001 1002 static void dec_io_migrations(struct cache *cache) 1003 { 1004 atomic_dec(&cache->nr_io_migrations); 1005 } 1006 1007 static bool discard_or_flush(struct bio *bio) 1008 { 1009 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf); 1010 } 1011 1012 static void calc_discard_block_range(struct cache *cache, struct bio *bio, 1013 dm_dblock_t *b, dm_dblock_t *e) 1014 { 1015 sector_t sb = bio->bi_iter.bi_sector; 1016 sector_t se = bio_end_sector(bio); 1017 1018 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size)); 1019 1020 if (se - sb < cache->discard_block_size) 1021 *e = *b; 1022 else 1023 *e = to_dblock(block_div(se, cache->discard_block_size)); 1024 } 1025 1026 /*----------------------------------------------------------------*/ 1027 1028 static void prevent_background_work(struct cache *cache) 1029 { 1030 spin_lock_irq(&cache->background_work_lock); 1031 cache->background_work_allowed = false; 1032 wait_event_lock_irq(cache->background_work_wait, 1033 cache->background_work_nr == 0, 1034 cache->background_work_lock); 1035 spin_unlock_irq(&cache->background_work_lock); 1036 } 1037 1038 static void allow_background_work(struct cache *cache) 1039 { 1040 spin_lock_irq(&cache->background_work_lock); 1041 cache->background_work_allowed = true; 1042 spin_unlock_irq(&cache->background_work_lock); 1043 } 1044 1045 static bool background_work_begin(struct cache *cache) 1046 { 1047 bool r; 1048 1049 spin_lock_irq(&cache->background_work_lock); 1050 r = cache->background_work_allowed; 1051 if (r) 1052 cache->background_work_nr++; 1053 spin_unlock_irq(&cache->background_work_lock); 1054 return r; 1055 } 1056 1057 static void background_work_end(struct cache *cache) 1058 { 1059 spin_lock_irq(&cache->background_work_lock); 1060 if (--cache->background_work_nr == 0) 1061 wake_up(&cache->background_work_wait); 1062 spin_unlock_irq(&cache->background_work_lock); 1063 } 1064 1065 /*----------------------------------------------------------------*/ 1066 1067 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) 1068 { 1069 return (bio_data_dir(bio) == WRITE) && 1070 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); 1071 } 1072 1073 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block) 1074 { 1075 return writeback_mode(cache) && 1076 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio)); 1077 } 1078 1079 static void quiesce(struct dm_cache_migration *mg, 1080 void (*continuation)(struct work_struct *)) 1081 { 1082 init_continuation(&mg->k, continuation); 1083 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws); 1084 } 1085 1086 static struct dm_cache_migration *ws_to_mg(struct work_struct *ws) 1087 { 1088 struct continuation *k = container_of(ws, struct continuation, ws); 1089 1090 return container_of(k, struct dm_cache_migration, k); 1091 } 1092 1093 static void copy_complete(int read_err, unsigned long write_err, void *context) 1094 { 1095 struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k); 1096 1097 if (read_err || write_err) 1098 mg->k.input = BLK_STS_IOERR; 1099 1100 queue_continuation(mg->cache->wq, &mg->k); 1101 } 1102 1103 static void copy(struct dm_cache_migration *mg, bool promote) 1104 { 1105 struct dm_io_region o_region, c_region; 1106 struct cache *cache = mg->cache; 1107 1108 o_region.bdev = cache->origin_dev->bdev; 1109 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block; 1110 o_region.count = cache->sectors_per_block; 1111 1112 c_region.bdev = cache->cache_dev->bdev; 1113 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block; 1114 c_region.count = cache->sectors_per_block; 1115 1116 if (promote) 1117 dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); 1118 else 1119 dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); 1120 } 1121 1122 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) 1123 { 1124 struct per_bio_data *pb = get_per_bio_data(bio); 1125 1126 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell)) 1127 free_prison_cell(cache, pb->cell); 1128 pb->cell = NULL; 1129 } 1130 1131 static void overwrite_endio(struct bio *bio) 1132 { 1133 struct dm_cache_migration *mg = bio->bi_private; 1134 struct cache *cache = mg->cache; 1135 struct per_bio_data *pb = get_per_bio_data(bio); 1136 1137 dm_unhook_bio(&pb->hook_info, bio); 1138 1139 if (bio->bi_status) 1140 mg->k.input = bio->bi_status; 1141 1142 queue_continuation(cache->wq, &mg->k); 1143 } 1144 1145 static void overwrite(struct dm_cache_migration *mg, 1146 void (*continuation)(struct work_struct *)) 1147 { 1148 struct bio *bio = mg->overwrite_bio; 1149 struct per_bio_data *pb = get_per_bio_data(bio); 1150 1151 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); 1152 1153 /* 1154 * The overwrite bio is part of the copy operation, as such it does 1155 * not set/clear discard or dirty flags. 1156 */ 1157 if (mg->op->op == POLICY_PROMOTE) 1158 remap_to_cache(mg->cache, bio, mg->op->cblock); 1159 else 1160 remap_to_origin(mg->cache, bio); 1161 1162 init_continuation(&mg->k, continuation); 1163 accounted_request(mg->cache, bio); 1164 } 1165 1166 /* 1167 * Migration steps: 1168 * 1169 * 1) exclusive lock preventing WRITEs 1170 * 2) quiesce 1171 * 3) copy or issue overwrite bio 1172 * 4) upgrade to exclusive lock preventing READs and WRITEs 1173 * 5) quiesce 1174 * 6) update metadata and commit 1175 * 7) unlock 1176 */ 1177 static void mg_complete(struct dm_cache_migration *mg, bool success) 1178 { 1179 struct bio_list bios; 1180 struct cache *cache = mg->cache; 1181 struct policy_work *op = mg->op; 1182 dm_cblock_t cblock = op->cblock; 1183 1184 if (success) 1185 update_stats(&cache->stats, op->op); 1186 1187 switch (op->op) { 1188 case POLICY_PROMOTE: 1189 clear_discard(cache, oblock_to_dblock(cache, op->oblock)); 1190 policy_complete_background_work(cache->policy, op, success); 1191 1192 if (mg->overwrite_bio) { 1193 if (success) 1194 force_set_dirty(cache, cblock); 1195 else if (mg->k.input) 1196 mg->overwrite_bio->bi_status = mg->k.input; 1197 else 1198 mg->overwrite_bio->bi_status = BLK_STS_IOERR; 1199 bio_endio(mg->overwrite_bio); 1200 } else { 1201 if (success) 1202 force_clear_dirty(cache, cblock); 1203 dec_io_migrations(cache); 1204 } 1205 break; 1206 1207 case POLICY_DEMOTE: 1208 /* 1209 * We clear dirty here to update the nr_dirty counter. 1210 */ 1211 if (success) 1212 force_clear_dirty(cache, cblock); 1213 policy_complete_background_work(cache->policy, op, success); 1214 dec_io_migrations(cache); 1215 break; 1216 1217 case POLICY_WRITEBACK: 1218 if (success) 1219 force_clear_dirty(cache, cblock); 1220 policy_complete_background_work(cache->policy, op, success); 1221 dec_io_migrations(cache); 1222 break; 1223 } 1224 1225 bio_list_init(&bios); 1226 if (mg->cell) { 1227 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1228 free_prison_cell(cache, mg->cell); 1229 } 1230 1231 free_migration(mg); 1232 defer_bios(cache, &bios); 1233 wake_migration_worker(cache); 1234 1235 background_work_end(cache); 1236 } 1237 1238 static void mg_success(struct work_struct *ws) 1239 { 1240 struct dm_cache_migration *mg = ws_to_mg(ws); 1241 1242 mg_complete(mg, mg->k.input == 0); 1243 } 1244 1245 static void mg_update_metadata(struct work_struct *ws) 1246 { 1247 int r; 1248 struct dm_cache_migration *mg = ws_to_mg(ws); 1249 struct cache *cache = mg->cache; 1250 struct policy_work *op = mg->op; 1251 1252 switch (op->op) { 1253 case POLICY_PROMOTE: 1254 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock); 1255 if (r) { 1256 DMERR_LIMIT("%s: migration failed; couldn't insert mapping", 1257 cache_device_name(cache)); 1258 metadata_operation_failed(cache, "dm_cache_insert_mapping", r); 1259 1260 mg_complete(mg, false); 1261 return; 1262 } 1263 mg_complete(mg, true); 1264 break; 1265 1266 case POLICY_DEMOTE: 1267 r = dm_cache_remove_mapping(cache->cmd, op->cblock); 1268 if (r) { 1269 DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata", 1270 cache_device_name(cache)); 1271 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1272 1273 mg_complete(mg, false); 1274 return; 1275 } 1276 1277 /* 1278 * It would be nice if we only had to commit when a REQ_FLUSH 1279 * comes through. But there's one scenario that we have to 1280 * look out for: 1281 * 1282 * - vblock x in a cache block 1283 * - domotion occurs 1284 * - cache block gets reallocated and over written 1285 * - crash 1286 * 1287 * When we recover, because there was no commit the cache will 1288 * rollback to having the data for vblock x in the cache block. 1289 * But the cache block has since been overwritten, so it'll end 1290 * up pointing to data that was never in 'x' during the history 1291 * of the device. 1292 * 1293 * To avoid this issue we require a commit as part of the 1294 * demotion operation. 1295 */ 1296 init_continuation(&mg->k, mg_success); 1297 continue_after_commit(&cache->committer, &mg->k); 1298 schedule_commit(&cache->committer); 1299 break; 1300 1301 case POLICY_WRITEBACK: 1302 mg_complete(mg, true); 1303 break; 1304 } 1305 } 1306 1307 static void mg_update_metadata_after_copy(struct work_struct *ws) 1308 { 1309 struct dm_cache_migration *mg = ws_to_mg(ws); 1310 1311 /* 1312 * Did the copy succeed? 1313 */ 1314 if (mg->k.input) 1315 mg_complete(mg, false); 1316 else 1317 mg_update_metadata(ws); 1318 } 1319 1320 static void mg_upgrade_lock(struct work_struct *ws) 1321 { 1322 int r; 1323 struct dm_cache_migration *mg = ws_to_mg(ws); 1324 1325 /* 1326 * Did the copy succeed? 1327 */ 1328 if (mg->k.input) 1329 mg_complete(mg, false); 1330 1331 else { 1332 /* 1333 * Now we want the lock to prevent both reads and writes. 1334 */ 1335 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell, 1336 READ_WRITE_LOCK_LEVEL); 1337 if (r < 0) 1338 mg_complete(mg, false); 1339 1340 else if (r) 1341 quiesce(mg, mg_update_metadata); 1342 1343 else 1344 mg_update_metadata(ws); 1345 } 1346 } 1347 1348 static void mg_full_copy(struct work_struct *ws) 1349 { 1350 struct dm_cache_migration *mg = ws_to_mg(ws); 1351 struct cache *cache = mg->cache; 1352 struct policy_work *op = mg->op; 1353 bool is_policy_promote = (op->op == POLICY_PROMOTE); 1354 1355 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) || 1356 is_discarded_oblock(cache, op->oblock)) { 1357 mg_upgrade_lock(ws); 1358 return; 1359 } 1360 1361 init_continuation(&mg->k, mg_upgrade_lock); 1362 copy(mg, is_policy_promote); 1363 } 1364 1365 static void mg_copy(struct work_struct *ws) 1366 { 1367 struct dm_cache_migration *mg = ws_to_mg(ws); 1368 1369 if (mg->overwrite_bio) { 1370 /* 1371 * No exclusive lock was held when we last checked if the bio 1372 * was optimisable. So we have to check again in case things 1373 * have changed (eg, the block may no longer be discarded). 1374 */ 1375 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) { 1376 /* 1377 * Fallback to a real full copy after doing some tidying up. 1378 */ 1379 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); 1380 1381 BUG_ON(rb); /* An exclusive lock must _not_ be held for this block */ 1382 mg->overwrite_bio = NULL; 1383 inc_io_migrations(mg->cache); 1384 mg_full_copy(ws); 1385 return; 1386 } 1387 1388 /* 1389 * It's safe to do this here, even though it's new data 1390 * because all IO has been locked out of the block. 1391 * 1392 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL 1393 * so _not_ using mg_upgrade_lock() as continutation. 1394 */ 1395 overwrite(mg, mg_update_metadata_after_copy); 1396 1397 } else 1398 mg_full_copy(ws); 1399 } 1400 1401 static int mg_lock_writes(struct dm_cache_migration *mg) 1402 { 1403 int r; 1404 struct dm_cell_key_v2 key; 1405 struct cache *cache = mg->cache; 1406 struct dm_bio_prison_cell_v2 *prealloc; 1407 1408 prealloc = alloc_prison_cell(cache); 1409 1410 /* 1411 * Prevent writes to the block, but allow reads to continue. 1412 * Unless we're using an overwrite bio, in which case we lock 1413 * everything. 1414 */ 1415 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key); 1416 r = dm_cell_lock_v2(cache->prison, &key, 1417 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL, 1418 prealloc, &mg->cell); 1419 if (r < 0) { 1420 free_prison_cell(cache, prealloc); 1421 mg_complete(mg, false); 1422 return r; 1423 } 1424 1425 if (mg->cell != prealloc) 1426 free_prison_cell(cache, prealloc); 1427 1428 if (r == 0) 1429 mg_copy(&mg->k.ws); 1430 else 1431 quiesce(mg, mg_copy); 1432 1433 return 0; 1434 } 1435 1436 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio) 1437 { 1438 struct dm_cache_migration *mg; 1439 1440 if (!background_work_begin(cache)) { 1441 policy_complete_background_work(cache->policy, op, false); 1442 return -EPERM; 1443 } 1444 1445 mg = alloc_migration(cache); 1446 1447 mg->op = op; 1448 mg->overwrite_bio = bio; 1449 1450 if (!bio) 1451 inc_io_migrations(cache); 1452 1453 return mg_lock_writes(mg); 1454 } 1455 1456 /* 1457 *-------------------------------------------------------------- 1458 * invalidation processing 1459 *-------------------------------------------------------------- 1460 */ 1461 1462 static void invalidate_complete(struct dm_cache_migration *mg, bool success) 1463 { 1464 struct bio_list bios; 1465 struct cache *cache = mg->cache; 1466 1467 if (success) 1468 atomic_inc(&cache->stats.demotion); 1469 1470 bio_list_init(&bios); 1471 if (mg->cell) { 1472 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1473 free_prison_cell(cache, mg->cell); 1474 } 1475 1476 if (mg->overwrite_bio) { 1477 // Set generic error if the bio hasn't been issued yet, 1478 // e.g., invalidation or metadata commit failed before bio 1479 // submission. Otherwise preserve the bio's own error status. 1480 if (!success && !mg->overwrite_bio->bi_status) 1481 mg->overwrite_bio->bi_status = BLK_STS_IOERR; 1482 bio_endio(mg->overwrite_bio); 1483 } 1484 1485 free_migration(mg); 1486 defer_bios(cache, &bios); 1487 1488 background_work_end(cache); 1489 } 1490 1491 static void invalidate_completed(struct work_struct *ws) 1492 { 1493 struct dm_cache_migration *mg = ws_to_mg(ws); 1494 1495 invalidate_complete(mg, !mg->k.input); 1496 } 1497 1498 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) 1499 { 1500 int r; 1501 1502 r = policy_invalidate_mapping(cache->policy, cblock); 1503 if (!r) { 1504 r = dm_cache_remove_mapping(cache->cmd, cblock); 1505 if (r) { 1506 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata", 1507 cache_device_name(cache)); 1508 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1509 } 1510 1511 } else if (r == -ENODATA) { 1512 /* 1513 * Harmless, already unmapped. 1514 */ 1515 r = 0; 1516 1517 } else 1518 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache)); 1519 1520 return r; 1521 } 1522 1523 static void invalidate_committed(struct work_struct *ws) 1524 { 1525 struct dm_cache_migration *mg = ws_to_mg(ws); 1526 struct cache *cache = mg->cache; 1527 struct bio *bio = mg->overwrite_bio; 1528 struct per_bio_data *pb = get_per_bio_data(bio); 1529 1530 if (mg->k.input) { 1531 invalidate_complete(mg, false); 1532 return; 1533 } 1534 1535 init_continuation(&mg->k, invalidate_completed); 1536 remap_to_origin_clear_discard(cache, bio, mg->invalidate_oblock); 1537 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); 1538 dm_submit_bio_remap(bio, NULL); 1539 } 1540 1541 static void invalidate_remove(struct work_struct *ws) 1542 { 1543 int r; 1544 struct dm_cache_migration *mg = ws_to_mg(ws); 1545 struct cache *cache = mg->cache; 1546 1547 r = invalidate_cblock(cache, mg->invalidate_cblock); 1548 if (r) { 1549 invalidate_complete(mg, false); 1550 return; 1551 } 1552 1553 init_continuation(&mg->k, invalidate_committed); 1554 continue_after_commit(&cache->committer, &mg->k); 1555 schedule_commit(&cache->committer); 1556 } 1557 1558 static int invalidate_lock(struct dm_cache_migration *mg) 1559 { 1560 int r; 1561 struct dm_cell_key_v2 key; 1562 struct cache *cache = mg->cache; 1563 struct dm_bio_prison_cell_v2 *prealloc; 1564 1565 prealloc = alloc_prison_cell(cache); 1566 1567 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key); 1568 r = dm_cell_lock_v2(cache->prison, &key, 1569 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell); 1570 if (r < 0) { 1571 free_prison_cell(cache, prealloc); 1572 1573 /* Defer the bio for retrying the cell lock */ 1574 if (mg->overwrite_bio) { 1575 struct bio *bio = mg->overwrite_bio; 1576 1577 mg->overwrite_bio = NULL; 1578 defer_bio(cache, bio); 1579 } 1580 1581 invalidate_complete(mg, false); 1582 return r; 1583 } 1584 1585 if (mg->cell != prealloc) 1586 free_prison_cell(cache, prealloc); 1587 1588 if (r) 1589 quiesce(mg, invalidate_remove); 1590 1591 else { 1592 /* 1593 * We can't call invalidate_remove() directly here because we 1594 * might still be in request context. 1595 */ 1596 init_continuation(&mg->k, invalidate_remove); 1597 queue_work(cache->wq, &mg->k.ws); 1598 } 1599 1600 return 0; 1601 } 1602 1603 static int invalidate_start(struct cache *cache, dm_cblock_t cblock, 1604 dm_oblock_t oblock, struct bio *bio) 1605 { 1606 struct dm_cache_migration *mg; 1607 1608 if (!background_work_begin(cache)) 1609 return -EPERM; 1610 1611 mg = alloc_migration(cache); 1612 1613 mg->overwrite_bio = bio; 1614 mg->invalidate_cblock = cblock; 1615 mg->invalidate_oblock = oblock; 1616 1617 return invalidate_lock(mg); 1618 } 1619 1620 /* 1621 *-------------------------------------------------------------- 1622 * bio processing 1623 *-------------------------------------------------------------- 1624 */ 1625 1626 enum busy { 1627 IDLE, 1628 BUSY 1629 }; 1630 1631 static enum busy spare_migration_bandwidth(struct cache *cache) 1632 { 1633 bool idle = dm_iot_idle_for(&cache->tracker, HZ); 1634 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) * 1635 cache->sectors_per_block; 1636 1637 if (idle && current_volume <= cache->migration_threshold) 1638 return IDLE; 1639 else 1640 return BUSY; 1641 } 1642 1643 static void inc_hit_counter(struct cache *cache, struct bio *bio) 1644 { 1645 atomic_inc(bio_data_dir(bio) == READ ? 1646 &cache->stats.read_hit : &cache->stats.write_hit); 1647 } 1648 1649 static void inc_miss_counter(struct cache *cache, struct bio *bio) 1650 { 1651 atomic_inc(bio_data_dir(bio) == READ ? 1652 &cache->stats.read_miss : &cache->stats.write_miss); 1653 } 1654 1655 /*----------------------------------------------------------------*/ 1656 1657 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block, 1658 bool *commit_needed) 1659 { 1660 int r, data_dir; 1661 bool rb, background_queued; 1662 dm_cblock_t cblock; 1663 1664 *commit_needed = false; 1665 1666 rb = bio_detain_shared(cache, block, bio); 1667 if (!rb) { 1668 /* 1669 * An exclusive lock is held for this block, so we have to 1670 * wait. We set the commit_needed flag so the current 1671 * transaction will be committed asap, allowing this lock 1672 * to be dropped. 1673 */ 1674 *commit_needed = true; 1675 return DM_MAPIO_SUBMITTED; 1676 } 1677 1678 data_dir = bio_data_dir(bio); 1679 1680 if (optimisable_bio(cache, bio, block)) { 1681 struct policy_work *op = NULL; 1682 1683 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op); 1684 if (unlikely(r && r != -ENOENT)) { 1685 DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d", 1686 cache_device_name(cache), r); 1687 bio_io_error(bio); 1688 return DM_MAPIO_SUBMITTED; 1689 } 1690 1691 if (r == -ENOENT && op) { 1692 bio_drop_shared_lock(cache, bio); 1693 BUG_ON(op->op != POLICY_PROMOTE); 1694 mg_start(cache, op, bio); 1695 return DM_MAPIO_SUBMITTED; 1696 } 1697 } else { 1698 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued); 1699 if (unlikely(r && r != -ENOENT)) { 1700 DMERR_LIMIT("%s: policy_lookup() failed with r = %d", 1701 cache_device_name(cache), r); 1702 bio_io_error(bio); 1703 return DM_MAPIO_SUBMITTED; 1704 } 1705 1706 if (background_queued) 1707 wake_migration_worker(cache); 1708 } 1709 1710 if (r == -ENOENT) { 1711 struct per_bio_data *pb = get_per_bio_data(bio); 1712 1713 /* 1714 * Miss. 1715 */ 1716 inc_miss_counter(cache, bio); 1717 if (pb->req_nr == 0) { 1718 accounted_begin(cache, bio); 1719 remap_to_origin_clear_discard(cache, bio, block); 1720 } else { 1721 /* 1722 * This is a duplicate writethrough io that is no 1723 * longer needed because the block has been demoted. 1724 */ 1725 bio_endio(bio); 1726 return DM_MAPIO_SUBMITTED; 1727 } 1728 } else { 1729 /* 1730 * Hit. 1731 */ 1732 inc_hit_counter(cache, bio); 1733 1734 /* 1735 * Passthrough always maps to the origin, invalidating any 1736 * cache blocks that are written to. 1737 */ 1738 if (passthrough_mode(cache)) { 1739 if (bio_data_dir(bio) == WRITE) { 1740 bio_drop_shared_lock(cache, bio); 1741 invalidate_start(cache, cblock, block, bio); 1742 return DM_MAPIO_SUBMITTED; 1743 } else 1744 remap_to_origin_clear_discard(cache, bio, block); 1745 } else { 1746 if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) && 1747 !is_dirty(cache, cblock)) { 1748 remap_to_origin_and_cache(cache, bio, block, cblock); 1749 accounted_begin(cache, bio); 1750 } else 1751 remap_to_cache_dirty(cache, bio, block, cblock); 1752 } 1753 } 1754 1755 /* 1756 * dm core turns FUA requests into a separate payload and FLUSH req. 1757 */ 1758 if (bio->bi_opf & REQ_FUA) { 1759 /* 1760 * issue_after_commit will call accounted_begin a second time. So 1761 * we call accounted_complete() to avoid double accounting. 1762 */ 1763 accounted_complete(cache, bio); 1764 issue_after_commit(&cache->committer, bio); 1765 *commit_needed = true; 1766 return DM_MAPIO_SUBMITTED; 1767 } 1768 1769 return DM_MAPIO_REMAPPED; 1770 } 1771 1772 static bool process_bio(struct cache *cache, struct bio *bio) 1773 { 1774 bool commit_needed; 1775 1776 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED) 1777 dm_submit_bio_remap(bio, NULL); 1778 1779 return commit_needed; 1780 } 1781 1782 /* 1783 * A non-zero return indicates read_only or fail_io mode. 1784 */ 1785 static int commit(struct cache *cache, bool clean_shutdown) 1786 { 1787 int r; 1788 1789 if (get_cache_mode(cache) >= CM_READ_ONLY) 1790 return -EINVAL; 1791 1792 atomic_inc(&cache->stats.commit_count); 1793 r = dm_cache_commit(cache->cmd, clean_shutdown); 1794 if (r) 1795 metadata_operation_failed(cache, "dm_cache_commit", r); 1796 1797 return r; 1798 } 1799 1800 /* 1801 * Used by the batcher. 1802 */ 1803 static blk_status_t commit_op(void *context) 1804 { 1805 struct cache *cache = context; 1806 1807 if (dm_cache_changed_this_transaction(cache->cmd)) 1808 return errno_to_blk_status(commit(cache, false)); 1809 1810 return 0; 1811 } 1812 1813 /*----------------------------------------------------------------*/ 1814 1815 static bool process_flush_bio(struct cache *cache, struct bio *bio) 1816 { 1817 struct per_bio_data *pb = get_per_bio_data(bio); 1818 1819 if (!pb->req_nr) 1820 remap_to_origin(cache, bio); 1821 else 1822 remap_to_cache(cache, bio, 0); 1823 1824 issue_after_commit(&cache->committer, bio); 1825 return true; 1826 } 1827 1828 static bool process_discard_bio(struct cache *cache, struct bio *bio) 1829 { 1830 dm_dblock_t b, e; 1831 1832 /* 1833 * FIXME: do we need to lock the region? Or can we just assume the 1834 * user wont be so foolish as to issue discard concurrently with 1835 * other IO? 1836 */ 1837 calc_discard_block_range(cache, bio, &b, &e); 1838 while (b != e) { 1839 set_discard(cache, b); 1840 b = to_dblock(from_dblock(b) + 1); 1841 } 1842 1843 if (cache->features.discard_passdown) { 1844 remap_to_origin(cache, bio); 1845 dm_submit_bio_remap(bio, NULL); 1846 } else 1847 bio_endio(bio); 1848 1849 return false; 1850 } 1851 1852 static void process_deferred_bios(struct work_struct *ws) 1853 { 1854 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker); 1855 1856 bool commit_needed = false; 1857 struct bio_list bios; 1858 struct bio *bio; 1859 1860 bio_list_init(&bios); 1861 1862 spin_lock_irq(&cache->lock); 1863 bio_list_merge_init(&bios, &cache->deferred_bios); 1864 spin_unlock_irq(&cache->lock); 1865 1866 while ((bio = bio_list_pop(&bios))) { 1867 if (bio->bi_opf & REQ_PREFLUSH) 1868 commit_needed = process_flush_bio(cache, bio) || commit_needed; 1869 1870 else if (bio_op(bio) == REQ_OP_DISCARD) 1871 commit_needed = process_discard_bio(cache, bio) || commit_needed; 1872 1873 else 1874 commit_needed = process_bio(cache, bio) || commit_needed; 1875 cond_resched(); 1876 } 1877 1878 if (commit_needed) 1879 schedule_commit(&cache->committer); 1880 } 1881 1882 /* 1883 *-------------------------------------------------------------- 1884 * Main worker loop 1885 *-------------------------------------------------------------- 1886 */ 1887 static void requeue_deferred_bios(struct cache *cache) 1888 { 1889 struct bio *bio; 1890 struct bio_list bios; 1891 1892 bio_list_init(&bios); 1893 bio_list_merge_init(&bios, &cache->deferred_bios); 1894 1895 while ((bio = bio_list_pop(&bios))) { 1896 bio->bi_status = BLK_STS_DM_REQUEUE; 1897 bio_endio(bio); 1898 cond_resched(); 1899 } 1900 } 1901 1902 /* 1903 * We want to commit periodically so that not too much 1904 * unwritten metadata builds up. 1905 */ 1906 static void do_waker(struct work_struct *ws) 1907 { 1908 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker); 1909 1910 policy_tick(cache->policy, true); 1911 wake_migration_worker(cache); 1912 schedule_commit(&cache->committer); 1913 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD); 1914 } 1915 1916 static void check_migrations(struct work_struct *ws) 1917 { 1918 int r; 1919 struct policy_work *op; 1920 struct cache *cache = container_of(ws, struct cache, migration_worker); 1921 enum busy b; 1922 1923 for (;;) { 1924 b = spare_migration_bandwidth(cache); 1925 1926 r = policy_get_background_work(cache->policy, b == IDLE, &op); 1927 if (r == -ENODATA) 1928 break; 1929 1930 if (r) { 1931 DMERR_LIMIT("%s: policy_background_work failed", 1932 cache_device_name(cache)); 1933 break; 1934 } 1935 1936 r = mg_start(cache, op, NULL); 1937 if (r) 1938 break; 1939 1940 cond_resched(); 1941 } 1942 } 1943 1944 /* 1945 *-------------------------------------------------------------- 1946 * Target methods 1947 *-------------------------------------------------------------- 1948 */ 1949 1950 /* 1951 * This function gets called on the error paths of the constructor, so we 1952 * have to cope with a partially initialised struct. 1953 */ 1954 static void __destroy(struct cache *cache) 1955 { 1956 mempool_exit(&cache->migration_pool); 1957 1958 if (cache->prison) 1959 dm_bio_prison_destroy_v2(cache->prison); 1960 1961 if (cache->wq) 1962 destroy_workqueue(cache->wq); 1963 1964 if (cache->dirty_bitset) 1965 free_bitset(cache->dirty_bitset); 1966 1967 if (cache->discard_bitset) 1968 free_bitset(cache->discard_bitset); 1969 1970 if (cache->invalid_bitset) 1971 free_bitset(cache->invalid_bitset); 1972 1973 if (cache->copier) 1974 dm_kcopyd_client_destroy(cache->copier); 1975 1976 if (cache->cmd) 1977 dm_cache_metadata_close(cache->cmd); 1978 1979 if (cache->metadata_dev) 1980 dm_put_device(cache->ti, cache->metadata_dev); 1981 1982 if (cache->origin_dev) 1983 dm_put_device(cache->ti, cache->origin_dev); 1984 1985 if (cache->cache_dev) 1986 dm_put_device(cache->ti, cache->cache_dev); 1987 1988 if (cache->policy) 1989 dm_cache_policy_destroy(cache->policy); 1990 1991 bioset_exit(&cache->bs); 1992 1993 kfree(cache); 1994 } 1995 1996 static void destroy(struct cache *cache) 1997 { 1998 unsigned int i; 1999 2000 cancel_delayed_work_sync(&cache->waker); 2001 2002 for (i = 0; i < cache->nr_ctr_args ; i++) 2003 kfree(cache->ctr_args[i]); 2004 kfree(cache->ctr_args); 2005 2006 __destroy(cache); 2007 } 2008 2009 static void cache_dtr(struct dm_target *ti) 2010 { 2011 struct cache *cache = ti->private; 2012 2013 destroy(cache); 2014 } 2015 2016 static sector_t get_dev_size(struct dm_dev *dev) 2017 { 2018 return bdev_nr_sectors(dev->bdev); 2019 } 2020 2021 /*----------------------------------------------------------------*/ 2022 2023 /* 2024 * Construct a cache device mapping. 2025 * 2026 * cache <metadata dev> <cache dev> <origin dev> <block size> 2027 * <#feature args> [<feature arg>]* 2028 * <policy> <#policy args> [<policy arg>]* 2029 * 2030 * metadata dev : fast device holding the persistent metadata 2031 * cache dev : fast device holding cached data blocks 2032 * origin dev : slow device holding original data blocks 2033 * block size : cache unit size in sectors 2034 * 2035 * #feature args : number of feature arguments passed 2036 * feature args : writethrough. (The default is writeback.) 2037 * 2038 * policy : the replacement policy to use 2039 * #policy args : an even number of policy arguments corresponding 2040 * to key/value pairs passed to the policy 2041 * policy args : key/value pairs passed to the policy 2042 * E.g. 'sequential_threshold 1024' 2043 * See cache-policies.txt for details. 2044 * 2045 * Optional feature arguments are: 2046 * writethrough : write through caching that prohibits cache block 2047 * content from being different from origin block content. 2048 * Without this argument, the default behaviour is to write 2049 * back cache block contents later for performance reasons, 2050 * so they may differ from the corresponding origin blocks. 2051 */ 2052 struct cache_args { 2053 struct dm_target *ti; 2054 2055 struct dm_dev *metadata_dev; 2056 2057 struct dm_dev *cache_dev; 2058 sector_t cache_sectors; 2059 2060 struct dm_dev *origin_dev; 2061 2062 uint32_t block_size; 2063 2064 const char *policy_name; 2065 int policy_argc; 2066 const char **policy_argv; 2067 2068 struct cache_features features; 2069 }; 2070 2071 static void destroy_cache_args(struct cache_args *ca) 2072 { 2073 if (ca->metadata_dev) 2074 dm_put_device(ca->ti, ca->metadata_dev); 2075 2076 if (ca->cache_dev) 2077 dm_put_device(ca->ti, ca->cache_dev); 2078 2079 if (ca->origin_dev) 2080 dm_put_device(ca->ti, ca->origin_dev); 2081 2082 kfree(ca); 2083 } 2084 2085 static bool at_least_one_arg(struct dm_arg_set *as, char **error) 2086 { 2087 if (!as->argc) { 2088 *error = "Insufficient args"; 2089 return false; 2090 } 2091 2092 return true; 2093 } 2094 2095 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as, 2096 char **error) 2097 { 2098 int r; 2099 sector_t metadata_dev_size; 2100 2101 if (!at_least_one_arg(as, error)) 2102 return -EINVAL; 2103 2104 r = dm_get_device(ca->ti, dm_shift_arg(as), 2105 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev); 2106 if (r) { 2107 *error = "Error opening metadata device"; 2108 return r; 2109 } 2110 2111 metadata_dev_size = get_dev_size(ca->metadata_dev); 2112 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING) 2113 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.", 2114 ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS); 2115 2116 return 0; 2117 } 2118 2119 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as, 2120 char **error) 2121 { 2122 int r; 2123 2124 if (!at_least_one_arg(as, error)) 2125 return -EINVAL; 2126 2127 r = dm_get_device(ca->ti, dm_shift_arg(as), 2128 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev); 2129 if (r) { 2130 *error = "Error opening cache device"; 2131 return r; 2132 } 2133 ca->cache_sectors = get_dev_size(ca->cache_dev); 2134 2135 return 0; 2136 } 2137 2138 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, 2139 char **error) 2140 { 2141 int r; 2142 2143 if (!at_least_one_arg(as, error)) 2144 return -EINVAL; 2145 2146 r = dm_get_device(ca->ti, dm_shift_arg(as), 2147 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev); 2148 if (r) { 2149 *error = "Error opening origin device"; 2150 return r; 2151 } 2152 2153 return 0; 2154 } 2155 2156 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as, 2157 char **error) 2158 { 2159 unsigned long block_size; 2160 2161 if (!at_least_one_arg(as, error)) 2162 return -EINVAL; 2163 2164 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size || 2165 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 2166 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 2167 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 2168 *error = "Invalid data block size"; 2169 return -EINVAL; 2170 } 2171 2172 if (block_size > ca->cache_sectors) { 2173 *error = "Data block size is larger than the cache device"; 2174 return -EINVAL; 2175 } 2176 2177 ca->block_size = block_size; 2178 2179 return 0; 2180 } 2181 2182 static void init_features(struct cache_features *cf) 2183 { 2184 cf->mode = CM_WRITE; 2185 cf->io_mode = CM_IO_WRITEBACK; 2186 cf->metadata_version = 1; 2187 cf->discard_passdown = true; 2188 } 2189 2190 static int parse_features(struct cache_args *ca, struct dm_arg_set *as, 2191 char **error) 2192 { 2193 static const struct dm_arg _args[] = { 2194 {0, 3, "Invalid number of cache feature arguments"}, 2195 }; 2196 2197 int r, mode_ctr = 0; 2198 unsigned int argc; 2199 const char *arg; 2200 struct cache_features *cf = &ca->features; 2201 2202 init_features(cf); 2203 2204 r = dm_read_arg_group(_args, as, &argc, error); 2205 if (r) 2206 return -EINVAL; 2207 2208 while (argc--) { 2209 arg = dm_shift_arg(as); 2210 2211 if (!strcasecmp(arg, "writeback")) { 2212 cf->io_mode = CM_IO_WRITEBACK; 2213 mode_ctr++; 2214 } 2215 2216 else if (!strcasecmp(arg, "writethrough")) { 2217 cf->io_mode = CM_IO_WRITETHROUGH; 2218 mode_ctr++; 2219 } 2220 2221 else if (!strcasecmp(arg, "passthrough")) { 2222 cf->io_mode = CM_IO_PASSTHROUGH; 2223 mode_ctr++; 2224 } 2225 2226 else if (!strcasecmp(arg, "metadata2")) 2227 cf->metadata_version = 2; 2228 2229 else if (!strcasecmp(arg, "no_discard_passdown")) 2230 cf->discard_passdown = false; 2231 2232 else { 2233 *error = "Unrecognised cache feature requested"; 2234 return -EINVAL; 2235 } 2236 } 2237 2238 if (mode_ctr > 1) { 2239 *error = "Duplicate cache io_mode features requested"; 2240 return -EINVAL; 2241 } 2242 2243 return 0; 2244 } 2245 2246 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as, 2247 char **error) 2248 { 2249 static const struct dm_arg _args[] = { 2250 {0, 1024, "Invalid number of policy arguments"}, 2251 }; 2252 2253 int r; 2254 2255 if (!at_least_one_arg(as, error)) 2256 return -EINVAL; 2257 2258 ca->policy_name = dm_shift_arg(as); 2259 2260 r = dm_read_arg_group(_args, as, &ca->policy_argc, error); 2261 if (r) 2262 return -EINVAL; 2263 2264 ca->policy_argv = (const char **)as->argv; 2265 dm_consume_args(as, ca->policy_argc); 2266 2267 return 0; 2268 } 2269 2270 static int parse_cache_args(struct cache_args *ca, int argc, char **argv, 2271 char **error) 2272 { 2273 int r; 2274 struct dm_arg_set as; 2275 2276 as.argc = argc; 2277 as.argv = argv; 2278 2279 r = parse_metadata_dev(ca, &as, error); 2280 if (r) 2281 return r; 2282 2283 r = parse_cache_dev(ca, &as, error); 2284 if (r) 2285 return r; 2286 2287 r = parse_origin_dev(ca, &as, error); 2288 if (r) 2289 return r; 2290 2291 r = parse_block_size(ca, &as, error); 2292 if (r) 2293 return r; 2294 2295 r = parse_features(ca, &as, error); 2296 if (r) 2297 return r; 2298 2299 r = parse_policy(ca, &as, error); 2300 if (r) 2301 return r; 2302 2303 return 0; 2304 } 2305 2306 /*----------------------------------------------------------------*/ 2307 2308 static struct kmem_cache *migration_cache = NULL; 2309 2310 #define NOT_CORE_OPTION 1 2311 2312 static int process_config_option(struct cache *cache, const char *key, const char *value) 2313 { 2314 unsigned long tmp; 2315 2316 if (!strcasecmp(key, "migration_threshold")) { 2317 if (kstrtoul(value, 10, &tmp)) 2318 return -EINVAL; 2319 2320 cache->migration_threshold = tmp; 2321 return 0; 2322 } 2323 2324 return NOT_CORE_OPTION; 2325 } 2326 2327 static int set_config_value(struct cache *cache, const char *key, const char *value) 2328 { 2329 int r = process_config_option(cache, key, value); 2330 2331 if (r == NOT_CORE_OPTION) 2332 r = policy_set_config_value(cache->policy, key, value); 2333 2334 if (r) 2335 DMWARN("bad config value for %s: %s", key, value); 2336 2337 return r; 2338 } 2339 2340 static int set_config_values(struct cache *cache, int argc, const char **argv) 2341 { 2342 int r = 0; 2343 2344 if (argc & 1) { 2345 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs."); 2346 return -EINVAL; 2347 } 2348 2349 while (argc) { 2350 r = set_config_value(cache, argv[0], argv[1]); 2351 if (r) 2352 break; 2353 2354 argc -= 2; 2355 argv += 2; 2356 } 2357 2358 return r; 2359 } 2360 2361 static int create_cache_policy(struct cache *cache, struct cache_args *ca, 2362 char **error) 2363 { 2364 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name, 2365 cache->cache_size, 2366 cache->origin_sectors, 2367 cache->sectors_per_block); 2368 if (IS_ERR(p)) { 2369 *error = "Error creating cache's policy"; 2370 return PTR_ERR(p); 2371 } 2372 cache->policy = p; 2373 BUG_ON(!cache->policy); 2374 2375 return 0; 2376 } 2377 2378 /* 2379 * We want the discard block size to be at least the size of the cache 2380 * block size and have no more than 2^14 discard blocks across the origin. 2381 */ 2382 #define MAX_DISCARD_BLOCKS (1 << 14) 2383 2384 static bool too_many_discard_blocks(sector_t discard_block_size, 2385 sector_t origin_size) 2386 { 2387 return div64_u64(origin_size, discard_block_size) > MAX_DISCARD_BLOCKS; 2388 } 2389 2390 static sector_t calculate_discard_block_size(sector_t cache_block_size, 2391 sector_t origin_size) 2392 { 2393 sector_t discard_block_size = cache_block_size; 2394 2395 if (origin_size) 2396 while (too_many_discard_blocks(discard_block_size, origin_size)) 2397 discard_block_size *= 2; 2398 2399 return discard_block_size; 2400 } 2401 2402 static void set_cache_size(struct cache *cache, dm_cblock_t size) 2403 { 2404 dm_block_t nr_blocks = from_cblock(size); 2405 2406 if (nr_blocks > (1 << 20) && cache->cache_size != size) 2407 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n" 2408 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n" 2409 "Please consider increasing the cache block size to reduce the overall cache block count.", 2410 (unsigned long long) nr_blocks); 2411 2412 cache->cache_size = size; 2413 } 2414 2415 #define DEFAULT_MIGRATION_THRESHOLD 2048 2416 2417 static int cache_create(struct cache_args *ca, struct cache **result) 2418 { 2419 int r = 0; 2420 char **error = &ca->ti->error; 2421 struct cache *cache; 2422 struct dm_target *ti = ca->ti; 2423 dm_block_t origin_blocks; 2424 sector_t discard_block_size; 2425 struct dm_cache_metadata *cmd; 2426 bool may_format = ca->features.mode == CM_WRITE; 2427 2428 cache = kzalloc_obj(*cache); 2429 if (!cache) 2430 return -ENOMEM; 2431 2432 cache->ti = ca->ti; 2433 ti->private = cache; 2434 ti->accounts_remapped_io = true; 2435 ti->num_flush_bios = 2; 2436 ti->flush_supported = true; 2437 2438 ti->num_discard_bios = 1; 2439 ti->discards_supported = true; 2440 2441 ti->per_io_data_size = sizeof(struct per_bio_data); 2442 2443 cache->features = ca->features; 2444 if (writethrough_mode(cache)) { 2445 /* Create bioset for writethrough bios issued to origin */ 2446 r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0); 2447 if (r) 2448 goto bad; 2449 } 2450 2451 cache->metadata_dev = ca->metadata_dev; 2452 cache->origin_dev = ca->origin_dev; 2453 cache->cache_dev = ca->cache_dev; 2454 2455 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL; 2456 2457 origin_blocks = cache->origin_sectors = ti->len; 2458 origin_blocks = block_div(origin_blocks, ca->block_size); 2459 cache->origin_blocks = to_oblock(origin_blocks); 2460 2461 cache->sectors_per_block = ca->block_size; 2462 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) { 2463 r = -EINVAL; 2464 goto bad; 2465 } 2466 2467 if (ca->block_size & (ca->block_size - 1)) { 2468 dm_block_t cache_size = ca->cache_sectors; 2469 2470 cache->sectors_per_block_shift = -1; 2471 cache_size = block_div(cache_size, ca->block_size); 2472 set_cache_size(cache, to_cblock(cache_size)); 2473 } else { 2474 cache->sectors_per_block_shift = __ffs(ca->block_size); 2475 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift)); 2476 } 2477 2478 r = create_cache_policy(cache, ca, error); 2479 if (r) 2480 goto bad; 2481 2482 cache->policy_nr_args = ca->policy_argc; 2483 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD; 2484 2485 r = set_config_values(cache, ca->policy_argc, ca->policy_argv); 2486 if (r) { 2487 *error = "Error setting cache policy's config values"; 2488 goto bad; 2489 } 2490 2491 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev, 2492 ca->block_size, may_format, 2493 dm_cache_policy_get_hint_size(cache->policy), 2494 ca->features.metadata_version); 2495 if (IS_ERR(cmd)) { 2496 *error = "Error creating metadata object"; 2497 r = PTR_ERR(cmd); 2498 goto bad; 2499 } 2500 cache->cmd = cmd; 2501 set_cache_mode(cache, CM_WRITE); 2502 if (get_cache_mode(cache) != CM_WRITE) { 2503 *error = "Unable to get write access to metadata, please check/repair metadata."; 2504 r = -EINVAL; 2505 goto bad; 2506 } 2507 2508 if (passthrough_mode(cache)) 2509 policy_allow_migrations(cache->policy, false); 2510 2511 spin_lock_init(&cache->lock); 2512 bio_list_init(&cache->deferred_bios); 2513 atomic_set(&cache->nr_io_migrations, 0); 2514 2515 r = -ENOMEM; 2516 atomic_set(&cache->nr_dirty, 0); 2517 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size)); 2518 if (!cache->dirty_bitset) { 2519 *error = "could not allocate dirty bitset"; 2520 goto bad; 2521 } 2522 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); 2523 2524 discard_block_size = 2525 calculate_discard_block_size(cache->sectors_per_block, 2526 cache->origin_sectors); 2527 if (discard_block_size > U32_MAX) { 2528 *error = "Discard block size exceeds 32 bits"; 2529 r = -EOVERFLOW; 2530 goto bad; 2531 } 2532 cache->discard_block_size = discard_block_size; 2533 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, 2534 cache->discard_block_size)); 2535 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); 2536 if (!cache->discard_bitset) { 2537 *error = "could not allocate discard bitset"; 2538 goto bad; 2539 } 2540 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 2541 2542 cache->invalid_bitset = alloc_bitset(from_cblock(cache->cache_size)); 2543 if (!cache->invalid_bitset) { 2544 *error = "could not allocate bitset for invalid blocks"; 2545 goto bad; 2546 } 2547 clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size)); 2548 2549 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2550 if (IS_ERR(cache->copier)) { 2551 *error = "could not create kcopyd client"; 2552 r = PTR_ERR(cache->copier); 2553 cache->copier = NULL; 2554 goto bad; 2555 } 2556 2557 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, 2558 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 2559 if (!cache->wq) { 2560 *error = "could not create workqueue for metadata object"; 2561 goto bad; 2562 } 2563 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios); 2564 INIT_WORK(&cache->migration_worker, check_migrations); 2565 INIT_DELAYED_WORK(&cache->waker, do_waker); 2566 2567 cache->prison = dm_bio_prison_create_v2(cache->wq); 2568 if (!cache->prison) { 2569 *error = "could not create bio prison"; 2570 goto bad; 2571 } 2572 2573 r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE, 2574 migration_cache); 2575 if (r) { 2576 *error = "Error creating cache's migration mempool"; 2577 goto bad; 2578 } 2579 2580 cache->need_tick_bio = true; 2581 cache->sized = false; 2582 cache->invalidate = false; 2583 cache->commit_requested = false; 2584 cache->loaded_mappings = false; 2585 cache->loaded_discards = false; 2586 2587 load_stats(cache); 2588 2589 atomic_set(&cache->stats.demotion, 0); 2590 atomic_set(&cache->stats.promotion, 0); 2591 atomic_set(&cache->stats.copies_avoided, 0); 2592 atomic_set(&cache->stats.cache_cell_clash, 0); 2593 atomic_set(&cache->stats.commit_count, 0); 2594 atomic_set(&cache->stats.discard_count, 0); 2595 2596 spin_lock_init(&cache->invalidation_lock); 2597 INIT_LIST_HEAD(&cache->invalidation_requests); 2598 2599 batcher_init(&cache->committer, commit_op, cache, 2600 issue_op, cache, cache->wq); 2601 dm_iot_init(&cache->tracker); 2602 2603 init_waitqueue_head(&cache->background_work_wait); 2604 spin_lock_init(&cache->background_work_lock); 2605 cache->background_work_allowed = false; 2606 cache->background_work_nr = 0; 2607 2608 *result = cache; 2609 return 0; 2610 bad: 2611 __destroy(cache); 2612 return r; 2613 } 2614 2615 static int copy_ctr_args(struct cache *cache, int argc, const char **argv) 2616 { 2617 unsigned int i; 2618 const char **copy; 2619 2620 copy = kzalloc_objs(*copy, argc); 2621 if (!copy) 2622 return -ENOMEM; 2623 for (i = 0; i < argc; i++) { 2624 copy[i] = kstrdup(argv[i], GFP_KERNEL); 2625 if (!copy[i]) { 2626 while (i--) 2627 kfree(copy[i]); 2628 kfree(copy); 2629 return -ENOMEM; 2630 } 2631 } 2632 2633 cache->nr_ctr_args = argc; 2634 cache->ctr_args = copy; 2635 2636 return 0; 2637 } 2638 2639 static int cache_ctr(struct dm_target *ti, unsigned int argc, char **argv) 2640 { 2641 int r = -EINVAL; 2642 struct cache_args *ca; 2643 struct cache *cache = NULL; 2644 2645 ca = kzalloc_obj(*ca); 2646 if (!ca) { 2647 ti->error = "Error allocating memory for cache"; 2648 return -ENOMEM; 2649 } 2650 ca->ti = ti; 2651 2652 r = parse_cache_args(ca, argc, argv, &ti->error); 2653 if (r) 2654 goto out; 2655 2656 r = cache_create(ca, &cache); 2657 if (r) 2658 goto out; 2659 2660 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3); 2661 if (r) { 2662 __destroy(cache); 2663 goto out; 2664 } 2665 2666 ti->private = cache; 2667 out: 2668 destroy_cache_args(ca); 2669 return r; 2670 } 2671 2672 /*----------------------------------------------------------------*/ 2673 2674 static int cache_map(struct dm_target *ti, struct bio *bio) 2675 { 2676 struct cache *cache = ti->private; 2677 2678 int r; 2679 bool commit_needed; 2680 dm_oblock_t block = get_bio_block(cache, bio); 2681 2682 init_per_bio_data(bio); 2683 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) { 2684 /* 2685 * This can only occur if the io goes to a partial block at 2686 * the end of the origin device. We don't cache these. 2687 * Just remap to the origin and carry on. 2688 */ 2689 remap_to_origin(cache, bio); 2690 accounted_begin(cache, bio); 2691 return DM_MAPIO_REMAPPED; 2692 } 2693 2694 if (discard_or_flush(bio)) { 2695 defer_bio(cache, bio); 2696 return DM_MAPIO_SUBMITTED; 2697 } 2698 2699 r = map_bio(cache, bio, block, &commit_needed); 2700 if (commit_needed) 2701 schedule_commit(&cache->committer); 2702 2703 return r; 2704 } 2705 2706 static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error) 2707 { 2708 struct cache *cache = ti->private; 2709 unsigned long flags; 2710 struct per_bio_data *pb = get_per_bio_data(bio); 2711 2712 if (pb->tick) { 2713 policy_tick(cache->policy, false); 2714 2715 spin_lock_irqsave(&cache->lock, flags); 2716 cache->need_tick_bio = true; 2717 spin_unlock_irqrestore(&cache->lock, flags); 2718 } 2719 2720 bio_drop_shared_lock(cache, bio); 2721 accounted_complete(cache, bio); 2722 2723 return DM_ENDIO_DONE; 2724 } 2725 2726 static int write_dirty_bitset(struct cache *cache) 2727 { 2728 int r; 2729 2730 if (get_cache_mode(cache) >= CM_READ_ONLY) 2731 return -EINVAL; 2732 2733 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset); 2734 if (r) 2735 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r); 2736 2737 return r; 2738 } 2739 2740 static int write_discard_bitset(struct cache *cache) 2741 { 2742 unsigned int i, r; 2743 2744 if (get_cache_mode(cache) >= CM_READ_ONLY) 2745 return -EINVAL; 2746 2747 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size, 2748 cache->discard_nr_blocks); 2749 if (r) { 2750 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache)); 2751 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r); 2752 return r; 2753 } 2754 2755 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) { 2756 r = dm_cache_set_discard(cache->cmd, to_dblock(i), 2757 is_discarded(cache, to_dblock(i))); 2758 if (r) { 2759 metadata_operation_failed(cache, "dm_cache_set_discard", r); 2760 return r; 2761 } 2762 } 2763 2764 return 0; 2765 } 2766 2767 static int write_hints(struct cache *cache) 2768 { 2769 int r; 2770 2771 if (get_cache_mode(cache) >= CM_READ_ONLY) 2772 return -EINVAL; 2773 2774 r = dm_cache_write_hints(cache->cmd, cache->policy); 2775 if (r) { 2776 metadata_operation_failed(cache, "dm_cache_write_hints", r); 2777 return r; 2778 } 2779 2780 return 0; 2781 } 2782 2783 /* 2784 * returns true on success 2785 */ 2786 static bool sync_metadata(struct cache *cache) 2787 { 2788 int r1, r2, r3, r4; 2789 2790 r1 = write_dirty_bitset(cache); 2791 if (r1) 2792 DMERR("%s: could not write dirty bitset", cache_device_name(cache)); 2793 2794 r2 = write_discard_bitset(cache); 2795 if (r2) 2796 DMERR("%s: could not write discard bitset", cache_device_name(cache)); 2797 2798 save_stats(cache); 2799 2800 r3 = write_hints(cache); 2801 if (r3) 2802 DMERR("%s: could not write hints", cache_device_name(cache)); 2803 2804 /* 2805 * If writing the above metadata failed, we still commit, but don't 2806 * set the clean shutdown flag. This will effectively force every 2807 * dirty bit to be set on reload. 2808 */ 2809 r4 = commit(cache, !r1 && !r2 && !r3); 2810 if (r4) 2811 DMERR("%s: could not write cache metadata", cache_device_name(cache)); 2812 2813 return !r1 && !r2 && !r3 && !r4; 2814 } 2815 2816 static void cache_postsuspend(struct dm_target *ti) 2817 { 2818 struct cache *cache = ti->private; 2819 2820 prevent_background_work(cache); 2821 BUG_ON(atomic_read(&cache->nr_io_migrations)); 2822 2823 cancel_delayed_work_sync(&cache->waker); 2824 drain_workqueue(cache->wq); 2825 WARN_ON(cache->tracker.in_flight); 2826 2827 /* 2828 * If it's a flush suspend there won't be any deferred bios, so this 2829 * call is harmless. 2830 */ 2831 requeue_deferred_bios(cache); 2832 2833 if (get_cache_mode(cache) == CM_WRITE) 2834 (void) sync_metadata(cache); 2835 } 2836 2837 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, 2838 bool dirty, uint32_t hint, bool hint_valid) 2839 { 2840 struct cache *cache = context; 2841 2842 if (dirty) { 2843 if (passthrough_mode(cache)) { 2844 DMERR("%s: cannot enter passthrough mode unless all blocks are clean", 2845 cache_device_name(cache)); 2846 return -EBUSY; 2847 } 2848 2849 set_bit(from_cblock(cblock), cache->dirty_bitset); 2850 atomic_inc(&cache->nr_dirty); 2851 } else 2852 clear_bit(from_cblock(cblock), cache->dirty_bitset); 2853 2854 return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid); 2855 } 2856 2857 static int load_filtered_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, 2858 bool dirty, uint32_t hint, bool hint_valid) 2859 { 2860 struct cache *cache = context; 2861 2862 if (from_oblock(oblock) >= from_oblock(cache->origin_blocks)) { 2863 if (dirty) { 2864 DMERR("%s: unable to shrink origin; cache block %u is dirty", 2865 cache_device_name(cache), from_cblock(cblock)); 2866 return -EFBIG; 2867 } 2868 set_bit(from_cblock(cblock), cache->invalid_bitset); 2869 return 0; 2870 } 2871 2872 return load_mapping(context, oblock, cblock, dirty, hint, hint_valid); 2873 } 2874 2875 /* 2876 * The discard block size in the on disk metadata is not 2877 * necessarily the same as we're currently using. So we have to 2878 * be careful to only set the discarded attribute if we know it 2879 * covers a complete block of the new size. 2880 */ 2881 struct discard_load_info { 2882 struct cache *cache; 2883 2884 /* 2885 * These blocks are sized using the on disk dblock size, rather 2886 * than the current one. 2887 */ 2888 dm_block_t block_size; 2889 dm_block_t discard_begin, discard_end; 2890 }; 2891 2892 static void discard_load_info_init(struct cache *cache, 2893 struct discard_load_info *li) 2894 { 2895 li->cache = cache; 2896 li->discard_begin = li->discard_end = 0; 2897 } 2898 2899 static void set_discard_range(struct discard_load_info *li) 2900 { 2901 sector_t b, e; 2902 2903 if (li->discard_begin == li->discard_end) 2904 return; 2905 2906 /* 2907 * Convert to sectors. 2908 */ 2909 b = li->discard_begin * li->block_size; 2910 e = li->discard_end * li->block_size; 2911 2912 /* 2913 * Then convert back to the current dblock size. 2914 */ 2915 b = dm_sector_div_up(b, li->cache->discard_block_size); 2916 sector_div(e, li->cache->discard_block_size); 2917 2918 /* 2919 * The origin may have shrunk, so we need to check we're still in 2920 * bounds. 2921 */ 2922 if (e > from_dblock(li->cache->discard_nr_blocks)) 2923 e = from_dblock(li->cache->discard_nr_blocks); 2924 2925 for (; b < e; b++) 2926 set_discard(li->cache, to_dblock(b)); 2927 } 2928 2929 static int load_discard(void *context, sector_t discard_block_size, 2930 dm_dblock_t dblock, bool discard) 2931 { 2932 struct discard_load_info *li = context; 2933 2934 li->block_size = discard_block_size; 2935 2936 if (discard) { 2937 if (from_dblock(dblock) == li->discard_end) 2938 /* 2939 * We're already in a discard range, just extend it. 2940 */ 2941 li->discard_end = li->discard_end + 1ULL; 2942 2943 else { 2944 /* 2945 * Emit the old range and start a new one. 2946 */ 2947 set_discard_range(li); 2948 li->discard_begin = from_dblock(dblock); 2949 li->discard_end = li->discard_begin + 1ULL; 2950 } 2951 } else { 2952 set_discard_range(li); 2953 li->discard_begin = li->discard_end = 0; 2954 } 2955 2956 return 0; 2957 } 2958 2959 static dm_cblock_t get_cache_dev_size(struct cache *cache) 2960 { 2961 sector_t size = get_dev_size(cache->cache_dev); 2962 (void) sector_div(size, cache->sectors_per_block); 2963 return to_cblock(size); 2964 } 2965 2966 static bool can_resume(struct cache *cache) 2967 { 2968 bool clean_when_opened; 2969 int r; 2970 2971 /* 2972 * Disallow retrying the resume operation for devices that failed the 2973 * first resume attempt, as the failure leaves the policy object partially 2974 * initialized. Retrying could trigger BUG_ON when loading cache mappings 2975 * into the incomplete policy object. 2976 */ 2977 if (cache->sized && !cache->loaded_mappings) { 2978 if (get_cache_mode(cache) != CM_WRITE) 2979 DMERR("%s: unable to resume a failed-loaded cache, please check metadata.", 2980 cache_device_name(cache)); 2981 else 2982 DMERR("%s: unable to resume cache due to missing proper cache table reload", 2983 cache_device_name(cache)); 2984 return false; 2985 } 2986 2987 if (passthrough_mode(cache)) { 2988 r = dm_cache_metadata_clean_when_opened(cache->cmd, &clean_when_opened); 2989 if (r) { 2990 DMERR("%s: failed to query metadata flags", cache_device_name(cache)); 2991 return false; 2992 } 2993 2994 if (!clean_when_opened) { 2995 DMERR("%s: unable to resume into passthrough mode after unclean shutdown", 2996 cache_device_name(cache)); 2997 return false; 2998 } 2999 } 3000 3001 return true; 3002 } 3003 3004 static bool can_resize(struct cache *cache, dm_cblock_t new_size) 3005 { 3006 if (from_cblock(new_size) > from_cblock(cache->cache_size)) { 3007 DMERR("%s: unable to extend cache due to missing cache table reload", 3008 cache_device_name(cache)); 3009 return false; 3010 } 3011 3012 /* 3013 * We can't drop a dirty block when shrinking the cache. 3014 */ 3015 if (cache->loaded_mappings) { 3016 new_size = to_cblock(find_next_bit(cache->dirty_bitset, 3017 from_cblock(cache->cache_size), 3018 from_cblock(new_size))); 3019 if (new_size != cache->cache_size) { 3020 DMERR("%s: unable to shrink cache; cache block %llu is dirty", 3021 cache_device_name(cache), 3022 (unsigned long long) from_cblock(new_size)); 3023 return false; 3024 } 3025 } 3026 3027 return true; 3028 } 3029 3030 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) 3031 { 3032 int r; 3033 3034 r = dm_cache_resize(cache->cmd, new_size); 3035 if (r) { 3036 DMERR("%s: could not resize cache metadata", cache_device_name(cache)); 3037 metadata_operation_failed(cache, "dm_cache_resize", r); 3038 return r; 3039 } 3040 3041 set_cache_size(cache, new_size); 3042 3043 return 0; 3044 } 3045 3046 static int truncate_oblocks(struct cache *cache) 3047 { 3048 uint32_t nr_blocks = from_cblock(cache->cache_size); 3049 uint32_t i; 3050 int r; 3051 3052 for_each_set_bit(i, cache->invalid_bitset, nr_blocks) { 3053 r = dm_cache_remove_mapping(cache->cmd, to_cblock(i)); 3054 if (r) { 3055 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata", 3056 cache_device_name(cache)); 3057 return r; 3058 } 3059 } 3060 3061 return 0; 3062 } 3063 3064 static int cache_preresume(struct dm_target *ti) 3065 { 3066 int r = 0; 3067 struct cache *cache = ti->private; 3068 dm_cblock_t csize = get_cache_dev_size(cache); 3069 3070 if (!can_resume(cache)) 3071 return -EINVAL; 3072 3073 /* 3074 * Check to see if the cache has resized. 3075 */ 3076 if (!cache->sized || csize != cache->cache_size) { 3077 if (!can_resize(cache, csize)) 3078 return -EINVAL; 3079 3080 r = resize_cache_dev(cache, csize); 3081 if (r) 3082 return r; 3083 3084 cache->sized = true; 3085 } 3086 3087 if (!cache->loaded_mappings) { 3088 /* 3089 * The fast device could have been resized since the last 3090 * failed preresume attempt. To be safe we start by a blank 3091 * bitset for cache blocks. 3092 */ 3093 clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size)); 3094 3095 r = dm_cache_load_mappings(cache->cmd, cache->policy, 3096 load_filtered_mapping, cache); 3097 if (r) { 3098 DMERR("%s: could not load cache mappings", cache_device_name(cache)); 3099 if (r != -EFBIG && r != -EBUSY) 3100 metadata_operation_failed(cache, "dm_cache_load_mappings", r); 3101 return r; 3102 } 3103 3104 r = truncate_oblocks(cache); 3105 if (r) { 3106 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 3107 return r; 3108 } 3109 3110 cache->loaded_mappings = true; 3111 } 3112 3113 if (!cache->loaded_discards) { 3114 struct discard_load_info li; 3115 3116 /* 3117 * The discard bitset could have been resized, or the 3118 * discard block size changed. To be safe we start by 3119 * setting every dblock to not discarded. 3120 */ 3121 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 3122 3123 discard_load_info_init(cache, &li); 3124 r = dm_cache_load_discards(cache->cmd, load_discard, &li); 3125 if (r) { 3126 DMERR("%s: could not load origin discards", cache_device_name(cache)); 3127 metadata_operation_failed(cache, "dm_cache_load_discards", r); 3128 return r; 3129 } 3130 set_discard_range(&li); 3131 3132 cache->loaded_discards = true; 3133 } 3134 3135 return r; 3136 } 3137 3138 static void cache_resume(struct dm_target *ti) 3139 { 3140 struct cache *cache = ti->private; 3141 3142 cache->need_tick_bio = true; 3143 allow_background_work(cache); 3144 do_waker(&cache->waker.work); 3145 } 3146 3147 static void emit_flags(struct cache *cache, char *result, 3148 unsigned int maxlen, ssize_t *sz_ptr) 3149 { 3150 ssize_t sz = *sz_ptr; 3151 struct cache_features *cf = &cache->features; 3152 unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1; 3153 3154 DMEMIT("%u ", count); 3155 3156 if (cf->metadata_version == 2) 3157 DMEMIT("metadata2 "); 3158 3159 if (writethrough_mode(cache)) 3160 DMEMIT("writethrough "); 3161 3162 else if (passthrough_mode(cache)) 3163 DMEMIT("passthrough "); 3164 3165 else if (writeback_mode(cache)) 3166 DMEMIT("writeback "); 3167 3168 else { 3169 DMEMIT("unknown "); 3170 DMERR("%s: internal error: unknown io mode: %d", 3171 cache_device_name(cache), (int) cf->io_mode); 3172 } 3173 3174 if (!cf->discard_passdown) 3175 DMEMIT("no_discard_passdown "); 3176 3177 *sz_ptr = sz; 3178 } 3179 3180 /* 3181 * Status format: 3182 * 3183 * <metadata block size> <#used metadata blocks>/<#total metadata blocks> 3184 * <cache block size> <#used cache blocks>/<#total cache blocks> 3185 * <#read hits> <#read misses> <#write hits> <#write misses> 3186 * <#demotions> <#promotions> <#dirty> 3187 * <#features> <features>* 3188 * <#core args> <core args> 3189 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check> 3190 */ 3191 static void cache_status(struct dm_target *ti, status_type_t type, 3192 unsigned int status_flags, char *result, unsigned int maxlen) 3193 { 3194 int r = 0; 3195 unsigned int i; 3196 ssize_t sz = 0; 3197 dm_block_t nr_free_blocks_metadata = 0; 3198 dm_block_t nr_blocks_metadata = 0; 3199 char buf[BDEVNAME_SIZE]; 3200 struct cache *cache = ti->private; 3201 dm_cblock_t residency; 3202 bool needs_check; 3203 3204 switch (type) { 3205 case STATUSTYPE_INFO: 3206 if (get_cache_mode(cache) == CM_FAIL) { 3207 DMEMIT("Fail"); 3208 break; 3209 } 3210 3211 /* Commit to ensure statistics aren't out-of-date */ 3212 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3213 (void) commit(cache, false); 3214 3215 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata); 3216 if (r) { 3217 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d", 3218 cache_device_name(cache), r); 3219 goto err; 3220 } 3221 3222 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata); 3223 if (r) { 3224 DMERR("%s: dm_cache_get_metadata_dev_size returned %d", 3225 cache_device_name(cache), r); 3226 goto err; 3227 } 3228 3229 residency = policy_residency(cache->policy); 3230 3231 DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ", 3232 (unsigned int)DM_CACHE_METADATA_BLOCK_SIZE, 3233 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3234 (unsigned long long)nr_blocks_metadata, 3235 (unsigned long long)cache->sectors_per_block, 3236 (unsigned long long) from_cblock(residency), 3237 (unsigned long long) from_cblock(cache->cache_size), 3238 (unsigned int) atomic_read(&cache->stats.read_hit), 3239 (unsigned int) atomic_read(&cache->stats.read_miss), 3240 (unsigned int) atomic_read(&cache->stats.write_hit), 3241 (unsigned int) atomic_read(&cache->stats.write_miss), 3242 (unsigned int) atomic_read(&cache->stats.demotion), 3243 (unsigned int) atomic_read(&cache->stats.promotion), 3244 (unsigned long) atomic_read(&cache->nr_dirty)); 3245 3246 emit_flags(cache, result, maxlen, &sz); 3247 3248 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold); 3249 3250 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy)); 3251 if (sz < maxlen) { 3252 r = policy_emit_config_values(cache->policy, result, maxlen, &sz); 3253 if (r) 3254 DMERR("%s: policy_emit_config_values returned %d", 3255 cache_device_name(cache), r); 3256 } 3257 3258 if (get_cache_mode(cache) == CM_READ_ONLY) 3259 DMEMIT("ro "); 3260 else 3261 DMEMIT("rw "); 3262 3263 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check); 3264 3265 if (r || needs_check) 3266 DMEMIT("needs_check "); 3267 else 3268 DMEMIT("- "); 3269 3270 break; 3271 3272 case STATUSTYPE_TABLE: 3273 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3274 DMEMIT("%s ", buf); 3275 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3276 DMEMIT("%s ", buf); 3277 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3278 DMEMIT("%s", buf); 3279 3280 for (i = 0; i < cache->nr_ctr_args - 1; i++) 3281 DMEMIT(" %s", cache->ctr_args[i]); 3282 if (cache->nr_ctr_args) 3283 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]); 3284 break; 3285 3286 case STATUSTYPE_IMA: 3287 DMEMIT_TARGET_NAME_VERSION(ti->type); 3288 if (get_cache_mode(cache) == CM_FAIL) 3289 DMEMIT(",metadata_mode=fail"); 3290 else if (get_cache_mode(cache) == CM_READ_ONLY) 3291 DMEMIT(",metadata_mode=ro"); 3292 else 3293 DMEMIT(",metadata_mode=rw"); 3294 3295 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3296 DMEMIT(",cache_metadata_device=%s", buf); 3297 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3298 DMEMIT(",cache_device=%s", buf); 3299 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3300 DMEMIT(",cache_origin_device=%s", buf); 3301 DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n'); 3302 DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n'); 3303 DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n'); 3304 DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n'); 3305 DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y'); 3306 DMEMIT(";"); 3307 break; 3308 } 3309 3310 return; 3311 3312 err: 3313 DMEMIT("Error"); 3314 } 3315 3316 /* 3317 * Defines a range of cblocks, begin to (end - 1) are in the range. end is 3318 * the one-past-the-end value. 3319 */ 3320 struct cblock_range { 3321 dm_cblock_t begin; 3322 dm_cblock_t end; 3323 }; 3324 3325 static inline dm_cblock_t cblock_succ(dm_cblock_t b) 3326 { 3327 return to_cblock(from_cblock(b) + 1); 3328 } 3329 3330 /* 3331 * A cache block range can take two forms: 3332 * 3333 * i) A single cblock, eg. '3456' 3334 * ii) A begin and end cblock with a dash between, eg. 123-234 3335 */ 3336 static int parse_cblock_range(struct cache *cache, char *str, 3337 struct cblock_range *result) 3338 { 3339 char *blocknr = strsep(&str, "-"); 3340 unsigned int b, e; 3341 int r; 3342 3343 r = kstrtouint(blocknr, 10, &b); 3344 if (r) 3345 goto bad; 3346 3347 result->begin = to_cblock(b); 3348 3349 if (str) { 3350 blocknr = str; 3351 3352 r = kstrtouint(blocknr, 10, &e); 3353 if (r) 3354 goto bad; 3355 3356 result->end = to_cblock(e); 3357 } else { 3358 result->end = cblock_succ(result->begin); 3359 } 3360 3361 return 0; 3362 3363 bad: 3364 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), blocknr); 3365 return -EINVAL; 3366 } 3367 3368 static int validate_cblock_range(struct cache *cache, struct cblock_range *range) 3369 { 3370 uint64_t b = from_cblock(range->begin); 3371 uint64_t e = from_cblock(range->end); 3372 uint64_t n = from_cblock(cache->cache_size); 3373 3374 if (b >= n) { 3375 DMERR("%s: begin cblock out of range: %llu >= %llu", 3376 cache_device_name(cache), b, n); 3377 return -EINVAL; 3378 } 3379 3380 if (e > n) { 3381 DMERR("%s: end cblock out of range: %llu > %llu", 3382 cache_device_name(cache), e, n); 3383 return -EINVAL; 3384 } 3385 3386 if (b >= e) { 3387 DMERR("%s: invalid cblock range: %llu >= %llu", 3388 cache_device_name(cache), b, e); 3389 return -EINVAL; 3390 } 3391 3392 return 0; 3393 } 3394 3395 static int request_invalidation(struct cache *cache, struct cblock_range *range) 3396 { 3397 int r = 0; 3398 3399 /* 3400 * We don't need to do any locking here because we know we're in 3401 * passthrough mode. There's is potential for a race between an 3402 * invalidation triggered by an io and an invalidation message. This 3403 * is harmless, we must not worry if the policy call fails. 3404 */ 3405 while (range->begin != range->end) { 3406 r = invalidate_cblock(cache, range->begin); 3407 if (r) 3408 return r; 3409 3410 range->begin = cblock_succ(range->begin); 3411 } 3412 3413 cache->commit_requested = true; 3414 return r; 3415 } 3416 3417 static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count, 3418 char **cblock_ranges) 3419 { 3420 int r = 0; 3421 unsigned int i; 3422 struct cblock_range range; 3423 3424 if (!passthrough_mode(cache)) { 3425 DMERR("%s: cache has to be in passthrough mode for invalidation", 3426 cache_device_name(cache)); 3427 return -EPERM; 3428 } 3429 3430 for (i = 0; i < count; i++) { 3431 r = parse_cblock_range(cache, cblock_ranges[i], &range); 3432 if (r) 3433 break; 3434 3435 r = validate_cblock_range(cache, &range); 3436 if (r) 3437 break; 3438 3439 /* 3440 * Pass begin and end origin blocks to the worker and wake it. 3441 */ 3442 r = request_invalidation(cache, &range); 3443 if (r) 3444 break; 3445 } 3446 3447 return r; 3448 } 3449 3450 /* 3451 * Supports 3452 * "<key> <value>" 3453 * and 3454 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]* 3455 * 3456 * The key migration_threshold is supported by the cache target core. 3457 */ 3458 static int cache_message(struct dm_target *ti, unsigned int argc, char **argv, 3459 char *result, unsigned int maxlen) 3460 { 3461 struct cache *cache = ti->private; 3462 3463 if (!argc) 3464 return -EINVAL; 3465 3466 if (get_cache_mode(cache) >= CM_READ_ONLY) { 3467 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode", 3468 cache_device_name(cache)); 3469 return -EOPNOTSUPP; 3470 } 3471 3472 if (!strcasecmp(argv[0], "invalidate_cblocks")) 3473 return process_invalidate_cblocks_message(cache, argc - 1, argv + 1); 3474 3475 if (argc != 2) 3476 return -EINVAL; 3477 3478 return set_config_value(cache, argv[0], argv[1]); 3479 } 3480 3481 static int cache_iterate_devices(struct dm_target *ti, 3482 iterate_devices_callout_fn fn, void *data) 3483 { 3484 int r = 0; 3485 struct cache *cache = ti->private; 3486 3487 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data); 3488 if (!r) 3489 r = fn(ti, cache->origin_dev, 0, ti->len, data); 3490 3491 return r; 3492 } 3493 3494 /* 3495 * If discard_passdown was enabled verify that the origin device 3496 * supports discards. Disable discard_passdown if not. 3497 */ 3498 static void disable_passdown_if_not_supported(struct cache *cache) 3499 { 3500 struct block_device *origin_bdev = cache->origin_dev->bdev; 3501 struct queue_limits *origin_limits = bdev_limits(origin_bdev); 3502 const char *reason = NULL; 3503 3504 if (!cache->features.discard_passdown) 3505 return; 3506 3507 if (!bdev_max_discard_sectors(origin_bdev)) 3508 reason = "discard unsupported"; 3509 3510 else if (origin_limits->max_discard_sectors < cache->sectors_per_block) 3511 reason = "max discard sectors smaller than a block"; 3512 3513 if (reason) { 3514 DMWARN("Origin device (%pg) %s: Disabling discard passdown.", 3515 origin_bdev, reason); 3516 cache->features.discard_passdown = false; 3517 } 3518 } 3519 3520 static void set_discard_limits(struct cache *cache, struct queue_limits *limits) 3521 { 3522 struct block_device *origin_bdev = cache->origin_dev->bdev; 3523 struct queue_limits *origin_limits = bdev_limits(origin_bdev); 3524 3525 if (!cache->features.discard_passdown) { 3526 /* No passdown is done so setting own virtual limits */ 3527 limits->max_hw_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024, 3528 cache->origin_sectors); 3529 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT; 3530 return; 3531 } 3532 3533 /* 3534 * cache_iterate_devices() is stacking both origin and fast device limits 3535 * but discards aren't passed to fast device, so inherit origin's limits. 3536 */ 3537 limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors; 3538 limits->discard_granularity = origin_limits->discard_granularity; 3539 limits->discard_alignment = origin_limits->discard_alignment; 3540 } 3541 3542 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits) 3543 { 3544 struct cache *cache = ti->private; 3545 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 3546 3547 /* 3548 * If the system-determined stacked limits are compatible with the 3549 * cache's blocksize (io_opt is a factor) do not override them. 3550 */ 3551 if (io_opt_sectors < cache->sectors_per_block || 3552 do_div(io_opt_sectors, cache->sectors_per_block)) { 3553 limits->io_min = cache->sectors_per_block << SECTOR_SHIFT; 3554 limits->io_opt = cache->sectors_per_block << SECTOR_SHIFT; 3555 } 3556 3557 disable_passdown_if_not_supported(cache); 3558 set_discard_limits(cache, limits); 3559 } 3560 3561 /*----------------------------------------------------------------*/ 3562 3563 static struct target_type cache_target = { 3564 .name = "cache", 3565 .version = {2, 4, 0}, 3566 .module = THIS_MODULE, 3567 .ctr = cache_ctr, 3568 .dtr = cache_dtr, 3569 .map = cache_map, 3570 .end_io = cache_end_io, 3571 .postsuspend = cache_postsuspend, 3572 .preresume = cache_preresume, 3573 .resume = cache_resume, 3574 .status = cache_status, 3575 .message = cache_message, 3576 .iterate_devices = cache_iterate_devices, 3577 .io_hints = cache_io_hints, 3578 }; 3579 3580 static int __init dm_cache_init(void) 3581 { 3582 int r; 3583 3584 migration_cache = KMEM_CACHE(dm_cache_migration, 0); 3585 if (!migration_cache) { 3586 r = -ENOMEM; 3587 goto err; 3588 } 3589 3590 btracker_work_cache = kmem_cache_create("dm_cache_bt_work", 3591 sizeof(struct bt_work), __alignof__(struct bt_work), 0, NULL); 3592 if (!btracker_work_cache) { 3593 r = -ENOMEM; 3594 goto err; 3595 } 3596 3597 r = dm_register_target(&cache_target); 3598 if (r) { 3599 goto err; 3600 } 3601 3602 return 0; 3603 3604 err: 3605 kmem_cache_destroy(migration_cache); 3606 kmem_cache_destroy(btracker_work_cache); 3607 return r; 3608 } 3609 3610 static void __exit dm_cache_exit(void) 3611 { 3612 dm_unregister_target(&cache_target); 3613 kmem_cache_destroy(migration_cache); 3614 kmem_cache_destroy(btracker_work_cache); 3615 } 3616 3617 module_init(dm_cache_init); 3618 module_exit(dm_cache_exit); 3619 3620 MODULE_DESCRIPTION(DM_NAME " cache target"); 3621 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>"); 3622 MODULE_LICENSE("GPL"); 3623