1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2012 Red Hat UK. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm-thin-metadata.h" 9 #include "dm-bio-prison-v1.h" 10 #include "dm.h" 11 12 #include <linux/device-mapper.h> 13 #include <linux/dm-io.h> 14 #include <linux/dm-kcopyd.h> 15 #include <linux/jiffies.h> 16 #include <linux/log2.h> 17 #include <linux/list.h> 18 #include <linux/rculist.h> 19 #include <linux/init.h> 20 #include <linux/module.h> 21 #include <linux/slab.h> 22 #include <linux/vmalloc.h> 23 #include <linux/sort.h> 24 #include <linux/rbtree.h> 25 26 #define DM_MSG_PREFIX "thin" 27 28 /* 29 * Tunable constants 30 */ 31 #define ENDIO_HOOK_POOL_SIZE 1024 32 #define MAPPING_POOL_SIZE 1024 33 #define COMMIT_PERIOD HZ 34 #define NO_SPACE_TIMEOUT_SECS 60 35 36 static unsigned int no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS; 37 38 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 39 "A percentage of time allocated for copy on write"); 40 41 /* 42 * The block size of the device holding pool data must be 43 * between 64KB and 1GB. 44 */ 45 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT) 46 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 47 48 /* 49 * Device id is restricted to 24 bits. 50 */ 51 #define MAX_DEV_ID ((1 << 24) - 1) 52 53 /* 54 * How do we handle breaking sharing of data blocks? 55 * ================================================= 56 * 57 * We use a standard copy-on-write btree to store the mappings for the 58 * devices (note I'm talking about copy-on-write of the metadata here, not 59 * the data). When you take an internal snapshot you clone the root node 60 * of the origin btree. After this there is no concept of an origin or a 61 * snapshot. They are just two device trees that happen to point to the 62 * same data blocks. 63 * 64 * When we get a write in we decide if it's to a shared data block using 65 * some timestamp magic. If it is, we have to break sharing. 66 * 67 * Let's say we write to a shared block in what was the origin. The 68 * steps are: 69 * 70 * i) plug io further to this physical block. (see bio_prison code). 71 * 72 * ii) quiesce any read io to that shared data block. Obviously 73 * including all devices that share this block. (see dm_deferred_set code) 74 * 75 * iii) copy the data block to a newly allocate block. This step can be 76 * missed out if the io covers the block. (schedule_copy). 77 * 78 * iv) insert the new mapping into the origin's btree 79 * (process_prepared_mapping). This act of inserting breaks some 80 * sharing of btree nodes between the two devices. Breaking sharing only 81 * effects the btree of that specific device. Btrees for the other 82 * devices that share the block never change. The btree for the origin 83 * device as it was after the last commit is untouched, ie. we're using 84 * persistent data structures in the functional programming sense. 85 * 86 * v) unplug io to this physical block, including the io that triggered 87 * the breaking of sharing. 88 * 89 * Steps (ii) and (iii) occur in parallel. 90 * 91 * The metadata _doesn't_ need to be committed before the io continues. We 92 * get away with this because the io is always written to a _new_ block. 93 * If there's a crash, then: 94 * 95 * - The origin mapping will point to the old origin block (the shared 96 * one). This will contain the data as it was before the io that triggered 97 * the breaking of sharing came in. 98 * 99 * - The snap mapping still points to the old block. As it would after 100 * the commit. 101 * 102 * The downside of this scheme is the timestamp magic isn't perfect, and 103 * will continue to think that data block in the snapshot device is shared 104 * even after the write to the origin has broken sharing. I suspect data 105 * blocks will typically be shared by many different devices, so we're 106 * breaking sharing n + 1 times, rather than n, where n is the number of 107 * devices that reference this data block. At the moment I think the 108 * benefits far, far outweigh the disadvantages. 109 */ 110 111 /*----------------------------------------------------------------*/ 112 113 /* 114 * Key building. 115 */ 116 enum lock_space { 117 VIRTUAL, 118 PHYSICAL 119 }; 120 121 static bool build_key(struct dm_thin_device *td, enum lock_space ls, 122 dm_block_t b, dm_block_t e, struct dm_cell_key *key) 123 { 124 key->virtual = (ls == VIRTUAL); 125 key->dev = dm_thin_dev_id(td); 126 key->block_begin = b; 127 key->block_end = e; 128 129 return dm_cell_key_has_valid_range(key); 130 } 131 132 static void build_data_key(struct dm_thin_device *td, dm_block_t b, 133 struct dm_cell_key *key) 134 { 135 (void) build_key(td, PHYSICAL, b, b + 1llu, key); 136 } 137 138 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b, 139 struct dm_cell_key *key) 140 { 141 (void) build_key(td, VIRTUAL, b, b + 1llu, key); 142 } 143 144 /*----------------------------------------------------------------*/ 145 146 #define THROTTLE_THRESHOLD (1 * HZ) 147 148 struct throttle { 149 struct rw_semaphore lock; 150 unsigned long threshold; 151 bool throttle_applied; 152 }; 153 154 static void throttle_init(struct throttle *t) 155 { 156 init_rwsem(&t->lock); 157 t->throttle_applied = false; 158 } 159 160 static void throttle_work_start(struct throttle *t) 161 { 162 t->threshold = jiffies + THROTTLE_THRESHOLD; 163 } 164 165 static void throttle_work_update(struct throttle *t) 166 { 167 if (!t->throttle_applied && time_is_before_jiffies(t->threshold)) { 168 down_write(&t->lock); 169 t->throttle_applied = true; 170 } 171 } 172 173 static void throttle_work_complete(struct throttle *t) 174 { 175 if (t->throttle_applied) { 176 t->throttle_applied = false; 177 up_write(&t->lock); 178 } 179 } 180 181 static void throttle_lock(struct throttle *t) 182 { 183 down_read(&t->lock); 184 } 185 186 static void throttle_unlock(struct throttle *t) 187 { 188 up_read(&t->lock); 189 } 190 191 /*----------------------------------------------------------------*/ 192 193 /* 194 * A pool device ties together a metadata device and a data device. It 195 * also provides the interface for creating and destroying internal 196 * devices. 197 */ 198 struct dm_thin_new_mapping; 199 200 /* 201 * The pool runs in various modes. Ordered in degraded order for comparisons. 202 */ 203 enum pool_mode { 204 PM_WRITE, /* metadata may be changed */ 205 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */ 206 207 /* 208 * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY. 209 */ 210 PM_OUT_OF_METADATA_SPACE, 211 PM_READ_ONLY, /* metadata may not be changed */ 212 213 PM_FAIL, /* all I/O fails */ 214 }; 215 216 struct pool_features { 217 enum pool_mode mode; 218 219 bool zero_new_blocks:1; 220 bool discard_enabled:1; 221 bool discard_passdown:1; 222 bool error_if_no_space:1; 223 }; 224 225 struct thin_c; 226 typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio); 227 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell); 228 typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m); 229 230 #define CELL_SORT_ARRAY_SIZE 8192 231 232 struct pool { 233 struct list_head list; 234 struct dm_target *ti; /* Only set if a pool target is bound */ 235 236 struct mapped_device *pool_md; 237 struct block_device *data_dev; 238 struct block_device *md_dev; 239 struct dm_pool_metadata *pmd; 240 241 dm_block_t low_water_blocks; 242 uint32_t sectors_per_block; 243 int sectors_per_block_shift; 244 245 struct pool_features pf; 246 bool low_water_triggered:1; /* A dm event has been sent */ 247 bool suspended:1; 248 bool out_of_data_space:1; 249 250 struct dm_bio_prison *prison; 251 struct dm_kcopyd_client *copier; 252 253 struct work_struct worker; 254 struct workqueue_struct *wq; 255 struct throttle throttle; 256 struct delayed_work waker; 257 struct delayed_work no_space_timeout; 258 259 unsigned long last_commit_jiffies; 260 unsigned int ref_count; 261 262 spinlock_t lock; 263 struct bio_list deferred_flush_bios; 264 struct bio_list deferred_flush_completions; 265 struct list_head prepared_mappings; 266 struct list_head prepared_discards; 267 struct list_head prepared_discards_pt2; 268 struct list_head active_thins; 269 270 struct dm_deferred_set *shared_read_ds; 271 struct dm_deferred_set *all_io_ds; 272 273 struct dm_thin_new_mapping *next_mapping; 274 275 process_bio_fn process_bio; 276 process_bio_fn process_discard; 277 278 process_cell_fn process_cell; 279 process_cell_fn process_discard_cell; 280 281 process_mapping_fn process_prepared_mapping; 282 process_mapping_fn process_prepared_discard; 283 process_mapping_fn process_prepared_discard_pt2; 284 285 struct dm_bio_prison_cell **cell_sort_array; 286 287 mempool_t mapping_pool; 288 }; 289 290 static void metadata_operation_failed(struct pool *pool, const char *op, int r); 291 292 static enum pool_mode get_pool_mode(struct pool *pool) 293 { 294 return pool->pf.mode; 295 } 296 297 static void notify_of_pool_mode_change(struct pool *pool) 298 { 299 static const char *descs[] = { 300 "write", 301 "out-of-data-space", 302 "read-only", 303 "read-only", 304 "fail" 305 }; 306 const char *extra_desc = NULL; 307 enum pool_mode mode = get_pool_mode(pool); 308 309 if (mode == PM_OUT_OF_DATA_SPACE) { 310 if (!pool->pf.error_if_no_space) 311 extra_desc = " (queue IO)"; 312 else 313 extra_desc = " (error IO)"; 314 } 315 316 dm_table_event(pool->ti->table); 317 DMINFO("%s: switching pool to %s%s mode", 318 dm_device_name(pool->pool_md), 319 descs[(int)mode], extra_desc ? : ""); 320 } 321 322 /* 323 * Target context for a pool. 324 */ 325 struct pool_c { 326 struct dm_target *ti; 327 struct pool *pool; 328 struct dm_dev *data_dev; 329 struct dm_dev *metadata_dev; 330 331 dm_block_t low_water_blocks; 332 struct pool_features requested_pf; /* Features requested during table load */ 333 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */ 334 }; 335 336 /* 337 * Target context for a thin. 338 */ 339 struct thin_c { 340 struct list_head list; 341 struct dm_dev *pool_dev; 342 struct dm_dev *origin_dev; 343 sector_t origin_size; 344 dm_thin_id dev_id; 345 346 struct pool *pool; 347 struct dm_thin_device *td; 348 struct mapped_device *thin_md; 349 350 bool requeue_mode:1; 351 spinlock_t lock; 352 struct list_head deferred_cells; 353 struct bio_list deferred_bio_list; 354 struct bio_list retry_on_resume_list; 355 struct rb_root sort_bio_list; /* sorted list of deferred bios */ 356 357 /* 358 * Ensures the thin is not destroyed until the worker has finished 359 * iterating the active_thins list. 360 */ 361 refcount_t refcount; 362 struct completion can_destroy; 363 }; 364 365 /*----------------------------------------------------------------*/ 366 367 static bool block_size_is_power_of_two(struct pool *pool) 368 { 369 return pool->sectors_per_block_shift >= 0; 370 } 371 372 static sector_t block_to_sectors(struct pool *pool, dm_block_t b) 373 { 374 return block_size_is_power_of_two(pool) ? 375 (b << pool->sectors_per_block_shift) : 376 (b * pool->sectors_per_block); 377 } 378 379 /*----------------------------------------------------------------*/ 380 381 struct discard_op { 382 struct thin_c *tc; 383 struct blk_plug plug; 384 struct bio *parent_bio; 385 struct bio *bio; 386 }; 387 388 static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent) 389 { 390 BUG_ON(!parent); 391 392 op->tc = tc; 393 blk_start_plug(&op->plug); 394 op->parent_bio = parent; 395 op->bio = NULL; 396 } 397 398 static void issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e) 399 { 400 struct thin_c *tc = op->tc; 401 sector_t s = block_to_sectors(tc->pool, data_b); 402 sector_t len = block_to_sectors(tc->pool, data_e - data_b); 403 404 __blkdev_issue_discard(tc->pool_dev->bdev, s, len, GFP_NOIO, &op->bio); 405 } 406 407 static void end_discard(struct discard_op *op, int r) 408 { 409 if (op->bio) { 410 /* 411 * Even if one of the calls to issue_discard failed, we 412 * need to wait for the chain to complete. 413 */ 414 bio_chain(op->bio, op->parent_bio); 415 op->bio->bi_opf = REQ_OP_DISCARD; 416 submit_bio(op->bio); 417 } 418 419 blk_finish_plug(&op->plug); 420 421 /* 422 * Even if r is set, there could be sub discards in flight that we 423 * need to wait for. 424 */ 425 if (r && !op->parent_bio->bi_status) 426 op->parent_bio->bi_status = errno_to_blk_status(r); 427 bio_endio(op->parent_bio); 428 } 429 430 /*----------------------------------------------------------------*/ 431 432 /* 433 * wake_worker() is used when new work is queued and when pool_resume is 434 * ready to continue deferred IO processing. 435 */ 436 static void wake_worker(struct pool *pool) 437 { 438 queue_work(pool->wq, &pool->worker); 439 } 440 441 /*----------------------------------------------------------------*/ 442 443 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio, 444 struct dm_bio_prison_cell **cell_result) 445 { 446 int r; 447 struct dm_bio_prison_cell *cell_prealloc; 448 449 /* 450 * Allocate a cell from the prison's mempool. 451 * This might block but it can't fail. 452 */ 453 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); 454 455 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); 456 if (r) { 457 /* 458 * We reused an old cell; we can get rid of 459 * the new one. 460 */ 461 dm_bio_prison_free_cell(pool->prison, cell_prealloc); 462 } 463 464 return r; 465 } 466 467 static void cell_release(struct pool *pool, 468 struct dm_bio_prison_cell *cell, 469 struct bio_list *bios) 470 { 471 dm_cell_release(pool->prison, cell, bios); 472 dm_bio_prison_free_cell(pool->prison, cell); 473 } 474 475 static void cell_visit_release(struct pool *pool, 476 void (*fn)(void *, struct dm_bio_prison_cell *), 477 void *context, 478 struct dm_bio_prison_cell *cell) 479 { 480 dm_cell_visit_release(pool->prison, fn, context, cell); 481 dm_bio_prison_free_cell(pool->prison, cell); 482 } 483 484 static void cell_release_no_holder(struct pool *pool, 485 struct dm_bio_prison_cell *cell, 486 struct bio_list *bios) 487 { 488 dm_cell_release_no_holder(pool->prison, cell, bios); 489 dm_bio_prison_free_cell(pool->prison, cell); 490 } 491 492 static void cell_error_with_code(struct pool *pool, 493 struct dm_bio_prison_cell *cell, blk_status_t error_code) 494 { 495 dm_cell_error(pool->prison, cell, error_code); 496 dm_bio_prison_free_cell(pool->prison, cell); 497 } 498 499 static blk_status_t get_pool_io_error_code(struct pool *pool) 500 { 501 return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR; 502 } 503 504 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell) 505 { 506 cell_error_with_code(pool, cell, get_pool_io_error_code(pool)); 507 } 508 509 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell) 510 { 511 cell_error_with_code(pool, cell, 0); 512 } 513 514 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell) 515 { 516 cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE); 517 } 518 519 /*----------------------------------------------------------------*/ 520 521 /* 522 * A global list of pools that uses a struct mapped_device as a key. 523 */ 524 static struct dm_thin_pool_table { 525 struct mutex mutex; 526 struct list_head pools; 527 } dm_thin_pool_table; 528 529 static void pool_table_init(void) 530 { 531 mutex_init(&dm_thin_pool_table.mutex); 532 INIT_LIST_HEAD(&dm_thin_pool_table.pools); 533 } 534 535 static void pool_table_exit(void) 536 { 537 mutex_destroy(&dm_thin_pool_table.mutex); 538 } 539 540 static void __pool_table_insert(struct pool *pool) 541 { 542 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 543 list_add(&pool->list, &dm_thin_pool_table.pools); 544 } 545 546 static void __pool_table_remove(struct pool *pool) 547 { 548 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 549 list_del(&pool->list); 550 } 551 552 static struct pool *__pool_table_lookup(struct mapped_device *md) 553 { 554 struct pool *pool = NULL, *tmp; 555 556 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 557 558 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 559 if (tmp->pool_md == md) { 560 pool = tmp; 561 break; 562 } 563 } 564 565 return pool; 566 } 567 568 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev) 569 { 570 struct pool *pool = NULL, *tmp; 571 572 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 573 574 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 575 if (tmp->md_dev == md_dev) { 576 pool = tmp; 577 break; 578 } 579 } 580 581 return pool; 582 } 583 584 /*----------------------------------------------------------------*/ 585 586 struct dm_thin_endio_hook { 587 struct thin_c *tc; 588 struct dm_deferred_entry *shared_read_entry; 589 struct dm_deferred_entry *all_io_entry; 590 struct dm_thin_new_mapping *overwrite_mapping; 591 struct rb_node rb_node; 592 struct dm_bio_prison_cell *cell; 593 }; 594 595 static void error_bio_list(struct bio_list *bios, blk_status_t error) 596 { 597 struct bio *bio; 598 599 while ((bio = bio_list_pop(bios))) { 600 bio->bi_status = error; 601 bio_endio(bio); 602 } 603 } 604 605 static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, 606 blk_status_t error) 607 { 608 struct bio_list bios; 609 610 bio_list_init(&bios); 611 612 spin_lock_irq(&tc->lock); 613 bio_list_merge_init(&bios, master); 614 spin_unlock_irq(&tc->lock); 615 616 error_bio_list(&bios, error); 617 } 618 619 static void requeue_deferred_cells(struct thin_c *tc) 620 { 621 struct pool *pool = tc->pool; 622 struct list_head cells; 623 struct dm_bio_prison_cell *cell, *tmp; 624 625 INIT_LIST_HEAD(&cells); 626 627 spin_lock_irq(&tc->lock); 628 list_splice_init(&tc->deferred_cells, &cells); 629 spin_unlock_irq(&tc->lock); 630 631 list_for_each_entry_safe(cell, tmp, &cells, user_list) 632 cell_requeue(pool, cell); 633 } 634 635 static void requeue_io(struct thin_c *tc) 636 { 637 struct bio_list bios; 638 639 bio_list_init(&bios); 640 641 spin_lock_irq(&tc->lock); 642 bio_list_merge_init(&bios, &tc->deferred_bio_list); 643 bio_list_merge_init(&bios, &tc->retry_on_resume_list); 644 spin_unlock_irq(&tc->lock); 645 646 error_bio_list(&bios, BLK_STS_DM_REQUEUE); 647 requeue_deferred_cells(tc); 648 } 649 650 static void error_retry_list_with_code(struct pool *pool, blk_status_t error) 651 { 652 struct thin_c *tc; 653 654 rcu_read_lock(); 655 list_for_each_entry_rcu(tc, &pool->active_thins, list) 656 error_thin_bio_list(tc, &tc->retry_on_resume_list, error); 657 rcu_read_unlock(); 658 } 659 660 static void error_retry_list(struct pool *pool) 661 { 662 error_retry_list_with_code(pool, get_pool_io_error_code(pool)); 663 } 664 665 /* 666 * This section of code contains the logic for processing a thin device's IO. 667 * Much of the code depends on pool object resources (lists, workqueues, etc) 668 * but most is exclusively called from the thin target rather than the thin-pool 669 * target. 670 */ 671 672 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio) 673 { 674 struct pool *pool = tc->pool; 675 sector_t block_nr = bio->bi_iter.bi_sector; 676 677 if (block_size_is_power_of_two(pool)) 678 block_nr >>= pool->sectors_per_block_shift; 679 else 680 (void) sector_div(block_nr, pool->sectors_per_block); 681 682 return block_nr; 683 } 684 685 /* 686 * Returns the _complete_ blocks that this bio covers. 687 */ 688 static void get_bio_block_range(struct thin_c *tc, struct bio *bio, 689 dm_block_t *begin, dm_block_t *end) 690 { 691 struct pool *pool = tc->pool; 692 sector_t b = bio->bi_iter.bi_sector; 693 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT); 694 695 b += pool->sectors_per_block - 1ull; /* so we round up */ 696 697 if (block_size_is_power_of_two(pool)) { 698 b >>= pool->sectors_per_block_shift; 699 e >>= pool->sectors_per_block_shift; 700 } else { 701 (void) sector_div(b, pool->sectors_per_block); 702 (void) sector_div(e, pool->sectors_per_block); 703 } 704 705 if (e < b) { 706 /* Can happen if the bio is within a single block. */ 707 e = b; 708 } 709 710 *begin = b; 711 *end = e; 712 } 713 714 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) 715 { 716 struct pool *pool = tc->pool; 717 sector_t bi_sector = bio->bi_iter.bi_sector; 718 719 bio_set_dev(bio, tc->pool_dev->bdev); 720 if (block_size_is_power_of_two(pool)) { 721 bio->bi_iter.bi_sector = 722 (block << pool->sectors_per_block_shift) | 723 (bi_sector & (pool->sectors_per_block - 1)); 724 } else { 725 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + 726 sector_div(bi_sector, pool->sectors_per_block); 727 } 728 } 729 730 static void remap_to_origin(struct thin_c *tc, struct bio *bio) 731 { 732 bio_set_dev(bio, tc->origin_dev->bdev); 733 } 734 735 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio) 736 { 737 return op_is_flush(bio->bi_opf) && 738 dm_thin_changed_this_transaction(tc->td); 739 } 740 741 static void inc_all_io_entry(struct pool *pool, struct bio *bio) 742 { 743 struct dm_thin_endio_hook *h; 744 745 if (bio_op(bio) == REQ_OP_DISCARD) 746 return; 747 748 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 749 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds); 750 } 751 752 static void issue(struct thin_c *tc, struct bio *bio) 753 { 754 struct pool *pool = tc->pool; 755 756 if (!bio_triggers_commit(tc, bio)) { 757 dm_submit_bio_remap(bio, NULL); 758 return; 759 } 760 761 /* 762 * Complete bio with an error if earlier I/O caused changes to 763 * the metadata that can't be committed e.g, due to I/O errors 764 * on the metadata device. 765 */ 766 if (dm_thin_aborted_changes(tc->td)) { 767 bio_io_error(bio); 768 return; 769 } 770 771 /* 772 * Batch together any bios that trigger commits and then issue a 773 * single commit for them in process_deferred_bios(). 774 */ 775 spin_lock_irq(&pool->lock); 776 bio_list_add(&pool->deferred_flush_bios, bio); 777 spin_unlock_irq(&pool->lock); 778 } 779 780 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio) 781 { 782 remap_to_origin(tc, bio); 783 issue(tc, bio); 784 } 785 786 static void remap_and_issue(struct thin_c *tc, struct bio *bio, 787 dm_block_t block) 788 { 789 remap(tc, bio, block); 790 issue(tc, bio); 791 } 792 793 /*----------------------------------------------------------------*/ 794 795 /* 796 * Bio endio functions. 797 */ 798 struct dm_thin_new_mapping { 799 struct list_head list; 800 801 bool pass_discard:1; 802 bool maybe_shared:1; 803 804 /* 805 * Track quiescing, copying and zeroing preparation actions. When this 806 * counter hits zero the block is prepared and can be inserted into the 807 * btree. 808 */ 809 atomic_t prepare_actions; 810 811 blk_status_t status; 812 struct thin_c *tc; 813 dm_block_t virt_begin, virt_end; 814 dm_block_t data_block; 815 struct dm_bio_prison_cell *cell; 816 817 /* 818 * If the bio covers the whole area of a block then we can avoid 819 * zeroing or copying. Instead this bio is hooked. The bio will 820 * still be in the cell, so care has to be taken to avoid issuing 821 * the bio twice. 822 */ 823 struct bio *bio; 824 bio_end_io_t *saved_bi_end_io; 825 }; 826 827 static void __complete_mapping_preparation(struct dm_thin_new_mapping *m) 828 { 829 struct pool *pool = m->tc->pool; 830 831 if (atomic_dec_and_test(&m->prepare_actions)) { 832 list_add_tail(&m->list, &pool->prepared_mappings); 833 wake_worker(pool); 834 } 835 } 836 837 static void complete_mapping_preparation(struct dm_thin_new_mapping *m) 838 { 839 unsigned long flags; 840 struct pool *pool = m->tc->pool; 841 842 spin_lock_irqsave(&pool->lock, flags); 843 __complete_mapping_preparation(m); 844 spin_unlock_irqrestore(&pool->lock, flags); 845 } 846 847 static void copy_complete(int read_err, unsigned long write_err, void *context) 848 { 849 struct dm_thin_new_mapping *m = context; 850 851 m->status = read_err || write_err ? BLK_STS_IOERR : 0; 852 complete_mapping_preparation(m); 853 } 854 855 static void overwrite_endio(struct bio *bio) 856 { 857 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 858 struct dm_thin_new_mapping *m = h->overwrite_mapping; 859 860 bio->bi_end_io = m->saved_bi_end_io; 861 862 m->status = bio->bi_status; 863 complete_mapping_preparation(m); 864 } 865 866 /*----------------------------------------------------------------*/ 867 868 /* 869 * Workqueue. 870 */ 871 872 /* 873 * Prepared mapping jobs. 874 */ 875 876 /* 877 * This sends the bios in the cell, except the original holder, back 878 * to the deferred_bios list. 879 */ 880 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell) 881 { 882 struct pool *pool = tc->pool; 883 unsigned long flags; 884 struct bio_list bios; 885 886 bio_list_init(&bios); 887 cell_release_no_holder(pool, cell, &bios); 888 889 if (!bio_list_empty(&bios)) { 890 spin_lock_irqsave(&tc->lock, flags); 891 bio_list_merge(&tc->deferred_bio_list, &bios); 892 spin_unlock_irqrestore(&tc->lock, flags); 893 wake_worker(pool); 894 } 895 } 896 897 static void thin_defer_bio(struct thin_c *tc, struct bio *bio); 898 899 struct remap_info { 900 struct thin_c *tc; 901 struct bio_list defer_bios; 902 struct bio_list issue_bios; 903 }; 904 905 static void __inc_remap_and_issue_cell(void *context, 906 struct dm_bio_prison_cell *cell) 907 { 908 struct remap_info *info = context; 909 struct bio *bio; 910 911 while ((bio = bio_list_pop(&cell->bios))) { 912 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) 913 bio_list_add(&info->defer_bios, bio); 914 else { 915 inc_all_io_entry(info->tc->pool, bio); 916 917 /* 918 * We can't issue the bios with the bio prison lock 919 * held, so we add them to a list to issue on 920 * return from this function. 921 */ 922 bio_list_add(&info->issue_bios, bio); 923 } 924 } 925 } 926 927 static void inc_remap_and_issue_cell(struct thin_c *tc, 928 struct dm_bio_prison_cell *cell, 929 dm_block_t block) 930 { 931 struct bio *bio; 932 struct remap_info info; 933 934 info.tc = tc; 935 bio_list_init(&info.defer_bios); 936 bio_list_init(&info.issue_bios); 937 938 /* 939 * We have to be careful to inc any bios we're about to issue 940 * before the cell is released, and avoid a race with new bios 941 * being added to the cell. 942 */ 943 cell_visit_release(tc->pool, __inc_remap_and_issue_cell, 944 &info, cell); 945 946 while ((bio = bio_list_pop(&info.defer_bios))) 947 thin_defer_bio(tc, bio); 948 949 while ((bio = bio_list_pop(&info.issue_bios))) 950 remap_and_issue(info.tc, bio, block); 951 } 952 953 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) 954 { 955 cell_error(m->tc->pool, m->cell); 956 list_del(&m->list); 957 mempool_free(m, &m->tc->pool->mapping_pool); 958 } 959 960 static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio) 961 { 962 struct pool *pool = tc->pool; 963 964 /* 965 * If the bio has the REQ_FUA flag set we must commit the metadata 966 * before signaling its completion. 967 */ 968 if (!bio_triggers_commit(tc, bio)) { 969 bio_endio(bio); 970 return; 971 } 972 973 /* 974 * Complete bio with an error if earlier I/O caused changes to the 975 * metadata that can't be committed, e.g, due to I/O errors on the 976 * metadata device. 977 */ 978 if (dm_thin_aborted_changes(tc->td)) { 979 bio_io_error(bio); 980 return; 981 } 982 983 /* 984 * Batch together any bios that trigger commits and then issue a 985 * single commit for them in process_deferred_bios(). 986 */ 987 spin_lock_irq(&pool->lock); 988 bio_list_add(&pool->deferred_flush_completions, bio); 989 spin_unlock_irq(&pool->lock); 990 } 991 992 static void process_prepared_mapping(struct dm_thin_new_mapping *m) 993 { 994 struct thin_c *tc = m->tc; 995 struct pool *pool = tc->pool; 996 struct bio *bio = m->bio; 997 int r; 998 999 if (m->status) { 1000 cell_error(pool, m->cell); 1001 goto out; 1002 } 1003 1004 /* 1005 * Commit the prepared block into the mapping btree. 1006 * Any I/O for this block arriving after this point will get 1007 * remapped to it directly. 1008 */ 1009 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block); 1010 if (r) { 1011 metadata_operation_failed(pool, "dm_thin_insert_block", r); 1012 cell_error(pool, m->cell); 1013 goto out; 1014 } 1015 1016 /* 1017 * Release any bios held while the block was being provisioned. 1018 * If we are processing a write bio that completely covers the block, 1019 * we already processed it so can ignore it now when processing 1020 * the bios in the cell. 1021 */ 1022 if (bio) { 1023 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1024 complete_overwrite_bio(tc, bio); 1025 } else { 1026 inc_all_io_entry(tc->pool, m->cell->holder); 1027 remap_and_issue(tc, m->cell->holder, m->data_block); 1028 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1029 } 1030 1031 out: 1032 list_del(&m->list); 1033 mempool_free(m, &pool->mapping_pool); 1034 } 1035 1036 /*----------------------------------------------------------------*/ 1037 1038 static void free_discard_mapping(struct dm_thin_new_mapping *m) 1039 { 1040 struct thin_c *tc = m->tc; 1041 1042 if (m->cell) 1043 cell_defer_no_holder(tc, m->cell); 1044 mempool_free(m, &tc->pool->mapping_pool); 1045 } 1046 1047 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) 1048 { 1049 bio_io_error(m->bio); 1050 free_discard_mapping(m); 1051 } 1052 1053 static void process_prepared_discard_success(struct dm_thin_new_mapping *m) 1054 { 1055 bio_endio(m->bio); 1056 free_discard_mapping(m); 1057 } 1058 1059 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m) 1060 { 1061 int r; 1062 struct thin_c *tc = m->tc; 1063 1064 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end); 1065 if (r) { 1066 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); 1067 bio_io_error(m->bio); 1068 } else 1069 bio_endio(m->bio); 1070 1071 cell_defer_no_holder(tc, m->cell); 1072 mempool_free(m, &tc->pool->mapping_pool); 1073 } 1074 1075 /*----------------------------------------------------------------*/ 1076 1077 static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m, 1078 struct bio *discard_parent) 1079 { 1080 /* 1081 * We've already unmapped this range of blocks, but before we 1082 * passdown we have to check that these blocks are now unused. 1083 */ 1084 int r = 0; 1085 bool shared = true; 1086 struct thin_c *tc = m->tc; 1087 struct pool *pool = tc->pool; 1088 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin; 1089 struct discard_op op; 1090 1091 begin_discard(&op, tc, discard_parent); 1092 while (b != end) { 1093 /* find start of unmapped run */ 1094 for (; b < end; b++) { 1095 r = dm_pool_block_is_shared(pool->pmd, b, &shared); 1096 if (r) 1097 goto out; 1098 1099 if (!shared) 1100 break; 1101 } 1102 1103 if (b == end) 1104 break; 1105 1106 /* find end of run */ 1107 for (e = b + 1; e != end; e++) { 1108 r = dm_pool_block_is_shared(pool->pmd, e, &shared); 1109 if (r) 1110 goto out; 1111 1112 if (shared) 1113 break; 1114 } 1115 1116 issue_discard(&op, b, e); 1117 1118 b = e; 1119 } 1120 out: 1121 end_discard(&op, r); 1122 } 1123 1124 static void queue_passdown_pt2(struct dm_thin_new_mapping *m) 1125 { 1126 unsigned long flags; 1127 struct pool *pool = m->tc->pool; 1128 1129 spin_lock_irqsave(&pool->lock, flags); 1130 list_add_tail(&m->list, &pool->prepared_discards_pt2); 1131 spin_unlock_irqrestore(&pool->lock, flags); 1132 wake_worker(pool); 1133 } 1134 1135 static void passdown_endio(struct bio *bio) 1136 { 1137 /* 1138 * It doesn't matter if the passdown discard failed, we still want 1139 * to unmap (we ignore err). 1140 */ 1141 queue_passdown_pt2(bio->bi_private); 1142 bio_put(bio); 1143 } 1144 1145 static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m) 1146 { 1147 int r; 1148 struct thin_c *tc = m->tc; 1149 struct pool *pool = tc->pool; 1150 struct bio *discard_parent; 1151 dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin); 1152 1153 /* 1154 * Only this thread allocates blocks, so we can be sure that the 1155 * newly unmapped blocks will not be allocated before the end of 1156 * the function. 1157 */ 1158 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end); 1159 if (r) { 1160 metadata_operation_failed(pool, "dm_thin_remove_range", r); 1161 bio_io_error(m->bio); 1162 cell_defer_no_holder(tc, m->cell); 1163 mempool_free(m, &pool->mapping_pool); 1164 return; 1165 } 1166 1167 /* 1168 * Increment the unmapped blocks. This prevents a race between the 1169 * passdown io and reallocation of freed blocks. 1170 */ 1171 r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end); 1172 if (r) { 1173 metadata_operation_failed(pool, "dm_pool_inc_data_range", r); 1174 bio_io_error(m->bio); 1175 cell_defer_no_holder(tc, m->cell); 1176 mempool_free(m, &pool->mapping_pool); 1177 return; 1178 } 1179 1180 discard_parent = bio_alloc(NULL, 1, 0, GFP_NOIO); 1181 discard_parent->bi_end_io = passdown_endio; 1182 discard_parent->bi_private = m; 1183 if (m->maybe_shared) 1184 passdown_double_checking_shared_status(m, discard_parent); 1185 else { 1186 struct discard_op op; 1187 1188 begin_discard(&op, tc, discard_parent); 1189 issue_discard(&op, m->data_block, data_end); 1190 end_discard(&op, 0); 1191 } 1192 } 1193 1194 static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m) 1195 { 1196 int r; 1197 struct thin_c *tc = m->tc; 1198 struct pool *pool = tc->pool; 1199 1200 /* 1201 * The passdown has completed, so now we can decrement all those 1202 * unmapped blocks. 1203 */ 1204 r = dm_pool_dec_data_range(pool->pmd, m->data_block, 1205 m->data_block + (m->virt_end - m->virt_begin)); 1206 if (r) { 1207 metadata_operation_failed(pool, "dm_pool_dec_data_range", r); 1208 bio_io_error(m->bio); 1209 } else 1210 bio_endio(m->bio); 1211 1212 cell_defer_no_holder(tc, m->cell); 1213 mempool_free(m, &pool->mapping_pool); 1214 } 1215 1216 static void process_prepared(struct pool *pool, struct list_head *head, 1217 process_mapping_fn *fn) 1218 { 1219 struct list_head maps; 1220 struct dm_thin_new_mapping *m, *tmp; 1221 1222 INIT_LIST_HEAD(&maps); 1223 spin_lock_irq(&pool->lock); 1224 list_splice_init(head, &maps); 1225 spin_unlock_irq(&pool->lock); 1226 1227 list_for_each_entry_safe(m, tmp, &maps, list) 1228 (*fn)(m); 1229 } 1230 1231 /* 1232 * Deferred bio jobs. 1233 */ 1234 static int io_overlaps_block(struct pool *pool, struct bio *bio) 1235 { 1236 return bio->bi_iter.bi_size == 1237 (pool->sectors_per_block << SECTOR_SHIFT); 1238 } 1239 1240 static int io_overwrites_block(struct pool *pool, struct bio *bio) 1241 { 1242 return (bio_data_dir(bio) == WRITE) && 1243 io_overlaps_block(pool, bio); 1244 } 1245 1246 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save, 1247 bio_end_io_t *fn) 1248 { 1249 *save = bio->bi_end_io; 1250 bio->bi_end_io = fn; 1251 } 1252 1253 static int ensure_next_mapping(struct pool *pool) 1254 { 1255 if (pool->next_mapping) 1256 return 0; 1257 1258 pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC); 1259 1260 return pool->next_mapping ? 0 : -ENOMEM; 1261 } 1262 1263 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool) 1264 { 1265 struct dm_thin_new_mapping *m = pool->next_mapping; 1266 1267 BUG_ON(!pool->next_mapping); 1268 1269 memset(m, 0, sizeof(struct dm_thin_new_mapping)); 1270 INIT_LIST_HEAD(&m->list); 1271 m->bio = NULL; 1272 1273 pool->next_mapping = NULL; 1274 1275 return m; 1276 } 1277 1278 static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m, 1279 sector_t begin, sector_t end) 1280 { 1281 struct dm_io_region to; 1282 1283 to.bdev = tc->pool_dev->bdev; 1284 to.sector = begin; 1285 to.count = end - begin; 1286 1287 dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); 1288 } 1289 1290 static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio, 1291 dm_block_t data_begin, 1292 struct dm_thin_new_mapping *m) 1293 { 1294 struct pool *pool = tc->pool; 1295 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1296 1297 h->overwrite_mapping = m; 1298 m->bio = bio; 1299 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio); 1300 inc_all_io_entry(pool, bio); 1301 remap_and_issue(tc, bio, data_begin); 1302 } 1303 1304 /* 1305 * A partial copy also needs to zero the uncopied region. 1306 */ 1307 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, 1308 struct dm_dev *origin, dm_block_t data_origin, 1309 dm_block_t data_dest, 1310 struct dm_bio_prison_cell *cell, struct bio *bio, 1311 sector_t len) 1312 { 1313 struct pool *pool = tc->pool; 1314 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1315 1316 m->tc = tc; 1317 m->virt_begin = virt_block; 1318 m->virt_end = virt_block + 1u; 1319 m->data_block = data_dest; 1320 m->cell = cell; 1321 1322 /* 1323 * quiesce action + copy action + an extra reference held for the 1324 * duration of this function (we may need to inc later for a 1325 * partial zero). 1326 */ 1327 atomic_set(&m->prepare_actions, 3); 1328 1329 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list)) 1330 complete_mapping_preparation(m); /* already quiesced */ 1331 1332 /* 1333 * IO to pool_dev remaps to the pool target's data_dev. 1334 * 1335 * If the whole block of data is being overwritten, we can issue the 1336 * bio immediately. Otherwise we use kcopyd to clone the data first. 1337 */ 1338 if (io_overwrites_block(pool, bio)) 1339 remap_and_issue_overwrite(tc, bio, data_dest, m); 1340 else { 1341 struct dm_io_region from, to; 1342 1343 from.bdev = origin->bdev; 1344 from.sector = data_origin * pool->sectors_per_block; 1345 from.count = len; 1346 1347 to.bdev = tc->pool_dev->bdev; 1348 to.sector = data_dest * pool->sectors_per_block; 1349 to.count = len; 1350 1351 dm_kcopyd_copy(pool->copier, &from, 1, &to, 1352 0, copy_complete, m); 1353 1354 /* 1355 * Do we need to zero a tail region? 1356 */ 1357 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) { 1358 atomic_inc(&m->prepare_actions); 1359 ll_zero(tc, m, 1360 data_dest * pool->sectors_per_block + len, 1361 (data_dest + 1) * pool->sectors_per_block); 1362 } 1363 } 1364 1365 complete_mapping_preparation(m); /* drop our ref */ 1366 } 1367 1368 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block, 1369 dm_block_t data_origin, dm_block_t data_dest, 1370 struct dm_bio_prison_cell *cell, struct bio *bio) 1371 { 1372 schedule_copy(tc, virt_block, tc->pool_dev, 1373 data_origin, data_dest, cell, bio, 1374 tc->pool->sectors_per_block); 1375 } 1376 1377 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, 1378 dm_block_t data_block, struct dm_bio_prison_cell *cell, 1379 struct bio *bio) 1380 { 1381 struct pool *pool = tc->pool; 1382 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1383 1384 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */ 1385 m->tc = tc; 1386 m->virt_begin = virt_block; 1387 m->virt_end = virt_block + 1u; 1388 m->data_block = data_block; 1389 m->cell = cell; 1390 1391 /* 1392 * If the whole block of data is being overwritten or we are not 1393 * zeroing pre-existing data, we can issue the bio immediately. 1394 * Otherwise we use kcopyd to zero the data first. 1395 */ 1396 if (pool->pf.zero_new_blocks) { 1397 if (io_overwrites_block(pool, bio)) 1398 remap_and_issue_overwrite(tc, bio, data_block, m); 1399 else { 1400 ll_zero(tc, m, data_block * pool->sectors_per_block, 1401 (data_block + 1) * pool->sectors_per_block); 1402 } 1403 } else 1404 process_prepared_mapping(m); 1405 } 1406 1407 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, 1408 dm_block_t data_dest, 1409 struct dm_bio_prison_cell *cell, struct bio *bio) 1410 { 1411 struct pool *pool = tc->pool; 1412 sector_t virt_block_begin = virt_block * pool->sectors_per_block; 1413 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; 1414 1415 if (virt_block_end <= tc->origin_size) { 1416 schedule_copy(tc, virt_block, tc->origin_dev, 1417 virt_block, data_dest, cell, bio, 1418 pool->sectors_per_block); 1419 1420 } else if (virt_block_begin < tc->origin_size) { 1421 schedule_copy(tc, virt_block, tc->origin_dev, 1422 virt_block, data_dest, cell, bio, 1423 tc->origin_size - virt_block_begin); 1424 1425 } else 1426 schedule_zero(tc, virt_block, data_dest, cell, bio); 1427 } 1428 1429 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode); 1430 1431 static void requeue_bios(struct pool *pool); 1432 1433 static bool is_read_only_pool_mode(enum pool_mode mode) 1434 { 1435 return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY); 1436 } 1437 1438 static bool is_read_only(struct pool *pool) 1439 { 1440 return is_read_only_pool_mode(get_pool_mode(pool)); 1441 } 1442 1443 static void check_for_metadata_space(struct pool *pool) 1444 { 1445 int r; 1446 const char *ooms_reason = NULL; 1447 dm_block_t nr_free; 1448 1449 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free); 1450 if (r) 1451 ooms_reason = "Could not get free metadata blocks"; 1452 else if (!nr_free) 1453 ooms_reason = "No free metadata blocks"; 1454 1455 if (ooms_reason && !is_read_only(pool)) { 1456 DMERR("%s", ooms_reason); 1457 set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE); 1458 } 1459 } 1460 1461 static void check_for_data_space(struct pool *pool) 1462 { 1463 int r; 1464 dm_block_t nr_free; 1465 1466 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE) 1467 return; 1468 1469 r = dm_pool_get_free_block_count(pool->pmd, &nr_free); 1470 if (r) 1471 return; 1472 1473 if (nr_free) { 1474 set_pool_mode(pool, PM_WRITE); 1475 requeue_bios(pool); 1476 } 1477 } 1478 1479 /* 1480 * A non-zero return indicates read_only or fail_io mode. 1481 * Many callers don't care about the return value. 1482 */ 1483 static int commit(struct pool *pool) 1484 { 1485 int r; 1486 1487 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) 1488 return -EINVAL; 1489 1490 r = dm_pool_commit_metadata(pool->pmd); 1491 if (r) 1492 metadata_operation_failed(pool, "dm_pool_commit_metadata", r); 1493 else { 1494 check_for_metadata_space(pool); 1495 check_for_data_space(pool); 1496 } 1497 1498 return r; 1499 } 1500 1501 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks) 1502 { 1503 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) { 1504 DMWARN("%s: reached low water mark for data device: sending event.", 1505 dm_device_name(pool->pool_md)); 1506 spin_lock_irq(&pool->lock); 1507 pool->low_water_triggered = true; 1508 spin_unlock_irq(&pool->lock); 1509 dm_table_event(pool->ti->table); 1510 } 1511 } 1512 1513 static int alloc_data_block(struct thin_c *tc, dm_block_t *result) 1514 { 1515 int r; 1516 dm_block_t free_blocks; 1517 struct pool *pool = tc->pool; 1518 1519 if (WARN_ON(get_pool_mode(pool) != PM_WRITE)) 1520 return -EINVAL; 1521 1522 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1523 if (r) { 1524 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1525 return r; 1526 } 1527 1528 check_low_water_mark(pool, free_blocks); 1529 1530 if (!free_blocks) { 1531 /* 1532 * Try to commit to see if that will free up some 1533 * more space. 1534 */ 1535 r = commit(pool); 1536 if (r) 1537 return r; 1538 1539 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1540 if (r) { 1541 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1542 return r; 1543 } 1544 1545 if (!free_blocks) { 1546 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); 1547 return -ENOSPC; 1548 } 1549 } 1550 1551 r = dm_pool_alloc_data_block(pool->pmd, result); 1552 if (r) { 1553 if (r == -ENOSPC) 1554 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); 1555 else 1556 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r); 1557 return r; 1558 } 1559 1560 r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks); 1561 if (r) { 1562 metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r); 1563 return r; 1564 } 1565 1566 if (!free_blocks) { 1567 /* Let's commit before we use up the metadata reserve. */ 1568 r = commit(pool); 1569 if (r) 1570 return r; 1571 } 1572 1573 return 0; 1574 } 1575 1576 /* 1577 * If we have run out of space, queue bios until the device is 1578 * resumed, presumably after having been reloaded with more space. 1579 */ 1580 static void retry_on_resume(struct bio *bio) 1581 { 1582 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1583 struct thin_c *tc = h->tc; 1584 1585 spin_lock_irq(&tc->lock); 1586 bio_list_add(&tc->retry_on_resume_list, bio); 1587 spin_unlock_irq(&tc->lock); 1588 } 1589 1590 static blk_status_t should_error_unserviceable_bio(struct pool *pool) 1591 { 1592 enum pool_mode m = get_pool_mode(pool); 1593 1594 switch (m) { 1595 case PM_WRITE: 1596 /* Shouldn't get here */ 1597 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode"); 1598 return BLK_STS_IOERR; 1599 1600 case PM_OUT_OF_DATA_SPACE: 1601 return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0; 1602 1603 case PM_OUT_OF_METADATA_SPACE: 1604 case PM_READ_ONLY: 1605 case PM_FAIL: 1606 return BLK_STS_IOERR; 1607 default: 1608 /* Shouldn't get here */ 1609 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode"); 1610 return BLK_STS_IOERR; 1611 } 1612 } 1613 1614 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio) 1615 { 1616 blk_status_t error = should_error_unserviceable_bio(pool); 1617 1618 if (error) { 1619 bio->bi_status = error; 1620 bio_endio(bio); 1621 } else 1622 retry_on_resume(bio); 1623 } 1624 1625 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell) 1626 { 1627 struct bio *bio; 1628 struct bio_list bios; 1629 blk_status_t error; 1630 1631 error = should_error_unserviceable_bio(pool); 1632 if (error) { 1633 cell_error_with_code(pool, cell, error); 1634 return; 1635 } 1636 1637 bio_list_init(&bios); 1638 cell_release(pool, cell, &bios); 1639 1640 while ((bio = bio_list_pop(&bios))) 1641 retry_on_resume(bio); 1642 } 1643 1644 static void process_discard_cell_no_passdown(struct thin_c *tc, 1645 struct dm_bio_prison_cell *virt_cell) 1646 { 1647 struct pool *pool = tc->pool; 1648 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1649 1650 /* 1651 * We don't need to lock the data blocks, since there's no 1652 * passdown. We only lock data blocks for allocation and breaking sharing. 1653 */ 1654 m->tc = tc; 1655 m->virt_begin = virt_cell->key.block_begin; 1656 m->virt_end = virt_cell->key.block_end; 1657 m->cell = virt_cell; 1658 m->bio = virt_cell->holder; 1659 1660 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1661 pool->process_prepared_discard(m); 1662 } 1663 1664 static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end, 1665 struct bio *bio) 1666 { 1667 struct pool *pool = tc->pool; 1668 1669 int r; 1670 bool maybe_shared; 1671 struct dm_cell_key data_key; 1672 struct dm_bio_prison_cell *data_cell; 1673 struct dm_thin_new_mapping *m; 1674 dm_block_t virt_begin, virt_end, data_begin, data_end; 1675 dm_block_t len, next_boundary; 1676 1677 while (begin != end) { 1678 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end, 1679 &data_begin, &maybe_shared); 1680 if (r) { 1681 /* 1682 * Silently fail, letting any mappings we've 1683 * created complete. 1684 */ 1685 break; 1686 } 1687 1688 data_end = data_begin + (virt_end - virt_begin); 1689 1690 /* 1691 * Make sure the data region obeys the bio prison restrictions. 1692 */ 1693 while (data_begin < data_end) { 1694 r = ensure_next_mapping(pool); 1695 if (r) 1696 return; /* we did our best */ 1697 1698 next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1) 1699 << BIO_PRISON_MAX_RANGE_SHIFT; 1700 len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin); 1701 1702 /* This key is certainly within range given the above splitting */ 1703 (void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key); 1704 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) { 1705 /* contention, we'll give up with this range */ 1706 virt_begin += len; 1707 data_begin += len; 1708 continue; 1709 } 1710 1711 /* 1712 * IO may still be going to the destination block. We must 1713 * quiesce before we can do the removal. 1714 */ 1715 m = get_next_mapping(pool); 1716 m->tc = tc; 1717 m->maybe_shared = maybe_shared; 1718 m->virt_begin = virt_begin; 1719 m->virt_end = virt_begin + len; 1720 m->data_block = data_begin; 1721 m->cell = data_cell; 1722 m->bio = bio; 1723 1724 /* 1725 * The parent bio must not complete before sub discard bios are 1726 * chained to it (see end_discard's bio_chain)! 1727 * 1728 * This per-mapping bi_remaining increment is paired with 1729 * the implicit decrement that occurs via bio_endio() in 1730 * end_discard(). 1731 */ 1732 bio_inc_remaining(bio); 1733 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1734 pool->process_prepared_discard(m); 1735 1736 virt_begin += len; 1737 data_begin += len; 1738 } 1739 1740 begin = virt_end; 1741 } 1742 } 1743 1744 static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell) 1745 { 1746 struct bio *bio = virt_cell->holder; 1747 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1748 1749 /* 1750 * The virt_cell will only get freed once the origin bio completes. 1751 * This means it will remain locked while all the individual 1752 * passdown bios are in flight. 1753 */ 1754 h->cell = virt_cell; 1755 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio); 1756 1757 /* 1758 * We complete the bio now, knowing that the bi_remaining field 1759 * will prevent completion until the sub range discards have 1760 * completed. 1761 */ 1762 bio_endio(bio); 1763 } 1764 1765 static void process_discard_bio(struct thin_c *tc, struct bio *bio) 1766 { 1767 dm_block_t begin, end; 1768 struct dm_cell_key virt_key; 1769 struct dm_bio_prison_cell *virt_cell; 1770 1771 get_bio_block_range(tc, bio, &begin, &end); 1772 if (begin == end) { 1773 /* 1774 * The discard covers less than a block. 1775 */ 1776 bio_endio(bio); 1777 return; 1778 } 1779 1780 if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) { 1781 DMERR_LIMIT("Discard doesn't respect bio prison limits"); 1782 bio_endio(bio); 1783 return; 1784 } 1785 1786 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) { 1787 /* 1788 * Potential starvation issue: We're relying on the 1789 * fs/application being well behaved, and not trying to 1790 * send IO to a region at the same time as discarding it. 1791 * If they do this persistently then it's possible this 1792 * cell will never be granted. 1793 */ 1794 return; 1795 } 1796 1797 tc->pool->process_discard_cell(tc, virt_cell); 1798 } 1799 1800 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block, 1801 struct dm_cell_key *key, 1802 struct dm_thin_lookup_result *lookup_result, 1803 struct dm_bio_prison_cell *cell) 1804 { 1805 int r; 1806 dm_block_t data_block; 1807 struct pool *pool = tc->pool; 1808 1809 r = alloc_data_block(tc, &data_block); 1810 switch (r) { 1811 case 0: 1812 schedule_internal_copy(tc, block, lookup_result->block, 1813 data_block, cell, bio); 1814 break; 1815 1816 case -ENOSPC: 1817 retry_bios_on_resume(pool, cell); 1818 break; 1819 1820 default: 1821 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1822 __func__, r); 1823 cell_error(pool, cell); 1824 break; 1825 } 1826 } 1827 1828 static void __remap_and_issue_shared_cell(void *context, 1829 struct dm_bio_prison_cell *cell) 1830 { 1831 struct remap_info *info = context; 1832 struct bio *bio; 1833 1834 while ((bio = bio_list_pop(&cell->bios))) { 1835 if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) || 1836 bio_op(bio) == REQ_OP_DISCARD) 1837 bio_list_add(&info->defer_bios, bio); 1838 else { 1839 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1840 1841 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds); 1842 inc_all_io_entry(info->tc->pool, bio); 1843 bio_list_add(&info->issue_bios, bio); 1844 } 1845 } 1846 } 1847 1848 static void remap_and_issue_shared_cell(struct thin_c *tc, 1849 struct dm_bio_prison_cell *cell, 1850 dm_block_t block) 1851 { 1852 struct bio *bio; 1853 struct remap_info info; 1854 1855 info.tc = tc; 1856 bio_list_init(&info.defer_bios); 1857 bio_list_init(&info.issue_bios); 1858 1859 cell_visit_release(tc->pool, __remap_and_issue_shared_cell, 1860 &info, cell); 1861 1862 while ((bio = bio_list_pop(&info.defer_bios))) 1863 thin_defer_bio(tc, bio); 1864 1865 while ((bio = bio_list_pop(&info.issue_bios))) 1866 remap_and_issue(tc, bio, block); 1867 } 1868 1869 static void process_shared_bio(struct thin_c *tc, struct bio *bio, 1870 dm_block_t block, 1871 struct dm_thin_lookup_result *lookup_result, 1872 struct dm_bio_prison_cell *virt_cell) 1873 { 1874 struct dm_bio_prison_cell *data_cell; 1875 struct pool *pool = tc->pool; 1876 struct dm_cell_key key; 1877 1878 /* 1879 * If cell is already occupied, then sharing is already in the process 1880 * of being broken so we have nothing further to do here. 1881 */ 1882 build_data_key(tc->td, lookup_result->block, &key); 1883 if (bio_detain(pool, &key, bio, &data_cell)) { 1884 cell_defer_no_holder(tc, virt_cell); 1885 return; 1886 } 1887 1888 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) { 1889 break_sharing(tc, bio, block, &key, lookup_result, data_cell); 1890 cell_defer_no_holder(tc, virt_cell); 1891 } else { 1892 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1893 1894 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds); 1895 inc_all_io_entry(pool, bio); 1896 remap_and_issue(tc, bio, lookup_result->block); 1897 1898 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block); 1899 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block); 1900 } 1901 } 1902 1903 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block, 1904 struct dm_bio_prison_cell *cell) 1905 { 1906 int r; 1907 dm_block_t data_block; 1908 struct pool *pool = tc->pool; 1909 1910 /* 1911 * Remap empty bios (flushes) immediately, without provisioning. 1912 */ 1913 if (!bio->bi_iter.bi_size) { 1914 inc_all_io_entry(pool, bio); 1915 cell_defer_no_holder(tc, cell); 1916 1917 remap_and_issue(tc, bio, 0); 1918 return; 1919 } 1920 1921 /* 1922 * Fill read bios with zeroes and complete them immediately. 1923 */ 1924 if (bio_data_dir(bio) == READ) { 1925 zero_fill_bio(bio); 1926 cell_defer_no_holder(tc, cell); 1927 bio_endio(bio); 1928 return; 1929 } 1930 1931 r = alloc_data_block(tc, &data_block); 1932 switch (r) { 1933 case 0: 1934 if (tc->origin_dev) 1935 schedule_external_copy(tc, block, data_block, cell, bio); 1936 else 1937 schedule_zero(tc, block, data_block, cell, bio); 1938 break; 1939 1940 case -ENOSPC: 1941 retry_bios_on_resume(pool, cell); 1942 break; 1943 1944 default: 1945 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1946 __func__, r); 1947 cell_error(pool, cell); 1948 break; 1949 } 1950 } 1951 1952 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1953 { 1954 int r; 1955 struct pool *pool = tc->pool; 1956 struct bio *bio = cell->holder; 1957 dm_block_t block = get_bio_block(tc, bio); 1958 struct dm_thin_lookup_result lookup_result; 1959 1960 if (tc->requeue_mode) { 1961 cell_requeue(pool, cell); 1962 return; 1963 } 1964 1965 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1966 switch (r) { 1967 case 0: 1968 if (lookup_result.shared) 1969 process_shared_bio(tc, bio, block, &lookup_result, cell); 1970 else { 1971 inc_all_io_entry(pool, bio); 1972 remap_and_issue(tc, bio, lookup_result.block); 1973 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 1974 } 1975 break; 1976 1977 case -ENODATA: 1978 if (bio_data_dir(bio) == READ && tc->origin_dev) { 1979 inc_all_io_entry(pool, bio); 1980 cell_defer_no_holder(tc, cell); 1981 1982 if (bio_end_sector(bio) <= tc->origin_size) 1983 remap_to_origin_and_issue(tc, bio); 1984 1985 else if (bio->bi_iter.bi_sector < tc->origin_size) { 1986 zero_fill_bio(bio); 1987 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT; 1988 remap_to_origin_and_issue(tc, bio); 1989 1990 } else { 1991 zero_fill_bio(bio); 1992 bio_endio(bio); 1993 } 1994 } else 1995 provision_block(tc, bio, block, cell); 1996 break; 1997 1998 default: 1999 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 2000 __func__, r); 2001 cell_defer_no_holder(tc, cell); 2002 bio_io_error(bio); 2003 break; 2004 } 2005 } 2006 2007 static void process_bio(struct thin_c *tc, struct bio *bio) 2008 { 2009 struct pool *pool = tc->pool; 2010 dm_block_t block = get_bio_block(tc, bio); 2011 struct dm_bio_prison_cell *cell; 2012 struct dm_cell_key key; 2013 2014 /* 2015 * If cell is already occupied, then the block is already 2016 * being provisioned so we have nothing further to do here. 2017 */ 2018 build_virtual_key(tc->td, block, &key); 2019 if (bio_detain(pool, &key, bio, &cell)) 2020 return; 2021 2022 process_cell(tc, cell); 2023 } 2024 2025 static void __process_bio_read_only(struct thin_c *tc, struct bio *bio, 2026 struct dm_bio_prison_cell *cell) 2027 { 2028 int r; 2029 int rw = bio_data_dir(bio); 2030 dm_block_t block = get_bio_block(tc, bio); 2031 struct dm_thin_lookup_result lookup_result; 2032 2033 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 2034 switch (r) { 2035 case 0: 2036 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) { 2037 handle_unserviceable_bio(tc->pool, bio); 2038 if (cell) 2039 cell_defer_no_holder(tc, cell); 2040 } else { 2041 inc_all_io_entry(tc->pool, bio); 2042 remap_and_issue(tc, bio, lookup_result.block); 2043 if (cell) 2044 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 2045 } 2046 break; 2047 2048 case -ENODATA: 2049 if (cell) 2050 cell_defer_no_holder(tc, cell); 2051 if (rw != READ) { 2052 handle_unserviceable_bio(tc->pool, bio); 2053 break; 2054 } 2055 2056 if (tc->origin_dev) { 2057 inc_all_io_entry(tc->pool, bio); 2058 remap_to_origin_and_issue(tc, bio); 2059 break; 2060 } 2061 2062 zero_fill_bio(bio); 2063 bio_endio(bio); 2064 break; 2065 2066 default: 2067 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 2068 __func__, r); 2069 if (cell) 2070 cell_defer_no_holder(tc, cell); 2071 bio_io_error(bio); 2072 break; 2073 } 2074 } 2075 2076 static void process_bio_read_only(struct thin_c *tc, struct bio *bio) 2077 { 2078 __process_bio_read_only(tc, bio, NULL); 2079 } 2080 2081 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2082 { 2083 __process_bio_read_only(tc, cell->holder, cell); 2084 } 2085 2086 static void process_bio_success(struct thin_c *tc, struct bio *bio) 2087 { 2088 bio_endio(bio); 2089 } 2090 2091 static void process_bio_fail(struct thin_c *tc, struct bio *bio) 2092 { 2093 bio_io_error(bio); 2094 } 2095 2096 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2097 { 2098 cell_success(tc->pool, cell); 2099 } 2100 2101 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2102 { 2103 cell_error(tc->pool, cell); 2104 } 2105 2106 /* 2107 * FIXME: should we also commit due to size of transaction, measured in 2108 * metadata blocks? 2109 */ 2110 static int need_commit_due_to_time(struct pool *pool) 2111 { 2112 return !time_in_range(jiffies, pool->last_commit_jiffies, 2113 pool->last_commit_jiffies + COMMIT_PERIOD); 2114 } 2115 2116 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node) 2117 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook)) 2118 2119 static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio) 2120 { 2121 struct rb_node **rbp, *parent; 2122 struct dm_thin_endio_hook *pbd; 2123 sector_t bi_sector = bio->bi_iter.bi_sector; 2124 2125 rbp = &tc->sort_bio_list.rb_node; 2126 parent = NULL; 2127 while (*rbp) { 2128 parent = *rbp; 2129 pbd = thin_pbd(parent); 2130 2131 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector) 2132 rbp = &(*rbp)->rb_left; 2133 else 2134 rbp = &(*rbp)->rb_right; 2135 } 2136 2137 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 2138 rb_link_node(&pbd->rb_node, parent, rbp); 2139 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list); 2140 } 2141 2142 static void __extract_sorted_bios(struct thin_c *tc) 2143 { 2144 struct rb_node *node; 2145 struct dm_thin_endio_hook *pbd; 2146 struct bio *bio; 2147 2148 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) { 2149 pbd = thin_pbd(node); 2150 bio = thin_bio(pbd); 2151 2152 bio_list_add(&tc->deferred_bio_list, bio); 2153 rb_erase(&pbd->rb_node, &tc->sort_bio_list); 2154 } 2155 2156 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list)); 2157 } 2158 2159 static void __sort_thin_deferred_bios(struct thin_c *tc) 2160 { 2161 struct bio *bio; 2162 struct bio_list bios; 2163 2164 bio_list_init(&bios); 2165 bio_list_merge(&bios, &tc->deferred_bio_list); 2166 bio_list_init(&tc->deferred_bio_list); 2167 2168 /* Sort deferred_bio_list using rb-tree */ 2169 while ((bio = bio_list_pop(&bios))) 2170 __thin_bio_rb_add(tc, bio); 2171 2172 /* 2173 * Transfer the sorted bios in sort_bio_list back to 2174 * deferred_bio_list to allow lockless submission of 2175 * all bios. 2176 */ 2177 __extract_sorted_bios(tc); 2178 } 2179 2180 static void process_thin_deferred_bios(struct thin_c *tc) 2181 { 2182 struct pool *pool = tc->pool; 2183 struct bio *bio; 2184 struct bio_list bios; 2185 struct blk_plug plug; 2186 unsigned int count = 0; 2187 2188 if (tc->requeue_mode) { 2189 error_thin_bio_list(tc, &tc->deferred_bio_list, 2190 BLK_STS_DM_REQUEUE); 2191 return; 2192 } 2193 2194 bio_list_init(&bios); 2195 2196 spin_lock_irq(&tc->lock); 2197 2198 if (bio_list_empty(&tc->deferred_bio_list)) { 2199 spin_unlock_irq(&tc->lock); 2200 return; 2201 } 2202 2203 __sort_thin_deferred_bios(tc); 2204 2205 bio_list_merge(&bios, &tc->deferred_bio_list); 2206 bio_list_init(&tc->deferred_bio_list); 2207 2208 spin_unlock_irq(&tc->lock); 2209 2210 blk_start_plug(&plug); 2211 while ((bio = bio_list_pop(&bios))) { 2212 /* 2213 * If we've got no free new_mapping structs, and processing 2214 * this bio might require one, we pause until there are some 2215 * prepared mappings to process. 2216 */ 2217 if (ensure_next_mapping(pool)) { 2218 spin_lock_irq(&tc->lock); 2219 bio_list_add(&tc->deferred_bio_list, bio); 2220 bio_list_merge(&tc->deferred_bio_list, &bios); 2221 spin_unlock_irq(&tc->lock); 2222 break; 2223 } 2224 2225 if (bio_op(bio) == REQ_OP_DISCARD) 2226 pool->process_discard(tc, bio); 2227 else 2228 pool->process_bio(tc, bio); 2229 2230 if ((count++ & 127) == 0) { 2231 throttle_work_update(&pool->throttle); 2232 dm_pool_issue_prefetches(pool->pmd); 2233 } 2234 cond_resched(); 2235 } 2236 blk_finish_plug(&plug); 2237 } 2238 2239 static int cmp_cells(const void *lhs, const void *rhs) 2240 { 2241 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs); 2242 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs); 2243 2244 BUG_ON(!lhs_cell->holder); 2245 BUG_ON(!rhs_cell->holder); 2246 2247 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector) 2248 return -1; 2249 2250 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector) 2251 return 1; 2252 2253 return 0; 2254 } 2255 2256 static unsigned int sort_cells(struct pool *pool, struct list_head *cells) 2257 { 2258 unsigned int count = 0; 2259 struct dm_bio_prison_cell *cell, *tmp; 2260 2261 list_for_each_entry_safe(cell, tmp, cells, user_list) { 2262 if (count >= CELL_SORT_ARRAY_SIZE) 2263 break; 2264 2265 pool->cell_sort_array[count++] = cell; 2266 list_del(&cell->user_list); 2267 } 2268 2269 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL); 2270 2271 return count; 2272 } 2273 2274 static void process_thin_deferred_cells(struct thin_c *tc) 2275 { 2276 struct pool *pool = tc->pool; 2277 struct list_head cells; 2278 struct dm_bio_prison_cell *cell; 2279 unsigned int i, j, count; 2280 2281 INIT_LIST_HEAD(&cells); 2282 2283 spin_lock_irq(&tc->lock); 2284 list_splice_init(&tc->deferred_cells, &cells); 2285 spin_unlock_irq(&tc->lock); 2286 2287 if (list_empty(&cells)) 2288 return; 2289 2290 do { 2291 count = sort_cells(tc->pool, &cells); 2292 2293 for (i = 0; i < count; i++) { 2294 cell = pool->cell_sort_array[i]; 2295 BUG_ON(!cell->holder); 2296 2297 /* 2298 * If we've got no free new_mapping structs, and processing 2299 * this bio might require one, we pause until there are some 2300 * prepared mappings to process. 2301 */ 2302 if (ensure_next_mapping(pool)) { 2303 for (j = i; j < count; j++) 2304 list_add(&pool->cell_sort_array[j]->user_list, &cells); 2305 2306 spin_lock_irq(&tc->lock); 2307 list_splice(&cells, &tc->deferred_cells); 2308 spin_unlock_irq(&tc->lock); 2309 return; 2310 } 2311 2312 if (bio_op(cell->holder) == REQ_OP_DISCARD) 2313 pool->process_discard_cell(tc, cell); 2314 else 2315 pool->process_cell(tc, cell); 2316 } 2317 cond_resched(); 2318 } while (!list_empty(&cells)); 2319 } 2320 2321 static void thin_get(struct thin_c *tc); 2322 static void thin_put(struct thin_c *tc); 2323 2324 /* 2325 * We can't hold rcu_read_lock() around code that can block. So we 2326 * find a thin with the rcu lock held; bump a refcount; then drop 2327 * the lock. 2328 */ 2329 static struct thin_c *get_first_thin(struct pool *pool) 2330 { 2331 struct thin_c *tc = NULL; 2332 2333 rcu_read_lock(); 2334 tc = list_first_or_null_rcu(&pool->active_thins, struct thin_c, list); 2335 if (tc) 2336 thin_get(tc); 2337 rcu_read_unlock(); 2338 2339 return tc; 2340 } 2341 2342 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc) 2343 { 2344 struct thin_c *old_tc = tc; 2345 2346 rcu_read_lock(); 2347 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) { 2348 thin_get(tc); 2349 thin_put(old_tc); 2350 rcu_read_unlock(); 2351 return tc; 2352 } 2353 thin_put(old_tc); 2354 rcu_read_unlock(); 2355 2356 return NULL; 2357 } 2358 2359 static void process_deferred_bios(struct pool *pool) 2360 { 2361 struct bio *bio; 2362 struct bio_list bios, bio_completions; 2363 struct thin_c *tc; 2364 2365 tc = get_first_thin(pool); 2366 while (tc) { 2367 process_thin_deferred_cells(tc); 2368 process_thin_deferred_bios(tc); 2369 tc = get_next_thin(pool, tc); 2370 } 2371 2372 /* 2373 * If there are any deferred flush bios, we must commit the metadata 2374 * before issuing them or signaling their completion. 2375 */ 2376 bio_list_init(&bios); 2377 bio_list_init(&bio_completions); 2378 2379 spin_lock_irq(&pool->lock); 2380 bio_list_merge(&bios, &pool->deferred_flush_bios); 2381 bio_list_init(&pool->deferred_flush_bios); 2382 2383 bio_list_merge(&bio_completions, &pool->deferred_flush_completions); 2384 bio_list_init(&pool->deferred_flush_completions); 2385 spin_unlock_irq(&pool->lock); 2386 2387 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) && 2388 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool))) 2389 return; 2390 2391 if (commit(pool)) { 2392 bio_list_merge(&bios, &bio_completions); 2393 2394 while ((bio = bio_list_pop(&bios))) 2395 bio_io_error(bio); 2396 return; 2397 } 2398 pool->last_commit_jiffies = jiffies; 2399 2400 while ((bio = bio_list_pop(&bio_completions))) 2401 bio_endio(bio); 2402 2403 while ((bio = bio_list_pop(&bios))) { 2404 /* 2405 * The data device was flushed as part of metadata commit, 2406 * so complete redundant flushes immediately. 2407 */ 2408 if (bio->bi_opf & REQ_PREFLUSH) 2409 bio_endio(bio); 2410 else 2411 dm_submit_bio_remap(bio, NULL); 2412 } 2413 } 2414 2415 static void do_worker(struct work_struct *ws) 2416 { 2417 struct pool *pool = container_of(ws, struct pool, worker); 2418 2419 throttle_work_start(&pool->throttle); 2420 dm_pool_issue_prefetches(pool->pmd); 2421 throttle_work_update(&pool->throttle); 2422 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping); 2423 throttle_work_update(&pool->throttle); 2424 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard); 2425 throttle_work_update(&pool->throttle); 2426 process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2); 2427 throttle_work_update(&pool->throttle); 2428 process_deferred_bios(pool); 2429 throttle_work_complete(&pool->throttle); 2430 } 2431 2432 /* 2433 * We want to commit periodically so that not too much 2434 * unwritten data builds up. 2435 */ 2436 static void do_waker(struct work_struct *ws) 2437 { 2438 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker); 2439 2440 wake_worker(pool); 2441 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD); 2442 } 2443 2444 /* 2445 * We're holding onto IO to allow userland time to react. After the 2446 * timeout either the pool will have been resized (and thus back in 2447 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space. 2448 */ 2449 static void do_no_space_timeout(struct work_struct *ws) 2450 { 2451 struct pool *pool = container_of(to_delayed_work(ws), struct pool, 2452 no_space_timeout); 2453 2454 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) { 2455 pool->pf.error_if_no_space = true; 2456 notify_of_pool_mode_change(pool); 2457 error_retry_list_with_code(pool, BLK_STS_NOSPC); 2458 } 2459 } 2460 2461 /*----------------------------------------------------------------*/ 2462 2463 struct pool_work { 2464 struct work_struct worker; 2465 struct completion complete; 2466 }; 2467 2468 static struct pool_work *to_pool_work(struct work_struct *ws) 2469 { 2470 return container_of(ws, struct pool_work, worker); 2471 } 2472 2473 static void pool_work_complete(struct pool_work *pw) 2474 { 2475 complete(&pw->complete); 2476 } 2477 2478 static void pool_work_wait(struct pool_work *pw, struct pool *pool, 2479 void (*fn)(struct work_struct *)) 2480 { 2481 INIT_WORK_ONSTACK(&pw->worker, fn); 2482 init_completion(&pw->complete); 2483 queue_work(pool->wq, &pw->worker); 2484 wait_for_completion(&pw->complete); 2485 destroy_work_on_stack(&pw->worker); 2486 } 2487 2488 /*----------------------------------------------------------------*/ 2489 2490 struct noflush_work { 2491 struct pool_work pw; 2492 struct thin_c *tc; 2493 }; 2494 2495 static struct noflush_work *to_noflush(struct work_struct *ws) 2496 { 2497 return container_of(to_pool_work(ws), struct noflush_work, pw); 2498 } 2499 2500 static void do_noflush_start(struct work_struct *ws) 2501 { 2502 struct noflush_work *w = to_noflush(ws); 2503 2504 w->tc->requeue_mode = true; 2505 requeue_io(w->tc); 2506 pool_work_complete(&w->pw); 2507 } 2508 2509 static void do_noflush_stop(struct work_struct *ws) 2510 { 2511 struct noflush_work *w = to_noflush(ws); 2512 2513 w->tc->requeue_mode = false; 2514 pool_work_complete(&w->pw); 2515 } 2516 2517 static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *)) 2518 { 2519 struct noflush_work w; 2520 2521 w.tc = tc; 2522 pool_work_wait(&w.pw, tc->pool, fn); 2523 } 2524 2525 /*----------------------------------------------------------------*/ 2526 2527 static void set_discard_callbacks(struct pool *pool) 2528 { 2529 struct pool_c *pt = pool->ti->private; 2530 2531 if (pt->adjusted_pf.discard_passdown) { 2532 pool->process_discard_cell = process_discard_cell_passdown; 2533 pool->process_prepared_discard = process_prepared_discard_passdown_pt1; 2534 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2; 2535 } else { 2536 pool->process_discard_cell = process_discard_cell_no_passdown; 2537 pool->process_prepared_discard = process_prepared_discard_no_passdown; 2538 } 2539 } 2540 2541 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) 2542 { 2543 struct pool_c *pt = pool->ti->private; 2544 bool needs_check = dm_pool_metadata_needs_check(pool->pmd); 2545 enum pool_mode old_mode = get_pool_mode(pool); 2546 unsigned long no_space_timeout = 2547 min_t(unsigned long, READ_ONCE(no_space_timeout_secs), 2548 MAX_JIFFY_OFFSET / HZ) * HZ; 2549 2550 /* 2551 * Never allow the pool to transition to PM_WRITE mode if user 2552 * intervention is required to verify metadata and data consistency. 2553 */ 2554 if (new_mode == PM_WRITE && needs_check) { 2555 DMERR("%s: unable to switch pool to write mode until repaired.", 2556 dm_device_name(pool->pool_md)); 2557 if (old_mode != new_mode) 2558 new_mode = old_mode; 2559 else 2560 new_mode = PM_READ_ONLY; 2561 } 2562 /* 2563 * If we were in PM_FAIL mode, rollback of metadata failed. We're 2564 * not going to recover without a thin_repair. So we never let the 2565 * pool move out of the old mode. 2566 */ 2567 if (old_mode == PM_FAIL) 2568 new_mode = old_mode; 2569 2570 switch (new_mode) { 2571 case PM_FAIL: 2572 dm_pool_metadata_read_only(pool->pmd); 2573 pool->process_bio = process_bio_fail; 2574 pool->process_discard = process_bio_fail; 2575 pool->process_cell = process_cell_fail; 2576 pool->process_discard_cell = process_cell_fail; 2577 pool->process_prepared_mapping = process_prepared_mapping_fail; 2578 pool->process_prepared_discard = process_prepared_discard_fail; 2579 2580 error_retry_list(pool); 2581 break; 2582 2583 case PM_OUT_OF_METADATA_SPACE: 2584 case PM_READ_ONLY: 2585 dm_pool_metadata_read_only(pool->pmd); 2586 pool->process_bio = process_bio_read_only; 2587 pool->process_discard = process_bio_success; 2588 pool->process_cell = process_cell_read_only; 2589 pool->process_discard_cell = process_cell_success; 2590 pool->process_prepared_mapping = process_prepared_mapping_fail; 2591 pool->process_prepared_discard = process_prepared_discard_success; 2592 2593 error_retry_list(pool); 2594 break; 2595 2596 case PM_OUT_OF_DATA_SPACE: 2597 /* 2598 * Ideally we'd never hit this state; the low water mark 2599 * would trigger userland to extend the pool before we 2600 * completely run out of data space. However, many small 2601 * IOs to unprovisioned space can consume data space at an 2602 * alarming rate. Adjust your low water mark if you're 2603 * frequently seeing this mode. 2604 */ 2605 pool->out_of_data_space = true; 2606 pool->process_bio = process_bio_read_only; 2607 pool->process_discard = process_discard_bio; 2608 pool->process_cell = process_cell_read_only; 2609 pool->process_prepared_mapping = process_prepared_mapping; 2610 set_discard_callbacks(pool); 2611 2612 if (!pool->pf.error_if_no_space && no_space_timeout) 2613 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout); 2614 break; 2615 2616 case PM_WRITE: 2617 if (old_mode == PM_OUT_OF_DATA_SPACE) 2618 cancel_delayed_work_sync(&pool->no_space_timeout); 2619 pool->out_of_data_space = false; 2620 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space; 2621 dm_pool_metadata_read_write(pool->pmd); 2622 pool->process_bio = process_bio; 2623 pool->process_discard = process_discard_bio; 2624 pool->process_cell = process_cell; 2625 pool->process_prepared_mapping = process_prepared_mapping; 2626 set_discard_callbacks(pool); 2627 break; 2628 } 2629 2630 pool->pf.mode = new_mode; 2631 /* 2632 * The pool mode may have changed, sync it so bind_control_target() 2633 * doesn't cause an unexpected mode transition on resume. 2634 */ 2635 pt->adjusted_pf.mode = new_mode; 2636 2637 if (old_mode != new_mode) 2638 notify_of_pool_mode_change(pool); 2639 } 2640 2641 static void abort_transaction(struct pool *pool) 2642 { 2643 const char *dev_name = dm_device_name(pool->pool_md); 2644 2645 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 2646 if (dm_pool_abort_metadata(pool->pmd)) { 2647 DMERR("%s: failed to abort metadata transaction", dev_name); 2648 set_pool_mode(pool, PM_FAIL); 2649 } 2650 2651 if (dm_pool_metadata_set_needs_check(pool->pmd)) { 2652 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 2653 set_pool_mode(pool, PM_FAIL); 2654 } 2655 } 2656 2657 static void metadata_operation_failed(struct pool *pool, const char *op, int r) 2658 { 2659 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 2660 dm_device_name(pool->pool_md), op, r); 2661 2662 abort_transaction(pool); 2663 set_pool_mode(pool, PM_READ_ONLY); 2664 } 2665 2666 /*----------------------------------------------------------------*/ 2667 2668 /* 2669 * Mapping functions. 2670 */ 2671 2672 /* 2673 * Called only while mapping a thin bio to hand it over to the workqueue. 2674 */ 2675 static void thin_defer_bio(struct thin_c *tc, struct bio *bio) 2676 { 2677 struct pool *pool = tc->pool; 2678 2679 spin_lock_irq(&tc->lock); 2680 bio_list_add(&tc->deferred_bio_list, bio); 2681 spin_unlock_irq(&tc->lock); 2682 2683 wake_worker(pool); 2684 } 2685 2686 static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio) 2687 { 2688 struct pool *pool = tc->pool; 2689 2690 throttle_lock(&pool->throttle); 2691 thin_defer_bio(tc, bio); 2692 throttle_unlock(&pool->throttle); 2693 } 2694 2695 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2696 { 2697 struct pool *pool = tc->pool; 2698 2699 throttle_lock(&pool->throttle); 2700 spin_lock_irq(&tc->lock); 2701 list_add_tail(&cell->user_list, &tc->deferred_cells); 2702 spin_unlock_irq(&tc->lock); 2703 throttle_unlock(&pool->throttle); 2704 2705 wake_worker(pool); 2706 } 2707 2708 static void thin_hook_bio(struct thin_c *tc, struct bio *bio) 2709 { 2710 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 2711 2712 h->tc = tc; 2713 h->shared_read_entry = NULL; 2714 h->all_io_entry = NULL; 2715 h->overwrite_mapping = NULL; 2716 h->cell = NULL; 2717 } 2718 2719 /* 2720 * Non-blocking function called from the thin target's map function. 2721 */ 2722 static int thin_bio_map(struct dm_target *ti, struct bio *bio) 2723 { 2724 int r; 2725 struct thin_c *tc = ti->private; 2726 dm_block_t block = get_bio_block(tc, bio); 2727 struct dm_thin_device *td = tc->td; 2728 struct dm_thin_lookup_result result; 2729 struct dm_bio_prison_cell *virt_cell, *data_cell; 2730 struct dm_cell_key key; 2731 2732 thin_hook_bio(tc, bio); 2733 2734 if (tc->requeue_mode) { 2735 bio->bi_status = BLK_STS_DM_REQUEUE; 2736 bio_endio(bio); 2737 return DM_MAPIO_SUBMITTED; 2738 } 2739 2740 if (get_pool_mode(tc->pool) == PM_FAIL) { 2741 bio_io_error(bio); 2742 return DM_MAPIO_SUBMITTED; 2743 } 2744 2745 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) { 2746 thin_defer_bio_with_throttle(tc, bio); 2747 return DM_MAPIO_SUBMITTED; 2748 } 2749 2750 /* 2751 * We must hold the virtual cell before doing the lookup, otherwise 2752 * there's a race with discard. 2753 */ 2754 build_virtual_key(tc->td, block, &key); 2755 if (bio_detain(tc->pool, &key, bio, &virt_cell)) 2756 return DM_MAPIO_SUBMITTED; 2757 2758 r = dm_thin_find_block(td, block, 0, &result); 2759 2760 /* 2761 * Note that we defer readahead too. 2762 */ 2763 switch (r) { 2764 case 0: 2765 if (unlikely(result.shared)) { 2766 /* 2767 * We have a race condition here between the 2768 * result.shared value returned by the lookup and 2769 * snapshot creation, which may cause new 2770 * sharing. 2771 * 2772 * To avoid this always quiesce the origin before 2773 * taking the snap. You want to do this anyway to 2774 * ensure a consistent application view 2775 * (i.e. lockfs). 2776 * 2777 * More distant ancestors are irrelevant. The 2778 * shared flag will be set in their case. 2779 */ 2780 thin_defer_cell(tc, virt_cell); 2781 return DM_MAPIO_SUBMITTED; 2782 } 2783 2784 build_data_key(tc->td, result.block, &key); 2785 if (bio_detain(tc->pool, &key, bio, &data_cell)) { 2786 cell_defer_no_holder(tc, virt_cell); 2787 return DM_MAPIO_SUBMITTED; 2788 } 2789 2790 inc_all_io_entry(tc->pool, bio); 2791 cell_defer_no_holder(tc, data_cell); 2792 cell_defer_no_holder(tc, virt_cell); 2793 2794 remap(tc, bio, result.block); 2795 return DM_MAPIO_REMAPPED; 2796 2797 case -ENODATA: 2798 case -EWOULDBLOCK: 2799 thin_defer_cell(tc, virt_cell); 2800 return DM_MAPIO_SUBMITTED; 2801 2802 default: 2803 /* 2804 * Must always call bio_io_error on failure. 2805 * dm_thin_find_block can fail with -EINVAL if the 2806 * pool is switched to fail-io mode. 2807 */ 2808 bio_io_error(bio); 2809 cell_defer_no_holder(tc, virt_cell); 2810 return DM_MAPIO_SUBMITTED; 2811 } 2812 } 2813 2814 static void requeue_bios(struct pool *pool) 2815 { 2816 struct thin_c *tc; 2817 2818 rcu_read_lock(); 2819 list_for_each_entry_rcu(tc, &pool->active_thins, list) { 2820 spin_lock_irq(&tc->lock); 2821 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list); 2822 bio_list_init(&tc->retry_on_resume_list); 2823 spin_unlock_irq(&tc->lock); 2824 } 2825 rcu_read_unlock(); 2826 } 2827 2828 /* 2829 *-------------------------------------------------------------- 2830 * Binding of control targets to a pool object 2831 *-------------------------------------------------------------- 2832 */ 2833 static bool is_factor(sector_t block_size, uint32_t n) 2834 { 2835 return !sector_div(block_size, n); 2836 } 2837 2838 /* 2839 * If discard_passdown was enabled verify that the data device 2840 * supports discards. Disable discard_passdown if not. 2841 */ 2842 static void disable_discard_passdown_if_not_supported(struct pool_c *pt) 2843 { 2844 struct pool *pool = pt->pool; 2845 struct block_device *data_bdev = pt->data_dev->bdev; 2846 struct queue_limits *data_limits = bdev_limits(data_bdev); 2847 const char *reason = NULL; 2848 2849 if (!pt->adjusted_pf.discard_passdown) 2850 return; 2851 2852 if (!bdev_max_discard_sectors(pt->data_dev->bdev)) 2853 reason = "discard unsupported"; 2854 2855 else if (data_limits->max_discard_sectors < pool->sectors_per_block) 2856 reason = "max discard sectors smaller than a block"; 2857 2858 if (reason) { 2859 DMWARN("Data device (%pg) %s: Disabling discard passdown.", data_bdev, reason); 2860 pt->adjusted_pf.discard_passdown = false; 2861 } 2862 } 2863 2864 static int bind_control_target(struct pool *pool, struct dm_target *ti) 2865 { 2866 struct pool_c *pt = ti->private; 2867 2868 /* 2869 * We want to make sure that a pool in PM_FAIL mode is never upgraded. 2870 */ 2871 enum pool_mode old_mode = get_pool_mode(pool); 2872 enum pool_mode new_mode = pt->adjusted_pf.mode; 2873 2874 /* 2875 * Don't change the pool's mode until set_pool_mode() below. 2876 * Otherwise the pool's process_* function pointers may 2877 * not match the desired pool mode. 2878 */ 2879 pt->adjusted_pf.mode = old_mode; 2880 2881 pool->ti = ti; 2882 pool->pf = pt->adjusted_pf; 2883 pool->low_water_blocks = pt->low_water_blocks; 2884 2885 set_pool_mode(pool, new_mode); 2886 2887 return 0; 2888 } 2889 2890 static void unbind_control_target(struct pool *pool, struct dm_target *ti) 2891 { 2892 if (pool->ti == ti) 2893 pool->ti = NULL; 2894 } 2895 2896 /* 2897 *-------------------------------------------------------------- 2898 * Pool creation 2899 *-------------------------------------------------------------- 2900 */ 2901 /* Initialize pool features. */ 2902 static void pool_features_init(struct pool_features *pf) 2903 { 2904 pf->mode = PM_WRITE; 2905 pf->zero_new_blocks = true; 2906 pf->discard_enabled = true; 2907 pf->discard_passdown = true; 2908 pf->error_if_no_space = false; 2909 } 2910 2911 static void __pool_destroy(struct pool *pool) 2912 { 2913 __pool_table_remove(pool); 2914 2915 vfree(pool->cell_sort_array); 2916 if (dm_pool_metadata_close(pool->pmd) < 0) 2917 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 2918 2919 dm_bio_prison_destroy(pool->prison); 2920 dm_kcopyd_client_destroy(pool->copier); 2921 2922 cancel_delayed_work_sync(&pool->waker); 2923 cancel_delayed_work_sync(&pool->no_space_timeout); 2924 if (pool->wq) 2925 destroy_workqueue(pool->wq); 2926 2927 if (pool->next_mapping) 2928 mempool_free(pool->next_mapping, &pool->mapping_pool); 2929 mempool_exit(&pool->mapping_pool); 2930 dm_deferred_set_destroy(pool->shared_read_ds); 2931 dm_deferred_set_destroy(pool->all_io_ds); 2932 kfree(pool); 2933 } 2934 2935 static struct kmem_cache *_new_mapping_cache; 2936 2937 static struct pool *pool_create(struct mapped_device *pool_md, 2938 struct block_device *metadata_dev, 2939 struct block_device *data_dev, 2940 unsigned long block_size, 2941 int read_only, char **error) 2942 { 2943 int r; 2944 void *err_p; 2945 struct pool *pool; 2946 struct dm_pool_metadata *pmd; 2947 bool format_device = read_only ? false : true; 2948 2949 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device); 2950 if (IS_ERR(pmd)) { 2951 *error = "Error creating metadata object"; 2952 return ERR_CAST(pmd); 2953 } 2954 2955 pool = kzalloc_obj(*pool); 2956 if (!pool) { 2957 *error = "Error allocating memory for pool"; 2958 err_p = ERR_PTR(-ENOMEM); 2959 goto bad_pool; 2960 } 2961 2962 pool->pmd = pmd; 2963 pool->sectors_per_block = block_size; 2964 if (block_size & (block_size - 1)) 2965 pool->sectors_per_block_shift = -1; 2966 else 2967 pool->sectors_per_block_shift = __ffs(block_size); 2968 pool->low_water_blocks = 0; 2969 pool_features_init(&pool->pf); 2970 pool->prison = dm_bio_prison_create(); 2971 if (!pool->prison) { 2972 *error = "Error creating pool's bio prison"; 2973 err_p = ERR_PTR(-ENOMEM); 2974 goto bad_prison; 2975 } 2976 2977 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2978 if (IS_ERR(pool->copier)) { 2979 r = PTR_ERR(pool->copier); 2980 *error = "Error creating pool's kcopyd client"; 2981 err_p = ERR_PTR(r); 2982 goto bad_kcopyd_client; 2983 } 2984 2985 /* 2986 * Create singlethreaded workqueue that will service all devices 2987 * that use this metadata. 2988 */ 2989 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); 2990 if (!pool->wq) { 2991 *error = "Error creating pool's workqueue"; 2992 err_p = ERR_PTR(-ENOMEM); 2993 goto bad_wq; 2994 } 2995 2996 throttle_init(&pool->throttle); 2997 INIT_WORK(&pool->worker, do_worker); 2998 INIT_DELAYED_WORK(&pool->waker, do_waker); 2999 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout); 3000 spin_lock_init(&pool->lock); 3001 bio_list_init(&pool->deferred_flush_bios); 3002 bio_list_init(&pool->deferred_flush_completions); 3003 INIT_LIST_HEAD(&pool->prepared_mappings); 3004 INIT_LIST_HEAD(&pool->prepared_discards); 3005 INIT_LIST_HEAD(&pool->prepared_discards_pt2); 3006 INIT_LIST_HEAD(&pool->active_thins); 3007 pool->low_water_triggered = false; 3008 pool->suspended = true; 3009 pool->out_of_data_space = false; 3010 3011 pool->shared_read_ds = dm_deferred_set_create(); 3012 if (!pool->shared_read_ds) { 3013 *error = "Error creating pool's shared read deferred set"; 3014 err_p = ERR_PTR(-ENOMEM); 3015 goto bad_shared_read_ds; 3016 } 3017 3018 pool->all_io_ds = dm_deferred_set_create(); 3019 if (!pool->all_io_ds) { 3020 *error = "Error creating pool's all io deferred set"; 3021 err_p = ERR_PTR(-ENOMEM); 3022 goto bad_all_io_ds; 3023 } 3024 3025 pool->next_mapping = NULL; 3026 r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE, 3027 _new_mapping_cache); 3028 if (r) { 3029 *error = "Error creating pool's mapping mempool"; 3030 err_p = ERR_PTR(r); 3031 goto bad_mapping_pool; 3032 } 3033 3034 pool->cell_sort_array = 3035 vmalloc_array(CELL_SORT_ARRAY_SIZE, 3036 sizeof(*pool->cell_sort_array)); 3037 if (!pool->cell_sort_array) { 3038 *error = "Error allocating cell sort array"; 3039 err_p = ERR_PTR(-ENOMEM); 3040 goto bad_sort_array; 3041 } 3042 3043 pool->ref_count = 1; 3044 pool->last_commit_jiffies = jiffies; 3045 pool->pool_md = pool_md; 3046 pool->md_dev = metadata_dev; 3047 pool->data_dev = data_dev; 3048 __pool_table_insert(pool); 3049 3050 return pool; 3051 3052 bad_sort_array: 3053 mempool_exit(&pool->mapping_pool); 3054 bad_mapping_pool: 3055 dm_deferred_set_destroy(pool->all_io_ds); 3056 bad_all_io_ds: 3057 dm_deferred_set_destroy(pool->shared_read_ds); 3058 bad_shared_read_ds: 3059 destroy_workqueue(pool->wq); 3060 bad_wq: 3061 dm_kcopyd_client_destroy(pool->copier); 3062 bad_kcopyd_client: 3063 dm_bio_prison_destroy(pool->prison); 3064 bad_prison: 3065 kfree(pool); 3066 bad_pool: 3067 if (dm_pool_metadata_close(pmd)) 3068 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 3069 3070 return err_p; 3071 } 3072 3073 static void __pool_inc(struct pool *pool) 3074 { 3075 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 3076 pool->ref_count++; 3077 } 3078 3079 static void __pool_dec(struct pool *pool) 3080 { 3081 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 3082 BUG_ON(!pool->ref_count); 3083 if (!--pool->ref_count) 3084 __pool_destroy(pool); 3085 } 3086 3087 static struct pool *__pool_find(struct mapped_device *pool_md, 3088 struct block_device *metadata_dev, 3089 struct block_device *data_dev, 3090 unsigned long block_size, int read_only, 3091 char **error, int *created) 3092 { 3093 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev); 3094 3095 if (pool) { 3096 if (pool->pool_md != pool_md) { 3097 *error = "metadata device already in use by a pool"; 3098 return ERR_PTR(-EBUSY); 3099 } 3100 if (pool->data_dev != data_dev) { 3101 *error = "data device already in use by a pool"; 3102 return ERR_PTR(-EBUSY); 3103 } 3104 __pool_inc(pool); 3105 3106 } else { 3107 pool = __pool_table_lookup(pool_md); 3108 if (pool) { 3109 if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) { 3110 *error = "different pool cannot replace a pool"; 3111 return ERR_PTR(-EINVAL); 3112 } 3113 __pool_inc(pool); 3114 3115 } else { 3116 pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error); 3117 *created = 1; 3118 } 3119 } 3120 3121 return pool; 3122 } 3123 3124 /* 3125 *-------------------------------------------------------------- 3126 * Pool target methods 3127 *-------------------------------------------------------------- 3128 */ 3129 static void pool_dtr(struct dm_target *ti) 3130 { 3131 struct pool_c *pt = ti->private; 3132 3133 mutex_lock(&dm_thin_pool_table.mutex); 3134 3135 unbind_control_target(pt->pool, ti); 3136 __pool_dec(pt->pool); 3137 dm_put_device(ti, pt->metadata_dev); 3138 dm_put_device(ti, pt->data_dev); 3139 kfree(pt); 3140 3141 mutex_unlock(&dm_thin_pool_table.mutex); 3142 } 3143 3144 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, 3145 struct dm_target *ti) 3146 { 3147 int r; 3148 unsigned int argc; 3149 const char *arg_name; 3150 3151 static const struct dm_arg _args[] = { 3152 {0, 5, "Invalid number of pool feature arguments"}, 3153 }; 3154 3155 /* 3156 * No feature arguments supplied. 3157 */ 3158 if (!as->argc) 3159 return 0; 3160 3161 r = dm_read_arg_group(_args, as, &argc, &ti->error); 3162 if (r) 3163 return -EINVAL; 3164 3165 while (argc && !r) { 3166 arg_name = dm_shift_arg(as); 3167 argc--; 3168 3169 if (!strcasecmp(arg_name, "skip_block_zeroing")) 3170 pf->zero_new_blocks = false; 3171 3172 else if (!strcasecmp(arg_name, "ignore_discard")) 3173 pf->discard_enabled = false; 3174 3175 else if (!strcasecmp(arg_name, "no_discard_passdown")) 3176 pf->discard_passdown = false; 3177 3178 else if (!strcasecmp(arg_name, "read_only")) 3179 pf->mode = PM_READ_ONLY; 3180 3181 else if (!strcasecmp(arg_name, "error_if_no_space")) 3182 pf->error_if_no_space = true; 3183 3184 else { 3185 ti->error = "Unrecognised pool feature requested"; 3186 r = -EINVAL; 3187 break; 3188 } 3189 } 3190 3191 return r; 3192 } 3193 3194 static void metadata_low_callback(void *context) 3195 { 3196 struct pool *pool = context; 3197 3198 DMWARN("%s: reached low water mark for metadata device: sending event.", 3199 dm_device_name(pool->pool_md)); 3200 3201 dm_table_event(pool->ti->table); 3202 } 3203 3204 /* 3205 * We need to flush the data device **before** committing the metadata. 3206 * 3207 * This ensures that the data blocks of any newly inserted mappings are 3208 * properly written to non-volatile storage and won't be lost in case of a 3209 * crash. 3210 * 3211 * Failure to do so can result in data corruption in the case of internal or 3212 * external snapshots and in the case of newly provisioned blocks, when block 3213 * zeroing is enabled. 3214 */ 3215 static int metadata_pre_commit_callback(void *context) 3216 { 3217 struct pool *pool = context; 3218 3219 return blkdev_issue_flush(pool->data_dev); 3220 } 3221 3222 static sector_t get_dev_size(struct block_device *bdev) 3223 { 3224 return bdev_nr_sectors(bdev); 3225 } 3226 3227 static void warn_if_metadata_device_too_big(struct block_device *bdev) 3228 { 3229 sector_t metadata_dev_size = get_dev_size(bdev); 3230 3231 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING) 3232 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.", 3233 bdev, THIN_METADATA_MAX_SECTORS); 3234 } 3235 3236 static sector_t get_metadata_dev_size(struct block_device *bdev) 3237 { 3238 sector_t metadata_dev_size = get_dev_size(bdev); 3239 3240 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS) 3241 metadata_dev_size = THIN_METADATA_MAX_SECTORS; 3242 3243 return metadata_dev_size; 3244 } 3245 3246 static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev) 3247 { 3248 sector_t metadata_dev_size = get_metadata_dev_size(bdev); 3249 3250 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE); 3251 3252 return metadata_dev_size; 3253 } 3254 3255 /* 3256 * When a metadata threshold is crossed a dm event is triggered, and 3257 * userland should respond by growing the metadata device. We could let 3258 * userland set the threshold, like we do with the data threshold, but I'm 3259 * not sure they know enough to do this well. 3260 */ 3261 static dm_block_t calc_metadata_threshold(struct pool_c *pt) 3262 { 3263 /* 3264 * 4M is ample for all ops with the possible exception of thin 3265 * device deletion which is harmless if it fails (just retry the 3266 * delete after you've grown the device). 3267 */ 3268 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4; 3269 3270 return min((dm_block_t)1024ULL /* 4M */, quarter); 3271 } 3272 3273 /* 3274 * thin-pool <metadata dev> <data dev> 3275 * <data block size (sectors)> 3276 * <low water mark (blocks)> 3277 * [<#feature args> [<arg>]*] 3278 * 3279 * Optional feature arguments are: 3280 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks. 3281 * ignore_discard: disable discard 3282 * no_discard_passdown: don't pass discards down to the data device 3283 * read_only: Don't allow any changes to be made to the pool metadata. 3284 * error_if_no_space: error IOs, instead of queueing, if no space. 3285 */ 3286 static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv) 3287 { 3288 int r, pool_created = 0; 3289 struct pool_c *pt; 3290 struct pool *pool; 3291 struct pool_features pf; 3292 struct dm_arg_set as; 3293 struct dm_dev *data_dev; 3294 unsigned long block_size; 3295 dm_block_t low_water_blocks; 3296 struct dm_dev *metadata_dev; 3297 blk_mode_t metadata_mode; 3298 3299 /* 3300 * FIXME Remove validation from scope of lock. 3301 */ 3302 mutex_lock(&dm_thin_pool_table.mutex); 3303 3304 if (argc < 4) { 3305 ti->error = "Invalid argument count"; 3306 r = -EINVAL; 3307 goto out_unlock; 3308 } 3309 3310 as.argc = argc; 3311 as.argv = argv; 3312 3313 /* make sure metadata and data are different devices */ 3314 if (!strcmp(argv[0], argv[1])) { 3315 ti->error = "Error setting metadata or data device"; 3316 r = -EINVAL; 3317 goto out_unlock; 3318 } 3319 3320 /* 3321 * Set default pool features. 3322 */ 3323 pool_features_init(&pf); 3324 3325 dm_consume_args(&as, 4); 3326 r = parse_pool_features(&as, &pf, ti); 3327 if (r) 3328 goto out_unlock; 3329 3330 metadata_mode = BLK_OPEN_READ | 3331 ((pf.mode == PM_READ_ONLY) ? 0 : BLK_OPEN_WRITE); 3332 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev); 3333 if (r) { 3334 ti->error = "Error opening metadata block device"; 3335 goto out_unlock; 3336 } 3337 warn_if_metadata_device_too_big(metadata_dev->bdev); 3338 3339 r = dm_get_device(ti, argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, &data_dev); 3340 if (r) { 3341 ti->error = "Error getting data device"; 3342 goto out_metadata; 3343 } 3344 3345 if (kstrtoul(argv[2], 10, &block_size) || !block_size || 3346 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 3347 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 3348 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 3349 ti->error = "Invalid block size"; 3350 r = -EINVAL; 3351 goto out; 3352 } 3353 3354 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) { 3355 ti->error = "Invalid low water mark"; 3356 r = -EINVAL; 3357 goto out; 3358 } 3359 3360 pt = kzalloc_obj(*pt); 3361 if (!pt) { 3362 r = -ENOMEM; 3363 goto out; 3364 } 3365 3366 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev, 3367 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created); 3368 if (IS_ERR(pool)) { 3369 r = PTR_ERR(pool); 3370 goto out_free_pt; 3371 } 3372 3373 /* 3374 * 'pool_created' reflects whether this is the first table load. 3375 * Top level discard support is not allowed to be changed after 3376 * initial load. This would require a pool reload to trigger thin 3377 * device changes. 3378 */ 3379 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) { 3380 ti->error = "Discard support cannot be disabled once enabled"; 3381 r = -EINVAL; 3382 goto out_flags_changed; 3383 } 3384 3385 pt->pool = pool; 3386 pt->ti = ti; 3387 pt->metadata_dev = metadata_dev; 3388 pt->data_dev = data_dev; 3389 pt->low_water_blocks = low_water_blocks; 3390 pt->adjusted_pf = pt->requested_pf = pf; 3391 ti->num_flush_bios = 1; 3392 ti->limit_swap_bios = true; 3393 3394 /* 3395 * Only need to enable discards if the pool should pass 3396 * them down to the data device. The thin device's discard 3397 * processing will cause mappings to be removed from the btree. 3398 */ 3399 if (pf.discard_enabled && pf.discard_passdown) { 3400 ti->num_discard_bios = 1; 3401 /* 3402 * Setting 'discards_supported' circumvents the normal 3403 * stacking of discard limits (this keeps the pool and 3404 * thin devices' discard limits consistent). 3405 */ 3406 ti->discards_supported = true; 3407 ti->max_discard_granularity = true; 3408 } 3409 ti->private = pt; 3410 3411 r = dm_pool_register_metadata_threshold(pt->pool->pmd, 3412 calc_metadata_threshold(pt), 3413 metadata_low_callback, 3414 pool); 3415 if (r) { 3416 ti->error = "Error registering metadata threshold"; 3417 goto out_flags_changed; 3418 } 3419 3420 dm_pool_register_pre_commit_callback(pool->pmd, 3421 metadata_pre_commit_callback, pool); 3422 3423 mutex_unlock(&dm_thin_pool_table.mutex); 3424 3425 return 0; 3426 3427 out_flags_changed: 3428 __pool_dec(pool); 3429 out_free_pt: 3430 kfree(pt); 3431 out: 3432 dm_put_device(ti, data_dev); 3433 out_metadata: 3434 dm_put_device(ti, metadata_dev); 3435 out_unlock: 3436 mutex_unlock(&dm_thin_pool_table.mutex); 3437 3438 return r; 3439 } 3440 3441 static int pool_map(struct dm_target *ti, struct bio *bio) 3442 { 3443 struct pool_c *pt = ti->private; 3444 3445 /* 3446 * As this is a singleton target, ti->begin is always zero. 3447 */ 3448 bio_set_dev(bio, pt->data_dev->bdev); 3449 3450 return DM_MAPIO_REMAPPED; 3451 } 3452 3453 static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit) 3454 { 3455 int r; 3456 struct pool_c *pt = ti->private; 3457 struct pool *pool = pt->pool; 3458 sector_t data_size = ti->len; 3459 dm_block_t sb_data_size; 3460 3461 *need_commit = false; 3462 3463 (void) sector_div(data_size, pool->sectors_per_block); 3464 3465 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size); 3466 if (r) { 3467 DMERR("%s: failed to retrieve data device size", 3468 dm_device_name(pool->pool_md)); 3469 return r; 3470 } 3471 3472 if (data_size < sb_data_size) { 3473 DMERR("%s: pool target (%llu blocks) too small: expected %llu", 3474 dm_device_name(pool->pool_md), 3475 (unsigned long long)data_size, sb_data_size); 3476 return -EINVAL; 3477 3478 } else if (data_size > sb_data_size) { 3479 if (dm_pool_metadata_needs_check(pool->pmd)) { 3480 DMERR("%s: unable to grow the data device until repaired.", 3481 dm_device_name(pool->pool_md)); 3482 return 0; 3483 } 3484 3485 if (sb_data_size) 3486 DMINFO("%s: growing the data device from %llu to %llu blocks", 3487 dm_device_name(pool->pool_md), 3488 sb_data_size, (unsigned long long)data_size); 3489 r = dm_pool_resize_data_dev(pool->pmd, data_size); 3490 if (r) { 3491 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r); 3492 return r; 3493 } 3494 3495 *need_commit = true; 3496 } 3497 3498 return 0; 3499 } 3500 3501 static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit) 3502 { 3503 int r; 3504 struct pool_c *pt = ti->private; 3505 struct pool *pool = pt->pool; 3506 dm_block_t metadata_dev_size, sb_metadata_dev_size; 3507 3508 *need_commit = false; 3509 3510 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev); 3511 3512 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size); 3513 if (r) { 3514 DMERR("%s: failed to retrieve metadata device size", 3515 dm_device_name(pool->pool_md)); 3516 return r; 3517 } 3518 3519 if (metadata_dev_size < sb_metadata_dev_size) { 3520 DMERR("%s: metadata device (%llu blocks) too small: expected %llu", 3521 dm_device_name(pool->pool_md), 3522 metadata_dev_size, sb_metadata_dev_size); 3523 return -EINVAL; 3524 3525 } else if (metadata_dev_size > sb_metadata_dev_size) { 3526 if (dm_pool_metadata_needs_check(pool->pmd)) { 3527 DMERR("%s: unable to grow the metadata device until repaired.", 3528 dm_device_name(pool->pool_md)); 3529 return 0; 3530 } 3531 3532 warn_if_metadata_device_too_big(pool->md_dev); 3533 DMINFO("%s: growing the metadata device from %llu to %llu blocks", 3534 dm_device_name(pool->pool_md), 3535 sb_metadata_dev_size, metadata_dev_size); 3536 3537 if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE) 3538 set_pool_mode(pool, PM_WRITE); 3539 3540 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size); 3541 if (r) { 3542 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r); 3543 return r; 3544 } 3545 3546 *need_commit = true; 3547 } 3548 3549 return 0; 3550 } 3551 3552 /* 3553 * Retrieves the number of blocks of the data device from 3554 * the superblock and compares it to the actual device size, 3555 * thus resizing the data device in case it has grown. 3556 * 3557 * This both copes with opening preallocated data devices in the ctr 3558 * being followed by a resume 3559 * -and- 3560 * calling the resume method individually after userspace has 3561 * grown the data device in reaction to a table event. 3562 */ 3563 static int pool_preresume(struct dm_target *ti) 3564 { 3565 int r; 3566 bool need_commit1, need_commit2; 3567 struct pool_c *pt = ti->private; 3568 struct pool *pool = pt->pool; 3569 3570 /* 3571 * Take control of the pool object. 3572 */ 3573 r = bind_control_target(pool, ti); 3574 if (r) 3575 goto out; 3576 3577 r = maybe_resize_data_dev(ti, &need_commit1); 3578 if (r) 3579 goto out; 3580 3581 r = maybe_resize_metadata_dev(ti, &need_commit2); 3582 if (r) 3583 goto out; 3584 3585 if (need_commit1 || need_commit2) 3586 (void) commit(pool); 3587 out: 3588 /* 3589 * When a thin-pool is PM_FAIL, it cannot be rebuilt if 3590 * bio is in deferred list. Therefore need to return 0 3591 * to allow pool_resume() to flush IO. 3592 */ 3593 if (r && get_pool_mode(pool) == PM_FAIL) 3594 r = 0; 3595 3596 return r; 3597 } 3598 3599 static void pool_suspend_active_thins(struct pool *pool) 3600 { 3601 struct thin_c *tc; 3602 3603 /* Suspend all active thin devices */ 3604 tc = get_first_thin(pool); 3605 while (tc) { 3606 dm_internal_suspend_noflush(tc->thin_md); 3607 tc = get_next_thin(pool, tc); 3608 } 3609 } 3610 3611 static void pool_resume_active_thins(struct pool *pool) 3612 { 3613 struct thin_c *tc; 3614 3615 /* Resume all active thin devices */ 3616 tc = get_first_thin(pool); 3617 while (tc) { 3618 dm_internal_resume(tc->thin_md); 3619 tc = get_next_thin(pool, tc); 3620 } 3621 } 3622 3623 static void pool_resume(struct dm_target *ti) 3624 { 3625 struct pool_c *pt = ti->private; 3626 struct pool *pool = pt->pool; 3627 3628 /* 3629 * Must requeue active_thins' bios and then resume 3630 * active_thins _before_ clearing 'suspend' flag. 3631 */ 3632 requeue_bios(pool); 3633 pool_resume_active_thins(pool); 3634 3635 spin_lock_irq(&pool->lock); 3636 pool->low_water_triggered = false; 3637 pool->suspended = false; 3638 spin_unlock_irq(&pool->lock); 3639 3640 do_waker(&pool->waker.work); 3641 } 3642 3643 static void pool_presuspend(struct dm_target *ti) 3644 { 3645 struct pool_c *pt = ti->private; 3646 struct pool *pool = pt->pool; 3647 3648 spin_lock_irq(&pool->lock); 3649 pool->suspended = true; 3650 spin_unlock_irq(&pool->lock); 3651 3652 pool_suspend_active_thins(pool); 3653 } 3654 3655 static void pool_presuspend_undo(struct dm_target *ti) 3656 { 3657 struct pool_c *pt = ti->private; 3658 struct pool *pool = pt->pool; 3659 3660 pool_resume_active_thins(pool); 3661 3662 spin_lock_irq(&pool->lock); 3663 pool->suspended = false; 3664 spin_unlock_irq(&pool->lock); 3665 } 3666 3667 static void pool_postsuspend(struct dm_target *ti) 3668 { 3669 struct pool_c *pt = ti->private; 3670 struct pool *pool = pt->pool; 3671 3672 cancel_delayed_work_sync(&pool->waker); 3673 cancel_delayed_work_sync(&pool->no_space_timeout); 3674 flush_workqueue(pool->wq); 3675 (void) commit(pool); 3676 } 3677 3678 static int check_arg_count(unsigned int argc, unsigned int args_required) 3679 { 3680 if (argc != args_required) { 3681 DMWARN("Message received with %u arguments instead of %u.", 3682 argc, args_required); 3683 return -EINVAL; 3684 } 3685 3686 return 0; 3687 } 3688 3689 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning) 3690 { 3691 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) && 3692 *dev_id <= MAX_DEV_ID) 3693 return 0; 3694 3695 if (warning) 3696 DMWARN("Message received with invalid device id: %s", arg); 3697 3698 return -EINVAL; 3699 } 3700 3701 static int process_create_thin_mesg(unsigned int argc, char **argv, struct pool *pool) 3702 { 3703 dm_thin_id dev_id; 3704 int r; 3705 3706 r = check_arg_count(argc, 2); 3707 if (r) 3708 return r; 3709 3710 r = read_dev_id(argv[1], &dev_id, 1); 3711 if (r) 3712 return r; 3713 3714 r = dm_pool_create_thin(pool->pmd, dev_id); 3715 if (r) { 3716 DMWARN("Creation of new thinly-provisioned device with id %s failed.", 3717 argv[1]); 3718 return r; 3719 } 3720 3721 return 0; 3722 } 3723 3724 static int process_create_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3725 { 3726 dm_thin_id dev_id; 3727 dm_thin_id origin_dev_id; 3728 int r; 3729 3730 r = check_arg_count(argc, 3); 3731 if (r) 3732 return r; 3733 3734 r = read_dev_id(argv[1], &dev_id, 1); 3735 if (r) 3736 return r; 3737 3738 r = read_dev_id(argv[2], &origin_dev_id, 1); 3739 if (r) 3740 return r; 3741 3742 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id); 3743 if (r) { 3744 DMWARN("Creation of new snapshot %s of device %s failed.", 3745 argv[1], argv[2]); 3746 return r; 3747 } 3748 3749 return 0; 3750 } 3751 3752 static int process_delete_mesg(unsigned int argc, char **argv, struct pool *pool) 3753 { 3754 dm_thin_id dev_id; 3755 int r; 3756 3757 r = check_arg_count(argc, 2); 3758 if (r) 3759 return r; 3760 3761 r = read_dev_id(argv[1], &dev_id, 1); 3762 if (r) 3763 return r; 3764 3765 r = dm_pool_delete_thin_device(pool->pmd, dev_id); 3766 if (r) 3767 DMWARN("Deletion of thin device %s failed.", argv[1]); 3768 3769 return r; 3770 } 3771 3772 static int process_set_transaction_id_mesg(unsigned int argc, char **argv, struct pool *pool) 3773 { 3774 dm_thin_id old_id, new_id; 3775 int r; 3776 3777 r = check_arg_count(argc, 3); 3778 if (r) 3779 return r; 3780 3781 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) { 3782 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]); 3783 return -EINVAL; 3784 } 3785 3786 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) { 3787 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]); 3788 return -EINVAL; 3789 } 3790 3791 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id); 3792 if (r) { 3793 DMWARN("Failed to change transaction id from %s to %s.", 3794 argv[1], argv[2]); 3795 return r; 3796 } 3797 3798 return 0; 3799 } 3800 3801 static int process_reserve_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3802 { 3803 int r; 3804 3805 r = check_arg_count(argc, 1); 3806 if (r) 3807 return r; 3808 3809 (void) commit(pool); 3810 3811 r = dm_pool_reserve_metadata_snap(pool->pmd); 3812 if (r) 3813 DMWARN("reserve_metadata_snap message failed."); 3814 3815 return r; 3816 } 3817 3818 static int process_release_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3819 { 3820 int r; 3821 3822 r = check_arg_count(argc, 1); 3823 if (r) 3824 return r; 3825 3826 r = dm_pool_release_metadata_snap(pool->pmd); 3827 if (r) 3828 DMWARN("release_metadata_snap message failed."); 3829 3830 return r; 3831 } 3832 3833 /* 3834 * Messages supported: 3835 * create_thin <dev_id> 3836 * create_snap <dev_id> <origin_id> 3837 * delete <dev_id> 3838 * set_transaction_id <current_trans_id> <new_trans_id> 3839 * reserve_metadata_snap 3840 * release_metadata_snap 3841 */ 3842 static int pool_message(struct dm_target *ti, unsigned int argc, char **argv, 3843 char *result, unsigned int maxlen) 3844 { 3845 int r = -EINVAL; 3846 struct pool_c *pt = ti->private; 3847 struct pool *pool = pt->pool; 3848 3849 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) { 3850 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode", 3851 dm_device_name(pool->pool_md)); 3852 return -EOPNOTSUPP; 3853 } 3854 3855 if (!strcasecmp(argv[0], "create_thin")) 3856 r = process_create_thin_mesg(argc, argv, pool); 3857 3858 else if (!strcasecmp(argv[0], "create_snap")) 3859 r = process_create_snap_mesg(argc, argv, pool); 3860 3861 else if (!strcasecmp(argv[0], "delete")) 3862 r = process_delete_mesg(argc, argv, pool); 3863 3864 else if (!strcasecmp(argv[0], "set_transaction_id")) 3865 r = process_set_transaction_id_mesg(argc, argv, pool); 3866 3867 else if (!strcasecmp(argv[0], "reserve_metadata_snap")) 3868 r = process_reserve_metadata_snap_mesg(argc, argv, pool); 3869 3870 else if (!strcasecmp(argv[0], "release_metadata_snap")) 3871 r = process_release_metadata_snap_mesg(argc, argv, pool); 3872 3873 else 3874 DMWARN("Unrecognised thin pool target message received: %s", argv[0]); 3875 3876 if (!r) 3877 (void) commit(pool); 3878 3879 return r; 3880 } 3881 3882 static void emit_flags(struct pool_features *pf, char *result, 3883 unsigned int sz, unsigned int maxlen) 3884 { 3885 unsigned int count = !pf->zero_new_blocks + !pf->discard_enabled + 3886 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) + 3887 pf->error_if_no_space; 3888 DMEMIT("%u ", count); 3889 3890 if (!pf->zero_new_blocks) 3891 DMEMIT("skip_block_zeroing "); 3892 3893 if (!pf->discard_enabled) 3894 DMEMIT("ignore_discard "); 3895 3896 if (!pf->discard_passdown) 3897 DMEMIT("no_discard_passdown "); 3898 3899 if (pf->mode == PM_READ_ONLY) 3900 DMEMIT("read_only "); 3901 3902 if (pf->error_if_no_space) 3903 DMEMIT("error_if_no_space "); 3904 } 3905 3906 /* 3907 * Status line is: 3908 * <transaction id> <used metadata sectors>/<total metadata sectors> 3909 * <used data sectors>/<total data sectors> <held metadata root> 3910 * <pool mode> <discard config> <no space config> <needs_check> 3911 */ 3912 static void pool_status(struct dm_target *ti, status_type_t type, 3913 unsigned int status_flags, char *result, unsigned int maxlen) 3914 { 3915 int r; 3916 unsigned int sz = 0; 3917 uint64_t transaction_id; 3918 dm_block_t nr_free_blocks_data; 3919 dm_block_t nr_free_blocks_metadata; 3920 dm_block_t nr_blocks_data; 3921 dm_block_t nr_blocks_metadata; 3922 dm_block_t held_root; 3923 enum pool_mode mode; 3924 char buf[BDEVNAME_SIZE]; 3925 char buf2[BDEVNAME_SIZE]; 3926 struct pool_c *pt = ti->private; 3927 struct pool *pool = pt->pool; 3928 3929 switch (type) { 3930 case STATUSTYPE_INFO: 3931 if (get_pool_mode(pool) == PM_FAIL) { 3932 DMEMIT("Fail"); 3933 break; 3934 } 3935 3936 /* Commit to ensure statistics aren't out-of-date */ 3937 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3938 (void) commit(pool); 3939 3940 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id); 3941 if (r) { 3942 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d", 3943 dm_device_name(pool->pool_md), r); 3944 goto err; 3945 } 3946 3947 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata); 3948 if (r) { 3949 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d", 3950 dm_device_name(pool->pool_md), r); 3951 goto err; 3952 } 3953 3954 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); 3955 if (r) { 3956 DMERR("%s: dm_pool_get_metadata_dev_size returned %d", 3957 dm_device_name(pool->pool_md), r); 3958 goto err; 3959 } 3960 3961 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data); 3962 if (r) { 3963 DMERR("%s: dm_pool_get_free_block_count returned %d", 3964 dm_device_name(pool->pool_md), r); 3965 goto err; 3966 } 3967 3968 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); 3969 if (r) { 3970 DMERR("%s: dm_pool_get_data_dev_size returned %d", 3971 dm_device_name(pool->pool_md), r); 3972 goto err; 3973 } 3974 3975 r = dm_pool_get_metadata_snap(pool->pmd, &held_root); 3976 if (r) { 3977 DMERR("%s: dm_pool_get_metadata_snap returned %d", 3978 dm_device_name(pool->pool_md), r); 3979 goto err; 3980 } 3981 3982 DMEMIT("%llu %llu/%llu %llu/%llu ", 3983 (unsigned long long)transaction_id, 3984 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3985 (unsigned long long)nr_blocks_metadata, 3986 (unsigned long long)(nr_blocks_data - nr_free_blocks_data), 3987 (unsigned long long)nr_blocks_data); 3988 3989 if (held_root) 3990 DMEMIT("%llu ", held_root); 3991 else 3992 DMEMIT("- "); 3993 3994 mode = get_pool_mode(pool); 3995 if (mode == PM_OUT_OF_DATA_SPACE) 3996 DMEMIT("out_of_data_space "); 3997 else if (is_read_only_pool_mode(mode)) 3998 DMEMIT("ro "); 3999 else 4000 DMEMIT("rw "); 4001 4002 if (!pool->pf.discard_enabled) 4003 DMEMIT("ignore_discard "); 4004 else if (pool->pf.discard_passdown) 4005 DMEMIT("discard_passdown "); 4006 else 4007 DMEMIT("no_discard_passdown "); 4008 4009 if (pool->pf.error_if_no_space) 4010 DMEMIT("error_if_no_space "); 4011 else 4012 DMEMIT("queue_if_no_space "); 4013 4014 if (dm_pool_metadata_needs_check(pool->pmd)) 4015 DMEMIT("needs_check "); 4016 else 4017 DMEMIT("- "); 4018 4019 DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt)); 4020 4021 break; 4022 4023 case STATUSTYPE_TABLE: 4024 DMEMIT("%s %s %lu %llu ", 4025 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev), 4026 format_dev_t(buf2, pt->data_dev->bdev->bd_dev), 4027 (unsigned long)pool->sectors_per_block, 4028 (unsigned long long)pt->low_water_blocks); 4029 emit_flags(&pt->requested_pf, result, sz, maxlen); 4030 break; 4031 4032 case STATUSTYPE_IMA: 4033 *result = '\0'; 4034 break; 4035 } 4036 return; 4037 4038 err: 4039 DMEMIT("Error"); 4040 } 4041 4042 static int pool_iterate_devices(struct dm_target *ti, 4043 iterate_devices_callout_fn fn, void *data) 4044 { 4045 struct pool_c *pt = ti->private; 4046 4047 return fn(ti, pt->data_dev, 0, ti->len, data); 4048 } 4049 4050 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) 4051 { 4052 struct pool_c *pt = ti->private; 4053 struct pool *pool = pt->pool; 4054 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 4055 4056 /* 4057 * If max_sectors is smaller than pool->sectors_per_block adjust it 4058 * to the highest possible power-of-2 factor of pool->sectors_per_block. 4059 * This is especially beneficial when the pool's data device is a RAID 4060 * device that has a full stripe width that matches pool->sectors_per_block 4061 * -- because even though partial RAID stripe-sized IOs will be issued to a 4062 * single RAID stripe; when aggregated they will end on a full RAID stripe 4063 * boundary.. which avoids additional partial RAID stripe writes cascading 4064 */ 4065 if (limits->max_sectors < pool->sectors_per_block) { 4066 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) { 4067 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0) 4068 limits->max_sectors--; 4069 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors); 4070 } 4071 } 4072 4073 /* 4074 * If the system-determined stacked limits are compatible with the 4075 * pool's blocksize (io_opt is a factor) do not override them. 4076 */ 4077 if (io_opt_sectors < pool->sectors_per_block || 4078 !is_factor(io_opt_sectors, pool->sectors_per_block)) { 4079 if (is_factor(pool->sectors_per_block, limits->max_sectors)) 4080 limits->io_min = limits->max_sectors << SECTOR_SHIFT; 4081 else 4082 limits->io_min = pool->sectors_per_block << SECTOR_SHIFT; 4083 limits->io_opt = pool->sectors_per_block << SECTOR_SHIFT; 4084 } 4085 4086 /* 4087 * pt->adjusted_pf is a staging area for the actual features to use. 4088 * They get transferred to the live pool in bind_control_target() 4089 * called from pool_preresume(). 4090 */ 4091 4092 if (pt->adjusted_pf.discard_enabled) { 4093 disable_discard_passdown_if_not_supported(pt); 4094 if (!pt->adjusted_pf.discard_passdown) 4095 limits->max_hw_discard_sectors = 0; 4096 /* 4097 * The pool uses the same discard limits as the underlying data 4098 * device. DM core has already set this up. 4099 */ 4100 } else { 4101 /* 4102 * Must explicitly disallow stacking discard limits otherwise the 4103 * block layer will stack them if pool's data device has support. 4104 */ 4105 limits->discard_granularity = 0; 4106 } 4107 } 4108 4109 static struct target_type pool_target = { 4110 .name = "thin-pool", 4111 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | 4112 DM_TARGET_IMMUTABLE | DM_TARGET_PASSES_CRYPTO, 4113 .version = {1, 24, 0}, 4114 .module = THIS_MODULE, 4115 .ctr = pool_ctr, 4116 .dtr = pool_dtr, 4117 .map = pool_map, 4118 .presuspend = pool_presuspend, 4119 .presuspend_undo = pool_presuspend_undo, 4120 .postsuspend = pool_postsuspend, 4121 .preresume = pool_preresume, 4122 .resume = pool_resume, 4123 .message = pool_message, 4124 .status = pool_status, 4125 .iterate_devices = pool_iterate_devices, 4126 .io_hints = pool_io_hints, 4127 }; 4128 4129 /* 4130 *-------------------------------------------------------------- 4131 * Thin target methods 4132 *-------------------------------------------------------------- 4133 */ 4134 static void thin_get(struct thin_c *tc) 4135 { 4136 refcount_inc(&tc->refcount); 4137 } 4138 4139 static void thin_put(struct thin_c *tc) 4140 { 4141 if (refcount_dec_and_test(&tc->refcount)) 4142 complete(&tc->can_destroy); 4143 } 4144 4145 static void thin_dtr(struct dm_target *ti) 4146 { 4147 struct thin_c *tc = ti->private; 4148 4149 spin_lock_irq(&tc->pool->lock); 4150 list_del_rcu(&tc->list); 4151 spin_unlock_irq(&tc->pool->lock); 4152 synchronize_rcu(); 4153 4154 thin_put(tc); 4155 wait_for_completion(&tc->can_destroy); 4156 4157 mutex_lock(&dm_thin_pool_table.mutex); 4158 4159 __pool_dec(tc->pool); 4160 dm_pool_close_thin_device(tc->td); 4161 dm_put_device(ti, tc->pool_dev); 4162 if (tc->origin_dev) 4163 dm_put_device(ti, tc->origin_dev); 4164 kfree(tc); 4165 4166 mutex_unlock(&dm_thin_pool_table.mutex); 4167 } 4168 4169 /* 4170 * Thin target parameters: 4171 * 4172 * <pool_dev> <dev_id> [origin_dev] 4173 * 4174 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool) 4175 * dev_id: the internal device identifier 4176 * origin_dev: a device external to the pool that should act as the origin 4177 * 4178 * If the pool device has discards disabled, they get disabled for the thin 4179 * device as well. 4180 */ 4181 static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 4182 { 4183 int r; 4184 struct thin_c *tc; 4185 struct dm_dev *pool_dev, *origin_dev; 4186 struct mapped_device *pool_md; 4187 4188 mutex_lock(&dm_thin_pool_table.mutex); 4189 4190 if (argc != 2 && argc != 3) { 4191 ti->error = "Invalid argument count"; 4192 r = -EINVAL; 4193 goto out_unlock; 4194 } 4195 4196 tc = ti->private = kzalloc_obj(*tc); 4197 if (!tc) { 4198 ti->error = "Out of memory"; 4199 r = -ENOMEM; 4200 goto out_unlock; 4201 } 4202 tc->thin_md = dm_table_get_md(ti->table); 4203 spin_lock_init(&tc->lock); 4204 INIT_LIST_HEAD(&tc->deferred_cells); 4205 bio_list_init(&tc->deferred_bio_list); 4206 bio_list_init(&tc->retry_on_resume_list); 4207 tc->sort_bio_list = RB_ROOT; 4208 4209 if (argc == 3) { 4210 if (!strcmp(argv[0], argv[2])) { 4211 ti->error = "Error setting origin device"; 4212 r = -EINVAL; 4213 goto bad_origin_dev; 4214 } 4215 4216 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &origin_dev); 4217 if (r) { 4218 ti->error = "Error opening origin device"; 4219 goto bad_origin_dev; 4220 } 4221 tc->origin_dev = origin_dev; 4222 } 4223 4224 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev); 4225 if (r) { 4226 ti->error = "Error opening pool device"; 4227 goto bad_pool_dev; 4228 } 4229 tc->pool_dev = pool_dev; 4230 4231 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) { 4232 ti->error = "Invalid device id"; 4233 r = -EINVAL; 4234 goto bad_common; 4235 } 4236 4237 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev); 4238 if (!pool_md) { 4239 ti->error = "Couldn't get pool mapped device"; 4240 r = -EINVAL; 4241 goto bad_common; 4242 } 4243 4244 tc->pool = __pool_table_lookup(pool_md); 4245 if (!tc->pool) { 4246 ti->error = "Couldn't find pool object"; 4247 r = -EINVAL; 4248 goto bad_pool_lookup; 4249 } 4250 __pool_inc(tc->pool); 4251 4252 if (get_pool_mode(tc->pool) == PM_FAIL) { 4253 ti->error = "Couldn't open thin device, Pool is in fail mode"; 4254 r = -EINVAL; 4255 goto bad_pool; 4256 } 4257 4258 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td); 4259 if (r) { 4260 ti->error = "Couldn't open thin internal device"; 4261 goto bad_pool; 4262 } 4263 4264 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block); 4265 if (r) 4266 goto bad; 4267 4268 ti->num_flush_bios = 1; 4269 ti->limit_swap_bios = true; 4270 ti->flush_supported = true; 4271 ti->accounts_remapped_io = true; 4272 ti->per_io_data_size = sizeof(struct dm_thin_endio_hook); 4273 4274 /* In case the pool supports discards, pass them on. */ 4275 if (tc->pool->pf.discard_enabled) { 4276 ti->discards_supported = true; 4277 ti->num_discard_bios = 1; 4278 ti->max_discard_granularity = true; 4279 } 4280 4281 mutex_unlock(&dm_thin_pool_table.mutex); 4282 4283 spin_lock_irq(&tc->pool->lock); 4284 if (tc->pool->suspended) { 4285 spin_unlock_irq(&tc->pool->lock); 4286 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */ 4287 ti->error = "Unable to activate thin device while pool is suspended"; 4288 r = -EINVAL; 4289 goto bad; 4290 } 4291 refcount_set(&tc->refcount, 1); 4292 init_completion(&tc->can_destroy); 4293 list_add_tail_rcu(&tc->list, &tc->pool->active_thins); 4294 spin_unlock_irq(&tc->pool->lock); 4295 /* 4296 * This synchronize_rcu() call is needed here otherwise we risk a 4297 * wake_worker() call finding no bios to process (because the newly 4298 * added tc isn't yet visible). So this reduces latency since we 4299 * aren't then dependent on the periodic commit to wake_worker(). 4300 */ 4301 synchronize_rcu(); 4302 4303 dm_put(pool_md); 4304 4305 return 0; 4306 4307 bad: 4308 dm_pool_close_thin_device(tc->td); 4309 bad_pool: 4310 __pool_dec(tc->pool); 4311 bad_pool_lookup: 4312 dm_put(pool_md); 4313 bad_common: 4314 dm_put_device(ti, tc->pool_dev); 4315 bad_pool_dev: 4316 if (tc->origin_dev) 4317 dm_put_device(ti, tc->origin_dev); 4318 bad_origin_dev: 4319 kfree(tc); 4320 out_unlock: 4321 mutex_unlock(&dm_thin_pool_table.mutex); 4322 4323 return r; 4324 } 4325 4326 static int thin_map(struct dm_target *ti, struct bio *bio) 4327 { 4328 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 4329 4330 return thin_bio_map(ti, bio); 4331 } 4332 4333 static int thin_endio(struct dm_target *ti, struct bio *bio, 4334 blk_status_t *err) 4335 { 4336 unsigned long flags; 4337 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 4338 struct list_head work; 4339 struct dm_thin_new_mapping *m, *tmp; 4340 struct pool *pool = h->tc->pool; 4341 4342 if (h->shared_read_entry) { 4343 INIT_LIST_HEAD(&work); 4344 dm_deferred_entry_dec(h->shared_read_entry, &work); 4345 4346 spin_lock_irqsave(&pool->lock, flags); 4347 list_for_each_entry_safe(m, tmp, &work, list) { 4348 list_del(&m->list); 4349 __complete_mapping_preparation(m); 4350 } 4351 spin_unlock_irqrestore(&pool->lock, flags); 4352 } 4353 4354 if (h->all_io_entry) { 4355 INIT_LIST_HEAD(&work); 4356 dm_deferred_entry_dec(h->all_io_entry, &work); 4357 if (!list_empty(&work)) { 4358 spin_lock_irqsave(&pool->lock, flags); 4359 list_for_each_entry_safe(m, tmp, &work, list) 4360 list_add_tail(&m->list, &pool->prepared_discards); 4361 spin_unlock_irqrestore(&pool->lock, flags); 4362 wake_worker(pool); 4363 } 4364 } 4365 4366 if (h->cell) 4367 cell_defer_no_holder(h->tc, h->cell); 4368 4369 return DM_ENDIO_DONE; 4370 } 4371 4372 static void thin_presuspend(struct dm_target *ti) 4373 { 4374 struct thin_c *tc = ti->private; 4375 4376 if (dm_noflush_suspending(ti)) 4377 noflush_work(tc, do_noflush_start); 4378 } 4379 4380 static void thin_postsuspend(struct dm_target *ti) 4381 { 4382 struct thin_c *tc = ti->private; 4383 4384 if (dm_noflush_suspending(ti)) 4385 noflush_work(tc, do_noflush_stop); 4386 } 4387 4388 static int thin_preresume(struct dm_target *ti) 4389 { 4390 struct thin_c *tc = ti->private; 4391 4392 if (tc->origin_dev) 4393 tc->origin_size = get_dev_size(tc->origin_dev->bdev); 4394 4395 return 0; 4396 } 4397 4398 /* 4399 * <nr mapped sectors> <highest mapped sector> 4400 */ 4401 static void thin_status(struct dm_target *ti, status_type_t type, 4402 unsigned int status_flags, char *result, unsigned int maxlen) 4403 { 4404 int r; 4405 ssize_t sz = 0; 4406 dm_block_t mapped, highest; 4407 char buf[BDEVNAME_SIZE]; 4408 struct thin_c *tc = ti->private; 4409 4410 if (get_pool_mode(tc->pool) == PM_FAIL) { 4411 DMEMIT("Fail"); 4412 return; 4413 } 4414 4415 if (!tc->td) 4416 DMEMIT("-"); 4417 else { 4418 switch (type) { 4419 case STATUSTYPE_INFO: 4420 r = dm_thin_get_mapped_count(tc->td, &mapped); 4421 if (r) { 4422 DMERR("dm_thin_get_mapped_count returned %d", r); 4423 goto err; 4424 } 4425 4426 r = dm_thin_get_highest_mapped_block(tc->td, &highest); 4427 if (r < 0) { 4428 DMERR("dm_thin_get_highest_mapped_block returned %d", r); 4429 goto err; 4430 } 4431 4432 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); 4433 if (r) 4434 DMEMIT("%llu", ((highest + 1) * 4435 tc->pool->sectors_per_block) - 1); 4436 else 4437 DMEMIT("-"); 4438 break; 4439 4440 case STATUSTYPE_TABLE: 4441 DMEMIT("%s %lu", 4442 format_dev_t(buf, tc->pool_dev->bdev->bd_dev), 4443 (unsigned long) tc->dev_id); 4444 if (tc->origin_dev) 4445 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev)); 4446 break; 4447 4448 case STATUSTYPE_IMA: 4449 *result = '\0'; 4450 break; 4451 } 4452 } 4453 4454 return; 4455 4456 err: 4457 DMEMIT("Error"); 4458 } 4459 4460 static int thin_iterate_devices(struct dm_target *ti, 4461 iterate_devices_callout_fn fn, void *data) 4462 { 4463 sector_t blocks; 4464 struct thin_c *tc = ti->private; 4465 struct pool *pool = tc->pool; 4466 4467 /* 4468 * We can't call dm_pool_get_data_dev_size() since that blocks. So 4469 * we follow a more convoluted path through to the pool's target. 4470 */ 4471 if (!pool->ti) 4472 return 0; /* nothing is bound */ 4473 4474 blocks = pool->ti->len; 4475 (void) sector_div(blocks, pool->sectors_per_block); 4476 if (blocks) 4477 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data); 4478 4479 return 0; 4480 } 4481 4482 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits) 4483 { 4484 struct thin_c *tc = ti->private; 4485 struct pool *pool = tc->pool; 4486 4487 if (pool->pf.discard_enabled) { 4488 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; 4489 limits->max_hw_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE; 4490 } 4491 } 4492 4493 static struct target_type thin_target = { 4494 .name = "thin", 4495 .features = DM_TARGET_PASSES_CRYPTO, 4496 .version = {1, 24, 0}, 4497 .module = THIS_MODULE, 4498 .ctr = thin_ctr, 4499 .dtr = thin_dtr, 4500 .map = thin_map, 4501 .end_io = thin_endio, 4502 .preresume = thin_preresume, 4503 .presuspend = thin_presuspend, 4504 .postsuspend = thin_postsuspend, 4505 .status = thin_status, 4506 .iterate_devices = thin_iterate_devices, 4507 .io_hints = thin_io_hints, 4508 }; 4509 4510 /*----------------------------------------------------------------*/ 4511 4512 static int __init dm_thin_init(void) 4513 { 4514 int r = -ENOMEM; 4515 4516 pool_table_init(); 4517 4518 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0); 4519 if (!_new_mapping_cache) 4520 return r; 4521 4522 r = dm_register_target(&thin_target); 4523 if (r) 4524 goto bad_new_mapping_cache; 4525 4526 r = dm_register_target(&pool_target); 4527 if (r) 4528 goto bad_thin_target; 4529 4530 return 0; 4531 4532 bad_thin_target: 4533 dm_unregister_target(&thin_target); 4534 bad_new_mapping_cache: 4535 kmem_cache_destroy(_new_mapping_cache); 4536 4537 return r; 4538 } 4539 4540 static void dm_thin_exit(void) 4541 { 4542 dm_unregister_target(&thin_target); 4543 dm_unregister_target(&pool_target); 4544 4545 kmem_cache_destroy(_new_mapping_cache); 4546 4547 pool_table_exit(); 4548 } 4549 4550 module_init(dm_thin_init); 4551 module_exit(dm_thin_exit); 4552 4553 module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644); 4554 MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds"); 4555 4556 MODULE_DESCRIPTION(DM_NAME " thin provisioning target"); 4557 MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>"); 4558 MODULE_LICENSE("GPL"); 4559