1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011-2012 Red Hat UK. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm-thin-metadata.h" 9 #include "dm-bio-prison-v1.h" 10 #include "dm.h" 11 12 #include <linux/device-mapper.h> 13 #include <linux/dm-io.h> 14 #include <linux/dm-kcopyd.h> 15 #include <linux/jiffies.h> 16 #include <linux/log2.h> 17 #include <linux/list.h> 18 #include <linux/rculist.h> 19 #include <linux/init.h> 20 #include <linux/module.h> 21 #include <linux/slab.h> 22 #include <linux/vmalloc.h> 23 #include <linux/sort.h> 24 #include <linux/rbtree.h> 25 26 #define DM_MSG_PREFIX "thin" 27 28 /* 29 * Tunable constants 30 */ 31 #define ENDIO_HOOK_POOL_SIZE 1024 32 #define MAPPING_POOL_SIZE 1024 33 #define COMMIT_PERIOD HZ 34 #define NO_SPACE_TIMEOUT_SECS 60 35 36 static unsigned int no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS; 37 38 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 39 "A percentage of time allocated for copy on write"); 40 41 /* 42 * The block size of the device holding pool data must be 43 * between 64KB and 1GB. 44 */ 45 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT) 46 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 47 48 /* 49 * Device id is restricted to 24 bits. 50 */ 51 #define MAX_DEV_ID ((1 << 24) - 1) 52 53 /* 54 * How do we handle breaking sharing of data blocks? 55 * ================================================= 56 * 57 * We use a standard copy-on-write btree to store the mappings for the 58 * devices (note I'm talking about copy-on-write of the metadata here, not 59 * the data). When you take an internal snapshot you clone the root node 60 * of the origin btree. After this there is no concept of an origin or a 61 * snapshot. They are just two device trees that happen to point to the 62 * same data blocks. 63 * 64 * When we get a write in we decide if it's to a shared data block using 65 * some timestamp magic. If it is, we have to break sharing. 66 * 67 * Let's say we write to a shared block in what was the origin. The 68 * steps are: 69 * 70 * i) plug io further to this physical block. (see bio_prison code). 71 * 72 * ii) quiesce any read io to that shared data block. Obviously 73 * including all devices that share this block. (see dm_deferred_set code) 74 * 75 * iii) copy the data block to a newly allocate block. This step can be 76 * missed out if the io covers the block. (schedule_copy). 77 * 78 * iv) insert the new mapping into the origin's btree 79 * (process_prepared_mapping). This act of inserting breaks some 80 * sharing of btree nodes between the two devices. Breaking sharing only 81 * effects the btree of that specific device. Btrees for the other 82 * devices that share the block never change. The btree for the origin 83 * device as it was after the last commit is untouched, ie. we're using 84 * persistent data structures in the functional programming sense. 85 * 86 * v) unplug io to this physical block, including the io that triggered 87 * the breaking of sharing. 88 * 89 * Steps (ii) and (iii) occur in parallel. 90 * 91 * The metadata _doesn't_ need to be committed before the io continues. We 92 * get away with this because the io is always written to a _new_ block. 93 * If there's a crash, then: 94 * 95 * - The origin mapping will point to the old origin block (the shared 96 * one). This will contain the data as it was before the io that triggered 97 * the breaking of sharing came in. 98 * 99 * - The snap mapping still points to the old block. As it would after 100 * the commit. 101 * 102 * The downside of this scheme is the timestamp magic isn't perfect, and 103 * will continue to think that data block in the snapshot device is shared 104 * even after the write to the origin has broken sharing. I suspect data 105 * blocks will typically be shared by many different devices, so we're 106 * breaking sharing n + 1 times, rather than n, where n is the number of 107 * devices that reference this data block. At the moment I think the 108 * benefits far, far outweigh the disadvantages. 109 */ 110 111 /*----------------------------------------------------------------*/ 112 113 /* 114 * Key building. 115 */ 116 enum lock_space { 117 VIRTUAL, 118 PHYSICAL 119 }; 120 121 static bool build_key(struct dm_thin_device *td, enum lock_space ls, 122 dm_block_t b, dm_block_t e, struct dm_cell_key *key) 123 { 124 key->virtual = (ls == VIRTUAL); 125 key->dev = dm_thin_dev_id(td); 126 key->block_begin = b; 127 key->block_end = e; 128 129 return dm_cell_key_has_valid_range(key); 130 } 131 132 static void build_data_key(struct dm_thin_device *td, dm_block_t b, 133 struct dm_cell_key *key) 134 { 135 (void) build_key(td, PHYSICAL, b, b + 1llu, key); 136 } 137 138 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b, 139 struct dm_cell_key *key) 140 { 141 (void) build_key(td, VIRTUAL, b, b + 1llu, key); 142 } 143 144 /*----------------------------------------------------------------*/ 145 146 #define THROTTLE_THRESHOLD (1 * HZ) 147 148 struct throttle { 149 struct rw_semaphore lock; 150 unsigned long threshold; 151 bool throttle_applied; 152 }; 153 154 static void throttle_init(struct throttle *t) 155 { 156 init_rwsem(&t->lock); 157 t->throttle_applied = false; 158 } 159 160 static void throttle_work_start(struct throttle *t) 161 { 162 t->threshold = jiffies + THROTTLE_THRESHOLD; 163 } 164 165 static void throttle_work_update(struct throttle *t) 166 { 167 if (!t->throttle_applied && time_is_before_jiffies(t->threshold)) { 168 down_write(&t->lock); 169 t->throttle_applied = true; 170 } 171 } 172 173 static void throttle_work_complete(struct throttle *t) 174 { 175 if (t->throttle_applied) { 176 t->throttle_applied = false; 177 up_write(&t->lock); 178 } 179 } 180 181 static void throttle_lock(struct throttle *t) 182 { 183 down_read(&t->lock); 184 } 185 186 static void throttle_unlock(struct throttle *t) 187 { 188 up_read(&t->lock); 189 } 190 191 /*----------------------------------------------------------------*/ 192 193 /* 194 * A pool device ties together a metadata device and a data device. It 195 * also provides the interface for creating and destroying internal 196 * devices. 197 */ 198 struct dm_thin_new_mapping; 199 200 /* 201 * The pool runs in various modes. Ordered in degraded order for comparisons. 202 */ 203 enum pool_mode { 204 PM_WRITE, /* metadata may be changed */ 205 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */ 206 207 /* 208 * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY. 209 */ 210 PM_OUT_OF_METADATA_SPACE, 211 PM_READ_ONLY, /* metadata may not be changed */ 212 213 PM_FAIL, /* all I/O fails */ 214 }; 215 216 struct pool_features { 217 enum pool_mode mode; 218 219 bool zero_new_blocks:1; 220 bool discard_enabled:1; 221 bool discard_passdown:1; 222 bool error_if_no_space:1; 223 }; 224 225 struct thin_c; 226 typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio); 227 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell); 228 typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m); 229 230 #define CELL_SORT_ARRAY_SIZE 8192 231 232 struct pool { 233 struct list_head list; 234 struct dm_target *ti; /* Only set if a pool target is bound */ 235 236 struct mapped_device *pool_md; 237 struct block_device *data_dev; 238 struct block_device *md_dev; 239 struct dm_pool_metadata *pmd; 240 241 dm_block_t low_water_blocks; 242 uint32_t sectors_per_block; 243 int sectors_per_block_shift; 244 245 struct pool_features pf; 246 bool low_water_triggered:1; /* A dm event has been sent */ 247 bool suspended:1; 248 bool out_of_data_space:1; 249 250 struct dm_bio_prison *prison; 251 struct dm_kcopyd_client *copier; 252 253 struct work_struct worker; 254 struct workqueue_struct *wq; 255 struct throttle throttle; 256 struct delayed_work waker; 257 struct delayed_work no_space_timeout; 258 259 unsigned long last_commit_jiffies; 260 unsigned int ref_count; 261 262 spinlock_t lock; 263 struct bio_list deferred_flush_bios; 264 struct bio_list deferred_flush_completions; 265 struct list_head prepared_mappings; 266 struct list_head prepared_discards; 267 struct list_head prepared_discards_pt2; 268 struct list_head active_thins; 269 270 struct dm_deferred_set *shared_read_ds; 271 struct dm_deferred_set *all_io_ds; 272 273 struct dm_thin_new_mapping *next_mapping; 274 275 process_bio_fn process_bio; 276 process_bio_fn process_discard; 277 278 process_cell_fn process_cell; 279 process_cell_fn process_discard_cell; 280 281 process_mapping_fn process_prepared_mapping; 282 process_mapping_fn process_prepared_discard; 283 process_mapping_fn process_prepared_discard_pt2; 284 285 struct dm_bio_prison_cell **cell_sort_array; 286 287 mempool_t mapping_pool; 288 }; 289 290 static void metadata_operation_failed(struct pool *pool, const char *op, int r); 291 292 static enum pool_mode get_pool_mode(struct pool *pool) 293 { 294 return pool->pf.mode; 295 } 296 297 static void notify_of_pool_mode_change(struct pool *pool) 298 { 299 static const char *descs[] = { 300 "write", 301 "out-of-data-space", 302 "read-only", 303 "read-only", 304 "fail" 305 }; 306 const char *extra_desc = NULL; 307 enum pool_mode mode = get_pool_mode(pool); 308 309 if (mode == PM_OUT_OF_DATA_SPACE) { 310 if (!pool->pf.error_if_no_space) 311 extra_desc = " (queue IO)"; 312 else 313 extra_desc = " (error IO)"; 314 } 315 316 dm_table_event(pool->ti->table); 317 DMINFO("%s: switching pool to %s%s mode", 318 dm_device_name(pool->pool_md), 319 descs[(int)mode], extra_desc ? : ""); 320 } 321 322 /* 323 * Target context for a pool. 324 */ 325 struct pool_c { 326 struct dm_target *ti; 327 struct pool *pool; 328 struct dm_dev *data_dev; 329 struct dm_dev *metadata_dev; 330 331 dm_block_t low_water_blocks; 332 struct pool_features requested_pf; /* Features requested during table load */ 333 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */ 334 }; 335 336 /* 337 * Target context for a thin. 338 */ 339 struct thin_c { 340 struct list_head list; 341 struct dm_dev *pool_dev; 342 struct dm_dev *origin_dev; 343 sector_t origin_size; 344 dm_thin_id dev_id; 345 346 struct pool *pool; 347 struct dm_thin_device *td; 348 struct mapped_device *thin_md; 349 350 bool requeue_mode:1; 351 spinlock_t lock; 352 struct list_head deferred_cells; 353 struct bio_list deferred_bio_list; 354 struct bio_list retry_on_resume_list; 355 struct rb_root sort_bio_list; /* sorted list of deferred bios */ 356 357 /* 358 * Ensures the thin is not destroyed until the worker has finished 359 * iterating the active_thins list. 360 */ 361 refcount_t refcount; 362 struct completion can_destroy; 363 }; 364 365 /*----------------------------------------------------------------*/ 366 367 static bool block_size_is_power_of_two(struct pool *pool) 368 { 369 return pool->sectors_per_block_shift >= 0; 370 } 371 372 static sector_t block_to_sectors(struct pool *pool, dm_block_t b) 373 { 374 return block_size_is_power_of_two(pool) ? 375 (b << pool->sectors_per_block_shift) : 376 (b * pool->sectors_per_block); 377 } 378 379 /*----------------------------------------------------------------*/ 380 381 struct discard_op { 382 struct thin_c *tc; 383 struct blk_plug plug; 384 struct bio *parent_bio; 385 struct bio *bio; 386 }; 387 388 static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent) 389 { 390 BUG_ON(!parent); 391 392 op->tc = tc; 393 blk_start_plug(&op->plug); 394 op->parent_bio = parent; 395 op->bio = NULL; 396 } 397 398 static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e) 399 { 400 struct thin_c *tc = op->tc; 401 sector_t s = block_to_sectors(tc->pool, data_b); 402 sector_t len = block_to_sectors(tc->pool, data_e - data_b); 403 404 return __blkdev_issue_discard(tc->pool_dev->bdev, s, len, GFP_NOIO, &op->bio); 405 } 406 407 static void end_discard(struct discard_op *op, int r) 408 { 409 if (op->bio) { 410 /* 411 * Even if one of the calls to issue_discard failed, we 412 * need to wait for the chain to complete. 413 */ 414 bio_chain(op->bio, op->parent_bio); 415 op->bio->bi_opf = REQ_OP_DISCARD; 416 submit_bio(op->bio); 417 } 418 419 blk_finish_plug(&op->plug); 420 421 /* 422 * Even if r is set, there could be sub discards in flight that we 423 * need to wait for. 424 */ 425 if (r && !op->parent_bio->bi_status) 426 op->parent_bio->bi_status = errno_to_blk_status(r); 427 bio_endio(op->parent_bio); 428 } 429 430 /*----------------------------------------------------------------*/ 431 432 /* 433 * wake_worker() is used when new work is queued and when pool_resume is 434 * ready to continue deferred IO processing. 435 */ 436 static void wake_worker(struct pool *pool) 437 { 438 queue_work(pool->wq, &pool->worker); 439 } 440 441 /*----------------------------------------------------------------*/ 442 443 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio, 444 struct dm_bio_prison_cell **cell_result) 445 { 446 int r; 447 struct dm_bio_prison_cell *cell_prealloc; 448 449 /* 450 * Allocate a cell from the prison's mempool. 451 * This might block but it can't fail. 452 */ 453 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); 454 455 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); 456 if (r) { 457 /* 458 * We reused an old cell; we can get rid of 459 * the new one. 460 */ 461 dm_bio_prison_free_cell(pool->prison, cell_prealloc); 462 } 463 464 return r; 465 } 466 467 static void cell_release(struct pool *pool, 468 struct dm_bio_prison_cell *cell, 469 struct bio_list *bios) 470 { 471 dm_cell_release(pool->prison, cell, bios); 472 dm_bio_prison_free_cell(pool->prison, cell); 473 } 474 475 static void cell_visit_release(struct pool *pool, 476 void (*fn)(void *, struct dm_bio_prison_cell *), 477 void *context, 478 struct dm_bio_prison_cell *cell) 479 { 480 dm_cell_visit_release(pool->prison, fn, context, cell); 481 dm_bio_prison_free_cell(pool->prison, cell); 482 } 483 484 static void cell_release_no_holder(struct pool *pool, 485 struct dm_bio_prison_cell *cell, 486 struct bio_list *bios) 487 { 488 dm_cell_release_no_holder(pool->prison, cell, bios); 489 dm_bio_prison_free_cell(pool->prison, cell); 490 } 491 492 static void cell_error_with_code(struct pool *pool, 493 struct dm_bio_prison_cell *cell, blk_status_t error_code) 494 { 495 dm_cell_error(pool->prison, cell, error_code); 496 dm_bio_prison_free_cell(pool->prison, cell); 497 } 498 499 static blk_status_t get_pool_io_error_code(struct pool *pool) 500 { 501 return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR; 502 } 503 504 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell) 505 { 506 cell_error_with_code(pool, cell, get_pool_io_error_code(pool)); 507 } 508 509 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell) 510 { 511 cell_error_with_code(pool, cell, 0); 512 } 513 514 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell) 515 { 516 cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE); 517 } 518 519 /*----------------------------------------------------------------*/ 520 521 /* 522 * A global list of pools that uses a struct mapped_device as a key. 523 */ 524 static struct dm_thin_pool_table { 525 struct mutex mutex; 526 struct list_head pools; 527 } dm_thin_pool_table; 528 529 static void pool_table_init(void) 530 { 531 mutex_init(&dm_thin_pool_table.mutex); 532 INIT_LIST_HEAD(&dm_thin_pool_table.pools); 533 } 534 535 static void pool_table_exit(void) 536 { 537 mutex_destroy(&dm_thin_pool_table.mutex); 538 } 539 540 static void __pool_table_insert(struct pool *pool) 541 { 542 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 543 list_add(&pool->list, &dm_thin_pool_table.pools); 544 } 545 546 static void __pool_table_remove(struct pool *pool) 547 { 548 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 549 list_del(&pool->list); 550 } 551 552 static struct pool *__pool_table_lookup(struct mapped_device *md) 553 { 554 struct pool *pool = NULL, *tmp; 555 556 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 557 558 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 559 if (tmp->pool_md == md) { 560 pool = tmp; 561 break; 562 } 563 } 564 565 return pool; 566 } 567 568 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev) 569 { 570 struct pool *pool = NULL, *tmp; 571 572 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 573 574 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 575 if (tmp->md_dev == md_dev) { 576 pool = tmp; 577 break; 578 } 579 } 580 581 return pool; 582 } 583 584 /*----------------------------------------------------------------*/ 585 586 struct dm_thin_endio_hook { 587 struct thin_c *tc; 588 struct dm_deferred_entry *shared_read_entry; 589 struct dm_deferred_entry *all_io_entry; 590 struct dm_thin_new_mapping *overwrite_mapping; 591 struct rb_node rb_node; 592 struct dm_bio_prison_cell *cell; 593 }; 594 595 static void error_bio_list(struct bio_list *bios, blk_status_t error) 596 { 597 struct bio *bio; 598 599 while ((bio = bio_list_pop(bios))) { 600 bio->bi_status = error; 601 bio_endio(bio); 602 } 603 } 604 605 static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, 606 blk_status_t error) 607 { 608 struct bio_list bios; 609 610 bio_list_init(&bios); 611 612 spin_lock_irq(&tc->lock); 613 bio_list_merge_init(&bios, master); 614 spin_unlock_irq(&tc->lock); 615 616 error_bio_list(&bios, error); 617 } 618 619 static void requeue_deferred_cells(struct thin_c *tc) 620 { 621 struct pool *pool = tc->pool; 622 struct list_head cells; 623 struct dm_bio_prison_cell *cell, *tmp; 624 625 INIT_LIST_HEAD(&cells); 626 627 spin_lock_irq(&tc->lock); 628 list_splice_init(&tc->deferred_cells, &cells); 629 spin_unlock_irq(&tc->lock); 630 631 list_for_each_entry_safe(cell, tmp, &cells, user_list) 632 cell_requeue(pool, cell); 633 } 634 635 static void requeue_io(struct thin_c *tc) 636 { 637 struct bio_list bios; 638 639 bio_list_init(&bios); 640 641 spin_lock_irq(&tc->lock); 642 bio_list_merge_init(&bios, &tc->deferred_bio_list); 643 bio_list_merge_init(&bios, &tc->retry_on_resume_list); 644 spin_unlock_irq(&tc->lock); 645 646 error_bio_list(&bios, BLK_STS_DM_REQUEUE); 647 requeue_deferred_cells(tc); 648 } 649 650 static void error_retry_list_with_code(struct pool *pool, blk_status_t error) 651 { 652 struct thin_c *tc; 653 654 rcu_read_lock(); 655 list_for_each_entry_rcu(tc, &pool->active_thins, list) 656 error_thin_bio_list(tc, &tc->retry_on_resume_list, error); 657 rcu_read_unlock(); 658 } 659 660 static void error_retry_list(struct pool *pool) 661 { 662 error_retry_list_with_code(pool, get_pool_io_error_code(pool)); 663 } 664 665 /* 666 * This section of code contains the logic for processing a thin device's IO. 667 * Much of the code depends on pool object resources (lists, workqueues, etc) 668 * but most is exclusively called from the thin target rather than the thin-pool 669 * target. 670 */ 671 672 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio) 673 { 674 struct pool *pool = tc->pool; 675 sector_t block_nr = bio->bi_iter.bi_sector; 676 677 if (block_size_is_power_of_two(pool)) 678 block_nr >>= pool->sectors_per_block_shift; 679 else 680 (void) sector_div(block_nr, pool->sectors_per_block); 681 682 return block_nr; 683 } 684 685 /* 686 * Returns the _complete_ blocks that this bio covers. 687 */ 688 static void get_bio_block_range(struct thin_c *tc, struct bio *bio, 689 dm_block_t *begin, dm_block_t *end) 690 { 691 struct pool *pool = tc->pool; 692 sector_t b = bio->bi_iter.bi_sector; 693 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT); 694 695 b += pool->sectors_per_block - 1ull; /* so we round up */ 696 697 if (block_size_is_power_of_two(pool)) { 698 b >>= pool->sectors_per_block_shift; 699 e >>= pool->sectors_per_block_shift; 700 } else { 701 (void) sector_div(b, pool->sectors_per_block); 702 (void) sector_div(e, pool->sectors_per_block); 703 } 704 705 if (e < b) { 706 /* Can happen if the bio is within a single block. */ 707 e = b; 708 } 709 710 *begin = b; 711 *end = e; 712 } 713 714 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) 715 { 716 struct pool *pool = tc->pool; 717 sector_t bi_sector = bio->bi_iter.bi_sector; 718 719 bio_set_dev(bio, tc->pool_dev->bdev); 720 if (block_size_is_power_of_two(pool)) { 721 bio->bi_iter.bi_sector = 722 (block << pool->sectors_per_block_shift) | 723 (bi_sector & (pool->sectors_per_block - 1)); 724 } else { 725 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + 726 sector_div(bi_sector, pool->sectors_per_block); 727 } 728 } 729 730 static void remap_to_origin(struct thin_c *tc, struct bio *bio) 731 { 732 bio_set_dev(bio, tc->origin_dev->bdev); 733 } 734 735 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio) 736 { 737 return op_is_flush(bio->bi_opf) && 738 dm_thin_changed_this_transaction(tc->td); 739 } 740 741 static void inc_all_io_entry(struct pool *pool, struct bio *bio) 742 { 743 struct dm_thin_endio_hook *h; 744 745 if (bio_op(bio) == REQ_OP_DISCARD) 746 return; 747 748 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 749 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds); 750 } 751 752 static void issue(struct thin_c *tc, struct bio *bio) 753 { 754 struct pool *pool = tc->pool; 755 756 if (!bio_triggers_commit(tc, bio)) { 757 dm_submit_bio_remap(bio, NULL); 758 return; 759 } 760 761 /* 762 * Complete bio with an error if earlier I/O caused changes to 763 * the metadata that can't be committed e.g, due to I/O errors 764 * on the metadata device. 765 */ 766 if (dm_thin_aborted_changes(tc->td)) { 767 bio_io_error(bio); 768 return; 769 } 770 771 /* 772 * Batch together any bios that trigger commits and then issue a 773 * single commit for them in process_deferred_bios(). 774 */ 775 spin_lock_irq(&pool->lock); 776 bio_list_add(&pool->deferred_flush_bios, bio); 777 spin_unlock_irq(&pool->lock); 778 } 779 780 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio) 781 { 782 remap_to_origin(tc, bio); 783 issue(tc, bio); 784 } 785 786 static void remap_and_issue(struct thin_c *tc, struct bio *bio, 787 dm_block_t block) 788 { 789 remap(tc, bio, block); 790 issue(tc, bio); 791 } 792 793 /*----------------------------------------------------------------*/ 794 795 /* 796 * Bio endio functions. 797 */ 798 struct dm_thin_new_mapping { 799 struct list_head list; 800 801 bool pass_discard:1; 802 bool maybe_shared:1; 803 804 /* 805 * Track quiescing, copying and zeroing preparation actions. When this 806 * counter hits zero the block is prepared and can be inserted into the 807 * btree. 808 */ 809 atomic_t prepare_actions; 810 811 blk_status_t status; 812 struct thin_c *tc; 813 dm_block_t virt_begin, virt_end; 814 dm_block_t data_block; 815 struct dm_bio_prison_cell *cell; 816 817 /* 818 * If the bio covers the whole area of a block then we can avoid 819 * zeroing or copying. Instead this bio is hooked. The bio will 820 * still be in the cell, so care has to be taken to avoid issuing 821 * the bio twice. 822 */ 823 struct bio *bio; 824 bio_end_io_t *saved_bi_end_io; 825 }; 826 827 static void __complete_mapping_preparation(struct dm_thin_new_mapping *m) 828 { 829 struct pool *pool = m->tc->pool; 830 831 if (atomic_dec_and_test(&m->prepare_actions)) { 832 list_add_tail(&m->list, &pool->prepared_mappings); 833 wake_worker(pool); 834 } 835 } 836 837 static void complete_mapping_preparation(struct dm_thin_new_mapping *m) 838 { 839 unsigned long flags; 840 struct pool *pool = m->tc->pool; 841 842 spin_lock_irqsave(&pool->lock, flags); 843 __complete_mapping_preparation(m); 844 spin_unlock_irqrestore(&pool->lock, flags); 845 } 846 847 static void copy_complete(int read_err, unsigned long write_err, void *context) 848 { 849 struct dm_thin_new_mapping *m = context; 850 851 m->status = read_err || write_err ? BLK_STS_IOERR : 0; 852 complete_mapping_preparation(m); 853 } 854 855 static void overwrite_endio(struct bio *bio) 856 { 857 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 858 struct dm_thin_new_mapping *m = h->overwrite_mapping; 859 860 bio->bi_end_io = m->saved_bi_end_io; 861 862 m->status = bio->bi_status; 863 complete_mapping_preparation(m); 864 } 865 866 /*----------------------------------------------------------------*/ 867 868 /* 869 * Workqueue. 870 */ 871 872 /* 873 * Prepared mapping jobs. 874 */ 875 876 /* 877 * This sends the bios in the cell, except the original holder, back 878 * to the deferred_bios list. 879 */ 880 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell) 881 { 882 struct pool *pool = tc->pool; 883 unsigned long flags; 884 struct bio_list bios; 885 886 bio_list_init(&bios); 887 cell_release_no_holder(pool, cell, &bios); 888 889 if (!bio_list_empty(&bios)) { 890 spin_lock_irqsave(&tc->lock, flags); 891 bio_list_merge(&tc->deferred_bio_list, &bios); 892 spin_unlock_irqrestore(&tc->lock, flags); 893 wake_worker(pool); 894 } 895 } 896 897 static void thin_defer_bio(struct thin_c *tc, struct bio *bio); 898 899 struct remap_info { 900 struct thin_c *tc; 901 struct bio_list defer_bios; 902 struct bio_list issue_bios; 903 }; 904 905 static void __inc_remap_and_issue_cell(void *context, 906 struct dm_bio_prison_cell *cell) 907 { 908 struct remap_info *info = context; 909 struct bio *bio; 910 911 while ((bio = bio_list_pop(&cell->bios))) { 912 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) 913 bio_list_add(&info->defer_bios, bio); 914 else { 915 inc_all_io_entry(info->tc->pool, bio); 916 917 /* 918 * We can't issue the bios with the bio prison lock 919 * held, so we add them to a list to issue on 920 * return from this function. 921 */ 922 bio_list_add(&info->issue_bios, bio); 923 } 924 } 925 } 926 927 static void inc_remap_and_issue_cell(struct thin_c *tc, 928 struct dm_bio_prison_cell *cell, 929 dm_block_t block) 930 { 931 struct bio *bio; 932 struct remap_info info; 933 934 info.tc = tc; 935 bio_list_init(&info.defer_bios); 936 bio_list_init(&info.issue_bios); 937 938 /* 939 * We have to be careful to inc any bios we're about to issue 940 * before the cell is released, and avoid a race with new bios 941 * being added to the cell. 942 */ 943 cell_visit_release(tc->pool, __inc_remap_and_issue_cell, 944 &info, cell); 945 946 while ((bio = bio_list_pop(&info.defer_bios))) 947 thin_defer_bio(tc, bio); 948 949 while ((bio = bio_list_pop(&info.issue_bios))) 950 remap_and_issue(info.tc, bio, block); 951 } 952 953 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) 954 { 955 cell_error(m->tc->pool, m->cell); 956 list_del(&m->list); 957 mempool_free(m, &m->tc->pool->mapping_pool); 958 } 959 960 static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio) 961 { 962 struct pool *pool = tc->pool; 963 964 /* 965 * If the bio has the REQ_FUA flag set we must commit the metadata 966 * before signaling its completion. 967 */ 968 if (!bio_triggers_commit(tc, bio)) { 969 bio_endio(bio); 970 return; 971 } 972 973 /* 974 * Complete bio with an error if earlier I/O caused changes to the 975 * metadata that can't be committed, e.g, due to I/O errors on the 976 * metadata device. 977 */ 978 if (dm_thin_aborted_changes(tc->td)) { 979 bio_io_error(bio); 980 return; 981 } 982 983 /* 984 * Batch together any bios that trigger commits and then issue a 985 * single commit for them in process_deferred_bios(). 986 */ 987 spin_lock_irq(&pool->lock); 988 bio_list_add(&pool->deferred_flush_completions, bio); 989 spin_unlock_irq(&pool->lock); 990 } 991 992 static void process_prepared_mapping(struct dm_thin_new_mapping *m) 993 { 994 struct thin_c *tc = m->tc; 995 struct pool *pool = tc->pool; 996 struct bio *bio = m->bio; 997 int r; 998 999 if (m->status) { 1000 cell_error(pool, m->cell); 1001 goto out; 1002 } 1003 1004 /* 1005 * Commit the prepared block into the mapping btree. 1006 * Any I/O for this block arriving after this point will get 1007 * remapped to it directly. 1008 */ 1009 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block); 1010 if (r) { 1011 metadata_operation_failed(pool, "dm_thin_insert_block", r); 1012 cell_error(pool, m->cell); 1013 goto out; 1014 } 1015 1016 /* 1017 * Release any bios held while the block was being provisioned. 1018 * If we are processing a write bio that completely covers the block, 1019 * we already processed it so can ignore it now when processing 1020 * the bios in the cell. 1021 */ 1022 if (bio) { 1023 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1024 complete_overwrite_bio(tc, bio); 1025 } else { 1026 inc_all_io_entry(tc->pool, m->cell->holder); 1027 remap_and_issue(tc, m->cell->holder, m->data_block); 1028 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1029 } 1030 1031 out: 1032 list_del(&m->list); 1033 mempool_free(m, &pool->mapping_pool); 1034 } 1035 1036 /*----------------------------------------------------------------*/ 1037 1038 static void free_discard_mapping(struct dm_thin_new_mapping *m) 1039 { 1040 struct thin_c *tc = m->tc; 1041 1042 if (m->cell) 1043 cell_defer_no_holder(tc, m->cell); 1044 mempool_free(m, &tc->pool->mapping_pool); 1045 } 1046 1047 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) 1048 { 1049 bio_io_error(m->bio); 1050 free_discard_mapping(m); 1051 } 1052 1053 static void process_prepared_discard_success(struct dm_thin_new_mapping *m) 1054 { 1055 bio_endio(m->bio); 1056 free_discard_mapping(m); 1057 } 1058 1059 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m) 1060 { 1061 int r; 1062 struct thin_c *tc = m->tc; 1063 1064 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end); 1065 if (r) { 1066 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); 1067 bio_io_error(m->bio); 1068 } else 1069 bio_endio(m->bio); 1070 1071 cell_defer_no_holder(tc, m->cell); 1072 mempool_free(m, &tc->pool->mapping_pool); 1073 } 1074 1075 /*----------------------------------------------------------------*/ 1076 1077 static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m, 1078 struct bio *discard_parent) 1079 { 1080 /* 1081 * We've already unmapped this range of blocks, but before we 1082 * passdown we have to check that these blocks are now unused. 1083 */ 1084 int r = 0; 1085 bool shared = true; 1086 struct thin_c *tc = m->tc; 1087 struct pool *pool = tc->pool; 1088 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin; 1089 struct discard_op op; 1090 1091 begin_discard(&op, tc, discard_parent); 1092 while (b != end) { 1093 /* find start of unmapped run */ 1094 for (; b < end; b++) { 1095 r = dm_pool_block_is_shared(pool->pmd, b, &shared); 1096 if (r) 1097 goto out; 1098 1099 if (!shared) 1100 break; 1101 } 1102 1103 if (b == end) 1104 break; 1105 1106 /* find end of run */ 1107 for (e = b + 1; e != end; e++) { 1108 r = dm_pool_block_is_shared(pool->pmd, e, &shared); 1109 if (r) 1110 goto out; 1111 1112 if (shared) 1113 break; 1114 } 1115 1116 r = issue_discard(&op, b, e); 1117 if (r) 1118 goto out; 1119 1120 b = e; 1121 } 1122 out: 1123 end_discard(&op, r); 1124 } 1125 1126 static void queue_passdown_pt2(struct dm_thin_new_mapping *m) 1127 { 1128 unsigned long flags; 1129 struct pool *pool = m->tc->pool; 1130 1131 spin_lock_irqsave(&pool->lock, flags); 1132 list_add_tail(&m->list, &pool->prepared_discards_pt2); 1133 spin_unlock_irqrestore(&pool->lock, flags); 1134 wake_worker(pool); 1135 } 1136 1137 static void passdown_endio(struct bio *bio) 1138 { 1139 /* 1140 * It doesn't matter if the passdown discard failed, we still want 1141 * to unmap (we ignore err). 1142 */ 1143 queue_passdown_pt2(bio->bi_private); 1144 bio_put(bio); 1145 } 1146 1147 static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m) 1148 { 1149 int r; 1150 struct thin_c *tc = m->tc; 1151 struct pool *pool = tc->pool; 1152 struct bio *discard_parent; 1153 dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin); 1154 1155 /* 1156 * Only this thread allocates blocks, so we can be sure that the 1157 * newly unmapped blocks will not be allocated before the end of 1158 * the function. 1159 */ 1160 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end); 1161 if (r) { 1162 metadata_operation_failed(pool, "dm_thin_remove_range", r); 1163 bio_io_error(m->bio); 1164 cell_defer_no_holder(tc, m->cell); 1165 mempool_free(m, &pool->mapping_pool); 1166 return; 1167 } 1168 1169 /* 1170 * Increment the unmapped blocks. This prevents a race between the 1171 * passdown io and reallocation of freed blocks. 1172 */ 1173 r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end); 1174 if (r) { 1175 metadata_operation_failed(pool, "dm_pool_inc_data_range", r); 1176 bio_io_error(m->bio); 1177 cell_defer_no_holder(tc, m->cell); 1178 mempool_free(m, &pool->mapping_pool); 1179 return; 1180 } 1181 1182 discard_parent = bio_alloc(NULL, 1, 0, GFP_NOIO); 1183 discard_parent->bi_end_io = passdown_endio; 1184 discard_parent->bi_private = m; 1185 if (m->maybe_shared) 1186 passdown_double_checking_shared_status(m, discard_parent); 1187 else { 1188 struct discard_op op; 1189 1190 begin_discard(&op, tc, discard_parent); 1191 r = issue_discard(&op, m->data_block, data_end); 1192 end_discard(&op, r); 1193 } 1194 } 1195 1196 static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m) 1197 { 1198 int r; 1199 struct thin_c *tc = m->tc; 1200 struct pool *pool = tc->pool; 1201 1202 /* 1203 * The passdown has completed, so now we can decrement all those 1204 * unmapped blocks. 1205 */ 1206 r = dm_pool_dec_data_range(pool->pmd, m->data_block, 1207 m->data_block + (m->virt_end - m->virt_begin)); 1208 if (r) { 1209 metadata_operation_failed(pool, "dm_pool_dec_data_range", r); 1210 bio_io_error(m->bio); 1211 } else 1212 bio_endio(m->bio); 1213 1214 cell_defer_no_holder(tc, m->cell); 1215 mempool_free(m, &pool->mapping_pool); 1216 } 1217 1218 static void process_prepared(struct pool *pool, struct list_head *head, 1219 process_mapping_fn *fn) 1220 { 1221 struct list_head maps; 1222 struct dm_thin_new_mapping *m, *tmp; 1223 1224 INIT_LIST_HEAD(&maps); 1225 spin_lock_irq(&pool->lock); 1226 list_splice_init(head, &maps); 1227 spin_unlock_irq(&pool->lock); 1228 1229 list_for_each_entry_safe(m, tmp, &maps, list) 1230 (*fn)(m); 1231 } 1232 1233 /* 1234 * Deferred bio jobs. 1235 */ 1236 static int io_overlaps_block(struct pool *pool, struct bio *bio) 1237 { 1238 return bio->bi_iter.bi_size == 1239 (pool->sectors_per_block << SECTOR_SHIFT); 1240 } 1241 1242 static int io_overwrites_block(struct pool *pool, struct bio *bio) 1243 { 1244 return (bio_data_dir(bio) == WRITE) && 1245 io_overlaps_block(pool, bio); 1246 } 1247 1248 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save, 1249 bio_end_io_t *fn) 1250 { 1251 *save = bio->bi_end_io; 1252 bio->bi_end_io = fn; 1253 } 1254 1255 static int ensure_next_mapping(struct pool *pool) 1256 { 1257 if (pool->next_mapping) 1258 return 0; 1259 1260 pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC); 1261 1262 return pool->next_mapping ? 0 : -ENOMEM; 1263 } 1264 1265 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool) 1266 { 1267 struct dm_thin_new_mapping *m = pool->next_mapping; 1268 1269 BUG_ON(!pool->next_mapping); 1270 1271 memset(m, 0, sizeof(struct dm_thin_new_mapping)); 1272 INIT_LIST_HEAD(&m->list); 1273 m->bio = NULL; 1274 1275 pool->next_mapping = NULL; 1276 1277 return m; 1278 } 1279 1280 static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m, 1281 sector_t begin, sector_t end) 1282 { 1283 struct dm_io_region to; 1284 1285 to.bdev = tc->pool_dev->bdev; 1286 to.sector = begin; 1287 to.count = end - begin; 1288 1289 dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); 1290 } 1291 1292 static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio, 1293 dm_block_t data_begin, 1294 struct dm_thin_new_mapping *m) 1295 { 1296 struct pool *pool = tc->pool; 1297 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1298 1299 h->overwrite_mapping = m; 1300 m->bio = bio; 1301 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio); 1302 inc_all_io_entry(pool, bio); 1303 remap_and_issue(tc, bio, data_begin); 1304 } 1305 1306 /* 1307 * A partial copy also needs to zero the uncopied region. 1308 */ 1309 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, 1310 struct dm_dev *origin, dm_block_t data_origin, 1311 dm_block_t data_dest, 1312 struct dm_bio_prison_cell *cell, struct bio *bio, 1313 sector_t len) 1314 { 1315 struct pool *pool = tc->pool; 1316 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1317 1318 m->tc = tc; 1319 m->virt_begin = virt_block; 1320 m->virt_end = virt_block + 1u; 1321 m->data_block = data_dest; 1322 m->cell = cell; 1323 1324 /* 1325 * quiesce action + copy action + an extra reference held for the 1326 * duration of this function (we may need to inc later for a 1327 * partial zero). 1328 */ 1329 atomic_set(&m->prepare_actions, 3); 1330 1331 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list)) 1332 complete_mapping_preparation(m); /* already quiesced */ 1333 1334 /* 1335 * IO to pool_dev remaps to the pool target's data_dev. 1336 * 1337 * If the whole block of data is being overwritten, we can issue the 1338 * bio immediately. Otherwise we use kcopyd to clone the data first. 1339 */ 1340 if (io_overwrites_block(pool, bio)) 1341 remap_and_issue_overwrite(tc, bio, data_dest, m); 1342 else { 1343 struct dm_io_region from, to; 1344 1345 from.bdev = origin->bdev; 1346 from.sector = data_origin * pool->sectors_per_block; 1347 from.count = len; 1348 1349 to.bdev = tc->pool_dev->bdev; 1350 to.sector = data_dest * pool->sectors_per_block; 1351 to.count = len; 1352 1353 dm_kcopyd_copy(pool->copier, &from, 1, &to, 1354 0, copy_complete, m); 1355 1356 /* 1357 * Do we need to zero a tail region? 1358 */ 1359 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) { 1360 atomic_inc(&m->prepare_actions); 1361 ll_zero(tc, m, 1362 data_dest * pool->sectors_per_block + len, 1363 (data_dest + 1) * pool->sectors_per_block); 1364 } 1365 } 1366 1367 complete_mapping_preparation(m); /* drop our ref */ 1368 } 1369 1370 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block, 1371 dm_block_t data_origin, dm_block_t data_dest, 1372 struct dm_bio_prison_cell *cell, struct bio *bio) 1373 { 1374 schedule_copy(tc, virt_block, tc->pool_dev, 1375 data_origin, data_dest, cell, bio, 1376 tc->pool->sectors_per_block); 1377 } 1378 1379 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, 1380 dm_block_t data_block, struct dm_bio_prison_cell *cell, 1381 struct bio *bio) 1382 { 1383 struct pool *pool = tc->pool; 1384 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1385 1386 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */ 1387 m->tc = tc; 1388 m->virt_begin = virt_block; 1389 m->virt_end = virt_block + 1u; 1390 m->data_block = data_block; 1391 m->cell = cell; 1392 1393 /* 1394 * If the whole block of data is being overwritten or we are not 1395 * zeroing pre-existing data, we can issue the bio immediately. 1396 * Otherwise we use kcopyd to zero the data first. 1397 */ 1398 if (pool->pf.zero_new_blocks) { 1399 if (io_overwrites_block(pool, bio)) 1400 remap_and_issue_overwrite(tc, bio, data_block, m); 1401 else { 1402 ll_zero(tc, m, data_block * pool->sectors_per_block, 1403 (data_block + 1) * pool->sectors_per_block); 1404 } 1405 } else 1406 process_prepared_mapping(m); 1407 } 1408 1409 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, 1410 dm_block_t data_dest, 1411 struct dm_bio_prison_cell *cell, struct bio *bio) 1412 { 1413 struct pool *pool = tc->pool; 1414 sector_t virt_block_begin = virt_block * pool->sectors_per_block; 1415 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; 1416 1417 if (virt_block_end <= tc->origin_size) { 1418 schedule_copy(tc, virt_block, tc->origin_dev, 1419 virt_block, data_dest, cell, bio, 1420 pool->sectors_per_block); 1421 1422 } else if (virt_block_begin < tc->origin_size) { 1423 schedule_copy(tc, virt_block, tc->origin_dev, 1424 virt_block, data_dest, cell, bio, 1425 tc->origin_size - virt_block_begin); 1426 1427 } else 1428 schedule_zero(tc, virt_block, data_dest, cell, bio); 1429 } 1430 1431 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode); 1432 1433 static void requeue_bios(struct pool *pool); 1434 1435 static bool is_read_only_pool_mode(enum pool_mode mode) 1436 { 1437 return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY); 1438 } 1439 1440 static bool is_read_only(struct pool *pool) 1441 { 1442 return is_read_only_pool_mode(get_pool_mode(pool)); 1443 } 1444 1445 static void check_for_metadata_space(struct pool *pool) 1446 { 1447 int r; 1448 const char *ooms_reason = NULL; 1449 dm_block_t nr_free; 1450 1451 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free); 1452 if (r) 1453 ooms_reason = "Could not get free metadata blocks"; 1454 else if (!nr_free) 1455 ooms_reason = "No free metadata blocks"; 1456 1457 if (ooms_reason && !is_read_only(pool)) { 1458 DMERR("%s", ooms_reason); 1459 set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE); 1460 } 1461 } 1462 1463 static void check_for_data_space(struct pool *pool) 1464 { 1465 int r; 1466 dm_block_t nr_free; 1467 1468 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE) 1469 return; 1470 1471 r = dm_pool_get_free_block_count(pool->pmd, &nr_free); 1472 if (r) 1473 return; 1474 1475 if (nr_free) { 1476 set_pool_mode(pool, PM_WRITE); 1477 requeue_bios(pool); 1478 } 1479 } 1480 1481 /* 1482 * A non-zero return indicates read_only or fail_io mode. 1483 * Many callers don't care about the return value. 1484 */ 1485 static int commit(struct pool *pool) 1486 { 1487 int r; 1488 1489 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) 1490 return -EINVAL; 1491 1492 r = dm_pool_commit_metadata(pool->pmd); 1493 if (r) 1494 metadata_operation_failed(pool, "dm_pool_commit_metadata", r); 1495 else { 1496 check_for_metadata_space(pool); 1497 check_for_data_space(pool); 1498 } 1499 1500 return r; 1501 } 1502 1503 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks) 1504 { 1505 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) { 1506 DMWARN("%s: reached low water mark for data device: sending event.", 1507 dm_device_name(pool->pool_md)); 1508 spin_lock_irq(&pool->lock); 1509 pool->low_water_triggered = true; 1510 spin_unlock_irq(&pool->lock); 1511 dm_table_event(pool->ti->table); 1512 } 1513 } 1514 1515 static int alloc_data_block(struct thin_c *tc, dm_block_t *result) 1516 { 1517 int r; 1518 dm_block_t free_blocks; 1519 struct pool *pool = tc->pool; 1520 1521 if (WARN_ON(get_pool_mode(pool) != PM_WRITE)) 1522 return -EINVAL; 1523 1524 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1525 if (r) { 1526 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1527 return r; 1528 } 1529 1530 check_low_water_mark(pool, free_blocks); 1531 1532 if (!free_blocks) { 1533 /* 1534 * Try to commit to see if that will free up some 1535 * more space. 1536 */ 1537 r = commit(pool); 1538 if (r) 1539 return r; 1540 1541 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1542 if (r) { 1543 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1544 return r; 1545 } 1546 1547 if (!free_blocks) { 1548 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); 1549 return -ENOSPC; 1550 } 1551 } 1552 1553 r = dm_pool_alloc_data_block(pool->pmd, result); 1554 if (r) { 1555 if (r == -ENOSPC) 1556 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); 1557 else 1558 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r); 1559 return r; 1560 } 1561 1562 r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks); 1563 if (r) { 1564 metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r); 1565 return r; 1566 } 1567 1568 if (!free_blocks) { 1569 /* Let's commit before we use up the metadata reserve. */ 1570 r = commit(pool); 1571 if (r) 1572 return r; 1573 } 1574 1575 return 0; 1576 } 1577 1578 /* 1579 * If we have run out of space, queue bios until the device is 1580 * resumed, presumably after having been reloaded with more space. 1581 */ 1582 static void retry_on_resume(struct bio *bio) 1583 { 1584 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1585 struct thin_c *tc = h->tc; 1586 1587 spin_lock_irq(&tc->lock); 1588 bio_list_add(&tc->retry_on_resume_list, bio); 1589 spin_unlock_irq(&tc->lock); 1590 } 1591 1592 static blk_status_t should_error_unserviceable_bio(struct pool *pool) 1593 { 1594 enum pool_mode m = get_pool_mode(pool); 1595 1596 switch (m) { 1597 case PM_WRITE: 1598 /* Shouldn't get here */ 1599 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode"); 1600 return BLK_STS_IOERR; 1601 1602 case PM_OUT_OF_DATA_SPACE: 1603 return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0; 1604 1605 case PM_OUT_OF_METADATA_SPACE: 1606 case PM_READ_ONLY: 1607 case PM_FAIL: 1608 return BLK_STS_IOERR; 1609 default: 1610 /* Shouldn't get here */ 1611 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode"); 1612 return BLK_STS_IOERR; 1613 } 1614 } 1615 1616 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio) 1617 { 1618 blk_status_t error = should_error_unserviceable_bio(pool); 1619 1620 if (error) { 1621 bio->bi_status = error; 1622 bio_endio(bio); 1623 } else 1624 retry_on_resume(bio); 1625 } 1626 1627 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell) 1628 { 1629 struct bio *bio; 1630 struct bio_list bios; 1631 blk_status_t error; 1632 1633 error = should_error_unserviceable_bio(pool); 1634 if (error) { 1635 cell_error_with_code(pool, cell, error); 1636 return; 1637 } 1638 1639 bio_list_init(&bios); 1640 cell_release(pool, cell, &bios); 1641 1642 while ((bio = bio_list_pop(&bios))) 1643 retry_on_resume(bio); 1644 } 1645 1646 static void process_discard_cell_no_passdown(struct thin_c *tc, 1647 struct dm_bio_prison_cell *virt_cell) 1648 { 1649 struct pool *pool = tc->pool; 1650 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1651 1652 /* 1653 * We don't need to lock the data blocks, since there's no 1654 * passdown. We only lock data blocks for allocation and breaking sharing. 1655 */ 1656 m->tc = tc; 1657 m->virt_begin = virt_cell->key.block_begin; 1658 m->virt_end = virt_cell->key.block_end; 1659 m->cell = virt_cell; 1660 m->bio = virt_cell->holder; 1661 1662 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1663 pool->process_prepared_discard(m); 1664 } 1665 1666 static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end, 1667 struct bio *bio) 1668 { 1669 struct pool *pool = tc->pool; 1670 1671 int r; 1672 bool maybe_shared; 1673 struct dm_cell_key data_key; 1674 struct dm_bio_prison_cell *data_cell; 1675 struct dm_thin_new_mapping *m; 1676 dm_block_t virt_begin, virt_end, data_begin, data_end; 1677 dm_block_t len, next_boundary; 1678 1679 while (begin != end) { 1680 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end, 1681 &data_begin, &maybe_shared); 1682 if (r) { 1683 /* 1684 * Silently fail, letting any mappings we've 1685 * created complete. 1686 */ 1687 break; 1688 } 1689 1690 data_end = data_begin + (virt_end - virt_begin); 1691 1692 /* 1693 * Make sure the data region obeys the bio prison restrictions. 1694 */ 1695 while (data_begin < data_end) { 1696 r = ensure_next_mapping(pool); 1697 if (r) 1698 return; /* we did our best */ 1699 1700 next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1) 1701 << BIO_PRISON_MAX_RANGE_SHIFT; 1702 len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin); 1703 1704 /* This key is certainly within range given the above splitting */ 1705 (void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key); 1706 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) { 1707 /* contention, we'll give up with this range */ 1708 data_begin += len; 1709 continue; 1710 } 1711 1712 /* 1713 * IO may still be going to the destination block. We must 1714 * quiesce before we can do the removal. 1715 */ 1716 m = get_next_mapping(pool); 1717 m->tc = tc; 1718 m->maybe_shared = maybe_shared; 1719 m->virt_begin = virt_begin; 1720 m->virt_end = virt_begin + len; 1721 m->data_block = data_begin; 1722 m->cell = data_cell; 1723 m->bio = bio; 1724 1725 /* 1726 * The parent bio must not complete before sub discard bios are 1727 * chained to it (see end_discard's bio_chain)! 1728 * 1729 * This per-mapping bi_remaining increment is paired with 1730 * the implicit decrement that occurs via bio_endio() in 1731 * end_discard(). 1732 */ 1733 bio_inc_remaining(bio); 1734 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1735 pool->process_prepared_discard(m); 1736 1737 virt_begin += len; 1738 data_begin += len; 1739 } 1740 1741 begin = virt_end; 1742 } 1743 } 1744 1745 static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell) 1746 { 1747 struct bio *bio = virt_cell->holder; 1748 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1749 1750 /* 1751 * The virt_cell will only get freed once the origin bio completes. 1752 * This means it will remain locked while all the individual 1753 * passdown bios are in flight. 1754 */ 1755 h->cell = virt_cell; 1756 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio); 1757 1758 /* 1759 * We complete the bio now, knowing that the bi_remaining field 1760 * will prevent completion until the sub range discards have 1761 * completed. 1762 */ 1763 bio_endio(bio); 1764 } 1765 1766 static void process_discard_bio(struct thin_c *tc, struct bio *bio) 1767 { 1768 dm_block_t begin, end; 1769 struct dm_cell_key virt_key; 1770 struct dm_bio_prison_cell *virt_cell; 1771 1772 get_bio_block_range(tc, bio, &begin, &end); 1773 if (begin == end) { 1774 /* 1775 * The discard covers less than a block. 1776 */ 1777 bio_endio(bio); 1778 return; 1779 } 1780 1781 if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) { 1782 DMERR_LIMIT("Discard doesn't respect bio prison limits"); 1783 bio_endio(bio); 1784 return; 1785 } 1786 1787 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) { 1788 /* 1789 * Potential starvation issue: We're relying on the 1790 * fs/application being well behaved, and not trying to 1791 * send IO to a region at the same time as discarding it. 1792 * If they do this persistently then it's possible this 1793 * cell will never be granted. 1794 */ 1795 return; 1796 } 1797 1798 tc->pool->process_discard_cell(tc, virt_cell); 1799 } 1800 1801 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block, 1802 struct dm_cell_key *key, 1803 struct dm_thin_lookup_result *lookup_result, 1804 struct dm_bio_prison_cell *cell) 1805 { 1806 int r; 1807 dm_block_t data_block; 1808 struct pool *pool = tc->pool; 1809 1810 r = alloc_data_block(tc, &data_block); 1811 switch (r) { 1812 case 0: 1813 schedule_internal_copy(tc, block, lookup_result->block, 1814 data_block, cell, bio); 1815 break; 1816 1817 case -ENOSPC: 1818 retry_bios_on_resume(pool, cell); 1819 break; 1820 1821 default: 1822 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1823 __func__, r); 1824 cell_error(pool, cell); 1825 break; 1826 } 1827 } 1828 1829 static void __remap_and_issue_shared_cell(void *context, 1830 struct dm_bio_prison_cell *cell) 1831 { 1832 struct remap_info *info = context; 1833 struct bio *bio; 1834 1835 while ((bio = bio_list_pop(&cell->bios))) { 1836 if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) || 1837 bio_op(bio) == REQ_OP_DISCARD) 1838 bio_list_add(&info->defer_bios, bio); 1839 else { 1840 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1841 1842 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds); 1843 inc_all_io_entry(info->tc->pool, bio); 1844 bio_list_add(&info->issue_bios, bio); 1845 } 1846 } 1847 } 1848 1849 static void remap_and_issue_shared_cell(struct thin_c *tc, 1850 struct dm_bio_prison_cell *cell, 1851 dm_block_t block) 1852 { 1853 struct bio *bio; 1854 struct remap_info info; 1855 1856 info.tc = tc; 1857 bio_list_init(&info.defer_bios); 1858 bio_list_init(&info.issue_bios); 1859 1860 cell_visit_release(tc->pool, __remap_and_issue_shared_cell, 1861 &info, cell); 1862 1863 while ((bio = bio_list_pop(&info.defer_bios))) 1864 thin_defer_bio(tc, bio); 1865 1866 while ((bio = bio_list_pop(&info.issue_bios))) 1867 remap_and_issue(tc, bio, block); 1868 } 1869 1870 static void process_shared_bio(struct thin_c *tc, struct bio *bio, 1871 dm_block_t block, 1872 struct dm_thin_lookup_result *lookup_result, 1873 struct dm_bio_prison_cell *virt_cell) 1874 { 1875 struct dm_bio_prison_cell *data_cell; 1876 struct pool *pool = tc->pool; 1877 struct dm_cell_key key; 1878 1879 /* 1880 * If cell is already occupied, then sharing is already in the process 1881 * of being broken so we have nothing further to do here. 1882 */ 1883 build_data_key(tc->td, lookup_result->block, &key); 1884 if (bio_detain(pool, &key, bio, &data_cell)) { 1885 cell_defer_no_holder(tc, virt_cell); 1886 return; 1887 } 1888 1889 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) { 1890 break_sharing(tc, bio, block, &key, lookup_result, data_cell); 1891 cell_defer_no_holder(tc, virt_cell); 1892 } else { 1893 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1894 1895 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds); 1896 inc_all_io_entry(pool, bio); 1897 remap_and_issue(tc, bio, lookup_result->block); 1898 1899 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block); 1900 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block); 1901 } 1902 } 1903 1904 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block, 1905 struct dm_bio_prison_cell *cell) 1906 { 1907 int r; 1908 dm_block_t data_block; 1909 struct pool *pool = tc->pool; 1910 1911 /* 1912 * Remap empty bios (flushes) immediately, without provisioning. 1913 */ 1914 if (!bio->bi_iter.bi_size) { 1915 inc_all_io_entry(pool, bio); 1916 cell_defer_no_holder(tc, cell); 1917 1918 remap_and_issue(tc, bio, 0); 1919 return; 1920 } 1921 1922 /* 1923 * Fill read bios with zeroes and complete them immediately. 1924 */ 1925 if (bio_data_dir(bio) == READ) { 1926 zero_fill_bio(bio); 1927 cell_defer_no_holder(tc, cell); 1928 bio_endio(bio); 1929 return; 1930 } 1931 1932 r = alloc_data_block(tc, &data_block); 1933 switch (r) { 1934 case 0: 1935 if (tc->origin_dev) 1936 schedule_external_copy(tc, block, data_block, cell, bio); 1937 else 1938 schedule_zero(tc, block, data_block, cell, bio); 1939 break; 1940 1941 case -ENOSPC: 1942 retry_bios_on_resume(pool, cell); 1943 break; 1944 1945 default: 1946 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1947 __func__, r); 1948 cell_error(pool, cell); 1949 break; 1950 } 1951 } 1952 1953 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1954 { 1955 int r; 1956 struct pool *pool = tc->pool; 1957 struct bio *bio = cell->holder; 1958 dm_block_t block = get_bio_block(tc, bio); 1959 struct dm_thin_lookup_result lookup_result; 1960 1961 if (tc->requeue_mode) { 1962 cell_requeue(pool, cell); 1963 return; 1964 } 1965 1966 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1967 switch (r) { 1968 case 0: 1969 if (lookup_result.shared) 1970 process_shared_bio(tc, bio, block, &lookup_result, cell); 1971 else { 1972 inc_all_io_entry(pool, bio); 1973 remap_and_issue(tc, bio, lookup_result.block); 1974 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 1975 } 1976 break; 1977 1978 case -ENODATA: 1979 if (bio_data_dir(bio) == READ && tc->origin_dev) { 1980 inc_all_io_entry(pool, bio); 1981 cell_defer_no_holder(tc, cell); 1982 1983 if (bio_end_sector(bio) <= tc->origin_size) 1984 remap_to_origin_and_issue(tc, bio); 1985 1986 else if (bio->bi_iter.bi_sector < tc->origin_size) { 1987 zero_fill_bio(bio); 1988 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT; 1989 remap_to_origin_and_issue(tc, bio); 1990 1991 } else { 1992 zero_fill_bio(bio); 1993 bio_endio(bio); 1994 } 1995 } else 1996 provision_block(tc, bio, block, cell); 1997 break; 1998 1999 default: 2000 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 2001 __func__, r); 2002 cell_defer_no_holder(tc, cell); 2003 bio_io_error(bio); 2004 break; 2005 } 2006 } 2007 2008 static void process_bio(struct thin_c *tc, struct bio *bio) 2009 { 2010 struct pool *pool = tc->pool; 2011 dm_block_t block = get_bio_block(tc, bio); 2012 struct dm_bio_prison_cell *cell; 2013 struct dm_cell_key key; 2014 2015 /* 2016 * If cell is already occupied, then the block is already 2017 * being provisioned so we have nothing further to do here. 2018 */ 2019 build_virtual_key(tc->td, block, &key); 2020 if (bio_detain(pool, &key, bio, &cell)) 2021 return; 2022 2023 process_cell(tc, cell); 2024 } 2025 2026 static void __process_bio_read_only(struct thin_c *tc, struct bio *bio, 2027 struct dm_bio_prison_cell *cell) 2028 { 2029 int r; 2030 int rw = bio_data_dir(bio); 2031 dm_block_t block = get_bio_block(tc, bio); 2032 struct dm_thin_lookup_result lookup_result; 2033 2034 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 2035 switch (r) { 2036 case 0: 2037 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) { 2038 handle_unserviceable_bio(tc->pool, bio); 2039 if (cell) 2040 cell_defer_no_holder(tc, cell); 2041 } else { 2042 inc_all_io_entry(tc->pool, bio); 2043 remap_and_issue(tc, bio, lookup_result.block); 2044 if (cell) 2045 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 2046 } 2047 break; 2048 2049 case -ENODATA: 2050 if (cell) 2051 cell_defer_no_holder(tc, cell); 2052 if (rw != READ) { 2053 handle_unserviceable_bio(tc->pool, bio); 2054 break; 2055 } 2056 2057 if (tc->origin_dev) { 2058 inc_all_io_entry(tc->pool, bio); 2059 remap_to_origin_and_issue(tc, bio); 2060 break; 2061 } 2062 2063 zero_fill_bio(bio); 2064 bio_endio(bio); 2065 break; 2066 2067 default: 2068 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 2069 __func__, r); 2070 if (cell) 2071 cell_defer_no_holder(tc, cell); 2072 bio_io_error(bio); 2073 break; 2074 } 2075 } 2076 2077 static void process_bio_read_only(struct thin_c *tc, struct bio *bio) 2078 { 2079 __process_bio_read_only(tc, bio, NULL); 2080 } 2081 2082 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2083 { 2084 __process_bio_read_only(tc, cell->holder, cell); 2085 } 2086 2087 static void process_bio_success(struct thin_c *tc, struct bio *bio) 2088 { 2089 bio_endio(bio); 2090 } 2091 2092 static void process_bio_fail(struct thin_c *tc, struct bio *bio) 2093 { 2094 bio_io_error(bio); 2095 } 2096 2097 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2098 { 2099 cell_success(tc->pool, cell); 2100 } 2101 2102 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2103 { 2104 cell_error(tc->pool, cell); 2105 } 2106 2107 /* 2108 * FIXME: should we also commit due to size of transaction, measured in 2109 * metadata blocks? 2110 */ 2111 static int need_commit_due_to_time(struct pool *pool) 2112 { 2113 return !time_in_range(jiffies, pool->last_commit_jiffies, 2114 pool->last_commit_jiffies + COMMIT_PERIOD); 2115 } 2116 2117 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node) 2118 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook)) 2119 2120 static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio) 2121 { 2122 struct rb_node **rbp, *parent; 2123 struct dm_thin_endio_hook *pbd; 2124 sector_t bi_sector = bio->bi_iter.bi_sector; 2125 2126 rbp = &tc->sort_bio_list.rb_node; 2127 parent = NULL; 2128 while (*rbp) { 2129 parent = *rbp; 2130 pbd = thin_pbd(parent); 2131 2132 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector) 2133 rbp = &(*rbp)->rb_left; 2134 else 2135 rbp = &(*rbp)->rb_right; 2136 } 2137 2138 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 2139 rb_link_node(&pbd->rb_node, parent, rbp); 2140 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list); 2141 } 2142 2143 static void __extract_sorted_bios(struct thin_c *tc) 2144 { 2145 struct rb_node *node; 2146 struct dm_thin_endio_hook *pbd; 2147 struct bio *bio; 2148 2149 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) { 2150 pbd = thin_pbd(node); 2151 bio = thin_bio(pbd); 2152 2153 bio_list_add(&tc->deferred_bio_list, bio); 2154 rb_erase(&pbd->rb_node, &tc->sort_bio_list); 2155 } 2156 2157 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list)); 2158 } 2159 2160 static void __sort_thin_deferred_bios(struct thin_c *tc) 2161 { 2162 struct bio *bio; 2163 struct bio_list bios; 2164 2165 bio_list_init(&bios); 2166 bio_list_merge(&bios, &tc->deferred_bio_list); 2167 bio_list_init(&tc->deferred_bio_list); 2168 2169 /* Sort deferred_bio_list using rb-tree */ 2170 while ((bio = bio_list_pop(&bios))) 2171 __thin_bio_rb_add(tc, bio); 2172 2173 /* 2174 * Transfer the sorted bios in sort_bio_list back to 2175 * deferred_bio_list to allow lockless submission of 2176 * all bios. 2177 */ 2178 __extract_sorted_bios(tc); 2179 } 2180 2181 static void process_thin_deferred_bios(struct thin_c *tc) 2182 { 2183 struct pool *pool = tc->pool; 2184 struct bio *bio; 2185 struct bio_list bios; 2186 struct blk_plug plug; 2187 unsigned int count = 0; 2188 2189 if (tc->requeue_mode) { 2190 error_thin_bio_list(tc, &tc->deferred_bio_list, 2191 BLK_STS_DM_REQUEUE); 2192 return; 2193 } 2194 2195 bio_list_init(&bios); 2196 2197 spin_lock_irq(&tc->lock); 2198 2199 if (bio_list_empty(&tc->deferred_bio_list)) { 2200 spin_unlock_irq(&tc->lock); 2201 return; 2202 } 2203 2204 __sort_thin_deferred_bios(tc); 2205 2206 bio_list_merge(&bios, &tc->deferred_bio_list); 2207 bio_list_init(&tc->deferred_bio_list); 2208 2209 spin_unlock_irq(&tc->lock); 2210 2211 blk_start_plug(&plug); 2212 while ((bio = bio_list_pop(&bios))) { 2213 /* 2214 * If we've got no free new_mapping structs, and processing 2215 * this bio might require one, we pause until there are some 2216 * prepared mappings to process. 2217 */ 2218 if (ensure_next_mapping(pool)) { 2219 spin_lock_irq(&tc->lock); 2220 bio_list_add(&tc->deferred_bio_list, bio); 2221 bio_list_merge(&tc->deferred_bio_list, &bios); 2222 spin_unlock_irq(&tc->lock); 2223 break; 2224 } 2225 2226 if (bio_op(bio) == REQ_OP_DISCARD) 2227 pool->process_discard(tc, bio); 2228 else 2229 pool->process_bio(tc, bio); 2230 2231 if ((count++ & 127) == 0) { 2232 throttle_work_update(&pool->throttle); 2233 dm_pool_issue_prefetches(pool->pmd); 2234 } 2235 cond_resched(); 2236 } 2237 blk_finish_plug(&plug); 2238 } 2239 2240 static int cmp_cells(const void *lhs, const void *rhs) 2241 { 2242 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs); 2243 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs); 2244 2245 BUG_ON(!lhs_cell->holder); 2246 BUG_ON(!rhs_cell->holder); 2247 2248 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector) 2249 return -1; 2250 2251 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector) 2252 return 1; 2253 2254 return 0; 2255 } 2256 2257 static unsigned int sort_cells(struct pool *pool, struct list_head *cells) 2258 { 2259 unsigned int count = 0; 2260 struct dm_bio_prison_cell *cell, *tmp; 2261 2262 list_for_each_entry_safe(cell, tmp, cells, user_list) { 2263 if (count >= CELL_SORT_ARRAY_SIZE) 2264 break; 2265 2266 pool->cell_sort_array[count++] = cell; 2267 list_del(&cell->user_list); 2268 } 2269 2270 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL); 2271 2272 return count; 2273 } 2274 2275 static void process_thin_deferred_cells(struct thin_c *tc) 2276 { 2277 struct pool *pool = tc->pool; 2278 struct list_head cells; 2279 struct dm_bio_prison_cell *cell; 2280 unsigned int i, j, count; 2281 2282 INIT_LIST_HEAD(&cells); 2283 2284 spin_lock_irq(&tc->lock); 2285 list_splice_init(&tc->deferred_cells, &cells); 2286 spin_unlock_irq(&tc->lock); 2287 2288 if (list_empty(&cells)) 2289 return; 2290 2291 do { 2292 count = sort_cells(tc->pool, &cells); 2293 2294 for (i = 0; i < count; i++) { 2295 cell = pool->cell_sort_array[i]; 2296 BUG_ON(!cell->holder); 2297 2298 /* 2299 * If we've got no free new_mapping structs, and processing 2300 * this bio might require one, we pause until there are some 2301 * prepared mappings to process. 2302 */ 2303 if (ensure_next_mapping(pool)) { 2304 for (j = i; j < count; j++) 2305 list_add(&pool->cell_sort_array[j]->user_list, &cells); 2306 2307 spin_lock_irq(&tc->lock); 2308 list_splice(&cells, &tc->deferred_cells); 2309 spin_unlock_irq(&tc->lock); 2310 return; 2311 } 2312 2313 if (bio_op(cell->holder) == REQ_OP_DISCARD) 2314 pool->process_discard_cell(tc, cell); 2315 else 2316 pool->process_cell(tc, cell); 2317 } 2318 cond_resched(); 2319 } while (!list_empty(&cells)); 2320 } 2321 2322 static void thin_get(struct thin_c *tc); 2323 static void thin_put(struct thin_c *tc); 2324 2325 /* 2326 * We can't hold rcu_read_lock() around code that can block. So we 2327 * find a thin with the rcu lock held; bump a refcount; then drop 2328 * the lock. 2329 */ 2330 static struct thin_c *get_first_thin(struct pool *pool) 2331 { 2332 struct thin_c *tc = NULL; 2333 2334 rcu_read_lock(); 2335 if (!list_empty(&pool->active_thins)) { 2336 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list); 2337 thin_get(tc); 2338 } 2339 rcu_read_unlock(); 2340 2341 return tc; 2342 } 2343 2344 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc) 2345 { 2346 struct thin_c *old_tc = tc; 2347 2348 rcu_read_lock(); 2349 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) { 2350 thin_get(tc); 2351 thin_put(old_tc); 2352 rcu_read_unlock(); 2353 return tc; 2354 } 2355 thin_put(old_tc); 2356 rcu_read_unlock(); 2357 2358 return NULL; 2359 } 2360 2361 static void process_deferred_bios(struct pool *pool) 2362 { 2363 struct bio *bio; 2364 struct bio_list bios, bio_completions; 2365 struct thin_c *tc; 2366 2367 tc = get_first_thin(pool); 2368 while (tc) { 2369 process_thin_deferred_cells(tc); 2370 process_thin_deferred_bios(tc); 2371 tc = get_next_thin(pool, tc); 2372 } 2373 2374 /* 2375 * If there are any deferred flush bios, we must commit the metadata 2376 * before issuing them or signaling their completion. 2377 */ 2378 bio_list_init(&bios); 2379 bio_list_init(&bio_completions); 2380 2381 spin_lock_irq(&pool->lock); 2382 bio_list_merge(&bios, &pool->deferred_flush_bios); 2383 bio_list_init(&pool->deferred_flush_bios); 2384 2385 bio_list_merge(&bio_completions, &pool->deferred_flush_completions); 2386 bio_list_init(&pool->deferred_flush_completions); 2387 spin_unlock_irq(&pool->lock); 2388 2389 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) && 2390 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool))) 2391 return; 2392 2393 if (commit(pool)) { 2394 bio_list_merge(&bios, &bio_completions); 2395 2396 while ((bio = bio_list_pop(&bios))) 2397 bio_io_error(bio); 2398 return; 2399 } 2400 pool->last_commit_jiffies = jiffies; 2401 2402 while ((bio = bio_list_pop(&bio_completions))) 2403 bio_endio(bio); 2404 2405 while ((bio = bio_list_pop(&bios))) { 2406 /* 2407 * The data device was flushed as part of metadata commit, 2408 * so complete redundant flushes immediately. 2409 */ 2410 if (bio->bi_opf & REQ_PREFLUSH) 2411 bio_endio(bio); 2412 else 2413 dm_submit_bio_remap(bio, NULL); 2414 } 2415 } 2416 2417 static void do_worker(struct work_struct *ws) 2418 { 2419 struct pool *pool = container_of(ws, struct pool, worker); 2420 2421 throttle_work_start(&pool->throttle); 2422 dm_pool_issue_prefetches(pool->pmd); 2423 throttle_work_update(&pool->throttle); 2424 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping); 2425 throttle_work_update(&pool->throttle); 2426 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard); 2427 throttle_work_update(&pool->throttle); 2428 process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2); 2429 throttle_work_update(&pool->throttle); 2430 process_deferred_bios(pool); 2431 throttle_work_complete(&pool->throttle); 2432 } 2433 2434 /* 2435 * We want to commit periodically so that not too much 2436 * unwritten data builds up. 2437 */ 2438 static void do_waker(struct work_struct *ws) 2439 { 2440 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker); 2441 2442 wake_worker(pool); 2443 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD); 2444 } 2445 2446 /* 2447 * We're holding onto IO to allow userland time to react. After the 2448 * timeout either the pool will have been resized (and thus back in 2449 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space. 2450 */ 2451 static void do_no_space_timeout(struct work_struct *ws) 2452 { 2453 struct pool *pool = container_of(to_delayed_work(ws), struct pool, 2454 no_space_timeout); 2455 2456 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) { 2457 pool->pf.error_if_no_space = true; 2458 notify_of_pool_mode_change(pool); 2459 error_retry_list_with_code(pool, BLK_STS_NOSPC); 2460 } 2461 } 2462 2463 /*----------------------------------------------------------------*/ 2464 2465 struct pool_work { 2466 struct work_struct worker; 2467 struct completion complete; 2468 }; 2469 2470 static struct pool_work *to_pool_work(struct work_struct *ws) 2471 { 2472 return container_of(ws, struct pool_work, worker); 2473 } 2474 2475 static void pool_work_complete(struct pool_work *pw) 2476 { 2477 complete(&pw->complete); 2478 } 2479 2480 static void pool_work_wait(struct pool_work *pw, struct pool *pool, 2481 void (*fn)(struct work_struct *)) 2482 { 2483 INIT_WORK_ONSTACK(&pw->worker, fn); 2484 init_completion(&pw->complete); 2485 queue_work(pool->wq, &pw->worker); 2486 wait_for_completion(&pw->complete); 2487 destroy_work_on_stack(&pw->worker); 2488 } 2489 2490 /*----------------------------------------------------------------*/ 2491 2492 struct noflush_work { 2493 struct pool_work pw; 2494 struct thin_c *tc; 2495 }; 2496 2497 static struct noflush_work *to_noflush(struct work_struct *ws) 2498 { 2499 return container_of(to_pool_work(ws), struct noflush_work, pw); 2500 } 2501 2502 static void do_noflush_start(struct work_struct *ws) 2503 { 2504 struct noflush_work *w = to_noflush(ws); 2505 2506 w->tc->requeue_mode = true; 2507 requeue_io(w->tc); 2508 pool_work_complete(&w->pw); 2509 } 2510 2511 static void do_noflush_stop(struct work_struct *ws) 2512 { 2513 struct noflush_work *w = to_noflush(ws); 2514 2515 w->tc->requeue_mode = false; 2516 pool_work_complete(&w->pw); 2517 } 2518 2519 static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *)) 2520 { 2521 struct noflush_work w; 2522 2523 w.tc = tc; 2524 pool_work_wait(&w.pw, tc->pool, fn); 2525 } 2526 2527 /*----------------------------------------------------------------*/ 2528 2529 static void set_discard_callbacks(struct pool *pool) 2530 { 2531 struct pool_c *pt = pool->ti->private; 2532 2533 if (pt->adjusted_pf.discard_passdown) { 2534 pool->process_discard_cell = process_discard_cell_passdown; 2535 pool->process_prepared_discard = process_prepared_discard_passdown_pt1; 2536 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2; 2537 } else { 2538 pool->process_discard_cell = process_discard_cell_no_passdown; 2539 pool->process_prepared_discard = process_prepared_discard_no_passdown; 2540 } 2541 } 2542 2543 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) 2544 { 2545 struct pool_c *pt = pool->ti->private; 2546 bool needs_check = dm_pool_metadata_needs_check(pool->pmd); 2547 enum pool_mode old_mode = get_pool_mode(pool); 2548 unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ; 2549 2550 /* 2551 * Never allow the pool to transition to PM_WRITE mode if user 2552 * intervention is required to verify metadata and data consistency. 2553 */ 2554 if (new_mode == PM_WRITE && needs_check) { 2555 DMERR("%s: unable to switch pool to write mode until repaired.", 2556 dm_device_name(pool->pool_md)); 2557 if (old_mode != new_mode) 2558 new_mode = old_mode; 2559 else 2560 new_mode = PM_READ_ONLY; 2561 } 2562 /* 2563 * If we were in PM_FAIL mode, rollback of metadata failed. We're 2564 * not going to recover without a thin_repair. So we never let the 2565 * pool move out of the old mode. 2566 */ 2567 if (old_mode == PM_FAIL) 2568 new_mode = old_mode; 2569 2570 switch (new_mode) { 2571 case PM_FAIL: 2572 dm_pool_metadata_read_only(pool->pmd); 2573 pool->process_bio = process_bio_fail; 2574 pool->process_discard = process_bio_fail; 2575 pool->process_cell = process_cell_fail; 2576 pool->process_discard_cell = process_cell_fail; 2577 pool->process_prepared_mapping = process_prepared_mapping_fail; 2578 pool->process_prepared_discard = process_prepared_discard_fail; 2579 2580 error_retry_list(pool); 2581 break; 2582 2583 case PM_OUT_OF_METADATA_SPACE: 2584 case PM_READ_ONLY: 2585 dm_pool_metadata_read_only(pool->pmd); 2586 pool->process_bio = process_bio_read_only; 2587 pool->process_discard = process_bio_success; 2588 pool->process_cell = process_cell_read_only; 2589 pool->process_discard_cell = process_cell_success; 2590 pool->process_prepared_mapping = process_prepared_mapping_fail; 2591 pool->process_prepared_discard = process_prepared_discard_success; 2592 2593 error_retry_list(pool); 2594 break; 2595 2596 case PM_OUT_OF_DATA_SPACE: 2597 /* 2598 * Ideally we'd never hit this state; the low water mark 2599 * would trigger userland to extend the pool before we 2600 * completely run out of data space. However, many small 2601 * IOs to unprovisioned space can consume data space at an 2602 * alarming rate. Adjust your low water mark if you're 2603 * frequently seeing this mode. 2604 */ 2605 pool->out_of_data_space = true; 2606 pool->process_bio = process_bio_read_only; 2607 pool->process_discard = process_discard_bio; 2608 pool->process_cell = process_cell_read_only; 2609 pool->process_prepared_mapping = process_prepared_mapping; 2610 set_discard_callbacks(pool); 2611 2612 if (!pool->pf.error_if_no_space && no_space_timeout) 2613 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout); 2614 break; 2615 2616 case PM_WRITE: 2617 if (old_mode == PM_OUT_OF_DATA_SPACE) 2618 cancel_delayed_work_sync(&pool->no_space_timeout); 2619 pool->out_of_data_space = false; 2620 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space; 2621 dm_pool_metadata_read_write(pool->pmd); 2622 pool->process_bio = process_bio; 2623 pool->process_discard = process_discard_bio; 2624 pool->process_cell = process_cell; 2625 pool->process_prepared_mapping = process_prepared_mapping; 2626 set_discard_callbacks(pool); 2627 break; 2628 } 2629 2630 pool->pf.mode = new_mode; 2631 /* 2632 * The pool mode may have changed, sync it so bind_control_target() 2633 * doesn't cause an unexpected mode transition on resume. 2634 */ 2635 pt->adjusted_pf.mode = new_mode; 2636 2637 if (old_mode != new_mode) 2638 notify_of_pool_mode_change(pool); 2639 } 2640 2641 static void abort_transaction(struct pool *pool) 2642 { 2643 const char *dev_name = dm_device_name(pool->pool_md); 2644 2645 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 2646 if (dm_pool_abort_metadata(pool->pmd)) { 2647 DMERR("%s: failed to abort metadata transaction", dev_name); 2648 set_pool_mode(pool, PM_FAIL); 2649 } 2650 2651 if (dm_pool_metadata_set_needs_check(pool->pmd)) { 2652 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 2653 set_pool_mode(pool, PM_FAIL); 2654 } 2655 } 2656 2657 static void metadata_operation_failed(struct pool *pool, const char *op, int r) 2658 { 2659 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 2660 dm_device_name(pool->pool_md), op, r); 2661 2662 abort_transaction(pool); 2663 set_pool_mode(pool, PM_READ_ONLY); 2664 } 2665 2666 /*----------------------------------------------------------------*/ 2667 2668 /* 2669 * Mapping functions. 2670 */ 2671 2672 /* 2673 * Called only while mapping a thin bio to hand it over to the workqueue. 2674 */ 2675 static void thin_defer_bio(struct thin_c *tc, struct bio *bio) 2676 { 2677 struct pool *pool = tc->pool; 2678 2679 spin_lock_irq(&tc->lock); 2680 bio_list_add(&tc->deferred_bio_list, bio); 2681 spin_unlock_irq(&tc->lock); 2682 2683 wake_worker(pool); 2684 } 2685 2686 static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio) 2687 { 2688 struct pool *pool = tc->pool; 2689 2690 throttle_lock(&pool->throttle); 2691 thin_defer_bio(tc, bio); 2692 throttle_unlock(&pool->throttle); 2693 } 2694 2695 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2696 { 2697 struct pool *pool = tc->pool; 2698 2699 throttle_lock(&pool->throttle); 2700 spin_lock_irq(&tc->lock); 2701 list_add_tail(&cell->user_list, &tc->deferred_cells); 2702 spin_unlock_irq(&tc->lock); 2703 throttle_unlock(&pool->throttle); 2704 2705 wake_worker(pool); 2706 } 2707 2708 static void thin_hook_bio(struct thin_c *tc, struct bio *bio) 2709 { 2710 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 2711 2712 h->tc = tc; 2713 h->shared_read_entry = NULL; 2714 h->all_io_entry = NULL; 2715 h->overwrite_mapping = NULL; 2716 h->cell = NULL; 2717 } 2718 2719 /* 2720 * Non-blocking function called from the thin target's map function. 2721 */ 2722 static int thin_bio_map(struct dm_target *ti, struct bio *bio) 2723 { 2724 int r; 2725 struct thin_c *tc = ti->private; 2726 dm_block_t block = get_bio_block(tc, bio); 2727 struct dm_thin_device *td = tc->td; 2728 struct dm_thin_lookup_result result; 2729 struct dm_bio_prison_cell *virt_cell, *data_cell; 2730 struct dm_cell_key key; 2731 2732 thin_hook_bio(tc, bio); 2733 2734 if (tc->requeue_mode) { 2735 bio->bi_status = BLK_STS_DM_REQUEUE; 2736 bio_endio(bio); 2737 return DM_MAPIO_SUBMITTED; 2738 } 2739 2740 if (get_pool_mode(tc->pool) == PM_FAIL) { 2741 bio_io_error(bio); 2742 return DM_MAPIO_SUBMITTED; 2743 } 2744 2745 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) { 2746 thin_defer_bio_with_throttle(tc, bio); 2747 return DM_MAPIO_SUBMITTED; 2748 } 2749 2750 /* 2751 * We must hold the virtual cell before doing the lookup, otherwise 2752 * there's a race with discard. 2753 */ 2754 build_virtual_key(tc->td, block, &key); 2755 if (bio_detain(tc->pool, &key, bio, &virt_cell)) 2756 return DM_MAPIO_SUBMITTED; 2757 2758 r = dm_thin_find_block(td, block, 0, &result); 2759 2760 /* 2761 * Note that we defer readahead too. 2762 */ 2763 switch (r) { 2764 case 0: 2765 if (unlikely(result.shared)) { 2766 /* 2767 * We have a race condition here between the 2768 * result.shared value returned by the lookup and 2769 * snapshot creation, which may cause new 2770 * sharing. 2771 * 2772 * To avoid this always quiesce the origin before 2773 * taking the snap. You want to do this anyway to 2774 * ensure a consistent application view 2775 * (i.e. lockfs). 2776 * 2777 * More distant ancestors are irrelevant. The 2778 * shared flag will be set in their case. 2779 */ 2780 thin_defer_cell(tc, virt_cell); 2781 return DM_MAPIO_SUBMITTED; 2782 } 2783 2784 build_data_key(tc->td, result.block, &key); 2785 if (bio_detain(tc->pool, &key, bio, &data_cell)) { 2786 cell_defer_no_holder(tc, virt_cell); 2787 return DM_MAPIO_SUBMITTED; 2788 } 2789 2790 inc_all_io_entry(tc->pool, bio); 2791 cell_defer_no_holder(tc, data_cell); 2792 cell_defer_no_holder(tc, virt_cell); 2793 2794 remap(tc, bio, result.block); 2795 return DM_MAPIO_REMAPPED; 2796 2797 case -ENODATA: 2798 case -EWOULDBLOCK: 2799 thin_defer_cell(tc, virt_cell); 2800 return DM_MAPIO_SUBMITTED; 2801 2802 default: 2803 /* 2804 * Must always call bio_io_error on failure. 2805 * dm_thin_find_block can fail with -EINVAL if the 2806 * pool is switched to fail-io mode. 2807 */ 2808 bio_io_error(bio); 2809 cell_defer_no_holder(tc, virt_cell); 2810 return DM_MAPIO_SUBMITTED; 2811 } 2812 } 2813 2814 static void requeue_bios(struct pool *pool) 2815 { 2816 struct thin_c *tc; 2817 2818 rcu_read_lock(); 2819 list_for_each_entry_rcu(tc, &pool->active_thins, list) { 2820 spin_lock_irq(&tc->lock); 2821 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list); 2822 bio_list_init(&tc->retry_on_resume_list); 2823 spin_unlock_irq(&tc->lock); 2824 } 2825 rcu_read_unlock(); 2826 } 2827 2828 /* 2829 *-------------------------------------------------------------- 2830 * Binding of control targets to a pool object 2831 *-------------------------------------------------------------- 2832 */ 2833 static bool is_factor(sector_t block_size, uint32_t n) 2834 { 2835 return !sector_div(block_size, n); 2836 } 2837 2838 /* 2839 * If discard_passdown was enabled verify that the data device 2840 * supports discards. Disable discard_passdown if not. 2841 */ 2842 static void disable_discard_passdown_if_not_supported(struct pool_c *pt) 2843 { 2844 struct pool *pool = pt->pool; 2845 struct block_device *data_bdev = pt->data_dev->bdev; 2846 struct queue_limits *data_limits = bdev_limits(data_bdev); 2847 const char *reason = NULL; 2848 2849 if (!pt->adjusted_pf.discard_passdown) 2850 return; 2851 2852 if (!bdev_max_discard_sectors(pt->data_dev->bdev)) 2853 reason = "discard unsupported"; 2854 2855 else if (data_limits->max_discard_sectors < pool->sectors_per_block) 2856 reason = "max discard sectors smaller than a block"; 2857 2858 if (reason) { 2859 DMWARN("Data device (%pg) %s: Disabling discard passdown.", data_bdev, reason); 2860 pt->adjusted_pf.discard_passdown = false; 2861 } 2862 } 2863 2864 static int bind_control_target(struct pool *pool, struct dm_target *ti) 2865 { 2866 struct pool_c *pt = ti->private; 2867 2868 /* 2869 * We want to make sure that a pool in PM_FAIL mode is never upgraded. 2870 */ 2871 enum pool_mode old_mode = get_pool_mode(pool); 2872 enum pool_mode new_mode = pt->adjusted_pf.mode; 2873 2874 /* 2875 * Don't change the pool's mode until set_pool_mode() below. 2876 * Otherwise the pool's process_* function pointers may 2877 * not match the desired pool mode. 2878 */ 2879 pt->adjusted_pf.mode = old_mode; 2880 2881 pool->ti = ti; 2882 pool->pf = pt->adjusted_pf; 2883 pool->low_water_blocks = pt->low_water_blocks; 2884 2885 set_pool_mode(pool, new_mode); 2886 2887 return 0; 2888 } 2889 2890 static void unbind_control_target(struct pool *pool, struct dm_target *ti) 2891 { 2892 if (pool->ti == ti) 2893 pool->ti = NULL; 2894 } 2895 2896 /* 2897 *-------------------------------------------------------------- 2898 * Pool creation 2899 *-------------------------------------------------------------- 2900 */ 2901 /* Initialize pool features. */ 2902 static void pool_features_init(struct pool_features *pf) 2903 { 2904 pf->mode = PM_WRITE; 2905 pf->zero_new_blocks = true; 2906 pf->discard_enabled = true; 2907 pf->discard_passdown = true; 2908 pf->error_if_no_space = false; 2909 } 2910 2911 static void __pool_destroy(struct pool *pool) 2912 { 2913 __pool_table_remove(pool); 2914 2915 vfree(pool->cell_sort_array); 2916 if (dm_pool_metadata_close(pool->pmd) < 0) 2917 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 2918 2919 dm_bio_prison_destroy(pool->prison); 2920 dm_kcopyd_client_destroy(pool->copier); 2921 2922 cancel_delayed_work_sync(&pool->waker); 2923 cancel_delayed_work_sync(&pool->no_space_timeout); 2924 if (pool->wq) 2925 destroy_workqueue(pool->wq); 2926 2927 if (pool->next_mapping) 2928 mempool_free(pool->next_mapping, &pool->mapping_pool); 2929 mempool_exit(&pool->mapping_pool); 2930 dm_deferred_set_destroy(pool->shared_read_ds); 2931 dm_deferred_set_destroy(pool->all_io_ds); 2932 kfree(pool); 2933 } 2934 2935 static struct kmem_cache *_new_mapping_cache; 2936 2937 static struct pool *pool_create(struct mapped_device *pool_md, 2938 struct block_device *metadata_dev, 2939 struct block_device *data_dev, 2940 unsigned long block_size, 2941 int read_only, char **error) 2942 { 2943 int r; 2944 void *err_p; 2945 struct pool *pool; 2946 struct dm_pool_metadata *pmd; 2947 bool format_device = read_only ? false : true; 2948 2949 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device); 2950 if (IS_ERR(pmd)) { 2951 *error = "Error creating metadata object"; 2952 return ERR_CAST(pmd); 2953 } 2954 2955 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 2956 if (!pool) { 2957 *error = "Error allocating memory for pool"; 2958 err_p = ERR_PTR(-ENOMEM); 2959 goto bad_pool; 2960 } 2961 2962 pool->pmd = pmd; 2963 pool->sectors_per_block = block_size; 2964 if (block_size & (block_size - 1)) 2965 pool->sectors_per_block_shift = -1; 2966 else 2967 pool->sectors_per_block_shift = __ffs(block_size); 2968 pool->low_water_blocks = 0; 2969 pool_features_init(&pool->pf); 2970 pool->prison = dm_bio_prison_create(); 2971 if (!pool->prison) { 2972 *error = "Error creating pool's bio prison"; 2973 err_p = ERR_PTR(-ENOMEM); 2974 goto bad_prison; 2975 } 2976 2977 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2978 if (IS_ERR(pool->copier)) { 2979 r = PTR_ERR(pool->copier); 2980 *error = "Error creating pool's kcopyd client"; 2981 err_p = ERR_PTR(r); 2982 goto bad_kcopyd_client; 2983 } 2984 2985 /* 2986 * Create singlethreaded workqueue that will service all devices 2987 * that use this metadata. 2988 */ 2989 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); 2990 if (!pool->wq) { 2991 *error = "Error creating pool's workqueue"; 2992 err_p = ERR_PTR(-ENOMEM); 2993 goto bad_wq; 2994 } 2995 2996 throttle_init(&pool->throttle); 2997 INIT_WORK(&pool->worker, do_worker); 2998 INIT_DELAYED_WORK(&pool->waker, do_waker); 2999 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout); 3000 spin_lock_init(&pool->lock); 3001 bio_list_init(&pool->deferred_flush_bios); 3002 bio_list_init(&pool->deferred_flush_completions); 3003 INIT_LIST_HEAD(&pool->prepared_mappings); 3004 INIT_LIST_HEAD(&pool->prepared_discards); 3005 INIT_LIST_HEAD(&pool->prepared_discards_pt2); 3006 INIT_LIST_HEAD(&pool->active_thins); 3007 pool->low_water_triggered = false; 3008 pool->suspended = true; 3009 pool->out_of_data_space = false; 3010 3011 pool->shared_read_ds = dm_deferred_set_create(); 3012 if (!pool->shared_read_ds) { 3013 *error = "Error creating pool's shared read deferred set"; 3014 err_p = ERR_PTR(-ENOMEM); 3015 goto bad_shared_read_ds; 3016 } 3017 3018 pool->all_io_ds = dm_deferred_set_create(); 3019 if (!pool->all_io_ds) { 3020 *error = "Error creating pool's all io deferred set"; 3021 err_p = ERR_PTR(-ENOMEM); 3022 goto bad_all_io_ds; 3023 } 3024 3025 pool->next_mapping = NULL; 3026 r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE, 3027 _new_mapping_cache); 3028 if (r) { 3029 *error = "Error creating pool's mapping mempool"; 3030 err_p = ERR_PTR(r); 3031 goto bad_mapping_pool; 3032 } 3033 3034 pool->cell_sort_array = 3035 vmalloc(array_size(CELL_SORT_ARRAY_SIZE, 3036 sizeof(*pool->cell_sort_array))); 3037 if (!pool->cell_sort_array) { 3038 *error = "Error allocating cell sort array"; 3039 err_p = ERR_PTR(-ENOMEM); 3040 goto bad_sort_array; 3041 } 3042 3043 pool->ref_count = 1; 3044 pool->last_commit_jiffies = jiffies; 3045 pool->pool_md = pool_md; 3046 pool->md_dev = metadata_dev; 3047 pool->data_dev = data_dev; 3048 __pool_table_insert(pool); 3049 3050 return pool; 3051 3052 bad_sort_array: 3053 mempool_exit(&pool->mapping_pool); 3054 bad_mapping_pool: 3055 dm_deferred_set_destroy(pool->all_io_ds); 3056 bad_all_io_ds: 3057 dm_deferred_set_destroy(pool->shared_read_ds); 3058 bad_shared_read_ds: 3059 destroy_workqueue(pool->wq); 3060 bad_wq: 3061 dm_kcopyd_client_destroy(pool->copier); 3062 bad_kcopyd_client: 3063 dm_bio_prison_destroy(pool->prison); 3064 bad_prison: 3065 kfree(pool); 3066 bad_pool: 3067 if (dm_pool_metadata_close(pmd)) 3068 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 3069 3070 return err_p; 3071 } 3072 3073 static void __pool_inc(struct pool *pool) 3074 { 3075 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 3076 pool->ref_count++; 3077 } 3078 3079 static void __pool_dec(struct pool *pool) 3080 { 3081 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 3082 BUG_ON(!pool->ref_count); 3083 if (!--pool->ref_count) 3084 __pool_destroy(pool); 3085 } 3086 3087 static struct pool *__pool_find(struct mapped_device *pool_md, 3088 struct block_device *metadata_dev, 3089 struct block_device *data_dev, 3090 unsigned long block_size, int read_only, 3091 char **error, int *created) 3092 { 3093 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev); 3094 3095 if (pool) { 3096 if (pool->pool_md != pool_md) { 3097 *error = "metadata device already in use by a pool"; 3098 return ERR_PTR(-EBUSY); 3099 } 3100 if (pool->data_dev != data_dev) { 3101 *error = "data device already in use by a pool"; 3102 return ERR_PTR(-EBUSY); 3103 } 3104 __pool_inc(pool); 3105 3106 } else { 3107 pool = __pool_table_lookup(pool_md); 3108 if (pool) { 3109 if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) { 3110 *error = "different pool cannot replace a pool"; 3111 return ERR_PTR(-EINVAL); 3112 } 3113 __pool_inc(pool); 3114 3115 } else { 3116 pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error); 3117 *created = 1; 3118 } 3119 } 3120 3121 return pool; 3122 } 3123 3124 /* 3125 *-------------------------------------------------------------- 3126 * Pool target methods 3127 *-------------------------------------------------------------- 3128 */ 3129 static void pool_dtr(struct dm_target *ti) 3130 { 3131 struct pool_c *pt = ti->private; 3132 3133 mutex_lock(&dm_thin_pool_table.mutex); 3134 3135 unbind_control_target(pt->pool, ti); 3136 __pool_dec(pt->pool); 3137 dm_put_device(ti, pt->metadata_dev); 3138 dm_put_device(ti, pt->data_dev); 3139 kfree(pt); 3140 3141 mutex_unlock(&dm_thin_pool_table.mutex); 3142 } 3143 3144 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, 3145 struct dm_target *ti) 3146 { 3147 int r; 3148 unsigned int argc; 3149 const char *arg_name; 3150 3151 static const struct dm_arg _args[] = { 3152 {0, 4, "Invalid number of pool feature arguments"}, 3153 }; 3154 3155 /* 3156 * No feature arguments supplied. 3157 */ 3158 if (!as->argc) 3159 return 0; 3160 3161 r = dm_read_arg_group(_args, as, &argc, &ti->error); 3162 if (r) 3163 return -EINVAL; 3164 3165 while (argc && !r) { 3166 arg_name = dm_shift_arg(as); 3167 argc--; 3168 3169 if (!strcasecmp(arg_name, "skip_block_zeroing")) 3170 pf->zero_new_blocks = false; 3171 3172 else if (!strcasecmp(arg_name, "ignore_discard")) 3173 pf->discard_enabled = false; 3174 3175 else if (!strcasecmp(arg_name, "no_discard_passdown")) 3176 pf->discard_passdown = false; 3177 3178 else if (!strcasecmp(arg_name, "read_only")) 3179 pf->mode = PM_READ_ONLY; 3180 3181 else if (!strcasecmp(arg_name, "error_if_no_space")) 3182 pf->error_if_no_space = true; 3183 3184 else { 3185 ti->error = "Unrecognised pool feature requested"; 3186 r = -EINVAL; 3187 break; 3188 } 3189 } 3190 3191 return r; 3192 } 3193 3194 static void metadata_low_callback(void *context) 3195 { 3196 struct pool *pool = context; 3197 3198 DMWARN("%s: reached low water mark for metadata device: sending event.", 3199 dm_device_name(pool->pool_md)); 3200 3201 dm_table_event(pool->ti->table); 3202 } 3203 3204 /* 3205 * We need to flush the data device **before** committing the metadata. 3206 * 3207 * This ensures that the data blocks of any newly inserted mappings are 3208 * properly written to non-volatile storage and won't be lost in case of a 3209 * crash. 3210 * 3211 * Failure to do so can result in data corruption in the case of internal or 3212 * external snapshots and in the case of newly provisioned blocks, when block 3213 * zeroing is enabled. 3214 */ 3215 static int metadata_pre_commit_callback(void *context) 3216 { 3217 struct pool *pool = context; 3218 3219 return blkdev_issue_flush(pool->data_dev); 3220 } 3221 3222 static sector_t get_dev_size(struct block_device *bdev) 3223 { 3224 return bdev_nr_sectors(bdev); 3225 } 3226 3227 static void warn_if_metadata_device_too_big(struct block_device *bdev) 3228 { 3229 sector_t metadata_dev_size = get_dev_size(bdev); 3230 3231 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING) 3232 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.", 3233 bdev, THIN_METADATA_MAX_SECTORS); 3234 } 3235 3236 static sector_t get_metadata_dev_size(struct block_device *bdev) 3237 { 3238 sector_t metadata_dev_size = get_dev_size(bdev); 3239 3240 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS) 3241 metadata_dev_size = THIN_METADATA_MAX_SECTORS; 3242 3243 return metadata_dev_size; 3244 } 3245 3246 static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev) 3247 { 3248 sector_t metadata_dev_size = get_metadata_dev_size(bdev); 3249 3250 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE); 3251 3252 return metadata_dev_size; 3253 } 3254 3255 /* 3256 * When a metadata threshold is crossed a dm event is triggered, and 3257 * userland should respond by growing the metadata device. We could let 3258 * userland set the threshold, like we do with the data threshold, but I'm 3259 * not sure they know enough to do this well. 3260 */ 3261 static dm_block_t calc_metadata_threshold(struct pool_c *pt) 3262 { 3263 /* 3264 * 4M is ample for all ops with the possible exception of thin 3265 * device deletion which is harmless if it fails (just retry the 3266 * delete after you've grown the device). 3267 */ 3268 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4; 3269 3270 return min((dm_block_t)1024ULL /* 4M */, quarter); 3271 } 3272 3273 /* 3274 * thin-pool <metadata dev> <data dev> 3275 * <data block size (sectors)> 3276 * <low water mark (blocks)> 3277 * [<#feature args> [<arg>]*] 3278 * 3279 * Optional feature arguments are: 3280 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks. 3281 * ignore_discard: disable discard 3282 * no_discard_passdown: don't pass discards down to the data device 3283 * read_only: Don't allow any changes to be made to the pool metadata. 3284 * error_if_no_space: error IOs, instead of queueing, if no space. 3285 */ 3286 static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv) 3287 { 3288 int r, pool_created = 0; 3289 struct pool_c *pt; 3290 struct pool *pool; 3291 struct pool_features pf; 3292 struct dm_arg_set as; 3293 struct dm_dev *data_dev; 3294 unsigned long block_size; 3295 dm_block_t low_water_blocks; 3296 struct dm_dev *metadata_dev; 3297 blk_mode_t metadata_mode; 3298 3299 /* 3300 * FIXME Remove validation from scope of lock. 3301 */ 3302 mutex_lock(&dm_thin_pool_table.mutex); 3303 3304 if (argc < 4) { 3305 ti->error = "Invalid argument count"; 3306 r = -EINVAL; 3307 goto out_unlock; 3308 } 3309 3310 as.argc = argc; 3311 as.argv = argv; 3312 3313 /* make sure metadata and data are different devices */ 3314 if (!strcmp(argv[0], argv[1])) { 3315 ti->error = "Error setting metadata or data device"; 3316 r = -EINVAL; 3317 goto out_unlock; 3318 } 3319 3320 /* 3321 * Set default pool features. 3322 */ 3323 pool_features_init(&pf); 3324 3325 dm_consume_args(&as, 4); 3326 r = parse_pool_features(&as, &pf, ti); 3327 if (r) 3328 goto out_unlock; 3329 3330 metadata_mode = BLK_OPEN_READ | 3331 ((pf.mode == PM_READ_ONLY) ? 0 : BLK_OPEN_WRITE); 3332 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev); 3333 if (r) { 3334 ti->error = "Error opening metadata block device"; 3335 goto out_unlock; 3336 } 3337 warn_if_metadata_device_too_big(metadata_dev->bdev); 3338 3339 r = dm_get_device(ti, argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, &data_dev); 3340 if (r) { 3341 ti->error = "Error getting data device"; 3342 goto out_metadata; 3343 } 3344 3345 if (kstrtoul(argv[2], 10, &block_size) || !block_size || 3346 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 3347 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 3348 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 3349 ti->error = "Invalid block size"; 3350 r = -EINVAL; 3351 goto out; 3352 } 3353 3354 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) { 3355 ti->error = "Invalid low water mark"; 3356 r = -EINVAL; 3357 goto out; 3358 } 3359 3360 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 3361 if (!pt) { 3362 r = -ENOMEM; 3363 goto out; 3364 } 3365 3366 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev, 3367 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created); 3368 if (IS_ERR(pool)) { 3369 r = PTR_ERR(pool); 3370 goto out_free_pt; 3371 } 3372 3373 /* 3374 * 'pool_created' reflects whether this is the first table load. 3375 * Top level discard support is not allowed to be changed after 3376 * initial load. This would require a pool reload to trigger thin 3377 * device changes. 3378 */ 3379 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) { 3380 ti->error = "Discard support cannot be disabled once enabled"; 3381 r = -EINVAL; 3382 goto out_flags_changed; 3383 } 3384 3385 pt->pool = pool; 3386 pt->ti = ti; 3387 pt->metadata_dev = metadata_dev; 3388 pt->data_dev = data_dev; 3389 pt->low_water_blocks = low_water_blocks; 3390 pt->adjusted_pf = pt->requested_pf = pf; 3391 ti->num_flush_bios = 1; 3392 ti->limit_swap_bios = true; 3393 3394 /* 3395 * Only need to enable discards if the pool should pass 3396 * them down to the data device. The thin device's discard 3397 * processing will cause mappings to be removed from the btree. 3398 */ 3399 if (pf.discard_enabled && pf.discard_passdown) { 3400 ti->num_discard_bios = 1; 3401 /* 3402 * Setting 'discards_supported' circumvents the normal 3403 * stacking of discard limits (this keeps the pool and 3404 * thin devices' discard limits consistent). 3405 */ 3406 ti->discards_supported = true; 3407 ti->max_discard_granularity = true; 3408 } 3409 ti->private = pt; 3410 3411 r = dm_pool_register_metadata_threshold(pt->pool->pmd, 3412 calc_metadata_threshold(pt), 3413 metadata_low_callback, 3414 pool); 3415 if (r) { 3416 ti->error = "Error registering metadata threshold"; 3417 goto out_flags_changed; 3418 } 3419 3420 dm_pool_register_pre_commit_callback(pool->pmd, 3421 metadata_pre_commit_callback, pool); 3422 3423 mutex_unlock(&dm_thin_pool_table.mutex); 3424 3425 return 0; 3426 3427 out_flags_changed: 3428 __pool_dec(pool); 3429 out_free_pt: 3430 kfree(pt); 3431 out: 3432 dm_put_device(ti, data_dev); 3433 out_metadata: 3434 dm_put_device(ti, metadata_dev); 3435 out_unlock: 3436 mutex_unlock(&dm_thin_pool_table.mutex); 3437 3438 return r; 3439 } 3440 3441 static int pool_map(struct dm_target *ti, struct bio *bio) 3442 { 3443 struct pool_c *pt = ti->private; 3444 struct pool *pool = pt->pool; 3445 3446 /* 3447 * As this is a singleton target, ti->begin is always zero. 3448 */ 3449 spin_lock_irq(&pool->lock); 3450 bio_set_dev(bio, pt->data_dev->bdev); 3451 spin_unlock_irq(&pool->lock); 3452 3453 return DM_MAPIO_REMAPPED; 3454 } 3455 3456 static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit) 3457 { 3458 int r; 3459 struct pool_c *pt = ti->private; 3460 struct pool *pool = pt->pool; 3461 sector_t data_size = ti->len; 3462 dm_block_t sb_data_size; 3463 3464 *need_commit = false; 3465 3466 (void) sector_div(data_size, pool->sectors_per_block); 3467 3468 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size); 3469 if (r) { 3470 DMERR("%s: failed to retrieve data device size", 3471 dm_device_name(pool->pool_md)); 3472 return r; 3473 } 3474 3475 if (data_size < sb_data_size) { 3476 DMERR("%s: pool target (%llu blocks) too small: expected %llu", 3477 dm_device_name(pool->pool_md), 3478 (unsigned long long)data_size, sb_data_size); 3479 return -EINVAL; 3480 3481 } else if (data_size > sb_data_size) { 3482 if (dm_pool_metadata_needs_check(pool->pmd)) { 3483 DMERR("%s: unable to grow the data device until repaired.", 3484 dm_device_name(pool->pool_md)); 3485 return 0; 3486 } 3487 3488 if (sb_data_size) 3489 DMINFO("%s: growing the data device from %llu to %llu blocks", 3490 dm_device_name(pool->pool_md), 3491 sb_data_size, (unsigned long long)data_size); 3492 r = dm_pool_resize_data_dev(pool->pmd, data_size); 3493 if (r) { 3494 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r); 3495 return r; 3496 } 3497 3498 *need_commit = true; 3499 } 3500 3501 return 0; 3502 } 3503 3504 static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit) 3505 { 3506 int r; 3507 struct pool_c *pt = ti->private; 3508 struct pool *pool = pt->pool; 3509 dm_block_t metadata_dev_size, sb_metadata_dev_size; 3510 3511 *need_commit = false; 3512 3513 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev); 3514 3515 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size); 3516 if (r) { 3517 DMERR("%s: failed to retrieve metadata device size", 3518 dm_device_name(pool->pool_md)); 3519 return r; 3520 } 3521 3522 if (metadata_dev_size < sb_metadata_dev_size) { 3523 DMERR("%s: metadata device (%llu blocks) too small: expected %llu", 3524 dm_device_name(pool->pool_md), 3525 metadata_dev_size, sb_metadata_dev_size); 3526 return -EINVAL; 3527 3528 } else if (metadata_dev_size > sb_metadata_dev_size) { 3529 if (dm_pool_metadata_needs_check(pool->pmd)) { 3530 DMERR("%s: unable to grow the metadata device until repaired.", 3531 dm_device_name(pool->pool_md)); 3532 return 0; 3533 } 3534 3535 warn_if_metadata_device_too_big(pool->md_dev); 3536 DMINFO("%s: growing the metadata device from %llu to %llu blocks", 3537 dm_device_name(pool->pool_md), 3538 sb_metadata_dev_size, metadata_dev_size); 3539 3540 if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE) 3541 set_pool_mode(pool, PM_WRITE); 3542 3543 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size); 3544 if (r) { 3545 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r); 3546 return r; 3547 } 3548 3549 *need_commit = true; 3550 } 3551 3552 return 0; 3553 } 3554 3555 /* 3556 * Retrieves the number of blocks of the data device from 3557 * the superblock and compares it to the actual device size, 3558 * thus resizing the data device in case it has grown. 3559 * 3560 * This both copes with opening preallocated data devices in the ctr 3561 * being followed by a resume 3562 * -and- 3563 * calling the resume method individually after userspace has 3564 * grown the data device in reaction to a table event. 3565 */ 3566 static int pool_preresume(struct dm_target *ti) 3567 { 3568 int r; 3569 bool need_commit1, need_commit2; 3570 struct pool_c *pt = ti->private; 3571 struct pool *pool = pt->pool; 3572 3573 /* 3574 * Take control of the pool object. 3575 */ 3576 r = bind_control_target(pool, ti); 3577 if (r) 3578 goto out; 3579 3580 r = maybe_resize_data_dev(ti, &need_commit1); 3581 if (r) 3582 goto out; 3583 3584 r = maybe_resize_metadata_dev(ti, &need_commit2); 3585 if (r) 3586 goto out; 3587 3588 if (need_commit1 || need_commit2) 3589 (void) commit(pool); 3590 out: 3591 /* 3592 * When a thin-pool is PM_FAIL, it cannot be rebuilt if 3593 * bio is in deferred list. Therefore need to return 0 3594 * to allow pool_resume() to flush IO. 3595 */ 3596 if (r && get_pool_mode(pool) == PM_FAIL) 3597 r = 0; 3598 3599 return r; 3600 } 3601 3602 static void pool_suspend_active_thins(struct pool *pool) 3603 { 3604 struct thin_c *tc; 3605 3606 /* Suspend all active thin devices */ 3607 tc = get_first_thin(pool); 3608 while (tc) { 3609 dm_internal_suspend_noflush(tc->thin_md); 3610 tc = get_next_thin(pool, tc); 3611 } 3612 } 3613 3614 static void pool_resume_active_thins(struct pool *pool) 3615 { 3616 struct thin_c *tc; 3617 3618 /* Resume all active thin devices */ 3619 tc = get_first_thin(pool); 3620 while (tc) { 3621 dm_internal_resume(tc->thin_md); 3622 tc = get_next_thin(pool, tc); 3623 } 3624 } 3625 3626 static void pool_resume(struct dm_target *ti) 3627 { 3628 struct pool_c *pt = ti->private; 3629 struct pool *pool = pt->pool; 3630 3631 /* 3632 * Must requeue active_thins' bios and then resume 3633 * active_thins _before_ clearing 'suspend' flag. 3634 */ 3635 requeue_bios(pool); 3636 pool_resume_active_thins(pool); 3637 3638 spin_lock_irq(&pool->lock); 3639 pool->low_water_triggered = false; 3640 pool->suspended = false; 3641 spin_unlock_irq(&pool->lock); 3642 3643 do_waker(&pool->waker.work); 3644 } 3645 3646 static void pool_presuspend(struct dm_target *ti) 3647 { 3648 struct pool_c *pt = ti->private; 3649 struct pool *pool = pt->pool; 3650 3651 spin_lock_irq(&pool->lock); 3652 pool->suspended = true; 3653 spin_unlock_irq(&pool->lock); 3654 3655 pool_suspend_active_thins(pool); 3656 } 3657 3658 static void pool_presuspend_undo(struct dm_target *ti) 3659 { 3660 struct pool_c *pt = ti->private; 3661 struct pool *pool = pt->pool; 3662 3663 pool_resume_active_thins(pool); 3664 3665 spin_lock_irq(&pool->lock); 3666 pool->suspended = false; 3667 spin_unlock_irq(&pool->lock); 3668 } 3669 3670 static void pool_postsuspend(struct dm_target *ti) 3671 { 3672 struct pool_c *pt = ti->private; 3673 struct pool *pool = pt->pool; 3674 3675 cancel_delayed_work_sync(&pool->waker); 3676 cancel_delayed_work_sync(&pool->no_space_timeout); 3677 flush_workqueue(pool->wq); 3678 (void) commit(pool); 3679 } 3680 3681 static int check_arg_count(unsigned int argc, unsigned int args_required) 3682 { 3683 if (argc != args_required) { 3684 DMWARN("Message received with %u arguments instead of %u.", 3685 argc, args_required); 3686 return -EINVAL; 3687 } 3688 3689 return 0; 3690 } 3691 3692 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning) 3693 { 3694 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) && 3695 *dev_id <= MAX_DEV_ID) 3696 return 0; 3697 3698 if (warning) 3699 DMWARN("Message received with invalid device id: %s", arg); 3700 3701 return -EINVAL; 3702 } 3703 3704 static int process_create_thin_mesg(unsigned int argc, char **argv, struct pool *pool) 3705 { 3706 dm_thin_id dev_id; 3707 int r; 3708 3709 r = check_arg_count(argc, 2); 3710 if (r) 3711 return r; 3712 3713 r = read_dev_id(argv[1], &dev_id, 1); 3714 if (r) 3715 return r; 3716 3717 r = dm_pool_create_thin(pool->pmd, dev_id); 3718 if (r) { 3719 DMWARN("Creation of new thinly-provisioned device with id %s failed.", 3720 argv[1]); 3721 return r; 3722 } 3723 3724 return 0; 3725 } 3726 3727 static int process_create_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3728 { 3729 dm_thin_id dev_id; 3730 dm_thin_id origin_dev_id; 3731 int r; 3732 3733 r = check_arg_count(argc, 3); 3734 if (r) 3735 return r; 3736 3737 r = read_dev_id(argv[1], &dev_id, 1); 3738 if (r) 3739 return r; 3740 3741 r = read_dev_id(argv[2], &origin_dev_id, 1); 3742 if (r) 3743 return r; 3744 3745 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id); 3746 if (r) { 3747 DMWARN("Creation of new snapshot %s of device %s failed.", 3748 argv[1], argv[2]); 3749 return r; 3750 } 3751 3752 return 0; 3753 } 3754 3755 static int process_delete_mesg(unsigned int argc, char **argv, struct pool *pool) 3756 { 3757 dm_thin_id dev_id; 3758 int r; 3759 3760 r = check_arg_count(argc, 2); 3761 if (r) 3762 return r; 3763 3764 r = read_dev_id(argv[1], &dev_id, 1); 3765 if (r) 3766 return r; 3767 3768 r = dm_pool_delete_thin_device(pool->pmd, dev_id); 3769 if (r) 3770 DMWARN("Deletion of thin device %s failed.", argv[1]); 3771 3772 return r; 3773 } 3774 3775 static int process_set_transaction_id_mesg(unsigned int argc, char **argv, struct pool *pool) 3776 { 3777 dm_thin_id old_id, new_id; 3778 int r; 3779 3780 r = check_arg_count(argc, 3); 3781 if (r) 3782 return r; 3783 3784 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) { 3785 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]); 3786 return -EINVAL; 3787 } 3788 3789 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) { 3790 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]); 3791 return -EINVAL; 3792 } 3793 3794 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id); 3795 if (r) { 3796 DMWARN("Failed to change transaction id from %s to %s.", 3797 argv[1], argv[2]); 3798 return r; 3799 } 3800 3801 return 0; 3802 } 3803 3804 static int process_reserve_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3805 { 3806 int r; 3807 3808 r = check_arg_count(argc, 1); 3809 if (r) 3810 return r; 3811 3812 (void) commit(pool); 3813 3814 r = dm_pool_reserve_metadata_snap(pool->pmd); 3815 if (r) 3816 DMWARN("reserve_metadata_snap message failed."); 3817 3818 return r; 3819 } 3820 3821 static int process_release_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool) 3822 { 3823 int r; 3824 3825 r = check_arg_count(argc, 1); 3826 if (r) 3827 return r; 3828 3829 r = dm_pool_release_metadata_snap(pool->pmd); 3830 if (r) 3831 DMWARN("release_metadata_snap message failed."); 3832 3833 return r; 3834 } 3835 3836 /* 3837 * Messages supported: 3838 * create_thin <dev_id> 3839 * create_snap <dev_id> <origin_id> 3840 * delete <dev_id> 3841 * set_transaction_id <current_trans_id> <new_trans_id> 3842 * reserve_metadata_snap 3843 * release_metadata_snap 3844 */ 3845 static int pool_message(struct dm_target *ti, unsigned int argc, char **argv, 3846 char *result, unsigned int maxlen) 3847 { 3848 int r = -EINVAL; 3849 struct pool_c *pt = ti->private; 3850 struct pool *pool = pt->pool; 3851 3852 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) { 3853 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode", 3854 dm_device_name(pool->pool_md)); 3855 return -EOPNOTSUPP; 3856 } 3857 3858 if (!strcasecmp(argv[0], "create_thin")) 3859 r = process_create_thin_mesg(argc, argv, pool); 3860 3861 else if (!strcasecmp(argv[0], "create_snap")) 3862 r = process_create_snap_mesg(argc, argv, pool); 3863 3864 else if (!strcasecmp(argv[0], "delete")) 3865 r = process_delete_mesg(argc, argv, pool); 3866 3867 else if (!strcasecmp(argv[0], "set_transaction_id")) 3868 r = process_set_transaction_id_mesg(argc, argv, pool); 3869 3870 else if (!strcasecmp(argv[0], "reserve_metadata_snap")) 3871 r = process_reserve_metadata_snap_mesg(argc, argv, pool); 3872 3873 else if (!strcasecmp(argv[0], "release_metadata_snap")) 3874 r = process_release_metadata_snap_mesg(argc, argv, pool); 3875 3876 else 3877 DMWARN("Unrecognised thin pool target message received: %s", argv[0]); 3878 3879 if (!r) 3880 (void) commit(pool); 3881 3882 return r; 3883 } 3884 3885 static void emit_flags(struct pool_features *pf, char *result, 3886 unsigned int sz, unsigned int maxlen) 3887 { 3888 unsigned int count = !pf->zero_new_blocks + !pf->discard_enabled + 3889 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) + 3890 pf->error_if_no_space; 3891 DMEMIT("%u ", count); 3892 3893 if (!pf->zero_new_blocks) 3894 DMEMIT("skip_block_zeroing "); 3895 3896 if (!pf->discard_enabled) 3897 DMEMIT("ignore_discard "); 3898 3899 if (!pf->discard_passdown) 3900 DMEMIT("no_discard_passdown "); 3901 3902 if (pf->mode == PM_READ_ONLY) 3903 DMEMIT("read_only "); 3904 3905 if (pf->error_if_no_space) 3906 DMEMIT("error_if_no_space "); 3907 } 3908 3909 /* 3910 * Status line is: 3911 * <transaction id> <used metadata sectors>/<total metadata sectors> 3912 * <used data sectors>/<total data sectors> <held metadata root> 3913 * <pool mode> <discard config> <no space config> <needs_check> 3914 */ 3915 static void pool_status(struct dm_target *ti, status_type_t type, 3916 unsigned int status_flags, char *result, unsigned int maxlen) 3917 { 3918 int r; 3919 unsigned int sz = 0; 3920 uint64_t transaction_id; 3921 dm_block_t nr_free_blocks_data; 3922 dm_block_t nr_free_blocks_metadata; 3923 dm_block_t nr_blocks_data; 3924 dm_block_t nr_blocks_metadata; 3925 dm_block_t held_root; 3926 enum pool_mode mode; 3927 char buf[BDEVNAME_SIZE]; 3928 char buf2[BDEVNAME_SIZE]; 3929 struct pool_c *pt = ti->private; 3930 struct pool *pool = pt->pool; 3931 3932 switch (type) { 3933 case STATUSTYPE_INFO: 3934 if (get_pool_mode(pool) == PM_FAIL) { 3935 DMEMIT("Fail"); 3936 break; 3937 } 3938 3939 /* Commit to ensure statistics aren't out-of-date */ 3940 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3941 (void) commit(pool); 3942 3943 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id); 3944 if (r) { 3945 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d", 3946 dm_device_name(pool->pool_md), r); 3947 goto err; 3948 } 3949 3950 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata); 3951 if (r) { 3952 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d", 3953 dm_device_name(pool->pool_md), r); 3954 goto err; 3955 } 3956 3957 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); 3958 if (r) { 3959 DMERR("%s: dm_pool_get_metadata_dev_size returned %d", 3960 dm_device_name(pool->pool_md), r); 3961 goto err; 3962 } 3963 3964 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data); 3965 if (r) { 3966 DMERR("%s: dm_pool_get_free_block_count returned %d", 3967 dm_device_name(pool->pool_md), r); 3968 goto err; 3969 } 3970 3971 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); 3972 if (r) { 3973 DMERR("%s: dm_pool_get_data_dev_size returned %d", 3974 dm_device_name(pool->pool_md), r); 3975 goto err; 3976 } 3977 3978 r = dm_pool_get_metadata_snap(pool->pmd, &held_root); 3979 if (r) { 3980 DMERR("%s: dm_pool_get_metadata_snap returned %d", 3981 dm_device_name(pool->pool_md), r); 3982 goto err; 3983 } 3984 3985 DMEMIT("%llu %llu/%llu %llu/%llu ", 3986 (unsigned long long)transaction_id, 3987 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3988 (unsigned long long)nr_blocks_metadata, 3989 (unsigned long long)(nr_blocks_data - nr_free_blocks_data), 3990 (unsigned long long)nr_blocks_data); 3991 3992 if (held_root) 3993 DMEMIT("%llu ", held_root); 3994 else 3995 DMEMIT("- "); 3996 3997 mode = get_pool_mode(pool); 3998 if (mode == PM_OUT_OF_DATA_SPACE) 3999 DMEMIT("out_of_data_space "); 4000 else if (is_read_only_pool_mode(mode)) 4001 DMEMIT("ro "); 4002 else 4003 DMEMIT("rw "); 4004 4005 if (!pool->pf.discard_enabled) 4006 DMEMIT("ignore_discard "); 4007 else if (pool->pf.discard_passdown) 4008 DMEMIT("discard_passdown "); 4009 else 4010 DMEMIT("no_discard_passdown "); 4011 4012 if (pool->pf.error_if_no_space) 4013 DMEMIT("error_if_no_space "); 4014 else 4015 DMEMIT("queue_if_no_space "); 4016 4017 if (dm_pool_metadata_needs_check(pool->pmd)) 4018 DMEMIT("needs_check "); 4019 else 4020 DMEMIT("- "); 4021 4022 DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt)); 4023 4024 break; 4025 4026 case STATUSTYPE_TABLE: 4027 DMEMIT("%s %s %lu %llu ", 4028 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev), 4029 format_dev_t(buf2, pt->data_dev->bdev->bd_dev), 4030 (unsigned long)pool->sectors_per_block, 4031 (unsigned long long)pt->low_water_blocks); 4032 emit_flags(&pt->requested_pf, result, sz, maxlen); 4033 break; 4034 4035 case STATUSTYPE_IMA: 4036 *result = '\0'; 4037 break; 4038 } 4039 return; 4040 4041 err: 4042 DMEMIT("Error"); 4043 } 4044 4045 static int pool_iterate_devices(struct dm_target *ti, 4046 iterate_devices_callout_fn fn, void *data) 4047 { 4048 struct pool_c *pt = ti->private; 4049 4050 return fn(ti, pt->data_dev, 0, ti->len, data); 4051 } 4052 4053 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) 4054 { 4055 struct pool_c *pt = ti->private; 4056 struct pool *pool = pt->pool; 4057 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 4058 4059 /* 4060 * If max_sectors is smaller than pool->sectors_per_block adjust it 4061 * to the highest possible power-of-2 factor of pool->sectors_per_block. 4062 * This is especially beneficial when the pool's data device is a RAID 4063 * device that has a full stripe width that matches pool->sectors_per_block 4064 * -- because even though partial RAID stripe-sized IOs will be issued to a 4065 * single RAID stripe; when aggregated they will end on a full RAID stripe 4066 * boundary.. which avoids additional partial RAID stripe writes cascading 4067 */ 4068 if (limits->max_sectors < pool->sectors_per_block) { 4069 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) { 4070 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0) 4071 limits->max_sectors--; 4072 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors); 4073 } 4074 } 4075 4076 /* 4077 * If the system-determined stacked limits are compatible with the 4078 * pool's blocksize (io_opt is a factor) do not override them. 4079 */ 4080 if (io_opt_sectors < pool->sectors_per_block || 4081 !is_factor(io_opt_sectors, pool->sectors_per_block)) { 4082 if (is_factor(pool->sectors_per_block, limits->max_sectors)) 4083 limits->io_min = limits->max_sectors << SECTOR_SHIFT; 4084 else 4085 limits->io_min = pool->sectors_per_block << SECTOR_SHIFT; 4086 limits->io_opt = pool->sectors_per_block << SECTOR_SHIFT; 4087 } 4088 4089 /* 4090 * pt->adjusted_pf is a staging area for the actual features to use. 4091 * They get transferred to the live pool in bind_control_target() 4092 * called from pool_preresume(). 4093 */ 4094 4095 if (pt->adjusted_pf.discard_enabled) { 4096 disable_discard_passdown_if_not_supported(pt); 4097 if (!pt->adjusted_pf.discard_passdown) 4098 limits->max_hw_discard_sectors = 0; 4099 /* 4100 * The pool uses the same discard limits as the underlying data 4101 * device. DM core has already set this up. 4102 */ 4103 } else { 4104 /* 4105 * Must explicitly disallow stacking discard limits otherwise the 4106 * block layer will stack them if pool's data device has support. 4107 */ 4108 limits->discard_granularity = 0; 4109 } 4110 } 4111 4112 static struct target_type pool_target = { 4113 .name = "thin-pool", 4114 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | 4115 DM_TARGET_IMMUTABLE, 4116 .version = {1, 23, 0}, 4117 .module = THIS_MODULE, 4118 .ctr = pool_ctr, 4119 .dtr = pool_dtr, 4120 .map = pool_map, 4121 .presuspend = pool_presuspend, 4122 .presuspend_undo = pool_presuspend_undo, 4123 .postsuspend = pool_postsuspend, 4124 .preresume = pool_preresume, 4125 .resume = pool_resume, 4126 .message = pool_message, 4127 .status = pool_status, 4128 .iterate_devices = pool_iterate_devices, 4129 .io_hints = pool_io_hints, 4130 }; 4131 4132 /* 4133 *-------------------------------------------------------------- 4134 * Thin target methods 4135 *-------------------------------------------------------------- 4136 */ 4137 static void thin_get(struct thin_c *tc) 4138 { 4139 refcount_inc(&tc->refcount); 4140 } 4141 4142 static void thin_put(struct thin_c *tc) 4143 { 4144 if (refcount_dec_and_test(&tc->refcount)) 4145 complete(&tc->can_destroy); 4146 } 4147 4148 static void thin_dtr(struct dm_target *ti) 4149 { 4150 struct thin_c *tc = ti->private; 4151 4152 spin_lock_irq(&tc->pool->lock); 4153 list_del_rcu(&tc->list); 4154 spin_unlock_irq(&tc->pool->lock); 4155 synchronize_rcu(); 4156 4157 thin_put(tc); 4158 wait_for_completion(&tc->can_destroy); 4159 4160 mutex_lock(&dm_thin_pool_table.mutex); 4161 4162 __pool_dec(tc->pool); 4163 dm_pool_close_thin_device(tc->td); 4164 dm_put_device(ti, tc->pool_dev); 4165 if (tc->origin_dev) 4166 dm_put_device(ti, tc->origin_dev); 4167 kfree(tc); 4168 4169 mutex_unlock(&dm_thin_pool_table.mutex); 4170 } 4171 4172 /* 4173 * Thin target parameters: 4174 * 4175 * <pool_dev> <dev_id> [origin_dev] 4176 * 4177 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool) 4178 * dev_id: the internal device identifier 4179 * origin_dev: a device external to the pool that should act as the origin 4180 * 4181 * If the pool device has discards disabled, they get disabled for the thin 4182 * device as well. 4183 */ 4184 static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 4185 { 4186 int r; 4187 struct thin_c *tc; 4188 struct dm_dev *pool_dev, *origin_dev; 4189 struct mapped_device *pool_md; 4190 4191 mutex_lock(&dm_thin_pool_table.mutex); 4192 4193 if (argc != 2 && argc != 3) { 4194 ti->error = "Invalid argument count"; 4195 r = -EINVAL; 4196 goto out_unlock; 4197 } 4198 4199 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL); 4200 if (!tc) { 4201 ti->error = "Out of memory"; 4202 r = -ENOMEM; 4203 goto out_unlock; 4204 } 4205 tc->thin_md = dm_table_get_md(ti->table); 4206 spin_lock_init(&tc->lock); 4207 INIT_LIST_HEAD(&tc->deferred_cells); 4208 bio_list_init(&tc->deferred_bio_list); 4209 bio_list_init(&tc->retry_on_resume_list); 4210 tc->sort_bio_list = RB_ROOT; 4211 4212 if (argc == 3) { 4213 if (!strcmp(argv[0], argv[2])) { 4214 ti->error = "Error setting origin device"; 4215 r = -EINVAL; 4216 goto bad_origin_dev; 4217 } 4218 4219 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &origin_dev); 4220 if (r) { 4221 ti->error = "Error opening origin device"; 4222 goto bad_origin_dev; 4223 } 4224 tc->origin_dev = origin_dev; 4225 } 4226 4227 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev); 4228 if (r) { 4229 ti->error = "Error opening pool device"; 4230 goto bad_pool_dev; 4231 } 4232 tc->pool_dev = pool_dev; 4233 4234 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) { 4235 ti->error = "Invalid device id"; 4236 r = -EINVAL; 4237 goto bad_common; 4238 } 4239 4240 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev); 4241 if (!pool_md) { 4242 ti->error = "Couldn't get pool mapped device"; 4243 r = -EINVAL; 4244 goto bad_common; 4245 } 4246 4247 tc->pool = __pool_table_lookup(pool_md); 4248 if (!tc->pool) { 4249 ti->error = "Couldn't find pool object"; 4250 r = -EINVAL; 4251 goto bad_pool_lookup; 4252 } 4253 __pool_inc(tc->pool); 4254 4255 if (get_pool_mode(tc->pool) == PM_FAIL) { 4256 ti->error = "Couldn't open thin device, Pool is in fail mode"; 4257 r = -EINVAL; 4258 goto bad_pool; 4259 } 4260 4261 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td); 4262 if (r) { 4263 ti->error = "Couldn't open thin internal device"; 4264 goto bad_pool; 4265 } 4266 4267 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block); 4268 if (r) 4269 goto bad; 4270 4271 ti->num_flush_bios = 1; 4272 ti->limit_swap_bios = true; 4273 ti->flush_supported = true; 4274 ti->accounts_remapped_io = true; 4275 ti->per_io_data_size = sizeof(struct dm_thin_endio_hook); 4276 4277 /* In case the pool supports discards, pass them on. */ 4278 if (tc->pool->pf.discard_enabled) { 4279 ti->discards_supported = true; 4280 ti->num_discard_bios = 1; 4281 ti->max_discard_granularity = true; 4282 } 4283 4284 mutex_unlock(&dm_thin_pool_table.mutex); 4285 4286 spin_lock_irq(&tc->pool->lock); 4287 if (tc->pool->suspended) { 4288 spin_unlock_irq(&tc->pool->lock); 4289 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */ 4290 ti->error = "Unable to activate thin device while pool is suspended"; 4291 r = -EINVAL; 4292 goto bad; 4293 } 4294 refcount_set(&tc->refcount, 1); 4295 init_completion(&tc->can_destroy); 4296 list_add_tail_rcu(&tc->list, &tc->pool->active_thins); 4297 spin_unlock_irq(&tc->pool->lock); 4298 /* 4299 * This synchronize_rcu() call is needed here otherwise we risk a 4300 * wake_worker() call finding no bios to process (because the newly 4301 * added tc isn't yet visible). So this reduces latency since we 4302 * aren't then dependent on the periodic commit to wake_worker(). 4303 */ 4304 synchronize_rcu(); 4305 4306 dm_put(pool_md); 4307 4308 return 0; 4309 4310 bad: 4311 dm_pool_close_thin_device(tc->td); 4312 bad_pool: 4313 __pool_dec(tc->pool); 4314 bad_pool_lookup: 4315 dm_put(pool_md); 4316 bad_common: 4317 dm_put_device(ti, tc->pool_dev); 4318 bad_pool_dev: 4319 if (tc->origin_dev) 4320 dm_put_device(ti, tc->origin_dev); 4321 bad_origin_dev: 4322 kfree(tc); 4323 out_unlock: 4324 mutex_unlock(&dm_thin_pool_table.mutex); 4325 4326 return r; 4327 } 4328 4329 static int thin_map(struct dm_target *ti, struct bio *bio) 4330 { 4331 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 4332 4333 return thin_bio_map(ti, bio); 4334 } 4335 4336 static int thin_endio(struct dm_target *ti, struct bio *bio, 4337 blk_status_t *err) 4338 { 4339 unsigned long flags; 4340 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 4341 struct list_head work; 4342 struct dm_thin_new_mapping *m, *tmp; 4343 struct pool *pool = h->tc->pool; 4344 4345 if (h->shared_read_entry) { 4346 INIT_LIST_HEAD(&work); 4347 dm_deferred_entry_dec(h->shared_read_entry, &work); 4348 4349 spin_lock_irqsave(&pool->lock, flags); 4350 list_for_each_entry_safe(m, tmp, &work, list) { 4351 list_del(&m->list); 4352 __complete_mapping_preparation(m); 4353 } 4354 spin_unlock_irqrestore(&pool->lock, flags); 4355 } 4356 4357 if (h->all_io_entry) { 4358 INIT_LIST_HEAD(&work); 4359 dm_deferred_entry_dec(h->all_io_entry, &work); 4360 if (!list_empty(&work)) { 4361 spin_lock_irqsave(&pool->lock, flags); 4362 list_for_each_entry_safe(m, tmp, &work, list) 4363 list_add_tail(&m->list, &pool->prepared_discards); 4364 spin_unlock_irqrestore(&pool->lock, flags); 4365 wake_worker(pool); 4366 } 4367 } 4368 4369 if (h->cell) 4370 cell_defer_no_holder(h->tc, h->cell); 4371 4372 return DM_ENDIO_DONE; 4373 } 4374 4375 static void thin_presuspend(struct dm_target *ti) 4376 { 4377 struct thin_c *tc = ti->private; 4378 4379 if (dm_noflush_suspending(ti)) 4380 noflush_work(tc, do_noflush_start); 4381 } 4382 4383 static void thin_postsuspend(struct dm_target *ti) 4384 { 4385 struct thin_c *tc = ti->private; 4386 4387 /* 4388 * The dm_noflush_suspending flag has been cleared by now, so 4389 * unfortunately we must always run this. 4390 */ 4391 noflush_work(tc, do_noflush_stop); 4392 } 4393 4394 static int thin_preresume(struct dm_target *ti) 4395 { 4396 struct thin_c *tc = ti->private; 4397 4398 if (tc->origin_dev) 4399 tc->origin_size = get_dev_size(tc->origin_dev->bdev); 4400 4401 return 0; 4402 } 4403 4404 /* 4405 * <nr mapped sectors> <highest mapped sector> 4406 */ 4407 static void thin_status(struct dm_target *ti, status_type_t type, 4408 unsigned int status_flags, char *result, unsigned int maxlen) 4409 { 4410 int r; 4411 ssize_t sz = 0; 4412 dm_block_t mapped, highest; 4413 char buf[BDEVNAME_SIZE]; 4414 struct thin_c *tc = ti->private; 4415 4416 if (get_pool_mode(tc->pool) == PM_FAIL) { 4417 DMEMIT("Fail"); 4418 return; 4419 } 4420 4421 if (!tc->td) 4422 DMEMIT("-"); 4423 else { 4424 switch (type) { 4425 case STATUSTYPE_INFO: 4426 r = dm_thin_get_mapped_count(tc->td, &mapped); 4427 if (r) { 4428 DMERR("dm_thin_get_mapped_count returned %d", r); 4429 goto err; 4430 } 4431 4432 r = dm_thin_get_highest_mapped_block(tc->td, &highest); 4433 if (r < 0) { 4434 DMERR("dm_thin_get_highest_mapped_block returned %d", r); 4435 goto err; 4436 } 4437 4438 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); 4439 if (r) 4440 DMEMIT("%llu", ((highest + 1) * 4441 tc->pool->sectors_per_block) - 1); 4442 else 4443 DMEMIT("-"); 4444 break; 4445 4446 case STATUSTYPE_TABLE: 4447 DMEMIT("%s %lu", 4448 format_dev_t(buf, tc->pool_dev->bdev->bd_dev), 4449 (unsigned long) tc->dev_id); 4450 if (tc->origin_dev) 4451 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev)); 4452 break; 4453 4454 case STATUSTYPE_IMA: 4455 *result = '\0'; 4456 break; 4457 } 4458 } 4459 4460 return; 4461 4462 err: 4463 DMEMIT("Error"); 4464 } 4465 4466 static int thin_iterate_devices(struct dm_target *ti, 4467 iterate_devices_callout_fn fn, void *data) 4468 { 4469 sector_t blocks; 4470 struct thin_c *tc = ti->private; 4471 struct pool *pool = tc->pool; 4472 4473 /* 4474 * We can't call dm_pool_get_data_dev_size() since that blocks. So 4475 * we follow a more convoluted path through to the pool's target. 4476 */ 4477 if (!pool->ti) 4478 return 0; /* nothing is bound */ 4479 4480 blocks = pool->ti->len; 4481 (void) sector_div(blocks, pool->sectors_per_block); 4482 if (blocks) 4483 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data); 4484 4485 return 0; 4486 } 4487 4488 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits) 4489 { 4490 struct thin_c *tc = ti->private; 4491 struct pool *pool = tc->pool; 4492 4493 if (pool->pf.discard_enabled) { 4494 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; 4495 limits->max_hw_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE; 4496 } 4497 } 4498 4499 static struct target_type thin_target = { 4500 .name = "thin", 4501 .version = {1, 23, 0}, 4502 .module = THIS_MODULE, 4503 .ctr = thin_ctr, 4504 .dtr = thin_dtr, 4505 .map = thin_map, 4506 .end_io = thin_endio, 4507 .preresume = thin_preresume, 4508 .presuspend = thin_presuspend, 4509 .postsuspend = thin_postsuspend, 4510 .status = thin_status, 4511 .iterate_devices = thin_iterate_devices, 4512 .io_hints = thin_io_hints, 4513 }; 4514 4515 /*----------------------------------------------------------------*/ 4516 4517 static int __init dm_thin_init(void) 4518 { 4519 int r = -ENOMEM; 4520 4521 pool_table_init(); 4522 4523 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0); 4524 if (!_new_mapping_cache) 4525 return r; 4526 4527 r = dm_register_target(&thin_target); 4528 if (r) 4529 goto bad_new_mapping_cache; 4530 4531 r = dm_register_target(&pool_target); 4532 if (r) 4533 goto bad_thin_target; 4534 4535 return 0; 4536 4537 bad_thin_target: 4538 dm_unregister_target(&thin_target); 4539 bad_new_mapping_cache: 4540 kmem_cache_destroy(_new_mapping_cache); 4541 4542 return r; 4543 } 4544 4545 static void dm_thin_exit(void) 4546 { 4547 dm_unregister_target(&thin_target); 4548 dm_unregister_target(&pool_target); 4549 4550 kmem_cache_destroy(_new_mapping_cache); 4551 4552 pool_table_exit(); 4553 } 4554 4555 module_init(dm_thin_init); 4556 module_exit(dm_thin_exit); 4557 4558 module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644); 4559 MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds"); 4560 4561 MODULE_DESCRIPTION(DM_NAME " thin provisioning target"); 4562 MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>"); 4563 MODULE_LICENSE("GPL"); 4564