xref: /linux/drivers/md/dm-cache-target.c (revision 87aecd24f58e15256870befed777bbffc5d4a9d5)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Red Hat. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm.h"
9 #include "dm-bio-prison-v2.h"
10 #include "dm-bio-record.h"
11 #include "dm-cache-metadata.h"
12 #include "dm-io-tracker.h"
13 #include "dm-cache-background-tracker.h"
14 
15 #include <linux/dm-io.h>
16 #include <linux/dm-kcopyd.h>
17 #include <linux/jiffies.h>
18 #include <linux/init.h>
19 #include <linux/kstrtox.h>
20 #include <linux/math64.h>
21 #include <linux/mempool.h>
22 #include <linux/module.h>
23 #include <linux/rwsem.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 
27 #define DM_MSG_PREFIX "cache"
28 
29 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
30 	"A percentage of time allocated for copying to and/or from cache");
31 
32 /*----------------------------------------------------------------*/
33 
34 /*
35  * Glossary:
36  *
37  * oblock: index of an origin block
38  * cblock: index of a cache block
39  * promotion: movement of a block from origin to cache
40  * demotion: movement of a block from cache to origin
41  * migration: movement of a block between the origin and cache device,
42  *	      either direction
43  */
44 
45 /*----------------------------------------------------------------*/
46 
47 /*
48  * Represents a chunk of future work.  'input' allows continuations to pass
49  * values between themselves, typically error values.
50  */
51 struct continuation {
52 	struct work_struct ws;
53 	blk_status_t input;
54 };
55 
56 static inline void init_continuation(struct continuation *k,
57 				     void (*fn)(struct work_struct *))
58 {
59 	INIT_WORK(&k->ws, fn);
60 	k->input = 0;
61 }
62 
63 static inline void queue_continuation(struct workqueue_struct *wq,
64 				      struct continuation *k)
65 {
66 	queue_work(wq, &k->ws);
67 }
68 
69 /*----------------------------------------------------------------*/
70 
71 /*
72  * The batcher collects together pieces of work that need a particular
73  * operation to occur before they can proceed (typically a commit).
74  */
75 struct batcher {
76 	/*
77 	 * The operation that everyone is waiting for.
78 	 */
79 	blk_status_t (*commit_op)(void *context);
80 	void *commit_context;
81 
82 	/*
83 	 * This is how bios should be issued once the commit op is complete
84 	 * (accounted_request).
85 	 */
86 	void (*issue_op)(struct bio *bio, void *context);
87 	void *issue_context;
88 
89 	/*
90 	 * Queued work gets put on here after commit.
91 	 */
92 	struct workqueue_struct *wq;
93 
94 	spinlock_t lock;
95 	struct list_head work_items;
96 	struct bio_list bios;
97 	struct work_struct commit_work;
98 
99 	bool commit_scheduled;
100 };
101 
102 static void __commit(struct work_struct *_ws)
103 {
104 	struct batcher *b = container_of(_ws, struct batcher, commit_work);
105 	blk_status_t r;
106 	struct list_head work_items;
107 	struct work_struct *ws, *tmp;
108 	struct continuation *k;
109 	struct bio *bio;
110 	struct bio_list bios;
111 
112 	INIT_LIST_HEAD(&work_items);
113 	bio_list_init(&bios);
114 
115 	/*
116 	 * We have to grab these before the commit_op to avoid a race
117 	 * condition.
118 	 */
119 	spin_lock_irq(&b->lock);
120 	list_splice_init(&b->work_items, &work_items);
121 	bio_list_merge_init(&bios, &b->bios);
122 	b->commit_scheduled = false;
123 	spin_unlock_irq(&b->lock);
124 
125 	r = b->commit_op(b->commit_context);
126 
127 	list_for_each_entry_safe(ws, tmp, &work_items, entry) {
128 		k = container_of(ws, struct continuation, ws);
129 		k->input = r;
130 		INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
131 		queue_work(b->wq, ws);
132 	}
133 
134 	while ((bio = bio_list_pop(&bios))) {
135 		if (r) {
136 			bio->bi_status = r;
137 			bio_endio(bio);
138 		} else
139 			b->issue_op(bio, b->issue_context);
140 	}
141 }
142 
143 static void batcher_init(struct batcher *b,
144 			 blk_status_t (*commit_op)(void *),
145 			 void *commit_context,
146 			 void (*issue_op)(struct bio *bio, void *),
147 			 void *issue_context,
148 			 struct workqueue_struct *wq)
149 {
150 	b->commit_op = commit_op;
151 	b->commit_context = commit_context;
152 	b->issue_op = issue_op;
153 	b->issue_context = issue_context;
154 	b->wq = wq;
155 
156 	spin_lock_init(&b->lock);
157 	INIT_LIST_HEAD(&b->work_items);
158 	bio_list_init(&b->bios);
159 	INIT_WORK(&b->commit_work, __commit);
160 	b->commit_scheduled = false;
161 }
162 
163 static void async_commit(struct batcher *b)
164 {
165 	queue_work(b->wq, &b->commit_work);
166 }
167 
168 static void continue_after_commit(struct batcher *b, struct continuation *k)
169 {
170 	bool commit_scheduled;
171 
172 	spin_lock_irq(&b->lock);
173 	commit_scheduled = b->commit_scheduled;
174 	list_add_tail(&k->ws.entry, &b->work_items);
175 	spin_unlock_irq(&b->lock);
176 
177 	if (commit_scheduled)
178 		async_commit(b);
179 }
180 
181 /*
182  * Bios are errored if commit failed.
183  */
184 static void issue_after_commit(struct batcher *b, struct bio *bio)
185 {
186 	bool commit_scheduled;
187 
188 	spin_lock_irq(&b->lock);
189 	commit_scheduled = b->commit_scheduled;
190 	bio_list_add(&b->bios, bio);
191 	spin_unlock_irq(&b->lock);
192 
193 	if (commit_scheduled)
194 		async_commit(b);
195 }
196 
197 /*
198  * Call this if some urgent work is waiting for the commit to complete.
199  */
200 static void schedule_commit(struct batcher *b)
201 {
202 	bool immediate;
203 
204 	spin_lock_irq(&b->lock);
205 	immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
206 	b->commit_scheduled = true;
207 	spin_unlock_irq(&b->lock);
208 
209 	if (immediate)
210 		async_commit(b);
211 }
212 
213 /*
214  * There are a couple of places where we let a bio run, but want to do some
215  * work before calling its endio function.  We do this by temporarily
216  * changing the endio fn.
217  */
218 struct dm_hook_info {
219 	bio_end_io_t *bi_end_io;
220 };
221 
222 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
223 			bio_end_io_t *bi_end_io, void *bi_private)
224 {
225 	h->bi_end_io = bio->bi_end_io;
226 
227 	bio->bi_end_io = bi_end_io;
228 	bio->bi_private = bi_private;
229 }
230 
231 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
232 {
233 	bio->bi_end_io = h->bi_end_io;
234 }
235 
236 /*----------------------------------------------------------------*/
237 
238 #define MIGRATION_POOL_SIZE 128
239 #define COMMIT_PERIOD HZ
240 #define MIGRATION_COUNT_WINDOW 10
241 
242 /*
243  * The block size of the device holding cache data must be
244  * between 32KB and 1GB.
245  */
246 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
247 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
248 
249 enum cache_metadata_mode {
250 	CM_WRITE,		/* metadata may be changed */
251 	CM_READ_ONLY,		/* metadata may not be changed */
252 	CM_FAIL
253 };
254 
255 enum cache_io_mode {
256 	/*
257 	 * Data is written to cached blocks only.  These blocks are marked
258 	 * dirty.  If you lose the cache device you will lose data.
259 	 * Potential performance increase for both reads and writes.
260 	 */
261 	CM_IO_WRITEBACK,
262 
263 	/*
264 	 * Data is written to both cache and origin.  Blocks are never
265 	 * dirty.  Potential performance benfit for reads only.
266 	 */
267 	CM_IO_WRITETHROUGH,
268 
269 	/*
270 	 * A degraded mode useful for various cache coherency situations
271 	 * (eg, rolling back snapshots).  Reads and writes always go to the
272 	 * origin.  If a write goes to a cached oblock, then the cache
273 	 * block is invalidated.
274 	 */
275 	CM_IO_PASSTHROUGH
276 };
277 
278 struct cache_features {
279 	enum cache_metadata_mode mode;
280 	enum cache_io_mode io_mode;
281 	unsigned int metadata_version;
282 	bool discard_passdown:1;
283 };
284 
285 struct cache_stats {
286 	atomic_t read_hit;
287 	atomic_t read_miss;
288 	atomic_t write_hit;
289 	atomic_t write_miss;
290 	atomic_t demotion;
291 	atomic_t promotion;
292 	atomic_t writeback;
293 	atomic_t copies_avoided;
294 	atomic_t cache_cell_clash;
295 	atomic_t commit_count;
296 	atomic_t discard_count;
297 };
298 
299 struct cache {
300 	struct dm_target *ti;
301 	spinlock_t lock;
302 
303 	/*
304 	 * Fields for converting from sectors to blocks.
305 	 */
306 	int sectors_per_block_shift;
307 	sector_t sectors_per_block;
308 
309 	struct dm_cache_metadata *cmd;
310 
311 	/*
312 	 * Metadata is written to this device.
313 	 */
314 	struct dm_dev *metadata_dev;
315 
316 	/*
317 	 * The slower of the two data devices.  Typically a spindle.
318 	 */
319 	struct dm_dev *origin_dev;
320 
321 	/*
322 	 * The faster of the two data devices.  Typically an SSD.
323 	 */
324 	struct dm_dev *cache_dev;
325 
326 	/*
327 	 * Size of the origin device in _complete_ blocks and native sectors.
328 	 */
329 	dm_oblock_t origin_blocks;
330 	sector_t origin_sectors;
331 
332 	/*
333 	 * Size of the cache device in blocks.
334 	 */
335 	dm_cblock_t cache_size;
336 
337 	/*
338 	 * Invalidation fields.
339 	 */
340 	spinlock_t invalidation_lock;
341 	struct list_head invalidation_requests;
342 
343 	sector_t migration_threshold;
344 
345 	/*
346 	 * The number of in flight migrations that are performing
347 	 * background io. eg, promotion, writeback.
348 	 */
349 	atomic_t nr_io_migrations;
350 
351 	struct bio_list deferred_bios;
352 
353 	struct rw_semaphore quiesce_lock;
354 
355 	/*
356 	 * origin_blocks entries, discarded if set.
357 	 */
358 	dm_dblock_t discard_nr_blocks;
359 	unsigned long *discard_bitset;
360 	uint32_t discard_block_size; /* a power of 2 times sectors per block */
361 
362 	/*
363 	 * Rather than reconstructing the table line for the status we just
364 	 * save it and regurgitate.
365 	 */
366 	unsigned int nr_ctr_args;
367 	const char **ctr_args;
368 
369 	struct dm_kcopyd_client *copier;
370 	struct work_struct deferred_bio_worker;
371 	struct work_struct migration_worker;
372 	struct workqueue_struct *wq;
373 	struct delayed_work waker;
374 	struct dm_bio_prison_v2 *prison;
375 
376 	/*
377 	 * cache_size entries, dirty if set
378 	 */
379 	unsigned long *dirty_bitset;
380 	atomic_t nr_dirty;
381 
382 	unsigned int policy_nr_args;
383 	struct dm_cache_policy *policy;
384 
385 	/*
386 	 * Cache features such as write-through.
387 	 */
388 	struct cache_features features;
389 
390 	struct cache_stats stats;
391 
392 	bool need_tick_bio:1;
393 	bool sized:1;
394 	bool invalidate:1;
395 	bool commit_requested:1;
396 	bool loaded_mappings:1;
397 	bool loaded_discards:1;
398 
399 	/* background work management */
400 	bool background_work_allowed;
401 	unsigned background_work_nr;
402 	spinlock_t background_work_lock;
403 	wait_queue_head_t background_work_wait;
404 
405 	struct batcher committer;
406 	struct work_struct commit_ws;
407 
408 	struct dm_io_tracker tracker;
409 
410 	mempool_t migration_pool;
411 
412 	struct bio_set bs;
413 
414 	/*
415 	 * Cache_size entries. Set bits indicate blocks mapped beyond the
416 	 * target length, which are marked for invalidation.
417 	 */
418 	unsigned long *invalid_bitset;
419 };
420 
421 struct per_bio_data {
422 	bool tick:1;
423 	unsigned int req_nr:2;
424 	struct dm_bio_prison_cell_v2 *cell;
425 	struct dm_hook_info hook_info;
426 	sector_t len;
427 };
428 
429 struct dm_cache_migration {
430 	struct continuation k;
431 	struct cache *cache;
432 
433 	struct policy_work *op;
434 	struct bio *overwrite_bio;
435 	struct dm_bio_prison_cell_v2 *cell;
436 
437 	dm_cblock_t invalidate_cblock;
438 	dm_oblock_t invalidate_oblock;
439 };
440 
441 /*----------------------------------------------------------------*/
442 
443 static bool writethrough_mode(struct cache *cache)
444 {
445 	return cache->features.io_mode == CM_IO_WRITETHROUGH;
446 }
447 
448 static bool writeback_mode(struct cache *cache)
449 {
450 	return cache->features.io_mode == CM_IO_WRITEBACK;
451 }
452 
453 static inline bool passthrough_mode(struct cache *cache)
454 {
455 	return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
456 }
457 
458 /*----------------------------------------------------------------*/
459 
460 static void wake_deferred_bio_worker(struct cache *cache)
461 {
462 	queue_work(cache->wq, &cache->deferred_bio_worker);
463 }
464 
465 static void wake_migration_worker(struct cache *cache)
466 {
467 	if (passthrough_mode(cache))
468 		return;
469 
470 	queue_work(cache->wq, &cache->migration_worker);
471 }
472 
473 /*----------------------------------------------------------------*/
474 
475 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
476 {
477 	return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);
478 }
479 
480 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
481 {
482 	dm_bio_prison_free_cell_v2(cache->prison, cell);
483 }
484 
485 static struct dm_cache_migration *alloc_migration(struct cache *cache)
486 {
487 	struct dm_cache_migration *mg;
488 
489 	mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);
490 
491 	memset(mg, 0, sizeof(*mg));
492 
493 	mg->cache = cache;
494 
495 	return mg;
496 }
497 
498 static void free_migration(struct dm_cache_migration *mg)
499 {
500 	mempool_free(mg, &mg->cache->migration_pool);
501 }
502 
503 /*----------------------------------------------------------------*/
504 
505 static inline dm_oblock_t oblock_succ(dm_oblock_t b)
506 {
507 	return to_oblock(from_oblock(b) + 1ull);
508 }
509 
510 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
511 {
512 	key->virtual = 0;
513 	key->dev = 0;
514 	key->block_begin = from_oblock(begin);
515 	key->block_end = from_oblock(end);
516 }
517 
518 /*
519  * We have two lock levels.  Level 0, which is used to prevent WRITEs, and
520  * level 1 which prevents *both* READs and WRITEs.
521  */
522 #define WRITE_LOCK_LEVEL 0
523 #define READ_WRITE_LOCK_LEVEL 1
524 
525 static unsigned int lock_level(struct bio *bio)
526 {
527 	return bio_data_dir(bio) == WRITE ?
528 		WRITE_LOCK_LEVEL :
529 		READ_WRITE_LOCK_LEVEL;
530 }
531 
532 /*
533  *--------------------------------------------------------------
534  * Per bio data
535  *--------------------------------------------------------------
536  */
537 
538 static struct per_bio_data *get_per_bio_data(struct bio *bio)
539 {
540 	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
541 
542 	BUG_ON(!pb);
543 	return pb;
544 }
545 
546 static struct per_bio_data *init_per_bio_data(struct bio *bio)
547 {
548 	struct per_bio_data *pb = get_per_bio_data(bio);
549 
550 	pb->tick = false;
551 	pb->req_nr = dm_bio_get_target_bio_nr(bio);
552 	pb->cell = NULL;
553 	pb->len = 0;
554 
555 	return pb;
556 }
557 
558 /*----------------------------------------------------------------*/
559 
560 static void defer_bio(struct cache *cache, struct bio *bio)
561 {
562 	spin_lock_irq(&cache->lock);
563 	bio_list_add(&cache->deferred_bios, bio);
564 	spin_unlock_irq(&cache->lock);
565 
566 	wake_deferred_bio_worker(cache);
567 }
568 
569 static void defer_bios(struct cache *cache, struct bio_list *bios)
570 {
571 	spin_lock_irq(&cache->lock);
572 	bio_list_merge_init(&cache->deferred_bios, bios);
573 	spin_unlock_irq(&cache->lock);
574 
575 	wake_deferred_bio_worker(cache);
576 }
577 
578 /*----------------------------------------------------------------*/
579 
580 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
581 {
582 	bool r;
583 	struct per_bio_data *pb;
584 	struct dm_cell_key_v2 key;
585 	dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
586 	struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
587 
588 	cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
589 
590 	build_key(oblock, end, &key);
591 	r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
592 	if (!r) {
593 		/*
594 		 * Failed to get the lock.
595 		 */
596 		free_prison_cell(cache, cell_prealloc);
597 		return r;
598 	}
599 
600 	if (cell != cell_prealloc)
601 		free_prison_cell(cache, cell_prealloc);
602 
603 	pb = get_per_bio_data(bio);
604 	pb->cell = cell;
605 
606 	return r;
607 }
608 
609 /*----------------------------------------------------------------*/
610 
611 static bool is_dirty(struct cache *cache, dm_cblock_t b)
612 {
613 	return test_bit(from_cblock(b), cache->dirty_bitset);
614 }
615 
616 static void set_dirty(struct cache *cache, dm_cblock_t cblock)
617 {
618 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
619 		atomic_inc(&cache->nr_dirty);
620 		policy_set_dirty(cache->policy, cblock);
621 	}
622 }
623 
624 /*
625  * These two are called when setting after migrations to force the policy
626  * and dirty bitset to be in sync.
627  */
628 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
629 {
630 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
631 		atomic_inc(&cache->nr_dirty);
632 	policy_set_dirty(cache->policy, cblock);
633 }
634 
635 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
636 {
637 	if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
638 		if (atomic_dec_return(&cache->nr_dirty) == 0)
639 			dm_table_event(cache->ti->table);
640 	}
641 
642 	policy_clear_dirty(cache->policy, cblock);
643 }
644 
645 /*----------------------------------------------------------------*/
646 
647 static bool block_size_is_power_of_two(struct cache *cache)
648 {
649 	return cache->sectors_per_block_shift >= 0;
650 }
651 
652 static dm_block_t block_div(dm_block_t b, uint32_t n)
653 {
654 	do_div(b, n);
655 
656 	return b;
657 }
658 
659 static dm_block_t oblocks_per_dblock(struct cache *cache)
660 {
661 	dm_block_t oblocks = cache->discard_block_size;
662 
663 	if (block_size_is_power_of_two(cache))
664 		oblocks >>= cache->sectors_per_block_shift;
665 	else
666 		oblocks = block_div(oblocks, cache->sectors_per_block);
667 
668 	return oblocks;
669 }
670 
671 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
672 {
673 	return to_dblock(block_div(from_oblock(oblock),
674 				   oblocks_per_dblock(cache)));
675 }
676 
677 static void set_discard(struct cache *cache, dm_dblock_t b)
678 {
679 	BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
680 	atomic_inc(&cache->stats.discard_count);
681 
682 	spin_lock_irq(&cache->lock);
683 	set_bit(from_dblock(b), cache->discard_bitset);
684 	spin_unlock_irq(&cache->lock);
685 }
686 
687 static void clear_discard(struct cache *cache, dm_dblock_t b)
688 {
689 	spin_lock_irq(&cache->lock);
690 	clear_bit(from_dblock(b), cache->discard_bitset);
691 	spin_unlock_irq(&cache->lock);
692 }
693 
694 static bool is_discarded(struct cache *cache, dm_dblock_t b)
695 {
696 	int r;
697 
698 	spin_lock_irq(&cache->lock);
699 	r = test_bit(from_dblock(b), cache->discard_bitset);
700 	spin_unlock_irq(&cache->lock);
701 
702 	return r;
703 }
704 
705 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
706 {
707 	int r;
708 
709 	spin_lock_irq(&cache->lock);
710 	r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
711 		     cache->discard_bitset);
712 	spin_unlock_irq(&cache->lock);
713 
714 	return r;
715 }
716 
717 /*
718  * -------------------------------------------------------------
719  * Remapping
720  *--------------------------------------------------------------
721  */
722 static void remap_to_origin(struct cache *cache, struct bio *bio)
723 {
724 	bio_set_dev(bio, cache->origin_dev->bdev);
725 }
726 
727 static void remap_to_cache(struct cache *cache, struct bio *bio,
728 			   dm_cblock_t cblock)
729 {
730 	sector_t bi_sector = bio->bi_iter.bi_sector;
731 	sector_t block = from_cblock(cblock);
732 
733 	bio_set_dev(bio, cache->cache_dev->bdev);
734 	if (!block_size_is_power_of_two(cache))
735 		bio->bi_iter.bi_sector =
736 			(block * cache->sectors_per_block) +
737 			sector_div(bi_sector, cache->sectors_per_block);
738 	else
739 		bio->bi_iter.bi_sector =
740 			(block << cache->sectors_per_block_shift) |
741 			(bi_sector & (cache->sectors_per_block - 1));
742 }
743 
744 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
745 {
746 	struct per_bio_data *pb;
747 
748 	spin_lock_irq(&cache->lock);
749 	if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
750 	    bio_op(bio) != REQ_OP_DISCARD) {
751 		pb = get_per_bio_data(bio);
752 		pb->tick = true;
753 		cache->need_tick_bio = false;
754 	}
755 	spin_unlock_irq(&cache->lock);
756 }
757 
758 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
759 					  dm_oblock_t oblock)
760 {
761 	// FIXME: check_if_tick_bio_needed() is called way too much through this interface
762 	check_if_tick_bio_needed(cache, bio);
763 	remap_to_origin(cache, bio);
764 	if (bio_data_dir(bio) == WRITE)
765 		clear_discard(cache, oblock_to_dblock(cache, oblock));
766 }
767 
768 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
769 				 dm_oblock_t oblock, dm_cblock_t cblock)
770 {
771 	check_if_tick_bio_needed(cache, bio);
772 	remap_to_cache(cache, bio, cblock);
773 	if (bio_data_dir(bio) == WRITE) {
774 		set_dirty(cache, cblock);
775 		clear_discard(cache, oblock_to_dblock(cache, oblock));
776 	}
777 }
778 
779 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
780 {
781 	sector_t block_nr = bio->bi_iter.bi_sector;
782 
783 	if (!block_size_is_power_of_two(cache))
784 		(void) sector_div(block_nr, cache->sectors_per_block);
785 	else
786 		block_nr >>= cache->sectors_per_block_shift;
787 
788 	return to_oblock(block_nr);
789 }
790 
791 static bool accountable_bio(struct cache *cache, struct bio *bio)
792 {
793 	return bio_op(bio) != REQ_OP_DISCARD;
794 }
795 
796 static void accounted_begin(struct cache *cache, struct bio *bio)
797 {
798 	struct per_bio_data *pb;
799 
800 	if (accountable_bio(cache, bio)) {
801 		pb = get_per_bio_data(bio);
802 		pb->len = bio_sectors(bio);
803 		dm_iot_io_begin(&cache->tracker, pb->len);
804 	}
805 }
806 
807 static void accounted_complete(struct cache *cache, struct bio *bio)
808 {
809 	struct per_bio_data *pb = get_per_bio_data(bio);
810 
811 	dm_iot_io_end(&cache->tracker, pb->len);
812 }
813 
814 static void accounted_request(struct cache *cache, struct bio *bio)
815 {
816 	accounted_begin(cache, bio);
817 	dm_submit_bio_remap(bio, NULL);
818 }
819 
820 static void issue_op(struct bio *bio, void *context)
821 {
822 	struct cache *cache = context;
823 
824 	accounted_request(cache, bio);
825 }
826 
827 /*
828  * When running in writethrough mode we need to send writes to clean blocks
829  * to both the cache and origin devices.  Clone the bio and send them in parallel.
830  */
831 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
832 				      dm_oblock_t oblock, dm_cblock_t cblock)
833 {
834 	struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio,
835 						 GFP_NOIO, &cache->bs);
836 
837 	BUG_ON(!origin_bio);
838 
839 	bio_chain(origin_bio, bio);
840 
841 	if (bio_data_dir(origin_bio) == WRITE)
842 		clear_discard(cache, oblock_to_dblock(cache, oblock));
843 	submit_bio(origin_bio);
844 
845 	remap_to_cache(cache, bio, cblock);
846 }
847 
848 /*
849  *--------------------------------------------------------------
850  * Failure modes
851  *--------------------------------------------------------------
852  */
853 static enum cache_metadata_mode get_cache_mode(struct cache *cache)
854 {
855 	return cache->features.mode;
856 }
857 
858 static const char *cache_device_name(struct cache *cache)
859 {
860 	return dm_table_device_name(cache->ti->table);
861 }
862 
863 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
864 {
865 	static const char *descs[] = {
866 		"write",
867 		"read-only",
868 		"fail"
869 	};
870 
871 	dm_table_event(cache->ti->table);
872 	DMINFO("%s: switching cache to %s mode",
873 	       cache_device_name(cache), descs[(int)mode]);
874 }
875 
876 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
877 {
878 	bool needs_check;
879 	enum cache_metadata_mode old_mode = get_cache_mode(cache);
880 
881 	if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
882 		DMERR("%s: unable to read needs_check flag, setting failure mode.",
883 		      cache_device_name(cache));
884 		new_mode = CM_FAIL;
885 	}
886 
887 	if (new_mode == CM_WRITE && needs_check) {
888 		DMERR("%s: unable to switch cache to write mode until repaired.",
889 		      cache_device_name(cache));
890 		if (old_mode != new_mode)
891 			new_mode = old_mode;
892 		else
893 			new_mode = CM_READ_ONLY;
894 	}
895 
896 	/* Never move out of fail mode */
897 	if (old_mode == CM_FAIL)
898 		new_mode = CM_FAIL;
899 
900 	switch (new_mode) {
901 	case CM_FAIL:
902 	case CM_READ_ONLY:
903 		dm_cache_metadata_set_read_only(cache->cmd);
904 		break;
905 
906 	case CM_WRITE:
907 		dm_cache_metadata_set_read_write(cache->cmd);
908 		break;
909 	}
910 
911 	cache->features.mode = new_mode;
912 
913 	if (new_mode != old_mode)
914 		notify_mode_switch(cache, new_mode);
915 }
916 
917 static void abort_transaction(struct cache *cache)
918 {
919 	const char *dev_name = cache_device_name(cache);
920 
921 	if (get_cache_mode(cache) >= CM_READ_ONLY)
922 		return;
923 
924 	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
925 	if (dm_cache_metadata_abort(cache->cmd)) {
926 		DMERR("%s: failed to abort metadata transaction", dev_name);
927 		set_cache_mode(cache, CM_FAIL);
928 	}
929 
930 	if (dm_cache_metadata_set_needs_check(cache->cmd)) {
931 		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
932 		set_cache_mode(cache, CM_FAIL);
933 	}
934 }
935 
936 static void metadata_operation_failed(struct cache *cache, const char *op, int r)
937 {
938 	DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
939 		    cache_device_name(cache), op, r);
940 	abort_transaction(cache);
941 	set_cache_mode(cache, CM_READ_ONLY);
942 }
943 
944 /*----------------------------------------------------------------*/
945 
946 static void load_stats(struct cache *cache)
947 {
948 	struct dm_cache_statistics stats;
949 
950 	dm_cache_metadata_get_stats(cache->cmd, &stats);
951 	atomic_set(&cache->stats.read_hit, stats.read_hits);
952 	atomic_set(&cache->stats.read_miss, stats.read_misses);
953 	atomic_set(&cache->stats.write_hit, stats.write_hits);
954 	atomic_set(&cache->stats.write_miss, stats.write_misses);
955 }
956 
957 static void save_stats(struct cache *cache)
958 {
959 	struct dm_cache_statistics stats;
960 
961 	if (get_cache_mode(cache) >= CM_READ_ONLY)
962 		return;
963 
964 	stats.read_hits = atomic_read(&cache->stats.read_hit);
965 	stats.read_misses = atomic_read(&cache->stats.read_miss);
966 	stats.write_hits = atomic_read(&cache->stats.write_hit);
967 	stats.write_misses = atomic_read(&cache->stats.write_miss);
968 
969 	dm_cache_metadata_set_stats(cache->cmd, &stats);
970 }
971 
972 static void update_stats(struct cache_stats *stats, enum policy_operation op)
973 {
974 	switch (op) {
975 	case POLICY_PROMOTE:
976 		atomic_inc(&stats->promotion);
977 		break;
978 
979 	case POLICY_DEMOTE:
980 		atomic_inc(&stats->demotion);
981 		break;
982 
983 	case POLICY_WRITEBACK:
984 		atomic_inc(&stats->writeback);
985 		break;
986 	}
987 }
988 
989 /*
990  *---------------------------------------------------------------------
991  * Migration processing
992  *
993  * Migration covers moving data from the origin device to the cache, or
994  * vice versa.
995  *---------------------------------------------------------------------
996  */
997 static void inc_io_migrations(struct cache *cache)
998 {
999 	atomic_inc(&cache->nr_io_migrations);
1000 }
1001 
1002 static void dec_io_migrations(struct cache *cache)
1003 {
1004 	atomic_dec(&cache->nr_io_migrations);
1005 }
1006 
1007 static bool discard_or_flush(struct bio *bio)
1008 {
1009 	return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
1010 }
1011 
1012 static void calc_discard_block_range(struct cache *cache, struct bio *bio,
1013 				     dm_dblock_t *b, dm_dblock_t *e)
1014 {
1015 	sector_t sb = bio->bi_iter.bi_sector;
1016 	sector_t se = bio_end_sector(bio);
1017 
1018 	*b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
1019 
1020 	if (se - sb < cache->discard_block_size)
1021 		*e = *b;
1022 	else
1023 		*e = to_dblock(block_div(se, cache->discard_block_size));
1024 }
1025 
1026 /*----------------------------------------------------------------*/
1027 
1028 static void prevent_background_work(struct cache *cache)
1029 {
1030 	spin_lock_irq(&cache->background_work_lock);
1031 	cache->background_work_allowed = false;
1032 	wait_event_lock_irq(cache->background_work_wait,
1033 			    cache->background_work_nr == 0,
1034 			    cache->background_work_lock);
1035 	spin_unlock_irq(&cache->background_work_lock);
1036 }
1037 
1038 static void allow_background_work(struct cache *cache)
1039 {
1040 	spin_lock_irq(&cache->background_work_lock);
1041 	cache->background_work_allowed = true;
1042 	spin_unlock_irq(&cache->background_work_lock);
1043 }
1044 
1045 static bool background_work_begin(struct cache *cache)
1046 {
1047 	bool r;
1048 
1049 	spin_lock_irq(&cache->background_work_lock);
1050 	r = cache->background_work_allowed;
1051 	if (r)
1052 		cache->background_work_nr++;
1053 	spin_unlock_irq(&cache->background_work_lock);
1054 	return r;
1055 }
1056 
1057 static void background_work_end(struct cache *cache)
1058 {
1059 	spin_lock_irq(&cache->background_work_lock);
1060 	if (--cache->background_work_nr == 0)
1061 		wake_up(&cache->background_work_wait);
1062 	spin_unlock_irq(&cache->background_work_lock);
1063 }
1064 
1065 /*----------------------------------------------------------------*/
1066 
1067 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1068 {
1069 	return (bio_data_dir(bio) == WRITE) &&
1070 		(bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1071 }
1072 
1073 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1074 {
1075 	return writeback_mode(cache) &&
1076 		(is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1077 }
1078 
1079 static void quiesce(struct dm_cache_migration *mg,
1080 		    void (*continuation)(struct work_struct *))
1081 {
1082 	init_continuation(&mg->k, continuation);
1083 	dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1084 }
1085 
1086 static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1087 {
1088 	struct continuation *k = container_of(ws, struct continuation, ws);
1089 
1090 	return container_of(k, struct dm_cache_migration, k);
1091 }
1092 
1093 static void copy_complete(int read_err, unsigned long write_err, void *context)
1094 {
1095 	struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1096 
1097 	if (read_err || write_err)
1098 		mg->k.input = BLK_STS_IOERR;
1099 
1100 	queue_continuation(mg->cache->wq, &mg->k);
1101 }
1102 
1103 static void copy(struct dm_cache_migration *mg, bool promote)
1104 {
1105 	struct dm_io_region o_region, c_region;
1106 	struct cache *cache = mg->cache;
1107 
1108 	o_region.bdev = cache->origin_dev->bdev;
1109 	o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1110 	o_region.count = cache->sectors_per_block;
1111 
1112 	c_region.bdev = cache->cache_dev->bdev;
1113 	c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1114 	c_region.count = cache->sectors_per_block;
1115 
1116 	if (promote)
1117 		dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1118 	else
1119 		dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1120 }
1121 
1122 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1123 {
1124 	struct per_bio_data *pb = get_per_bio_data(bio);
1125 
1126 	if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1127 		free_prison_cell(cache, pb->cell);
1128 	pb->cell = NULL;
1129 }
1130 
1131 static void overwrite_endio(struct bio *bio)
1132 {
1133 	struct dm_cache_migration *mg = bio->bi_private;
1134 	struct cache *cache = mg->cache;
1135 	struct per_bio_data *pb = get_per_bio_data(bio);
1136 
1137 	dm_unhook_bio(&pb->hook_info, bio);
1138 
1139 	if (bio->bi_status)
1140 		mg->k.input = bio->bi_status;
1141 
1142 	queue_continuation(cache->wq, &mg->k);
1143 }
1144 
1145 static void overwrite(struct dm_cache_migration *mg,
1146 		      void (*continuation)(struct work_struct *))
1147 {
1148 	struct bio *bio = mg->overwrite_bio;
1149 	struct per_bio_data *pb = get_per_bio_data(bio);
1150 
1151 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1152 
1153 	/*
1154 	 * The overwrite bio is part of the copy operation, as such it does
1155 	 * not set/clear discard or dirty flags.
1156 	 */
1157 	if (mg->op->op == POLICY_PROMOTE)
1158 		remap_to_cache(mg->cache, bio, mg->op->cblock);
1159 	else
1160 		remap_to_origin(mg->cache, bio);
1161 
1162 	init_continuation(&mg->k, continuation);
1163 	accounted_request(mg->cache, bio);
1164 }
1165 
1166 /*
1167  * Migration steps:
1168  *
1169  * 1) exclusive lock preventing WRITEs
1170  * 2) quiesce
1171  * 3) copy or issue overwrite bio
1172  * 4) upgrade to exclusive lock preventing READs and WRITEs
1173  * 5) quiesce
1174  * 6) update metadata and commit
1175  * 7) unlock
1176  */
1177 static void mg_complete(struct dm_cache_migration *mg, bool success)
1178 {
1179 	struct bio_list bios;
1180 	struct cache *cache = mg->cache;
1181 	struct policy_work *op = mg->op;
1182 	dm_cblock_t cblock = op->cblock;
1183 
1184 	if (success)
1185 		update_stats(&cache->stats, op->op);
1186 
1187 	switch (op->op) {
1188 	case POLICY_PROMOTE:
1189 		clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1190 		policy_complete_background_work(cache->policy, op, success);
1191 
1192 		if (mg->overwrite_bio) {
1193 			if (success)
1194 				force_set_dirty(cache, cblock);
1195 			else if (mg->k.input)
1196 				mg->overwrite_bio->bi_status = mg->k.input;
1197 			else
1198 				mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1199 			bio_endio(mg->overwrite_bio);
1200 		} else {
1201 			if (success)
1202 				force_clear_dirty(cache, cblock);
1203 			dec_io_migrations(cache);
1204 		}
1205 		break;
1206 
1207 	case POLICY_DEMOTE:
1208 		/*
1209 		 * We clear dirty here to update the nr_dirty counter.
1210 		 */
1211 		if (success)
1212 			force_clear_dirty(cache, cblock);
1213 		policy_complete_background_work(cache->policy, op, success);
1214 		dec_io_migrations(cache);
1215 		break;
1216 
1217 	case POLICY_WRITEBACK:
1218 		if (success)
1219 			force_clear_dirty(cache, cblock);
1220 		policy_complete_background_work(cache->policy, op, success);
1221 		dec_io_migrations(cache);
1222 		break;
1223 	}
1224 
1225 	bio_list_init(&bios);
1226 	if (mg->cell) {
1227 		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1228 			free_prison_cell(cache, mg->cell);
1229 	}
1230 
1231 	free_migration(mg);
1232 	defer_bios(cache, &bios);
1233 	wake_migration_worker(cache);
1234 
1235 	background_work_end(cache);
1236 }
1237 
1238 static void mg_success(struct work_struct *ws)
1239 {
1240 	struct dm_cache_migration *mg = ws_to_mg(ws);
1241 
1242 	mg_complete(mg, mg->k.input == 0);
1243 }
1244 
1245 static void mg_update_metadata(struct work_struct *ws)
1246 {
1247 	int r;
1248 	struct dm_cache_migration *mg = ws_to_mg(ws);
1249 	struct cache *cache = mg->cache;
1250 	struct policy_work *op = mg->op;
1251 
1252 	switch (op->op) {
1253 	case POLICY_PROMOTE:
1254 		r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1255 		if (r) {
1256 			DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1257 				    cache_device_name(cache));
1258 			metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1259 
1260 			mg_complete(mg, false);
1261 			return;
1262 		}
1263 		mg_complete(mg, true);
1264 		break;
1265 
1266 	case POLICY_DEMOTE:
1267 		r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1268 		if (r) {
1269 			DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1270 				    cache_device_name(cache));
1271 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1272 
1273 			mg_complete(mg, false);
1274 			return;
1275 		}
1276 
1277 		/*
1278 		 * It would be nice if we only had to commit when a REQ_FLUSH
1279 		 * comes through.  But there's one scenario that we have to
1280 		 * look out for:
1281 		 *
1282 		 * - vblock x in a cache block
1283 		 * - domotion occurs
1284 		 * - cache block gets reallocated and over written
1285 		 * - crash
1286 		 *
1287 		 * When we recover, because there was no commit the cache will
1288 		 * rollback to having the data for vblock x in the cache block.
1289 		 * But the cache block has since been overwritten, so it'll end
1290 		 * up pointing to data that was never in 'x' during the history
1291 		 * of the device.
1292 		 *
1293 		 * To avoid this issue we require a commit as part of the
1294 		 * demotion operation.
1295 		 */
1296 		init_continuation(&mg->k, mg_success);
1297 		continue_after_commit(&cache->committer, &mg->k);
1298 		schedule_commit(&cache->committer);
1299 		break;
1300 
1301 	case POLICY_WRITEBACK:
1302 		mg_complete(mg, true);
1303 		break;
1304 	}
1305 }
1306 
1307 static void mg_update_metadata_after_copy(struct work_struct *ws)
1308 {
1309 	struct dm_cache_migration *mg = ws_to_mg(ws);
1310 
1311 	/*
1312 	 * Did the copy succeed?
1313 	 */
1314 	if (mg->k.input)
1315 		mg_complete(mg, false);
1316 	else
1317 		mg_update_metadata(ws);
1318 }
1319 
1320 static void mg_upgrade_lock(struct work_struct *ws)
1321 {
1322 	int r;
1323 	struct dm_cache_migration *mg = ws_to_mg(ws);
1324 
1325 	/*
1326 	 * Did the copy succeed?
1327 	 */
1328 	if (mg->k.input)
1329 		mg_complete(mg, false);
1330 
1331 	else {
1332 		/*
1333 		 * Now we want the lock to prevent both reads and writes.
1334 		 */
1335 		r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1336 					    READ_WRITE_LOCK_LEVEL);
1337 		if (r < 0)
1338 			mg_complete(mg, false);
1339 
1340 		else if (r)
1341 			quiesce(mg, mg_update_metadata);
1342 
1343 		else
1344 			mg_update_metadata(ws);
1345 	}
1346 }
1347 
1348 static void mg_full_copy(struct work_struct *ws)
1349 {
1350 	struct dm_cache_migration *mg = ws_to_mg(ws);
1351 	struct cache *cache = mg->cache;
1352 	struct policy_work *op = mg->op;
1353 	bool is_policy_promote = (op->op == POLICY_PROMOTE);
1354 
1355 	if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1356 	    is_discarded_oblock(cache, op->oblock)) {
1357 		mg_upgrade_lock(ws);
1358 		return;
1359 	}
1360 
1361 	init_continuation(&mg->k, mg_upgrade_lock);
1362 	copy(mg, is_policy_promote);
1363 }
1364 
1365 static void mg_copy(struct work_struct *ws)
1366 {
1367 	struct dm_cache_migration *mg = ws_to_mg(ws);
1368 
1369 	if (mg->overwrite_bio) {
1370 		/*
1371 		 * No exclusive lock was held when we last checked if the bio
1372 		 * was optimisable.  So we have to check again in case things
1373 		 * have changed (eg, the block may no longer be discarded).
1374 		 */
1375 		if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1376 			/*
1377 			 * Fallback to a real full copy after doing some tidying up.
1378 			 */
1379 			bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
1380 
1381 			BUG_ON(rb); /* An exclusive lock must _not_ be held for this block */
1382 			mg->overwrite_bio = NULL;
1383 			inc_io_migrations(mg->cache);
1384 			mg_full_copy(ws);
1385 			return;
1386 		}
1387 
1388 		/*
1389 		 * It's safe to do this here, even though it's new data
1390 		 * because all IO has been locked out of the block.
1391 		 *
1392 		 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1393 		 * so _not_ using mg_upgrade_lock() as continutation.
1394 		 */
1395 		overwrite(mg, mg_update_metadata_after_copy);
1396 
1397 	} else
1398 		mg_full_copy(ws);
1399 }
1400 
1401 static int mg_lock_writes(struct dm_cache_migration *mg)
1402 {
1403 	int r;
1404 	struct dm_cell_key_v2 key;
1405 	struct cache *cache = mg->cache;
1406 	struct dm_bio_prison_cell_v2 *prealloc;
1407 
1408 	prealloc = alloc_prison_cell(cache);
1409 
1410 	/*
1411 	 * Prevent writes to the block, but allow reads to continue.
1412 	 * Unless we're using an overwrite bio, in which case we lock
1413 	 * everything.
1414 	 */
1415 	build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1416 	r = dm_cell_lock_v2(cache->prison, &key,
1417 			    mg->overwrite_bio ?  READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1418 			    prealloc, &mg->cell);
1419 	if (r < 0) {
1420 		free_prison_cell(cache, prealloc);
1421 		mg_complete(mg, false);
1422 		return r;
1423 	}
1424 
1425 	if (mg->cell != prealloc)
1426 		free_prison_cell(cache, prealloc);
1427 
1428 	if (r == 0)
1429 		mg_copy(&mg->k.ws);
1430 	else
1431 		quiesce(mg, mg_copy);
1432 
1433 	return 0;
1434 }
1435 
1436 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1437 {
1438 	struct dm_cache_migration *mg;
1439 
1440 	if (!background_work_begin(cache)) {
1441 		policy_complete_background_work(cache->policy, op, false);
1442 		return -EPERM;
1443 	}
1444 
1445 	mg = alloc_migration(cache);
1446 
1447 	mg->op = op;
1448 	mg->overwrite_bio = bio;
1449 
1450 	if (!bio)
1451 		inc_io_migrations(cache);
1452 
1453 	return mg_lock_writes(mg);
1454 }
1455 
1456 /*
1457  *--------------------------------------------------------------
1458  * invalidation processing
1459  *--------------------------------------------------------------
1460  */
1461 
1462 static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1463 {
1464 	struct bio_list bios;
1465 	struct cache *cache = mg->cache;
1466 
1467 	if (success)
1468 		atomic_inc(&cache->stats.demotion);
1469 
1470 	bio_list_init(&bios);
1471 	if (mg->cell) {
1472 		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1473 			free_prison_cell(cache, mg->cell);
1474 	}
1475 
1476 	if (mg->overwrite_bio) {
1477 		// Set generic error if the bio hasn't been issued yet,
1478 		// e.g., invalidation or metadata commit failed before bio
1479 		// submission. Otherwise preserve the bio's own error status.
1480 		if (!success && !mg->overwrite_bio->bi_status)
1481 			mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1482 		bio_endio(mg->overwrite_bio);
1483 	}
1484 
1485 	free_migration(mg);
1486 	defer_bios(cache, &bios);
1487 
1488 	background_work_end(cache);
1489 }
1490 
1491 static void invalidate_completed(struct work_struct *ws)
1492 {
1493 	struct dm_cache_migration *mg = ws_to_mg(ws);
1494 
1495 	invalidate_complete(mg, !mg->k.input);
1496 }
1497 
1498 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1499 {
1500 	int r;
1501 
1502 	r = policy_invalidate_mapping(cache->policy, cblock);
1503 	if (!r) {
1504 		r = dm_cache_remove_mapping(cache->cmd, cblock);
1505 		if (r) {
1506 			DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1507 				    cache_device_name(cache));
1508 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1509 		}
1510 
1511 	} else if (r == -ENODATA) {
1512 		/*
1513 		 * Harmless, already unmapped.
1514 		 */
1515 		r = 0;
1516 
1517 	} else
1518 		DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1519 
1520 	return r;
1521 }
1522 
1523 static void invalidate_committed(struct work_struct *ws)
1524 {
1525 	struct dm_cache_migration *mg = ws_to_mg(ws);
1526 	struct cache *cache = mg->cache;
1527 	struct bio *bio = mg->overwrite_bio;
1528 	struct per_bio_data *pb = get_per_bio_data(bio);
1529 
1530 	if (mg->k.input) {
1531 		invalidate_complete(mg, false);
1532 		return;
1533 	}
1534 
1535 	init_continuation(&mg->k, invalidate_completed);
1536 	remap_to_origin_clear_discard(cache, bio, mg->invalidate_oblock);
1537 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1538 	dm_submit_bio_remap(bio, NULL);
1539 }
1540 
1541 static void invalidate_remove(struct work_struct *ws)
1542 {
1543 	int r;
1544 	struct dm_cache_migration *mg = ws_to_mg(ws);
1545 	struct cache *cache = mg->cache;
1546 
1547 	r = invalidate_cblock(cache, mg->invalidate_cblock);
1548 	if (r) {
1549 		invalidate_complete(mg, false);
1550 		return;
1551 	}
1552 
1553 	init_continuation(&mg->k, invalidate_committed);
1554 	continue_after_commit(&cache->committer, &mg->k);
1555 	schedule_commit(&cache->committer);
1556 }
1557 
1558 static int invalidate_lock(struct dm_cache_migration *mg)
1559 {
1560 	int r;
1561 	struct dm_cell_key_v2 key;
1562 	struct cache *cache = mg->cache;
1563 	struct dm_bio_prison_cell_v2 *prealloc;
1564 
1565 	prealloc = alloc_prison_cell(cache);
1566 
1567 	build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1568 	r = dm_cell_lock_v2(cache->prison, &key,
1569 			    READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1570 	if (r < 0) {
1571 		free_prison_cell(cache, prealloc);
1572 
1573 		/* Defer the bio for retrying the cell lock */
1574 		if (mg->overwrite_bio) {
1575 			struct bio *bio = mg->overwrite_bio;
1576 
1577 			mg->overwrite_bio = NULL;
1578 			defer_bio(cache, bio);
1579 		}
1580 
1581 		invalidate_complete(mg, false);
1582 		return r;
1583 	}
1584 
1585 	if (mg->cell != prealloc)
1586 		free_prison_cell(cache, prealloc);
1587 
1588 	if (r)
1589 		quiesce(mg, invalidate_remove);
1590 
1591 	else {
1592 		/*
1593 		 * We can't call invalidate_remove() directly here because we
1594 		 * might still be in request context.
1595 		 */
1596 		init_continuation(&mg->k, invalidate_remove);
1597 		queue_work(cache->wq, &mg->k.ws);
1598 	}
1599 
1600 	return 0;
1601 }
1602 
1603 static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1604 			    dm_oblock_t oblock, struct bio *bio)
1605 {
1606 	struct dm_cache_migration *mg;
1607 
1608 	if (!background_work_begin(cache))
1609 		return -EPERM;
1610 
1611 	mg = alloc_migration(cache);
1612 
1613 	mg->overwrite_bio = bio;
1614 	mg->invalidate_cblock = cblock;
1615 	mg->invalidate_oblock = oblock;
1616 
1617 	return invalidate_lock(mg);
1618 }
1619 
1620 /*
1621  *--------------------------------------------------------------
1622  * bio processing
1623  *--------------------------------------------------------------
1624  */
1625 
1626 enum busy {
1627 	IDLE,
1628 	BUSY
1629 };
1630 
1631 static enum busy spare_migration_bandwidth(struct cache *cache)
1632 {
1633 	bool idle = dm_iot_idle_for(&cache->tracker, HZ);
1634 	sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1635 		cache->sectors_per_block;
1636 
1637 	if (idle && current_volume <= cache->migration_threshold)
1638 		return IDLE;
1639 	else
1640 		return BUSY;
1641 }
1642 
1643 static void inc_hit_counter(struct cache *cache, struct bio *bio)
1644 {
1645 	atomic_inc(bio_data_dir(bio) == READ ?
1646 		   &cache->stats.read_hit : &cache->stats.write_hit);
1647 }
1648 
1649 static void inc_miss_counter(struct cache *cache, struct bio *bio)
1650 {
1651 	atomic_inc(bio_data_dir(bio) == READ ?
1652 		   &cache->stats.read_miss : &cache->stats.write_miss);
1653 }
1654 
1655 /*----------------------------------------------------------------*/
1656 
1657 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1658 		   bool *commit_needed)
1659 {
1660 	int r, data_dir;
1661 	bool rb, background_queued;
1662 	dm_cblock_t cblock;
1663 
1664 	*commit_needed = false;
1665 
1666 	rb = bio_detain_shared(cache, block, bio);
1667 	if (!rb) {
1668 		/*
1669 		 * An exclusive lock is held for this block, so we have to
1670 		 * wait.  We set the commit_needed flag so the current
1671 		 * transaction will be committed asap, allowing this lock
1672 		 * to be dropped.
1673 		 */
1674 		*commit_needed = true;
1675 		return DM_MAPIO_SUBMITTED;
1676 	}
1677 
1678 	data_dir = bio_data_dir(bio);
1679 
1680 	if (optimisable_bio(cache, bio, block)) {
1681 		struct policy_work *op = NULL;
1682 
1683 		r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1684 		if (unlikely(r && r != -ENOENT)) {
1685 			DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1686 				    cache_device_name(cache), r);
1687 			bio_io_error(bio);
1688 			return DM_MAPIO_SUBMITTED;
1689 		}
1690 
1691 		if (r == -ENOENT && op) {
1692 			bio_drop_shared_lock(cache, bio);
1693 			BUG_ON(op->op != POLICY_PROMOTE);
1694 			mg_start(cache, op, bio);
1695 			return DM_MAPIO_SUBMITTED;
1696 		}
1697 	} else {
1698 		r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1699 		if (unlikely(r && r != -ENOENT)) {
1700 			DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1701 				    cache_device_name(cache), r);
1702 			bio_io_error(bio);
1703 			return DM_MAPIO_SUBMITTED;
1704 		}
1705 
1706 		if (background_queued)
1707 			wake_migration_worker(cache);
1708 	}
1709 
1710 	if (r == -ENOENT) {
1711 		struct per_bio_data *pb = get_per_bio_data(bio);
1712 
1713 		/*
1714 		 * Miss.
1715 		 */
1716 		inc_miss_counter(cache, bio);
1717 		if (pb->req_nr == 0) {
1718 			accounted_begin(cache, bio);
1719 			remap_to_origin_clear_discard(cache, bio, block);
1720 		} else {
1721 			/*
1722 			 * This is a duplicate writethrough io that is no
1723 			 * longer needed because the block has been demoted.
1724 			 */
1725 			bio_endio(bio);
1726 			return DM_MAPIO_SUBMITTED;
1727 		}
1728 	} else {
1729 		/*
1730 		 * Hit.
1731 		 */
1732 		inc_hit_counter(cache, bio);
1733 
1734 		/*
1735 		 * Passthrough always maps to the origin, invalidating any
1736 		 * cache blocks that are written to.
1737 		 */
1738 		if (passthrough_mode(cache)) {
1739 			if (bio_data_dir(bio) == WRITE) {
1740 				bio_drop_shared_lock(cache, bio);
1741 				invalidate_start(cache, cblock, block, bio);
1742 				return DM_MAPIO_SUBMITTED;
1743 			} else
1744 				remap_to_origin_clear_discard(cache, bio, block);
1745 		} else {
1746 			if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
1747 			    !is_dirty(cache, cblock)) {
1748 				remap_to_origin_and_cache(cache, bio, block, cblock);
1749 				accounted_begin(cache, bio);
1750 			} else
1751 				remap_to_cache_dirty(cache, bio, block, cblock);
1752 		}
1753 	}
1754 
1755 	/*
1756 	 * dm core turns FUA requests into a separate payload and FLUSH req.
1757 	 */
1758 	if (bio->bi_opf & REQ_FUA) {
1759 		/*
1760 		 * issue_after_commit will call accounted_begin a second time.  So
1761 		 * we call accounted_complete() to avoid double accounting.
1762 		 */
1763 		accounted_complete(cache, bio);
1764 		issue_after_commit(&cache->committer, bio);
1765 		*commit_needed = true;
1766 		return DM_MAPIO_SUBMITTED;
1767 	}
1768 
1769 	return DM_MAPIO_REMAPPED;
1770 }
1771 
1772 static bool process_bio(struct cache *cache, struct bio *bio)
1773 {
1774 	bool commit_needed;
1775 
1776 	if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1777 		dm_submit_bio_remap(bio, NULL);
1778 
1779 	return commit_needed;
1780 }
1781 
1782 /*
1783  * A non-zero return indicates read_only or fail_io mode.
1784  */
1785 static int commit(struct cache *cache, bool clean_shutdown)
1786 {
1787 	int r;
1788 
1789 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1790 		return -EINVAL;
1791 
1792 	atomic_inc(&cache->stats.commit_count);
1793 	r = dm_cache_commit(cache->cmd, clean_shutdown);
1794 	if (r)
1795 		metadata_operation_failed(cache, "dm_cache_commit", r);
1796 
1797 	return r;
1798 }
1799 
1800 /*
1801  * Used by the batcher.
1802  */
1803 static blk_status_t commit_op(void *context)
1804 {
1805 	struct cache *cache = context;
1806 
1807 	if (dm_cache_changed_this_transaction(cache->cmd))
1808 		return errno_to_blk_status(commit(cache, false));
1809 
1810 	return 0;
1811 }
1812 
1813 /*----------------------------------------------------------------*/
1814 
1815 static bool process_flush_bio(struct cache *cache, struct bio *bio)
1816 {
1817 	struct per_bio_data *pb = get_per_bio_data(bio);
1818 
1819 	if (!pb->req_nr)
1820 		remap_to_origin(cache, bio);
1821 	else
1822 		remap_to_cache(cache, bio, 0);
1823 
1824 	issue_after_commit(&cache->committer, bio);
1825 	return true;
1826 }
1827 
1828 static bool process_discard_bio(struct cache *cache, struct bio *bio)
1829 {
1830 	dm_dblock_t b, e;
1831 
1832 	/*
1833 	 * FIXME: do we need to lock the region?  Or can we just assume the
1834 	 * user wont be so foolish as to issue discard concurrently with
1835 	 * other IO?
1836 	 */
1837 	calc_discard_block_range(cache, bio, &b, &e);
1838 	while (b != e) {
1839 		set_discard(cache, b);
1840 		b = to_dblock(from_dblock(b) + 1);
1841 	}
1842 
1843 	if (cache->features.discard_passdown) {
1844 		remap_to_origin(cache, bio);
1845 		dm_submit_bio_remap(bio, NULL);
1846 	} else
1847 		bio_endio(bio);
1848 
1849 	return false;
1850 }
1851 
1852 static void process_deferred_bios(struct work_struct *ws)
1853 {
1854 	struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1855 
1856 	bool commit_needed = false;
1857 	struct bio_list bios;
1858 	struct bio *bio;
1859 
1860 	bio_list_init(&bios);
1861 
1862 	spin_lock_irq(&cache->lock);
1863 	bio_list_merge_init(&bios, &cache->deferred_bios);
1864 	spin_unlock_irq(&cache->lock);
1865 
1866 	while ((bio = bio_list_pop(&bios))) {
1867 		if (bio->bi_opf & REQ_PREFLUSH)
1868 			commit_needed = process_flush_bio(cache, bio) || commit_needed;
1869 
1870 		else if (bio_op(bio) == REQ_OP_DISCARD)
1871 			commit_needed = process_discard_bio(cache, bio) || commit_needed;
1872 
1873 		else
1874 			commit_needed = process_bio(cache, bio) || commit_needed;
1875 		cond_resched();
1876 	}
1877 
1878 	if (commit_needed)
1879 		schedule_commit(&cache->committer);
1880 }
1881 
1882 /*
1883  *--------------------------------------------------------------
1884  * Main worker loop
1885  *--------------------------------------------------------------
1886  */
1887 static void requeue_deferred_bios(struct cache *cache)
1888 {
1889 	struct bio *bio;
1890 	struct bio_list bios;
1891 
1892 	bio_list_init(&bios);
1893 	bio_list_merge_init(&bios, &cache->deferred_bios);
1894 
1895 	while ((bio = bio_list_pop(&bios))) {
1896 		bio->bi_status = BLK_STS_DM_REQUEUE;
1897 		bio_endio(bio);
1898 		cond_resched();
1899 	}
1900 }
1901 
1902 /*
1903  * We want to commit periodically so that not too much
1904  * unwritten metadata builds up.
1905  */
1906 static void do_waker(struct work_struct *ws)
1907 {
1908 	struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1909 
1910 	policy_tick(cache->policy, true);
1911 	wake_migration_worker(cache);
1912 	schedule_commit(&cache->committer);
1913 	queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1914 }
1915 
1916 static void check_migrations(struct work_struct *ws)
1917 {
1918 	int r;
1919 	struct policy_work *op;
1920 	struct cache *cache = container_of(ws, struct cache, migration_worker);
1921 	enum busy b;
1922 
1923 	for (;;) {
1924 		b = spare_migration_bandwidth(cache);
1925 
1926 		r = policy_get_background_work(cache->policy, b == IDLE, &op);
1927 		if (r == -ENODATA)
1928 			break;
1929 
1930 		if (r) {
1931 			DMERR_LIMIT("%s: policy_background_work failed",
1932 				    cache_device_name(cache));
1933 			break;
1934 		}
1935 
1936 		r = mg_start(cache, op, NULL);
1937 		if (r)
1938 			break;
1939 
1940 		cond_resched();
1941 	}
1942 }
1943 
1944 /*
1945  *--------------------------------------------------------------
1946  * Target methods
1947  *--------------------------------------------------------------
1948  */
1949 
1950 /*
1951  * This function gets called on the error paths of the constructor, so we
1952  * have to cope with a partially initialised struct.
1953  */
1954 static void __destroy(struct cache *cache)
1955 {
1956 	mempool_exit(&cache->migration_pool);
1957 
1958 	if (cache->prison)
1959 		dm_bio_prison_destroy_v2(cache->prison);
1960 
1961 	if (cache->wq)
1962 		destroy_workqueue(cache->wq);
1963 
1964 	if (cache->dirty_bitset)
1965 		free_bitset(cache->dirty_bitset);
1966 
1967 	if (cache->discard_bitset)
1968 		free_bitset(cache->discard_bitset);
1969 
1970 	if (cache->invalid_bitset)
1971 		free_bitset(cache->invalid_bitset);
1972 
1973 	if (cache->copier)
1974 		dm_kcopyd_client_destroy(cache->copier);
1975 
1976 	if (cache->cmd)
1977 		dm_cache_metadata_close(cache->cmd);
1978 
1979 	if (cache->metadata_dev)
1980 		dm_put_device(cache->ti, cache->metadata_dev);
1981 
1982 	if (cache->origin_dev)
1983 		dm_put_device(cache->ti, cache->origin_dev);
1984 
1985 	if (cache->cache_dev)
1986 		dm_put_device(cache->ti, cache->cache_dev);
1987 
1988 	if (cache->policy)
1989 		dm_cache_policy_destroy(cache->policy);
1990 
1991 	bioset_exit(&cache->bs);
1992 
1993 	kfree(cache);
1994 }
1995 
1996 static void destroy(struct cache *cache)
1997 {
1998 	unsigned int i;
1999 
2000 	cancel_delayed_work_sync(&cache->waker);
2001 
2002 	for (i = 0; i < cache->nr_ctr_args ; i++)
2003 		kfree(cache->ctr_args[i]);
2004 	kfree(cache->ctr_args);
2005 
2006 	__destroy(cache);
2007 }
2008 
2009 static void cache_dtr(struct dm_target *ti)
2010 {
2011 	struct cache *cache = ti->private;
2012 
2013 	destroy(cache);
2014 }
2015 
2016 static sector_t get_dev_size(struct dm_dev *dev)
2017 {
2018 	return bdev_nr_sectors(dev->bdev);
2019 }
2020 
2021 /*----------------------------------------------------------------*/
2022 
2023 /*
2024  * Construct a cache device mapping.
2025  *
2026  * cache <metadata dev> <cache dev> <origin dev> <block size>
2027  *       <#feature args> [<feature arg>]*
2028  *       <policy> <#policy args> [<policy arg>]*
2029  *
2030  * metadata dev    : fast device holding the persistent metadata
2031  * cache dev	   : fast device holding cached data blocks
2032  * origin dev	   : slow device holding original data blocks
2033  * block size	   : cache unit size in sectors
2034  *
2035  * #feature args   : number of feature arguments passed
2036  * feature args    : writethrough.  (The default is writeback.)
2037  *
2038  * policy	   : the replacement policy to use
2039  * #policy args    : an even number of policy arguments corresponding
2040  *		     to key/value pairs passed to the policy
2041  * policy args	   : key/value pairs passed to the policy
2042  *		     E.g. 'sequential_threshold 1024'
2043  *		     See cache-policies.txt for details.
2044  *
2045  * Optional feature arguments are:
2046  *   writethrough  : write through caching that prohibits cache block
2047  *		     content from being different from origin block content.
2048  *		     Without this argument, the default behaviour is to write
2049  *		     back cache block contents later for performance reasons,
2050  *		     so they may differ from the corresponding origin blocks.
2051  */
2052 struct cache_args {
2053 	struct dm_target *ti;
2054 
2055 	struct dm_dev *metadata_dev;
2056 
2057 	struct dm_dev *cache_dev;
2058 	sector_t cache_sectors;
2059 
2060 	struct dm_dev *origin_dev;
2061 
2062 	uint32_t block_size;
2063 
2064 	const char *policy_name;
2065 	int policy_argc;
2066 	const char **policy_argv;
2067 
2068 	struct cache_features features;
2069 };
2070 
2071 static void destroy_cache_args(struct cache_args *ca)
2072 {
2073 	if (ca->metadata_dev)
2074 		dm_put_device(ca->ti, ca->metadata_dev);
2075 
2076 	if (ca->cache_dev)
2077 		dm_put_device(ca->ti, ca->cache_dev);
2078 
2079 	if (ca->origin_dev)
2080 		dm_put_device(ca->ti, ca->origin_dev);
2081 
2082 	kfree(ca);
2083 }
2084 
2085 static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2086 {
2087 	if (!as->argc) {
2088 		*error = "Insufficient args";
2089 		return false;
2090 	}
2091 
2092 	return true;
2093 }
2094 
2095 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2096 			      char **error)
2097 {
2098 	int r;
2099 	sector_t metadata_dev_size;
2100 
2101 	if (!at_least_one_arg(as, error))
2102 		return -EINVAL;
2103 
2104 	r = dm_get_device(ca->ti, dm_shift_arg(as),
2105 			  BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev);
2106 	if (r) {
2107 		*error = "Error opening metadata device";
2108 		return r;
2109 	}
2110 
2111 	metadata_dev_size = get_dev_size(ca->metadata_dev);
2112 	if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2113 		DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
2114 		       ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS);
2115 
2116 	return 0;
2117 }
2118 
2119 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2120 			   char **error)
2121 {
2122 	int r;
2123 
2124 	if (!at_least_one_arg(as, error))
2125 		return -EINVAL;
2126 
2127 	r = dm_get_device(ca->ti, dm_shift_arg(as),
2128 			  BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev);
2129 	if (r) {
2130 		*error = "Error opening cache device";
2131 		return r;
2132 	}
2133 	ca->cache_sectors = get_dev_size(ca->cache_dev);
2134 
2135 	return 0;
2136 }
2137 
2138 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2139 			    char **error)
2140 {
2141 	int r;
2142 
2143 	if (!at_least_one_arg(as, error))
2144 		return -EINVAL;
2145 
2146 	r = dm_get_device(ca->ti, dm_shift_arg(as),
2147 			  BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev);
2148 	if (r) {
2149 		*error = "Error opening origin device";
2150 		return r;
2151 	}
2152 
2153 	return 0;
2154 }
2155 
2156 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2157 			    char **error)
2158 {
2159 	unsigned long block_size;
2160 
2161 	if (!at_least_one_arg(as, error))
2162 		return -EINVAL;
2163 
2164 	if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
2165 	    block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2166 	    block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
2167 	    block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2168 		*error = "Invalid data block size";
2169 		return -EINVAL;
2170 	}
2171 
2172 	if (block_size > ca->cache_sectors) {
2173 		*error = "Data block size is larger than the cache device";
2174 		return -EINVAL;
2175 	}
2176 
2177 	ca->block_size = block_size;
2178 
2179 	return 0;
2180 }
2181 
2182 static void init_features(struct cache_features *cf)
2183 {
2184 	cf->mode = CM_WRITE;
2185 	cf->io_mode = CM_IO_WRITEBACK;
2186 	cf->metadata_version = 1;
2187 	cf->discard_passdown = true;
2188 }
2189 
2190 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2191 			  char **error)
2192 {
2193 	static const struct dm_arg _args[] = {
2194 		{0, 3, "Invalid number of cache feature arguments"},
2195 	};
2196 
2197 	int r, mode_ctr = 0;
2198 	unsigned int argc;
2199 	const char *arg;
2200 	struct cache_features *cf = &ca->features;
2201 
2202 	init_features(cf);
2203 
2204 	r = dm_read_arg_group(_args, as, &argc, error);
2205 	if (r)
2206 		return -EINVAL;
2207 
2208 	while (argc--) {
2209 		arg = dm_shift_arg(as);
2210 
2211 		if (!strcasecmp(arg, "writeback")) {
2212 			cf->io_mode = CM_IO_WRITEBACK;
2213 			mode_ctr++;
2214 		}
2215 
2216 		else if (!strcasecmp(arg, "writethrough")) {
2217 			cf->io_mode = CM_IO_WRITETHROUGH;
2218 			mode_ctr++;
2219 		}
2220 
2221 		else if (!strcasecmp(arg, "passthrough")) {
2222 			cf->io_mode = CM_IO_PASSTHROUGH;
2223 			mode_ctr++;
2224 		}
2225 
2226 		else if (!strcasecmp(arg, "metadata2"))
2227 			cf->metadata_version = 2;
2228 
2229 		else if (!strcasecmp(arg, "no_discard_passdown"))
2230 			cf->discard_passdown = false;
2231 
2232 		else {
2233 			*error = "Unrecognised cache feature requested";
2234 			return -EINVAL;
2235 		}
2236 	}
2237 
2238 	if (mode_ctr > 1) {
2239 		*error = "Duplicate cache io_mode features requested";
2240 		return -EINVAL;
2241 	}
2242 
2243 	return 0;
2244 }
2245 
2246 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2247 			char **error)
2248 {
2249 	static const struct dm_arg _args[] = {
2250 		{0, 1024, "Invalid number of policy arguments"},
2251 	};
2252 
2253 	int r;
2254 
2255 	if (!at_least_one_arg(as, error))
2256 		return -EINVAL;
2257 
2258 	ca->policy_name = dm_shift_arg(as);
2259 
2260 	r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2261 	if (r)
2262 		return -EINVAL;
2263 
2264 	ca->policy_argv = (const char **)as->argv;
2265 	dm_consume_args(as, ca->policy_argc);
2266 
2267 	return 0;
2268 }
2269 
2270 static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2271 			    char **error)
2272 {
2273 	int r;
2274 	struct dm_arg_set as;
2275 
2276 	as.argc = argc;
2277 	as.argv = argv;
2278 
2279 	r = parse_metadata_dev(ca, &as, error);
2280 	if (r)
2281 		return r;
2282 
2283 	r = parse_cache_dev(ca, &as, error);
2284 	if (r)
2285 		return r;
2286 
2287 	r = parse_origin_dev(ca, &as, error);
2288 	if (r)
2289 		return r;
2290 
2291 	r = parse_block_size(ca, &as, error);
2292 	if (r)
2293 		return r;
2294 
2295 	r = parse_features(ca, &as, error);
2296 	if (r)
2297 		return r;
2298 
2299 	r = parse_policy(ca, &as, error);
2300 	if (r)
2301 		return r;
2302 
2303 	return 0;
2304 }
2305 
2306 /*----------------------------------------------------------------*/
2307 
2308 static struct kmem_cache *migration_cache = NULL;
2309 
2310 #define NOT_CORE_OPTION 1
2311 
2312 static int process_config_option(struct cache *cache, const char *key, const char *value)
2313 {
2314 	unsigned long tmp;
2315 
2316 	if (!strcasecmp(key, "migration_threshold")) {
2317 		if (kstrtoul(value, 10, &tmp))
2318 			return -EINVAL;
2319 
2320 		cache->migration_threshold = tmp;
2321 		return 0;
2322 	}
2323 
2324 	return NOT_CORE_OPTION;
2325 }
2326 
2327 static int set_config_value(struct cache *cache, const char *key, const char *value)
2328 {
2329 	int r = process_config_option(cache, key, value);
2330 
2331 	if (r == NOT_CORE_OPTION)
2332 		r = policy_set_config_value(cache->policy, key, value);
2333 
2334 	if (r)
2335 		DMWARN("bad config value for %s: %s", key, value);
2336 
2337 	return r;
2338 }
2339 
2340 static int set_config_values(struct cache *cache, int argc, const char **argv)
2341 {
2342 	int r = 0;
2343 
2344 	if (argc & 1) {
2345 		DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2346 		return -EINVAL;
2347 	}
2348 
2349 	while (argc) {
2350 		r = set_config_value(cache, argv[0], argv[1]);
2351 		if (r)
2352 			break;
2353 
2354 		argc -= 2;
2355 		argv += 2;
2356 	}
2357 
2358 	return r;
2359 }
2360 
2361 static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2362 			       char **error)
2363 {
2364 	struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2365 							   cache->cache_size,
2366 							   cache->origin_sectors,
2367 							   cache->sectors_per_block);
2368 	if (IS_ERR(p)) {
2369 		*error = "Error creating cache's policy";
2370 		return PTR_ERR(p);
2371 	}
2372 	cache->policy = p;
2373 	BUG_ON(!cache->policy);
2374 
2375 	return 0;
2376 }
2377 
2378 /*
2379  * We want the discard block size to be at least the size of the cache
2380  * block size and have no more than 2^14 discard blocks across the origin.
2381  */
2382 #define MAX_DISCARD_BLOCKS (1 << 14)
2383 
2384 static bool too_many_discard_blocks(sector_t discard_block_size,
2385 				    sector_t origin_size)
2386 {
2387 	return div64_u64(origin_size, discard_block_size) > MAX_DISCARD_BLOCKS;
2388 }
2389 
2390 static sector_t calculate_discard_block_size(sector_t cache_block_size,
2391 					     sector_t origin_size)
2392 {
2393 	sector_t discard_block_size = cache_block_size;
2394 
2395 	if (origin_size)
2396 		while (too_many_discard_blocks(discard_block_size, origin_size))
2397 			discard_block_size *= 2;
2398 
2399 	return discard_block_size;
2400 }
2401 
2402 static void set_cache_size(struct cache *cache, dm_cblock_t size)
2403 {
2404 	dm_block_t nr_blocks = from_cblock(size);
2405 
2406 	if (nr_blocks > (1 << 20) && cache->cache_size != size)
2407 		DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2408 			     "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2409 			     "Please consider increasing the cache block size to reduce the overall cache block count.",
2410 			     (unsigned long long) nr_blocks);
2411 
2412 	cache->cache_size = size;
2413 }
2414 
2415 #define DEFAULT_MIGRATION_THRESHOLD 2048
2416 
2417 static int cache_create(struct cache_args *ca, struct cache **result)
2418 {
2419 	int r = 0;
2420 	char **error = &ca->ti->error;
2421 	struct cache *cache;
2422 	struct dm_target *ti = ca->ti;
2423 	dm_block_t origin_blocks;
2424 	sector_t discard_block_size;
2425 	struct dm_cache_metadata *cmd;
2426 	bool may_format = ca->features.mode == CM_WRITE;
2427 
2428 	cache = kzalloc_obj(*cache);
2429 	if (!cache)
2430 		return -ENOMEM;
2431 
2432 	cache->ti = ca->ti;
2433 	ti->private = cache;
2434 	ti->accounts_remapped_io = true;
2435 	ti->num_flush_bios = 2;
2436 	ti->flush_supported = true;
2437 
2438 	ti->num_discard_bios = 1;
2439 	ti->discards_supported = true;
2440 
2441 	ti->per_io_data_size = sizeof(struct per_bio_data);
2442 
2443 	cache->features = ca->features;
2444 	if (writethrough_mode(cache)) {
2445 		/* Create bioset for writethrough bios issued to origin */
2446 		r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
2447 		if (r)
2448 			goto bad;
2449 	}
2450 
2451 	cache->metadata_dev = ca->metadata_dev;
2452 	cache->origin_dev = ca->origin_dev;
2453 	cache->cache_dev = ca->cache_dev;
2454 
2455 	ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2456 
2457 	origin_blocks = cache->origin_sectors = ti->len;
2458 	origin_blocks = block_div(origin_blocks, ca->block_size);
2459 	cache->origin_blocks = to_oblock(origin_blocks);
2460 
2461 	cache->sectors_per_block = ca->block_size;
2462 	if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2463 		r = -EINVAL;
2464 		goto bad;
2465 	}
2466 
2467 	if (ca->block_size & (ca->block_size - 1)) {
2468 		dm_block_t cache_size = ca->cache_sectors;
2469 
2470 		cache->sectors_per_block_shift = -1;
2471 		cache_size = block_div(cache_size, ca->block_size);
2472 		set_cache_size(cache, to_cblock(cache_size));
2473 	} else {
2474 		cache->sectors_per_block_shift = __ffs(ca->block_size);
2475 		set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2476 	}
2477 
2478 	r = create_cache_policy(cache, ca, error);
2479 	if (r)
2480 		goto bad;
2481 
2482 	cache->policy_nr_args = ca->policy_argc;
2483 	cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
2484 
2485 	r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
2486 	if (r) {
2487 		*error = "Error setting cache policy's config values";
2488 		goto bad;
2489 	}
2490 
2491 	cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2492 				     ca->block_size, may_format,
2493 				     dm_cache_policy_get_hint_size(cache->policy),
2494 				     ca->features.metadata_version);
2495 	if (IS_ERR(cmd)) {
2496 		*error = "Error creating metadata object";
2497 		r = PTR_ERR(cmd);
2498 		goto bad;
2499 	}
2500 	cache->cmd = cmd;
2501 	set_cache_mode(cache, CM_WRITE);
2502 	if (get_cache_mode(cache) != CM_WRITE) {
2503 		*error = "Unable to get write access to metadata, please check/repair metadata.";
2504 		r = -EINVAL;
2505 		goto bad;
2506 	}
2507 
2508 	if (passthrough_mode(cache))
2509 		policy_allow_migrations(cache->policy, false);
2510 
2511 	spin_lock_init(&cache->lock);
2512 	bio_list_init(&cache->deferred_bios);
2513 	atomic_set(&cache->nr_io_migrations, 0);
2514 
2515 	r = -ENOMEM;
2516 	atomic_set(&cache->nr_dirty, 0);
2517 	cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2518 	if (!cache->dirty_bitset) {
2519 		*error = "could not allocate dirty bitset";
2520 		goto bad;
2521 	}
2522 	clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2523 
2524 	discard_block_size =
2525 		calculate_discard_block_size(cache->sectors_per_block,
2526 					     cache->origin_sectors);
2527 	if (discard_block_size > U32_MAX) {
2528 		*error = "Discard block size exceeds 32 bits";
2529 		r = -EOVERFLOW;
2530 		goto bad;
2531 	}
2532 	cache->discard_block_size = discard_block_size;
2533 	cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
2534 							      cache->discard_block_size));
2535 	cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2536 	if (!cache->discard_bitset) {
2537 		*error = "could not allocate discard bitset";
2538 		goto bad;
2539 	}
2540 	clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2541 
2542 	cache->invalid_bitset = alloc_bitset(from_cblock(cache->cache_size));
2543 	if (!cache->invalid_bitset) {
2544 		*error = "could not allocate bitset for invalid blocks";
2545 		goto bad;
2546 	}
2547 	clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size));
2548 
2549 	cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2550 	if (IS_ERR(cache->copier)) {
2551 		*error = "could not create kcopyd client";
2552 		r = PTR_ERR(cache->copier);
2553 		cache->copier = NULL;
2554 		goto bad;
2555 	}
2556 
2557 	cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX,
2558 				    WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2559 	if (!cache->wq) {
2560 		*error = "could not create workqueue for metadata object";
2561 		goto bad;
2562 	}
2563 	INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2564 	INIT_WORK(&cache->migration_worker, check_migrations);
2565 	INIT_DELAYED_WORK(&cache->waker, do_waker);
2566 
2567 	cache->prison = dm_bio_prison_create_v2(cache->wq);
2568 	if (!cache->prison) {
2569 		*error = "could not create bio prison";
2570 		goto bad;
2571 	}
2572 
2573 	r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
2574 				   migration_cache);
2575 	if (r) {
2576 		*error = "Error creating cache's migration mempool";
2577 		goto bad;
2578 	}
2579 
2580 	cache->need_tick_bio = true;
2581 	cache->sized = false;
2582 	cache->invalidate = false;
2583 	cache->commit_requested = false;
2584 	cache->loaded_mappings = false;
2585 	cache->loaded_discards = false;
2586 
2587 	load_stats(cache);
2588 
2589 	atomic_set(&cache->stats.demotion, 0);
2590 	atomic_set(&cache->stats.promotion, 0);
2591 	atomic_set(&cache->stats.copies_avoided, 0);
2592 	atomic_set(&cache->stats.cache_cell_clash, 0);
2593 	atomic_set(&cache->stats.commit_count, 0);
2594 	atomic_set(&cache->stats.discard_count, 0);
2595 
2596 	spin_lock_init(&cache->invalidation_lock);
2597 	INIT_LIST_HEAD(&cache->invalidation_requests);
2598 
2599 	batcher_init(&cache->committer, commit_op, cache,
2600 		     issue_op, cache, cache->wq);
2601 	dm_iot_init(&cache->tracker);
2602 
2603 	init_waitqueue_head(&cache->background_work_wait);
2604 	spin_lock_init(&cache->background_work_lock);
2605 	cache->background_work_allowed = false;
2606 	cache->background_work_nr = 0;
2607 
2608 	*result = cache;
2609 	return 0;
2610 bad:
2611 	__destroy(cache);
2612 	return r;
2613 }
2614 
2615 static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2616 {
2617 	unsigned int i;
2618 	const char **copy;
2619 
2620 	copy = kzalloc_objs(*copy, argc);
2621 	if (!copy)
2622 		return -ENOMEM;
2623 	for (i = 0; i < argc; i++) {
2624 		copy[i] = kstrdup(argv[i], GFP_KERNEL);
2625 		if (!copy[i]) {
2626 			while (i--)
2627 				kfree(copy[i]);
2628 			kfree(copy);
2629 			return -ENOMEM;
2630 		}
2631 	}
2632 
2633 	cache->nr_ctr_args = argc;
2634 	cache->ctr_args = copy;
2635 
2636 	return 0;
2637 }
2638 
2639 static int cache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2640 {
2641 	int r = -EINVAL;
2642 	struct cache_args *ca;
2643 	struct cache *cache = NULL;
2644 
2645 	ca = kzalloc_obj(*ca);
2646 	if (!ca) {
2647 		ti->error = "Error allocating memory for cache";
2648 		return -ENOMEM;
2649 	}
2650 	ca->ti = ti;
2651 
2652 	r = parse_cache_args(ca, argc, argv, &ti->error);
2653 	if (r)
2654 		goto out;
2655 
2656 	r = cache_create(ca, &cache);
2657 	if (r)
2658 		goto out;
2659 
2660 	r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2661 	if (r) {
2662 		__destroy(cache);
2663 		goto out;
2664 	}
2665 
2666 	ti->private = cache;
2667 out:
2668 	destroy_cache_args(ca);
2669 	return r;
2670 }
2671 
2672 /*----------------------------------------------------------------*/
2673 
2674 static int cache_map(struct dm_target *ti, struct bio *bio)
2675 {
2676 	struct cache *cache = ti->private;
2677 
2678 	int r;
2679 	bool commit_needed;
2680 	dm_oblock_t block = get_bio_block(cache, bio);
2681 
2682 	init_per_bio_data(bio);
2683 	if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2684 		/*
2685 		 * This can only occur if the io goes to a partial block at
2686 		 * the end of the origin device.  We don't cache these.
2687 		 * Just remap to the origin and carry on.
2688 		 */
2689 		remap_to_origin(cache, bio);
2690 		accounted_begin(cache, bio);
2691 		return DM_MAPIO_REMAPPED;
2692 	}
2693 
2694 	if (discard_or_flush(bio)) {
2695 		defer_bio(cache, bio);
2696 		return DM_MAPIO_SUBMITTED;
2697 	}
2698 
2699 	r = map_bio(cache, bio, block, &commit_needed);
2700 	if (commit_needed)
2701 		schedule_commit(&cache->committer);
2702 
2703 	return r;
2704 }
2705 
2706 static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
2707 {
2708 	struct cache *cache = ti->private;
2709 	unsigned long flags;
2710 	struct per_bio_data *pb = get_per_bio_data(bio);
2711 
2712 	if (pb->tick) {
2713 		policy_tick(cache->policy, false);
2714 
2715 		spin_lock_irqsave(&cache->lock, flags);
2716 		cache->need_tick_bio = true;
2717 		spin_unlock_irqrestore(&cache->lock, flags);
2718 	}
2719 
2720 	bio_drop_shared_lock(cache, bio);
2721 	accounted_complete(cache, bio);
2722 
2723 	return DM_ENDIO_DONE;
2724 }
2725 
2726 static int write_dirty_bitset(struct cache *cache)
2727 {
2728 	int r;
2729 
2730 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2731 		return -EINVAL;
2732 
2733 	r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2734 	if (r)
2735 		metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2736 
2737 	return r;
2738 }
2739 
2740 static int write_discard_bitset(struct cache *cache)
2741 {
2742 	unsigned int i, r;
2743 
2744 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2745 		return -EINVAL;
2746 
2747 	r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2748 					   cache->discard_nr_blocks);
2749 	if (r) {
2750 		DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2751 		metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2752 		return r;
2753 	}
2754 
2755 	for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2756 		r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2757 					 is_discarded(cache, to_dblock(i)));
2758 		if (r) {
2759 			metadata_operation_failed(cache, "dm_cache_set_discard", r);
2760 			return r;
2761 		}
2762 	}
2763 
2764 	return 0;
2765 }
2766 
2767 static int write_hints(struct cache *cache)
2768 {
2769 	int r;
2770 
2771 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2772 		return -EINVAL;
2773 
2774 	r = dm_cache_write_hints(cache->cmd, cache->policy);
2775 	if (r) {
2776 		metadata_operation_failed(cache, "dm_cache_write_hints", r);
2777 		return r;
2778 	}
2779 
2780 	return 0;
2781 }
2782 
2783 /*
2784  * returns true on success
2785  */
2786 static bool sync_metadata(struct cache *cache)
2787 {
2788 	int r1, r2, r3, r4;
2789 
2790 	r1 = write_dirty_bitset(cache);
2791 	if (r1)
2792 		DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2793 
2794 	r2 = write_discard_bitset(cache);
2795 	if (r2)
2796 		DMERR("%s: could not write discard bitset", cache_device_name(cache));
2797 
2798 	save_stats(cache);
2799 
2800 	r3 = write_hints(cache);
2801 	if (r3)
2802 		DMERR("%s: could not write hints", cache_device_name(cache));
2803 
2804 	/*
2805 	 * If writing the above metadata failed, we still commit, but don't
2806 	 * set the clean shutdown flag.  This will effectively force every
2807 	 * dirty bit to be set on reload.
2808 	 */
2809 	r4 = commit(cache, !r1 && !r2 && !r3);
2810 	if (r4)
2811 		DMERR("%s: could not write cache metadata", cache_device_name(cache));
2812 
2813 	return !r1 && !r2 && !r3 && !r4;
2814 }
2815 
2816 static void cache_postsuspend(struct dm_target *ti)
2817 {
2818 	struct cache *cache = ti->private;
2819 
2820 	prevent_background_work(cache);
2821 	BUG_ON(atomic_read(&cache->nr_io_migrations));
2822 
2823 	cancel_delayed_work_sync(&cache->waker);
2824 	drain_workqueue(cache->wq);
2825 	WARN_ON(cache->tracker.in_flight);
2826 
2827 	/*
2828 	 * If it's a flush suspend there won't be any deferred bios, so this
2829 	 * call is harmless.
2830 	 */
2831 	requeue_deferred_bios(cache);
2832 
2833 	if (get_cache_mode(cache) == CM_WRITE)
2834 		(void) sync_metadata(cache);
2835 }
2836 
2837 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2838 			bool dirty, uint32_t hint, bool hint_valid)
2839 {
2840 	struct cache *cache = context;
2841 
2842 	if (dirty) {
2843 		if (passthrough_mode(cache)) {
2844 			DMERR("%s: cannot enter passthrough mode unless all blocks are clean",
2845 			      cache_device_name(cache));
2846 			return -EBUSY;
2847 		}
2848 
2849 		set_bit(from_cblock(cblock), cache->dirty_bitset);
2850 		atomic_inc(&cache->nr_dirty);
2851 	} else
2852 		clear_bit(from_cblock(cblock), cache->dirty_bitset);
2853 
2854 	return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
2855 }
2856 
2857 static int load_filtered_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2858 				 bool dirty, uint32_t hint, bool hint_valid)
2859 {
2860 	struct cache *cache = context;
2861 
2862 	if (from_oblock(oblock) >= from_oblock(cache->origin_blocks)) {
2863 		if (dirty) {
2864 			DMERR("%s: unable to shrink origin; cache block %u is dirty",
2865 			      cache_device_name(cache), from_cblock(cblock));
2866 			return -EFBIG;
2867 		}
2868 		set_bit(from_cblock(cblock), cache->invalid_bitset);
2869 		return 0;
2870 	}
2871 
2872 	return load_mapping(context, oblock, cblock, dirty, hint, hint_valid);
2873 }
2874 
2875 /*
2876  * The discard block size in the on disk metadata is not
2877  * necessarily the same as we're currently using.  So we have to
2878  * be careful to only set the discarded attribute if we know it
2879  * covers a complete block of the new size.
2880  */
2881 struct discard_load_info {
2882 	struct cache *cache;
2883 
2884 	/*
2885 	 * These blocks are sized using the on disk dblock size, rather
2886 	 * than the current one.
2887 	 */
2888 	dm_block_t block_size;
2889 	dm_block_t discard_begin, discard_end;
2890 };
2891 
2892 static void discard_load_info_init(struct cache *cache,
2893 				   struct discard_load_info *li)
2894 {
2895 	li->cache = cache;
2896 	li->discard_begin = li->discard_end = 0;
2897 }
2898 
2899 static void set_discard_range(struct discard_load_info *li)
2900 {
2901 	sector_t b, e;
2902 
2903 	if (li->discard_begin == li->discard_end)
2904 		return;
2905 
2906 	/*
2907 	 * Convert to sectors.
2908 	 */
2909 	b = li->discard_begin * li->block_size;
2910 	e = li->discard_end * li->block_size;
2911 
2912 	/*
2913 	 * Then convert back to the current dblock size.
2914 	 */
2915 	b = dm_sector_div_up(b, li->cache->discard_block_size);
2916 	sector_div(e, li->cache->discard_block_size);
2917 
2918 	/*
2919 	 * The origin may have shrunk, so we need to check we're still in
2920 	 * bounds.
2921 	 */
2922 	if (e > from_dblock(li->cache->discard_nr_blocks))
2923 		e = from_dblock(li->cache->discard_nr_blocks);
2924 
2925 	for (; b < e; b++)
2926 		set_discard(li->cache, to_dblock(b));
2927 }
2928 
2929 static int load_discard(void *context, sector_t discard_block_size,
2930 			dm_dblock_t dblock, bool discard)
2931 {
2932 	struct discard_load_info *li = context;
2933 
2934 	li->block_size = discard_block_size;
2935 
2936 	if (discard) {
2937 		if (from_dblock(dblock) == li->discard_end)
2938 			/*
2939 			 * We're already in a discard range, just extend it.
2940 			 */
2941 			li->discard_end = li->discard_end + 1ULL;
2942 
2943 		else {
2944 			/*
2945 			 * Emit the old range and start a new one.
2946 			 */
2947 			set_discard_range(li);
2948 			li->discard_begin = from_dblock(dblock);
2949 			li->discard_end = li->discard_begin + 1ULL;
2950 		}
2951 	} else {
2952 		set_discard_range(li);
2953 		li->discard_begin = li->discard_end = 0;
2954 	}
2955 
2956 	return 0;
2957 }
2958 
2959 static dm_cblock_t get_cache_dev_size(struct cache *cache)
2960 {
2961 	sector_t size = get_dev_size(cache->cache_dev);
2962 	(void) sector_div(size, cache->sectors_per_block);
2963 	return to_cblock(size);
2964 }
2965 
2966 static bool can_resume(struct cache *cache)
2967 {
2968 	bool clean_when_opened;
2969 	int r;
2970 
2971 	/*
2972 	 * Disallow retrying the resume operation for devices that failed the
2973 	 * first resume attempt, as the failure leaves the policy object partially
2974 	 * initialized. Retrying could trigger BUG_ON when loading cache mappings
2975 	 * into the incomplete policy object.
2976 	 */
2977 	if (cache->sized && !cache->loaded_mappings) {
2978 		if (get_cache_mode(cache) != CM_WRITE)
2979 			DMERR("%s: unable to resume a failed-loaded cache, please check metadata.",
2980 			      cache_device_name(cache));
2981 		else
2982 			DMERR("%s: unable to resume cache due to missing proper cache table reload",
2983 			      cache_device_name(cache));
2984 		return false;
2985 	}
2986 
2987 	if (passthrough_mode(cache)) {
2988 		r = dm_cache_metadata_clean_when_opened(cache->cmd, &clean_when_opened);
2989 		if (r) {
2990 			DMERR("%s: failed to query metadata flags", cache_device_name(cache));
2991 			return false;
2992 		}
2993 
2994 		if (!clean_when_opened) {
2995 			DMERR("%s: unable to resume into passthrough mode after unclean shutdown",
2996 			      cache_device_name(cache));
2997 			return false;
2998 		}
2999 	}
3000 
3001 	return true;
3002 }
3003 
3004 static bool can_resize(struct cache *cache, dm_cblock_t new_size)
3005 {
3006 	if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
3007 		DMERR("%s: unable to extend cache due to missing cache table reload",
3008 		      cache_device_name(cache));
3009 		return false;
3010 	}
3011 
3012 	/*
3013 	 * We can't drop a dirty block when shrinking the cache.
3014 	 */
3015 	if (cache->loaded_mappings) {
3016 		new_size = to_cblock(find_next_bit(cache->dirty_bitset,
3017 						   from_cblock(cache->cache_size),
3018 						   from_cblock(new_size)));
3019 		if (new_size != cache->cache_size) {
3020 			DMERR("%s: unable to shrink cache; cache block %llu is dirty",
3021 			      cache_device_name(cache),
3022 			      (unsigned long long) from_cblock(new_size));
3023 			return false;
3024 		}
3025 	}
3026 
3027 	return true;
3028 }
3029 
3030 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
3031 {
3032 	int r;
3033 
3034 	r = dm_cache_resize(cache->cmd, new_size);
3035 	if (r) {
3036 		DMERR("%s: could not resize cache metadata", cache_device_name(cache));
3037 		metadata_operation_failed(cache, "dm_cache_resize", r);
3038 		return r;
3039 	}
3040 
3041 	set_cache_size(cache, new_size);
3042 
3043 	return 0;
3044 }
3045 
3046 static int truncate_oblocks(struct cache *cache)
3047 {
3048 	uint32_t nr_blocks = from_cblock(cache->cache_size);
3049 	uint32_t i;
3050 	int r;
3051 
3052 	for_each_set_bit(i, cache->invalid_bitset, nr_blocks) {
3053 		r = dm_cache_remove_mapping(cache->cmd, to_cblock(i));
3054 		if (r) {
3055 			DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
3056 				    cache_device_name(cache));
3057 			return r;
3058 		}
3059 	}
3060 
3061 	return 0;
3062 }
3063 
3064 static int cache_preresume(struct dm_target *ti)
3065 {
3066 	int r = 0;
3067 	struct cache *cache = ti->private;
3068 	dm_cblock_t csize = get_cache_dev_size(cache);
3069 
3070 	if (!can_resume(cache))
3071 		return -EINVAL;
3072 
3073 	/*
3074 	 * Check to see if the cache has resized.
3075 	 */
3076 	if (!cache->sized || csize != cache->cache_size) {
3077 		if (!can_resize(cache, csize))
3078 			return -EINVAL;
3079 
3080 		r = resize_cache_dev(cache, csize);
3081 		if (r)
3082 			return r;
3083 
3084 		cache->sized = true;
3085 	}
3086 
3087 	if (!cache->loaded_mappings) {
3088 		/*
3089 		 * The fast device could have been resized since the last
3090 		 * failed preresume attempt.  To be safe we start by a blank
3091 		 * bitset for cache blocks.
3092 		 */
3093 		clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size));
3094 
3095 		r = dm_cache_load_mappings(cache->cmd, cache->policy,
3096 					   load_filtered_mapping, cache);
3097 		if (r) {
3098 			DMERR("%s: could not load cache mappings", cache_device_name(cache));
3099 			if (r != -EFBIG && r != -EBUSY)
3100 				metadata_operation_failed(cache, "dm_cache_load_mappings", r);
3101 			return r;
3102 		}
3103 
3104 		r = truncate_oblocks(cache);
3105 		if (r) {
3106 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
3107 			return r;
3108 		}
3109 
3110 		cache->loaded_mappings = true;
3111 	}
3112 
3113 	if (!cache->loaded_discards) {
3114 		struct discard_load_info li;
3115 
3116 		/*
3117 		 * The discard bitset could have been resized, or the
3118 		 * discard block size changed.  To be safe we start by
3119 		 * setting every dblock to not discarded.
3120 		 */
3121 		clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
3122 
3123 		discard_load_info_init(cache, &li);
3124 		r = dm_cache_load_discards(cache->cmd, load_discard, &li);
3125 		if (r) {
3126 			DMERR("%s: could not load origin discards", cache_device_name(cache));
3127 			metadata_operation_failed(cache, "dm_cache_load_discards", r);
3128 			return r;
3129 		}
3130 		set_discard_range(&li);
3131 
3132 		cache->loaded_discards = true;
3133 	}
3134 
3135 	return r;
3136 }
3137 
3138 static void cache_resume(struct dm_target *ti)
3139 {
3140 	struct cache *cache = ti->private;
3141 
3142 	cache->need_tick_bio = true;
3143 	allow_background_work(cache);
3144 	do_waker(&cache->waker.work);
3145 }
3146 
3147 static void emit_flags(struct cache *cache, char *result,
3148 		       unsigned int maxlen, ssize_t *sz_ptr)
3149 {
3150 	ssize_t sz = *sz_ptr;
3151 	struct cache_features *cf = &cache->features;
3152 	unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
3153 
3154 	DMEMIT("%u ", count);
3155 
3156 	if (cf->metadata_version == 2)
3157 		DMEMIT("metadata2 ");
3158 
3159 	if (writethrough_mode(cache))
3160 		DMEMIT("writethrough ");
3161 
3162 	else if (passthrough_mode(cache))
3163 		DMEMIT("passthrough ");
3164 
3165 	else if (writeback_mode(cache))
3166 		DMEMIT("writeback ");
3167 
3168 	else {
3169 		DMEMIT("unknown ");
3170 		DMERR("%s: internal error: unknown io mode: %d",
3171 		      cache_device_name(cache), (int) cf->io_mode);
3172 	}
3173 
3174 	if (!cf->discard_passdown)
3175 		DMEMIT("no_discard_passdown ");
3176 
3177 	*sz_ptr = sz;
3178 }
3179 
3180 /*
3181  * Status format:
3182  *
3183  * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3184  * <cache block size> <#used cache blocks>/<#total cache blocks>
3185  * <#read hits> <#read misses> <#write hits> <#write misses>
3186  * <#demotions> <#promotions> <#dirty>
3187  * <#features> <features>*
3188  * <#core args> <core args>
3189  * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3190  */
3191 static void cache_status(struct dm_target *ti, status_type_t type,
3192 			 unsigned int status_flags, char *result, unsigned int maxlen)
3193 {
3194 	int r = 0;
3195 	unsigned int i;
3196 	ssize_t sz = 0;
3197 	dm_block_t nr_free_blocks_metadata = 0;
3198 	dm_block_t nr_blocks_metadata = 0;
3199 	char buf[BDEVNAME_SIZE];
3200 	struct cache *cache = ti->private;
3201 	dm_cblock_t residency;
3202 	bool needs_check;
3203 
3204 	switch (type) {
3205 	case STATUSTYPE_INFO:
3206 		if (get_cache_mode(cache) == CM_FAIL) {
3207 			DMEMIT("Fail");
3208 			break;
3209 		}
3210 
3211 		/* Commit to ensure statistics aren't out-of-date */
3212 		if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3213 			(void) commit(cache, false);
3214 
3215 		r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3216 		if (r) {
3217 			DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3218 			      cache_device_name(cache), r);
3219 			goto err;
3220 		}
3221 
3222 		r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3223 		if (r) {
3224 			DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3225 			      cache_device_name(cache), r);
3226 			goto err;
3227 		}
3228 
3229 		residency = policy_residency(cache->policy);
3230 
3231 		DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
3232 		       (unsigned int)DM_CACHE_METADATA_BLOCK_SIZE,
3233 		       (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3234 		       (unsigned long long)nr_blocks_metadata,
3235 		       (unsigned long long)cache->sectors_per_block,
3236 		       (unsigned long long) from_cblock(residency),
3237 		       (unsigned long long) from_cblock(cache->cache_size),
3238 		       (unsigned int) atomic_read(&cache->stats.read_hit),
3239 		       (unsigned int) atomic_read(&cache->stats.read_miss),
3240 		       (unsigned int) atomic_read(&cache->stats.write_hit),
3241 		       (unsigned int) atomic_read(&cache->stats.write_miss),
3242 		       (unsigned int) atomic_read(&cache->stats.demotion),
3243 		       (unsigned int) atomic_read(&cache->stats.promotion),
3244 		       (unsigned long) atomic_read(&cache->nr_dirty));
3245 
3246 		emit_flags(cache, result, maxlen, &sz);
3247 
3248 		DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
3249 
3250 		DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3251 		if (sz < maxlen) {
3252 			r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3253 			if (r)
3254 				DMERR("%s: policy_emit_config_values returned %d",
3255 				      cache_device_name(cache), r);
3256 		}
3257 
3258 		if (get_cache_mode(cache) == CM_READ_ONLY)
3259 			DMEMIT("ro ");
3260 		else
3261 			DMEMIT("rw ");
3262 
3263 		r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3264 
3265 		if (r || needs_check)
3266 			DMEMIT("needs_check ");
3267 		else
3268 			DMEMIT("- ");
3269 
3270 		break;
3271 
3272 	case STATUSTYPE_TABLE:
3273 		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3274 		DMEMIT("%s ", buf);
3275 		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3276 		DMEMIT("%s ", buf);
3277 		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3278 		DMEMIT("%s", buf);
3279 
3280 		for (i = 0; i < cache->nr_ctr_args - 1; i++)
3281 			DMEMIT(" %s", cache->ctr_args[i]);
3282 		if (cache->nr_ctr_args)
3283 			DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
3284 		break;
3285 
3286 	case STATUSTYPE_IMA:
3287 		DMEMIT_TARGET_NAME_VERSION(ti->type);
3288 		if (get_cache_mode(cache) == CM_FAIL)
3289 			DMEMIT(",metadata_mode=fail");
3290 		else if (get_cache_mode(cache) == CM_READ_ONLY)
3291 			DMEMIT(",metadata_mode=ro");
3292 		else
3293 			DMEMIT(",metadata_mode=rw");
3294 
3295 		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3296 		DMEMIT(",cache_metadata_device=%s", buf);
3297 		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3298 		DMEMIT(",cache_device=%s", buf);
3299 		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3300 		DMEMIT(",cache_origin_device=%s", buf);
3301 		DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n');
3302 		DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n');
3303 		DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n');
3304 		DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n');
3305 		DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y');
3306 		DMEMIT(";");
3307 		break;
3308 	}
3309 
3310 	return;
3311 
3312 err:
3313 	DMEMIT("Error");
3314 }
3315 
3316 /*
3317  * Defines a range of cblocks, begin to (end - 1) are in the range.  end is
3318  * the one-past-the-end value.
3319  */
3320 struct cblock_range {
3321 	dm_cblock_t begin;
3322 	dm_cblock_t end;
3323 };
3324 
3325 static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3326 {
3327 	return to_cblock(from_cblock(b) + 1);
3328 }
3329 
3330 /*
3331  * A cache block range can take two forms:
3332  *
3333  * i) A single cblock, eg. '3456'
3334  * ii) A begin and end cblock with a dash between, eg. 123-234
3335  */
3336 static int parse_cblock_range(struct cache *cache, char *str,
3337 			      struct cblock_range *result)
3338 {
3339 	char *blocknr = strsep(&str, "-");
3340 	unsigned int b, e;
3341 	int r;
3342 
3343 	r = kstrtouint(blocknr, 10, &b);
3344 	if (r)
3345 		goto bad;
3346 
3347 	result->begin = to_cblock(b);
3348 
3349 	if (str) {
3350 		blocknr = str;
3351 
3352 		r = kstrtouint(blocknr, 10, &e);
3353 		if (r)
3354 			goto bad;
3355 
3356 		result->end = to_cblock(e);
3357 	} else {
3358 		result->end = cblock_succ(result->begin);
3359 	}
3360 
3361 	return 0;
3362 
3363 bad:
3364 	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), blocknr);
3365 	return -EINVAL;
3366 }
3367 
3368 static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
3369 {
3370 	uint64_t b = from_cblock(range->begin);
3371 	uint64_t e = from_cblock(range->end);
3372 	uint64_t n = from_cblock(cache->cache_size);
3373 
3374 	if (b >= n) {
3375 		DMERR("%s: begin cblock out of range: %llu >= %llu",
3376 		      cache_device_name(cache), b, n);
3377 		return -EINVAL;
3378 	}
3379 
3380 	if (e > n) {
3381 		DMERR("%s: end cblock out of range: %llu > %llu",
3382 		      cache_device_name(cache), e, n);
3383 		return -EINVAL;
3384 	}
3385 
3386 	if (b >= e) {
3387 		DMERR("%s: invalid cblock range: %llu >= %llu",
3388 		      cache_device_name(cache), b, e);
3389 		return -EINVAL;
3390 	}
3391 
3392 	return 0;
3393 }
3394 
3395 static int request_invalidation(struct cache *cache, struct cblock_range *range)
3396 {
3397 	int r = 0;
3398 
3399 	/*
3400 	 * We don't need to do any locking here because we know we're in
3401 	 * passthrough mode.  There's is potential for a race between an
3402 	 * invalidation triggered by an io and an invalidation message.  This
3403 	 * is harmless, we must not worry if the policy call fails.
3404 	 */
3405 	while (range->begin != range->end) {
3406 		r = invalidate_cblock(cache, range->begin);
3407 		if (r)
3408 			return r;
3409 
3410 		range->begin = cblock_succ(range->begin);
3411 	}
3412 
3413 	cache->commit_requested = true;
3414 	return r;
3415 }
3416 
3417 static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count,
3418 					      char **cblock_ranges)
3419 {
3420 	int r = 0;
3421 	unsigned int i;
3422 	struct cblock_range range;
3423 
3424 	if (!passthrough_mode(cache)) {
3425 		DMERR("%s: cache has to be in passthrough mode for invalidation",
3426 		      cache_device_name(cache));
3427 		return -EPERM;
3428 	}
3429 
3430 	for (i = 0; i < count; i++) {
3431 		r = parse_cblock_range(cache, cblock_ranges[i], &range);
3432 		if (r)
3433 			break;
3434 
3435 		r = validate_cblock_range(cache, &range);
3436 		if (r)
3437 			break;
3438 
3439 		/*
3440 		 * Pass begin and end origin blocks to the worker and wake it.
3441 		 */
3442 		r = request_invalidation(cache, &range);
3443 		if (r)
3444 			break;
3445 	}
3446 
3447 	return r;
3448 }
3449 
3450 /*
3451  * Supports
3452  *	"<key> <value>"
3453  * and
3454  *     "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3455  *
3456  * The key migration_threshold is supported by the cache target core.
3457  */
3458 static int cache_message(struct dm_target *ti, unsigned int argc, char **argv,
3459 			 char *result, unsigned int maxlen)
3460 {
3461 	struct cache *cache = ti->private;
3462 
3463 	if (!argc)
3464 		return -EINVAL;
3465 
3466 	if (get_cache_mode(cache) >= CM_READ_ONLY) {
3467 		DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3468 		      cache_device_name(cache));
3469 		return -EOPNOTSUPP;
3470 	}
3471 
3472 	if (!strcasecmp(argv[0], "invalidate_cblocks"))
3473 		return process_invalidate_cblocks_message(cache, argc - 1, argv + 1);
3474 
3475 	if (argc != 2)
3476 		return -EINVAL;
3477 
3478 	return set_config_value(cache, argv[0], argv[1]);
3479 }
3480 
3481 static int cache_iterate_devices(struct dm_target *ti,
3482 				 iterate_devices_callout_fn fn, void *data)
3483 {
3484 	int r = 0;
3485 	struct cache *cache = ti->private;
3486 
3487 	r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3488 	if (!r)
3489 		r = fn(ti, cache->origin_dev, 0, ti->len, data);
3490 
3491 	return r;
3492 }
3493 
3494 /*
3495  * If discard_passdown was enabled verify that the origin device
3496  * supports discards.  Disable discard_passdown if not.
3497  */
3498 static void disable_passdown_if_not_supported(struct cache *cache)
3499 {
3500 	struct block_device *origin_bdev = cache->origin_dev->bdev;
3501 	struct queue_limits *origin_limits = bdev_limits(origin_bdev);
3502 	const char *reason = NULL;
3503 
3504 	if (!cache->features.discard_passdown)
3505 		return;
3506 
3507 	if (!bdev_max_discard_sectors(origin_bdev))
3508 		reason = "discard unsupported";
3509 
3510 	else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
3511 		reason = "max discard sectors smaller than a block";
3512 
3513 	if (reason) {
3514 		DMWARN("Origin device (%pg) %s: Disabling discard passdown.",
3515 		       origin_bdev, reason);
3516 		cache->features.discard_passdown = false;
3517 	}
3518 }
3519 
3520 static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3521 {
3522 	struct block_device *origin_bdev = cache->origin_dev->bdev;
3523 	struct queue_limits *origin_limits = bdev_limits(origin_bdev);
3524 
3525 	if (!cache->features.discard_passdown) {
3526 		/* No passdown is done so setting own virtual limits */
3527 		limits->max_hw_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3528 						       cache->origin_sectors);
3529 		limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3530 		return;
3531 	}
3532 
3533 	/*
3534 	 * cache_iterate_devices() is stacking both origin and fast device limits
3535 	 * but discards aren't passed to fast device, so inherit origin's limits.
3536 	 */
3537 	limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
3538 	limits->discard_granularity = origin_limits->discard_granularity;
3539 	limits->discard_alignment = origin_limits->discard_alignment;
3540 }
3541 
3542 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3543 {
3544 	struct cache *cache = ti->private;
3545 	uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3546 
3547 	/*
3548 	 * If the system-determined stacked limits are compatible with the
3549 	 * cache's blocksize (io_opt is a factor) do not override them.
3550 	 */
3551 	if (io_opt_sectors < cache->sectors_per_block ||
3552 	    do_div(io_opt_sectors, cache->sectors_per_block)) {
3553 		limits->io_min = cache->sectors_per_block << SECTOR_SHIFT;
3554 		limits->io_opt = cache->sectors_per_block << SECTOR_SHIFT;
3555 	}
3556 
3557 	disable_passdown_if_not_supported(cache);
3558 	set_discard_limits(cache, limits);
3559 }
3560 
3561 /*----------------------------------------------------------------*/
3562 
3563 static struct target_type cache_target = {
3564 	.name = "cache",
3565 	.version = {2, 4, 0},
3566 	.module = THIS_MODULE,
3567 	.ctr = cache_ctr,
3568 	.dtr = cache_dtr,
3569 	.map = cache_map,
3570 	.end_io = cache_end_io,
3571 	.postsuspend = cache_postsuspend,
3572 	.preresume = cache_preresume,
3573 	.resume = cache_resume,
3574 	.status = cache_status,
3575 	.message = cache_message,
3576 	.iterate_devices = cache_iterate_devices,
3577 	.io_hints = cache_io_hints,
3578 };
3579 
3580 static int __init dm_cache_init(void)
3581 {
3582 	int r;
3583 
3584 	migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3585 	if (!migration_cache) {
3586 		r = -ENOMEM;
3587 		goto err;
3588 	}
3589 
3590 	btracker_work_cache = kmem_cache_create("dm_cache_bt_work",
3591 		sizeof(struct bt_work), __alignof__(struct bt_work), 0, NULL);
3592 	if (!btracker_work_cache) {
3593 		r = -ENOMEM;
3594 		goto err;
3595 	}
3596 
3597 	r = dm_register_target(&cache_target);
3598 	if (r) {
3599 		goto err;
3600 	}
3601 
3602 	return 0;
3603 
3604 err:
3605 	kmem_cache_destroy(migration_cache);
3606 	kmem_cache_destroy(btracker_work_cache);
3607 	return r;
3608 }
3609 
3610 static void __exit dm_cache_exit(void)
3611 {
3612 	dm_unregister_target(&cache_target);
3613 	kmem_cache_destroy(migration_cache);
3614 	kmem_cache_destroy(btracker_work_cache);
3615 }
3616 
3617 module_init(dm_cache_init);
3618 module_exit(dm_cache_exit);
3619 
3620 MODULE_DESCRIPTION(DM_NAME " cache target");
3621 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
3622 MODULE_LICENSE("GPL");
3623