xref: /linux/drivers/md/dm.c (revision 670c12ce09a8180ebe0c608d8a789a7e58fe7d8e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
4  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5  *
6  * This file is released under the GPL.
7  */
8 
9 #include "dm-core.h"
10 #include "dm-rq.h"
11 #include "dm-uevent.h"
12 #include "dm-ima.h"
13 
14 #include <linux/bio-integrity.h>
15 #include <linux/init.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/sched/mm.h>
19 #include <linux/sched/signal.h>
20 #include <linux/blkpg.h>
21 #include <linux/bio.h>
22 #include <linux/mempool.h>
23 #include <linux/dax.h>
24 #include <linux/slab.h>
25 #include <linux/idr.h>
26 #include <linux/uio.h>
27 #include <linux/hdreg.h>
28 #include <linux/delay.h>
29 #include <linux/wait.h>
30 #include <linux/pr.h>
31 #include <linux/refcount.h>
32 #include <linux/part_stat.h>
33 #include <linux/blk-crypto.h>
34 #include <linux/blk-crypto-profile.h>
35 
36 #define DM_MSG_PREFIX "core"
37 
38 /*
39  * Cookies are numeric values sent with CHANGE and REMOVE
40  * uevents while resuming, removing or renaming the device.
41  */
42 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
43 #define DM_COOKIE_LENGTH 24
44 
45 /*
46  * For REQ_POLLED fs bio, this flag is set if we link mapped underlying
47  * dm_io into one list, and reuse bio->bi_private as the list head. Before
48  * ending this fs bio, we will recover its ->bi_private.
49  */
50 #define REQ_DM_POLL_LIST	REQ_DRV
51 
52 static const char *_name = DM_NAME;
53 
54 static unsigned int major;
55 static unsigned int _major;
56 
57 static DEFINE_IDR(_minor_idr);
58 
59 static DEFINE_SPINLOCK(_minor_lock);
60 
61 static void do_deferred_remove(struct work_struct *w);
62 
63 static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
64 
65 static struct workqueue_struct *deferred_remove_workqueue;
66 
67 atomic_t dm_global_event_nr = ATOMIC_INIT(0);
68 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
69 
dm_issue_global_event(void)70 void dm_issue_global_event(void)
71 {
72 	atomic_inc(&dm_global_event_nr);
73 	wake_up(&dm_global_eventq);
74 }
75 
76 DEFINE_STATIC_KEY_FALSE(stats_enabled);
77 DEFINE_STATIC_KEY_FALSE(swap_bios_enabled);
78 DEFINE_STATIC_KEY_FALSE(zoned_enabled);
79 
80 /*
81  * One of these is allocated (on-stack) per original bio.
82  */
83 struct clone_info {
84 	struct dm_table *map;
85 	struct bio *bio;
86 	struct dm_io *io;
87 	sector_t sector;
88 	unsigned int sector_count;
89 	bool is_abnormal_io:1;
90 	bool submit_as_polled:1;
91 };
92 
clone_to_tio(struct bio * clone)93 static inline struct dm_target_io *clone_to_tio(struct bio *clone)
94 {
95 	return container_of(clone, struct dm_target_io, clone);
96 }
97 
dm_per_bio_data(struct bio * bio,size_t data_size)98 void *dm_per_bio_data(struct bio *bio, size_t data_size)
99 {
100 	if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO))
101 		return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
102 	return (char *)bio - DM_IO_BIO_OFFSET - data_size;
103 }
104 EXPORT_SYMBOL_GPL(dm_per_bio_data);
105 
dm_bio_from_per_bio_data(void * data,size_t data_size)106 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
107 {
108 	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
109 
110 	if (io->magic == DM_IO_MAGIC)
111 		return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
112 	BUG_ON(io->magic != DM_TIO_MAGIC);
113 	return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
114 }
115 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
116 
dm_bio_get_target_bio_nr(const struct bio * bio)117 unsigned int dm_bio_get_target_bio_nr(const struct bio *bio)
118 {
119 	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
120 }
121 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
122 
123 #define MINOR_ALLOCED ((void *)-1)
124 
125 #define DM_NUMA_NODE NUMA_NO_NODE
126 static int dm_numa_node = DM_NUMA_NODE;
127 
128 #define DEFAULT_SWAP_BIOS	(8 * 1048576 / PAGE_SIZE)
129 static int swap_bios = DEFAULT_SWAP_BIOS;
get_swap_bios(void)130 static int get_swap_bios(void)
131 {
132 	int latch = READ_ONCE(swap_bios);
133 
134 	if (unlikely(latch <= 0))
135 		latch = DEFAULT_SWAP_BIOS;
136 	return latch;
137 }
138 
139 struct table_device {
140 	struct list_head list;
141 	refcount_t count;
142 	struct dm_dev dm_dev;
143 };
144 
145 /*
146  * Bio-based DM's mempools' reserved IOs set by the user.
147  */
148 #define RESERVED_BIO_BASED_IOS		16
149 static unsigned int reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
150 
__dm_get_module_param_int(int * module_param,int min,int max)151 static int __dm_get_module_param_int(int *module_param, int min, int max)
152 {
153 	int param = READ_ONCE(*module_param);
154 	int modified_param = 0;
155 	bool modified = true;
156 
157 	if (param < min)
158 		modified_param = min;
159 	else if (param > max)
160 		modified_param = max;
161 	else
162 		modified = false;
163 
164 	if (modified) {
165 		(void)cmpxchg(module_param, param, modified_param);
166 		param = modified_param;
167 	}
168 
169 	return param;
170 }
171 
__dm_get_module_param(unsigned int * module_param,unsigned int def,unsigned int max)172 unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max)
173 {
174 	unsigned int param = READ_ONCE(*module_param);
175 	unsigned int modified_param = 0;
176 
177 	if (!param)
178 		modified_param = def;
179 	else if (param > max)
180 		modified_param = max;
181 
182 	if (modified_param) {
183 		(void)cmpxchg(module_param, param, modified_param);
184 		param = modified_param;
185 	}
186 
187 	return param;
188 }
189 
dm_get_reserved_bio_based_ios(void)190 unsigned int dm_get_reserved_bio_based_ios(void)
191 {
192 	return __dm_get_module_param(&reserved_bio_based_ios,
193 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
194 }
195 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
196 
dm_get_numa_node(void)197 static unsigned int dm_get_numa_node(void)
198 {
199 	return __dm_get_module_param_int(&dm_numa_node,
200 					 DM_NUMA_NODE, num_online_nodes() - 1);
201 }
202 
local_init(void)203 static int __init local_init(void)
204 {
205 	int r;
206 
207 	r = dm_uevent_init();
208 	if (r)
209 		return r;
210 
211 	deferred_remove_workqueue = alloc_ordered_workqueue("kdmremove", 0);
212 	if (!deferred_remove_workqueue) {
213 		r = -ENOMEM;
214 		goto out_uevent_exit;
215 	}
216 
217 	_major = major;
218 	r = register_blkdev(_major, _name);
219 	if (r < 0)
220 		goto out_free_workqueue;
221 
222 	if (!_major)
223 		_major = r;
224 
225 	return 0;
226 
227 out_free_workqueue:
228 	destroy_workqueue(deferred_remove_workqueue);
229 out_uevent_exit:
230 	dm_uevent_exit();
231 
232 	return r;
233 }
234 
local_exit(void)235 static void local_exit(void)
236 {
237 	destroy_workqueue(deferred_remove_workqueue);
238 
239 	unregister_blkdev(_major, _name);
240 	dm_uevent_exit();
241 
242 	_major = 0;
243 
244 	DMINFO("cleaned up");
245 }
246 
247 static int (*_inits[])(void) __initdata = {
248 	local_init,
249 	dm_target_init,
250 	dm_linear_init,
251 	dm_stripe_init,
252 	dm_io_init,
253 	dm_kcopyd_init,
254 	dm_interface_init,
255 	dm_statistics_init,
256 };
257 
258 static void (*_exits[])(void) = {
259 	local_exit,
260 	dm_target_exit,
261 	dm_linear_exit,
262 	dm_stripe_exit,
263 	dm_io_exit,
264 	dm_kcopyd_exit,
265 	dm_interface_exit,
266 	dm_statistics_exit,
267 };
268 
dm_init(void)269 static int __init dm_init(void)
270 {
271 	const int count = ARRAY_SIZE(_inits);
272 	int r, i;
273 
274 #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
275 	DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
276 	       " Duplicate IMA measurements will not be recorded in the IMA log.");
277 #endif
278 
279 	for (i = 0; i < count; i++) {
280 		r = _inits[i]();
281 		if (r)
282 			goto bad;
283 	}
284 
285 	return 0;
286 bad:
287 	while (i--)
288 		_exits[i]();
289 
290 	return r;
291 }
292 
dm_exit(void)293 static void __exit dm_exit(void)
294 {
295 	int i = ARRAY_SIZE(_exits);
296 
297 	while (i--)
298 		_exits[i]();
299 
300 	/*
301 	 * Should be empty by this point.
302 	 */
303 	idr_destroy(&_minor_idr);
304 }
305 
306 /*
307  * Block device functions
308  */
dm_deleting_md(struct mapped_device * md)309 int dm_deleting_md(struct mapped_device *md)
310 {
311 	return test_bit(DMF_DELETING, &md->flags);
312 }
313 
dm_blk_open(struct gendisk * disk,blk_mode_t mode)314 static int dm_blk_open(struct gendisk *disk, blk_mode_t mode)
315 {
316 	struct mapped_device *md;
317 
318 	spin_lock(&_minor_lock);
319 
320 	md = disk->private_data;
321 	if (!md)
322 		goto out;
323 
324 	if (test_bit(DMF_FREEING, &md->flags) ||
325 	    dm_deleting_md(md)) {
326 		md = NULL;
327 		goto out;
328 	}
329 
330 	dm_get(md);
331 	atomic_inc(&md->open_count);
332 out:
333 	spin_unlock(&_minor_lock);
334 
335 	return md ? 0 : -ENXIO;
336 }
337 
dm_blk_close(struct gendisk * disk)338 static void dm_blk_close(struct gendisk *disk)
339 {
340 	struct mapped_device *md;
341 
342 	spin_lock(&_minor_lock);
343 
344 	md = disk->private_data;
345 	if (WARN_ON(!md))
346 		goto out;
347 
348 	if (atomic_dec_and_test(&md->open_count) &&
349 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
350 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
351 
352 	dm_put(md);
353 out:
354 	spin_unlock(&_minor_lock);
355 }
356 
dm_open_count(struct mapped_device * md)357 int dm_open_count(struct mapped_device *md)
358 {
359 	return atomic_read(&md->open_count);
360 }
361 
362 /*
363  * Guarantees nothing is using the device before it's deleted.
364  */
dm_lock_for_deletion(struct mapped_device * md,bool mark_deferred,bool only_deferred)365 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
366 {
367 	int r = 0;
368 
369 	spin_lock(&_minor_lock);
370 
371 	if (dm_open_count(md)) {
372 		r = -EBUSY;
373 		if (mark_deferred)
374 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
375 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
376 		r = -EEXIST;
377 	else
378 		set_bit(DMF_DELETING, &md->flags);
379 
380 	spin_unlock(&_minor_lock);
381 
382 	return r;
383 }
384 
dm_cancel_deferred_remove(struct mapped_device * md)385 int dm_cancel_deferred_remove(struct mapped_device *md)
386 {
387 	int r = 0;
388 
389 	spin_lock(&_minor_lock);
390 
391 	if (test_bit(DMF_DELETING, &md->flags))
392 		r = -EBUSY;
393 	else
394 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
395 
396 	spin_unlock(&_minor_lock);
397 
398 	return r;
399 }
400 
do_deferred_remove(struct work_struct * w)401 static void do_deferred_remove(struct work_struct *w)
402 {
403 	dm_deferred_remove();
404 }
405 
dm_blk_getgeo(struct block_device * bdev,struct hd_geometry * geo)406 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
407 {
408 	struct mapped_device *md = bdev->bd_disk->private_data;
409 
410 	return dm_get_geometry(md, geo);
411 }
412 
dm_prepare_ioctl(struct mapped_device * md,int * srcu_idx,struct block_device ** bdev)413 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
414 			    struct block_device **bdev)
415 {
416 	struct dm_target *ti;
417 	struct dm_table *map;
418 	int r;
419 
420 retry:
421 	r = -ENOTTY;
422 	map = dm_get_live_table(md, srcu_idx);
423 	if (!map || !dm_table_get_size(map))
424 		return r;
425 
426 	/* We only support devices that have a single target */
427 	if (map->num_targets != 1)
428 		return r;
429 
430 	ti = dm_table_get_target(map, 0);
431 	if (!ti->type->prepare_ioctl)
432 		return r;
433 
434 	if (dm_suspended_md(md))
435 		return -EAGAIN;
436 
437 	r = ti->type->prepare_ioctl(ti, bdev);
438 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
439 		dm_put_live_table(md, *srcu_idx);
440 		fsleep(10000);
441 		goto retry;
442 	}
443 
444 	return r;
445 }
446 
dm_unprepare_ioctl(struct mapped_device * md,int srcu_idx)447 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
448 {
449 	dm_put_live_table(md, srcu_idx);
450 }
451 
dm_blk_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)452 static int dm_blk_ioctl(struct block_device *bdev, blk_mode_t mode,
453 			unsigned int cmd, unsigned long arg)
454 {
455 	struct mapped_device *md = bdev->bd_disk->private_data;
456 	int r, srcu_idx;
457 
458 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
459 	if (r < 0)
460 		goto out;
461 
462 	if (r > 0) {
463 		/*
464 		 * Target determined this ioctl is being issued against a
465 		 * subset of the parent bdev; require extra privileges.
466 		 */
467 		if (!capable(CAP_SYS_RAWIO)) {
468 			DMDEBUG_LIMIT(
469 	"%s: sending ioctl %x to DM device without required privilege.",
470 				current->comm, cmd);
471 			r = -ENOIOCTLCMD;
472 			goto out;
473 		}
474 	}
475 
476 	if (!bdev->bd_disk->fops->ioctl)
477 		r = -ENOTTY;
478 	else
479 		r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
480 out:
481 	dm_unprepare_ioctl(md, srcu_idx);
482 	return r;
483 }
484 
dm_start_time_ns_from_clone(struct bio * bio)485 u64 dm_start_time_ns_from_clone(struct bio *bio)
486 {
487 	return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
488 }
489 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
490 
bio_is_flush_with_data(struct bio * bio)491 static inline bool bio_is_flush_with_data(struct bio *bio)
492 {
493 	return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
494 }
495 
dm_io_sectors(struct dm_io * io,struct bio * bio)496 static inline unsigned int dm_io_sectors(struct dm_io *io, struct bio *bio)
497 {
498 	/*
499 	 * If REQ_PREFLUSH set, don't account payload, it will be
500 	 * submitted (and accounted) after this flush completes.
501 	 */
502 	if (bio_is_flush_with_data(bio))
503 		return 0;
504 	if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT)))
505 		return io->sectors;
506 	return bio_sectors(bio);
507 }
508 
dm_io_acct(struct dm_io * io,bool end)509 static void dm_io_acct(struct dm_io *io, bool end)
510 {
511 	struct bio *bio = io->orig_bio;
512 
513 	if (dm_io_flagged(io, DM_IO_BLK_STAT)) {
514 		if (!end)
515 			bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
516 					   io->start_time);
517 		else
518 			bdev_end_io_acct(bio->bi_bdev, bio_op(bio),
519 					 dm_io_sectors(io, bio),
520 					 io->start_time);
521 	}
522 
523 	if (static_branch_unlikely(&stats_enabled) &&
524 	    unlikely(dm_stats_used(&io->md->stats))) {
525 		sector_t sector;
526 
527 		if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT)))
528 			sector = bio_end_sector(bio) - io->sector_offset;
529 		else
530 			sector = bio->bi_iter.bi_sector;
531 
532 		dm_stats_account_io(&io->md->stats, bio_data_dir(bio),
533 				    sector, dm_io_sectors(io, bio),
534 				    end, io->start_time, &io->stats_aux);
535 	}
536 }
537 
__dm_start_io_acct(struct dm_io * io)538 static void __dm_start_io_acct(struct dm_io *io)
539 {
540 	dm_io_acct(io, false);
541 }
542 
dm_start_io_acct(struct dm_io * io,struct bio * clone)543 static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
544 {
545 	/*
546 	 * Ensure IO accounting is only ever started once.
547 	 */
548 	if (dm_io_flagged(io, DM_IO_ACCOUNTED))
549 		return;
550 
551 	/* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */
552 	if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) {
553 		dm_io_set_flag(io, DM_IO_ACCOUNTED);
554 	} else {
555 		unsigned long flags;
556 		/* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
557 		spin_lock_irqsave(&io->lock, flags);
558 		if (dm_io_flagged(io, DM_IO_ACCOUNTED)) {
559 			spin_unlock_irqrestore(&io->lock, flags);
560 			return;
561 		}
562 		dm_io_set_flag(io, DM_IO_ACCOUNTED);
563 		spin_unlock_irqrestore(&io->lock, flags);
564 	}
565 
566 	__dm_start_io_acct(io);
567 }
568 
dm_end_io_acct(struct dm_io * io)569 static void dm_end_io_acct(struct dm_io *io)
570 {
571 	dm_io_acct(io, true);
572 }
573 
alloc_io(struct mapped_device * md,struct bio * bio,gfp_t gfp_mask)574 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio, gfp_t gfp_mask)
575 {
576 	struct dm_io *io;
577 	struct dm_target_io *tio;
578 	struct bio *clone;
579 
580 	clone = bio_alloc_clone(NULL, bio, gfp_mask, &md->mempools->io_bs);
581 	if (unlikely(!clone))
582 		return NULL;
583 	tio = clone_to_tio(clone);
584 	tio->flags = 0;
585 	dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
586 	tio->io = NULL;
587 
588 	io = container_of(tio, struct dm_io, tio);
589 	io->magic = DM_IO_MAGIC;
590 	io->status = BLK_STS_OK;
591 
592 	/* one ref is for submission, the other is for completion */
593 	atomic_set(&io->io_count, 2);
594 	this_cpu_inc(*md->pending_io);
595 	io->orig_bio = bio;
596 	io->md = md;
597 	spin_lock_init(&io->lock);
598 	io->start_time = jiffies;
599 	io->flags = 0;
600 	if (blk_queue_io_stat(md->queue))
601 		dm_io_set_flag(io, DM_IO_BLK_STAT);
602 
603 	if (static_branch_unlikely(&stats_enabled) &&
604 	    unlikely(dm_stats_used(&md->stats)))
605 		dm_stats_record_start(&md->stats, &io->stats_aux);
606 
607 	return io;
608 }
609 
free_io(struct dm_io * io)610 static void free_io(struct dm_io *io)
611 {
612 	bio_put(&io->tio.clone);
613 }
614 
alloc_tio(struct clone_info * ci,struct dm_target * ti,unsigned int target_bio_nr,unsigned int * len,gfp_t gfp_mask)615 static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
616 			     unsigned int target_bio_nr, unsigned int *len, gfp_t gfp_mask)
617 {
618 	struct mapped_device *md = ci->io->md;
619 	struct dm_target_io *tio;
620 	struct bio *clone;
621 
622 	if (!ci->io->tio.io) {
623 		/* the dm_target_io embedded in ci->io is available */
624 		tio = &ci->io->tio;
625 		/* alloc_io() already initialized embedded clone */
626 		clone = &tio->clone;
627 	} else {
628 		clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,
629 					&md->mempools->bs);
630 		if (!clone)
631 			return NULL;
632 
633 		/* REQ_DM_POLL_LIST shouldn't be inherited */
634 		clone->bi_opf &= ~REQ_DM_POLL_LIST;
635 
636 		tio = clone_to_tio(clone);
637 		tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */
638 	}
639 
640 	tio->magic = DM_TIO_MAGIC;
641 	tio->io = ci->io;
642 	tio->ti = ti;
643 	tio->target_bio_nr = target_bio_nr;
644 	tio->len_ptr = len;
645 	tio->old_sector = 0;
646 
647 	/* Set default bdev, but target must bio_set_dev() before issuing IO */
648 	clone->bi_bdev = md->disk->part0;
649 	if (likely(ti != NULL) && unlikely(ti->needs_bio_set_dev))
650 		bio_set_dev(clone, md->disk->part0);
651 
652 	if (len) {
653 		clone->bi_iter.bi_size = to_bytes(*len);
654 		if (bio_integrity(clone))
655 			bio_integrity_trim(clone);
656 	}
657 
658 	return clone;
659 }
660 
free_tio(struct bio * clone)661 static void free_tio(struct bio *clone)
662 {
663 	if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO))
664 		return;
665 	bio_put(clone);
666 }
667 
668 /*
669  * Add the bio to the list of deferred io.
670  */
queue_io(struct mapped_device * md,struct bio * bio)671 static void queue_io(struct mapped_device *md, struct bio *bio)
672 {
673 	unsigned long flags;
674 
675 	spin_lock_irqsave(&md->deferred_lock, flags);
676 	bio_list_add(&md->deferred, bio);
677 	spin_unlock_irqrestore(&md->deferred_lock, flags);
678 	queue_work(md->wq, &md->work);
679 }
680 
681 /*
682  * Everyone (including functions in this file), should use this
683  * function to access the md->map field, and make sure they call
684  * dm_put_live_table() when finished.
685  */
dm_get_live_table(struct mapped_device * md,int * srcu_idx)686 struct dm_table *dm_get_live_table(struct mapped_device *md,
687 				   int *srcu_idx) __acquires(md->io_barrier)
688 {
689 	*srcu_idx = srcu_read_lock(&md->io_barrier);
690 
691 	return srcu_dereference(md->map, &md->io_barrier);
692 }
693 
dm_put_live_table(struct mapped_device * md,int srcu_idx)694 void dm_put_live_table(struct mapped_device *md,
695 		       int srcu_idx) __releases(md->io_barrier)
696 {
697 	srcu_read_unlock(&md->io_barrier, srcu_idx);
698 }
699 
dm_sync_table(struct mapped_device * md)700 void dm_sync_table(struct mapped_device *md)
701 {
702 	synchronize_srcu(&md->io_barrier);
703 	synchronize_rcu_expedited();
704 }
705 
706 /*
707  * A fast alternative to dm_get_live_table/dm_put_live_table.
708  * The caller must not block between these two functions.
709  */
dm_get_live_table_fast(struct mapped_device * md)710 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
711 {
712 	rcu_read_lock();
713 	return rcu_dereference(md->map);
714 }
715 
dm_put_live_table_fast(struct mapped_device * md)716 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
717 {
718 	rcu_read_unlock();
719 }
720 
721 static char *_dm_claim_ptr = "I belong to device-mapper";
722 
723 /*
724  * Open a table device so we can use it as a map destination.
725  */
open_table_device(struct mapped_device * md,dev_t dev,blk_mode_t mode)726 static struct table_device *open_table_device(struct mapped_device *md,
727 		dev_t dev, blk_mode_t mode)
728 {
729 	struct table_device *td;
730 	struct file *bdev_file;
731 	struct block_device *bdev;
732 	u64 part_off;
733 	int r;
734 
735 	td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
736 	if (!td)
737 		return ERR_PTR(-ENOMEM);
738 	refcount_set(&td->count, 1);
739 
740 	bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL);
741 	if (IS_ERR(bdev_file)) {
742 		r = PTR_ERR(bdev_file);
743 		goto out_free_td;
744 	}
745 
746 	bdev = file_bdev(bdev_file);
747 
748 	/*
749 	 * We can be called before the dm disk is added.  In that case we can't
750 	 * register the holder relation here.  It will be done once add_disk was
751 	 * called.
752 	 */
753 	if (md->disk->slave_dir) {
754 		r = bd_link_disk_holder(bdev, md->disk);
755 		if (r)
756 			goto out_blkdev_put;
757 	}
758 
759 	td->dm_dev.mode = mode;
760 	td->dm_dev.bdev = bdev;
761 	td->dm_dev.bdev_file = bdev_file;
762 	td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off,
763 						NULL, NULL);
764 	format_dev_t(td->dm_dev.name, dev);
765 	list_add(&td->list, &md->table_devices);
766 	return td;
767 
768 out_blkdev_put:
769 	__fput_sync(bdev_file);
770 out_free_td:
771 	kfree(td);
772 	return ERR_PTR(r);
773 }
774 
775 /*
776  * Close a table device that we've been using.
777  */
close_table_device(struct table_device * td,struct mapped_device * md)778 static void close_table_device(struct table_device *td, struct mapped_device *md)
779 {
780 	if (md->disk->slave_dir)
781 		bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
782 
783 	/* Leverage async fput() if DMF_DEFERRED_REMOVE set */
784 	if (unlikely(test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
785 		fput(td->dm_dev.bdev_file);
786 	else
787 		__fput_sync(td->dm_dev.bdev_file);
788 
789 	put_dax(td->dm_dev.dax_dev);
790 	list_del(&td->list);
791 	kfree(td);
792 }
793 
find_table_device(struct list_head * l,dev_t dev,blk_mode_t mode)794 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
795 					      blk_mode_t mode)
796 {
797 	struct table_device *td;
798 
799 	list_for_each_entry(td, l, list)
800 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
801 			return td;
802 
803 	return NULL;
804 }
805 
dm_get_table_device(struct mapped_device * md,dev_t dev,blk_mode_t mode,struct dm_dev ** result)806 int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode,
807 			struct dm_dev **result)
808 {
809 	struct table_device *td;
810 
811 	mutex_lock(&md->table_devices_lock);
812 	td = find_table_device(&md->table_devices, dev, mode);
813 	if (!td) {
814 		td = open_table_device(md, dev, mode);
815 		if (IS_ERR(td)) {
816 			mutex_unlock(&md->table_devices_lock);
817 			return PTR_ERR(td);
818 		}
819 	} else {
820 		refcount_inc(&td->count);
821 	}
822 	mutex_unlock(&md->table_devices_lock);
823 
824 	*result = &td->dm_dev;
825 	return 0;
826 }
827 
dm_put_table_device(struct mapped_device * md,struct dm_dev * d)828 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
829 {
830 	struct table_device *td = container_of(d, struct table_device, dm_dev);
831 
832 	mutex_lock(&md->table_devices_lock);
833 	if (refcount_dec_and_test(&td->count))
834 		close_table_device(td, md);
835 	mutex_unlock(&md->table_devices_lock);
836 }
837 
838 /*
839  * Get the geometry associated with a dm device
840  */
dm_get_geometry(struct mapped_device * md,struct hd_geometry * geo)841 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
842 {
843 	*geo = md->geometry;
844 
845 	return 0;
846 }
847 
848 /*
849  * Set the geometry of a device.
850  */
dm_set_geometry(struct mapped_device * md,struct hd_geometry * geo)851 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
852 {
853 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
854 
855 	if (geo->start > sz) {
856 		DMERR("Start sector is beyond the geometry limits.");
857 		return -EINVAL;
858 	}
859 
860 	md->geometry = *geo;
861 
862 	return 0;
863 }
864 
__noflush_suspending(struct mapped_device * md)865 static int __noflush_suspending(struct mapped_device *md)
866 {
867 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
868 }
869 
dm_requeue_add_io(struct dm_io * io,bool first_stage)870 static void dm_requeue_add_io(struct dm_io *io, bool first_stage)
871 {
872 	struct mapped_device *md = io->md;
873 
874 	if (first_stage) {
875 		struct dm_io *next = md->requeue_list;
876 
877 		md->requeue_list = io;
878 		io->next = next;
879 	} else {
880 		bio_list_add_head(&md->deferred, io->orig_bio);
881 	}
882 }
883 
dm_kick_requeue(struct mapped_device * md,bool first_stage)884 static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
885 {
886 	if (first_stage)
887 		queue_work(md->wq, &md->requeue_work);
888 	else
889 		queue_work(md->wq, &md->work);
890 }
891 
892 /*
893  * Return true if the dm_io's original bio is requeued.
894  * io->status is updated with error if requeue disallowed.
895  */
dm_handle_requeue(struct dm_io * io,bool first_stage)896 static bool dm_handle_requeue(struct dm_io *io, bool first_stage)
897 {
898 	struct bio *bio = io->orig_bio;
899 	bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE);
900 	bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) &&
901 				     (bio->bi_opf & REQ_POLLED));
902 	struct mapped_device *md = io->md;
903 	bool requeued = false;
904 
905 	if (handle_requeue || handle_polled_eagain) {
906 		unsigned long flags;
907 
908 		if (bio->bi_opf & REQ_POLLED) {
909 			/*
910 			 * Upper layer won't help us poll split bio
911 			 * (io->orig_bio may only reflect a subset of the
912 			 * pre-split original) so clear REQ_POLLED.
913 			 */
914 			bio_clear_polled(bio);
915 		}
916 
917 		/*
918 		 * Target requested pushing back the I/O or
919 		 * polled IO hit BLK_STS_AGAIN.
920 		 */
921 		spin_lock_irqsave(&md->deferred_lock, flags);
922 		if ((__noflush_suspending(md) &&
923 		     !WARN_ON_ONCE(dm_is_zone_write(md, bio))) ||
924 		    handle_polled_eagain || first_stage) {
925 			dm_requeue_add_io(io, first_stage);
926 			requeued = true;
927 		} else {
928 			/*
929 			 * noflush suspend was interrupted or this is
930 			 * a write to a zoned target.
931 			 */
932 			io->status = BLK_STS_IOERR;
933 		}
934 		spin_unlock_irqrestore(&md->deferred_lock, flags);
935 	}
936 
937 	if (requeued)
938 		dm_kick_requeue(md, first_stage);
939 
940 	return requeued;
941 }
942 
__dm_io_complete(struct dm_io * io,bool first_stage)943 static void __dm_io_complete(struct dm_io *io, bool first_stage)
944 {
945 	struct bio *bio = io->orig_bio;
946 	struct mapped_device *md = io->md;
947 	blk_status_t io_error;
948 	bool requeued;
949 
950 	requeued = dm_handle_requeue(io, first_stage);
951 	if (requeued && first_stage)
952 		return;
953 
954 	io_error = io->status;
955 	if (dm_io_flagged(io, DM_IO_ACCOUNTED))
956 		dm_end_io_acct(io);
957 	else if (!io_error) {
958 		/*
959 		 * Must handle target that DM_MAPIO_SUBMITTED only to
960 		 * then bio_endio() rather than dm_submit_bio_remap()
961 		 */
962 		__dm_start_io_acct(io);
963 		dm_end_io_acct(io);
964 	}
965 	free_io(io);
966 	smp_wmb();
967 	this_cpu_dec(*md->pending_io);
968 
969 	/* nudge anyone waiting on suspend queue */
970 	if (unlikely(wq_has_sleeper(&md->wait)))
971 		wake_up(&md->wait);
972 
973 	/* Return early if the original bio was requeued */
974 	if (requeued)
975 		return;
976 
977 	if (bio_is_flush_with_data(bio)) {
978 		/*
979 		 * Preflush done for flush with data, reissue
980 		 * without REQ_PREFLUSH.
981 		 */
982 		bio->bi_opf &= ~REQ_PREFLUSH;
983 		queue_io(md, bio);
984 	} else {
985 		/* done with normal IO or empty flush */
986 		if (io_error)
987 			bio->bi_status = io_error;
988 		bio_endio(bio);
989 	}
990 }
991 
dm_wq_requeue_work(struct work_struct * work)992 static void dm_wq_requeue_work(struct work_struct *work)
993 {
994 	struct mapped_device *md = container_of(work, struct mapped_device,
995 						requeue_work);
996 	unsigned long flags;
997 	struct dm_io *io;
998 
999 	/* reuse deferred lock to simplify dm_handle_requeue */
1000 	spin_lock_irqsave(&md->deferred_lock, flags);
1001 	io = md->requeue_list;
1002 	md->requeue_list = NULL;
1003 	spin_unlock_irqrestore(&md->deferred_lock, flags);
1004 
1005 	while (io) {
1006 		struct dm_io *next = io->next;
1007 
1008 		dm_io_rewind(io, &md->disk->bio_split);
1009 
1010 		io->next = NULL;
1011 		__dm_io_complete(io, false);
1012 		io = next;
1013 		cond_resched();
1014 	}
1015 }
1016 
1017 /*
1018  * Two staged requeue:
1019  *
1020  * 1) io->orig_bio points to the real original bio, and the part mapped to
1021  *    this io must be requeued, instead of other parts of the original bio.
1022  *
1023  * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io.
1024  */
dm_io_complete(struct dm_io * io)1025 static void dm_io_complete(struct dm_io *io)
1026 {
1027 	bool first_requeue;
1028 
1029 	/*
1030 	 * Only dm_io that has been split needs two stage requeue, otherwise
1031 	 * we may run into long bio clone chain during suspend and OOM could
1032 	 * be triggered.
1033 	 *
1034 	 * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they
1035 	 * also aren't handled via the first stage requeue.
1036 	 */
1037 	if (dm_io_flagged(io, DM_IO_WAS_SPLIT))
1038 		first_requeue = true;
1039 	else
1040 		first_requeue = false;
1041 
1042 	__dm_io_complete(io, first_requeue);
1043 }
1044 
1045 /*
1046  * Decrements the number of outstanding ios that a bio has been
1047  * cloned into, completing the original io if necc.
1048  */
__dm_io_dec_pending(struct dm_io * io)1049 static inline void __dm_io_dec_pending(struct dm_io *io)
1050 {
1051 	if (atomic_dec_and_test(&io->io_count))
1052 		dm_io_complete(io);
1053 }
1054 
dm_io_set_error(struct dm_io * io,blk_status_t error)1055 static void dm_io_set_error(struct dm_io *io, blk_status_t error)
1056 {
1057 	unsigned long flags;
1058 
1059 	/* Push-back supersedes any I/O errors */
1060 	spin_lock_irqsave(&io->lock, flags);
1061 	if (!(io->status == BLK_STS_DM_REQUEUE &&
1062 	      __noflush_suspending(io->md))) {
1063 		io->status = error;
1064 	}
1065 	spin_unlock_irqrestore(&io->lock, flags);
1066 }
1067 
dm_io_dec_pending(struct dm_io * io,blk_status_t error)1068 static void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
1069 {
1070 	if (unlikely(error))
1071 		dm_io_set_error(io, error);
1072 
1073 	__dm_io_dec_pending(io);
1074 }
1075 
1076 /*
1077  * The queue_limits are only valid as long as you have a reference
1078  * count on 'md'. But _not_ imposing verification to avoid atomic_read(),
1079  */
dm_get_queue_limits(struct mapped_device * md)1080 static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1081 {
1082 	return &md->queue->limits;
1083 }
1084 
disable_discard(struct mapped_device * md)1085 void disable_discard(struct mapped_device *md)
1086 {
1087 	struct queue_limits *limits = dm_get_queue_limits(md);
1088 
1089 	/* device doesn't really support DISCARD, disable it */
1090 	limits->max_hw_discard_sectors = 0;
1091 }
1092 
disable_write_zeroes(struct mapped_device * md)1093 void disable_write_zeroes(struct mapped_device *md)
1094 {
1095 	struct queue_limits *limits = dm_get_queue_limits(md);
1096 
1097 	/* device doesn't really support WRITE ZEROES, disable it */
1098 	limits->max_write_zeroes_sectors = 0;
1099 }
1100 
swap_bios_limit(struct dm_target * ti,struct bio * bio)1101 static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
1102 {
1103 	return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
1104 }
1105 
clone_endio(struct bio * bio)1106 static void clone_endio(struct bio *bio)
1107 {
1108 	blk_status_t error = bio->bi_status;
1109 	struct dm_target_io *tio = clone_to_tio(bio);
1110 	struct dm_target *ti = tio->ti;
1111 	dm_endio_fn endio = likely(ti != NULL) ? ti->type->end_io : NULL;
1112 	struct dm_io *io = tio->io;
1113 	struct mapped_device *md = io->md;
1114 
1115 	if (unlikely(error == BLK_STS_TARGET)) {
1116 		if (bio_op(bio) == REQ_OP_DISCARD &&
1117 		    !bdev_max_discard_sectors(bio->bi_bdev))
1118 			disable_discard(md);
1119 		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
1120 			 !bdev_write_zeroes_sectors(bio->bi_bdev))
1121 			disable_write_zeroes(md);
1122 	}
1123 
1124 	if (static_branch_unlikely(&zoned_enabled) &&
1125 	    unlikely(bdev_is_zoned(bio->bi_bdev)))
1126 		dm_zone_endio(io, bio);
1127 
1128 	if (endio) {
1129 		int r = endio(ti, bio, &error);
1130 
1131 		switch (r) {
1132 		case DM_ENDIO_REQUEUE:
1133 			if (static_branch_unlikely(&zoned_enabled)) {
1134 				/*
1135 				 * Requeuing writes to a sequential zone of a zoned
1136 				 * target will break the sequential write pattern:
1137 				 * fail such IO.
1138 				 */
1139 				if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
1140 					error = BLK_STS_IOERR;
1141 				else
1142 					error = BLK_STS_DM_REQUEUE;
1143 			} else
1144 				error = BLK_STS_DM_REQUEUE;
1145 			fallthrough;
1146 		case DM_ENDIO_DONE:
1147 			break;
1148 		case DM_ENDIO_INCOMPLETE:
1149 			/* The target will handle the io */
1150 			return;
1151 		default:
1152 			DMCRIT("unimplemented target endio return value: %d", r);
1153 			BUG();
1154 		}
1155 	}
1156 
1157 	if (static_branch_unlikely(&swap_bios_enabled) &&
1158 	    likely(ti != NULL) && unlikely(swap_bios_limit(ti, bio)))
1159 		up(&md->swap_bios_semaphore);
1160 
1161 	free_tio(bio);
1162 	dm_io_dec_pending(io, error);
1163 }
1164 
1165 /*
1166  * Return maximum size of I/O possible at the supplied sector up to the current
1167  * target boundary.
1168  */
max_io_len_target_boundary(struct dm_target * ti,sector_t target_offset)1169 static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
1170 						  sector_t target_offset)
1171 {
1172 	return ti->len - target_offset;
1173 }
1174 
__max_io_len(struct dm_target * ti,sector_t sector,unsigned int max_granularity,unsigned int max_sectors)1175 static sector_t __max_io_len(struct dm_target *ti, sector_t sector,
1176 			     unsigned int max_granularity,
1177 			     unsigned int max_sectors)
1178 {
1179 	sector_t target_offset = dm_target_offset(ti, sector);
1180 	sector_t len = max_io_len_target_boundary(ti, target_offset);
1181 
1182 	/*
1183 	 * Does the target need to split IO even further?
1184 	 * - varied (per target) IO splitting is a tenet of DM; this
1185 	 *   explains why stacked chunk_sectors based splitting via
1186 	 *   bio_split_to_limits() isn't possible here.
1187 	 */
1188 	if (!max_granularity)
1189 		return len;
1190 	return min_t(sector_t, len,
1191 		min(max_sectors ? : queue_max_sectors(ti->table->md->queue),
1192 		    blk_boundary_sectors_left(target_offset, max_granularity)));
1193 }
1194 
max_io_len(struct dm_target * ti,sector_t sector)1195 static inline sector_t max_io_len(struct dm_target *ti, sector_t sector)
1196 {
1197 	return __max_io_len(ti, sector, ti->max_io_len, 0);
1198 }
1199 
dm_set_target_max_io_len(struct dm_target * ti,sector_t len)1200 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1201 {
1202 	if (len > UINT_MAX) {
1203 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1204 		      (unsigned long long)len, UINT_MAX);
1205 		ti->error = "Maximum size of target IO is too large";
1206 		return -EINVAL;
1207 	}
1208 
1209 	ti->max_io_len = (uint32_t) len;
1210 
1211 	return 0;
1212 }
1213 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1214 
dm_dax_get_live_target(struct mapped_device * md,sector_t sector,int * srcu_idx)1215 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1216 						sector_t sector, int *srcu_idx)
1217 	__acquires(md->io_barrier)
1218 {
1219 	struct dm_table *map;
1220 	struct dm_target *ti;
1221 
1222 	map = dm_get_live_table(md, srcu_idx);
1223 	if (!map)
1224 		return NULL;
1225 
1226 	ti = dm_table_find_target(map, sector);
1227 	if (!ti)
1228 		return NULL;
1229 
1230 	return ti;
1231 }
1232 
dm_dax_direct_access(struct dax_device * dax_dev,pgoff_t pgoff,long nr_pages,enum dax_access_mode mode,void ** kaddr,pfn_t * pfn)1233 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1234 		long nr_pages, enum dax_access_mode mode, void **kaddr,
1235 		pfn_t *pfn)
1236 {
1237 	struct mapped_device *md = dax_get_private(dax_dev);
1238 	sector_t sector = pgoff * PAGE_SECTORS;
1239 	struct dm_target *ti;
1240 	long len, ret = -EIO;
1241 	int srcu_idx;
1242 
1243 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1244 
1245 	if (!ti)
1246 		goto out;
1247 	if (!ti->type->direct_access)
1248 		goto out;
1249 	len = max_io_len(ti, sector) / PAGE_SECTORS;
1250 	if (len < 1)
1251 		goto out;
1252 	nr_pages = min(len, nr_pages);
1253 	ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn);
1254 
1255  out:
1256 	dm_put_live_table(md, srcu_idx);
1257 
1258 	return ret;
1259 }
1260 
dm_dax_zero_page_range(struct dax_device * dax_dev,pgoff_t pgoff,size_t nr_pages)1261 static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1262 				  size_t nr_pages)
1263 {
1264 	struct mapped_device *md = dax_get_private(dax_dev);
1265 	sector_t sector = pgoff * PAGE_SECTORS;
1266 	struct dm_target *ti;
1267 	int ret = -EIO;
1268 	int srcu_idx;
1269 
1270 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1271 
1272 	if (!ti)
1273 		goto out;
1274 	if (WARN_ON(!ti->type->dax_zero_page_range)) {
1275 		/*
1276 		 * ->zero_page_range() is mandatory dax operation. If we are
1277 		 *  here, something is wrong.
1278 		 */
1279 		goto out;
1280 	}
1281 	ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1282  out:
1283 	dm_put_live_table(md, srcu_idx);
1284 
1285 	return ret;
1286 }
1287 
dm_dax_recovery_write(struct dax_device * dax_dev,pgoff_t pgoff,void * addr,size_t bytes,struct iov_iter * i)1288 static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
1289 		void *addr, size_t bytes, struct iov_iter *i)
1290 {
1291 	struct mapped_device *md = dax_get_private(dax_dev);
1292 	sector_t sector = pgoff * PAGE_SECTORS;
1293 	struct dm_target *ti;
1294 	int srcu_idx;
1295 	long ret = 0;
1296 
1297 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1298 	if (!ti || !ti->type->dax_recovery_write)
1299 		goto out;
1300 
1301 	ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i);
1302 out:
1303 	dm_put_live_table(md, srcu_idx);
1304 	return ret;
1305 }
1306 
1307 /*
1308  * A target may call dm_accept_partial_bio only from the map routine.  It is
1309  * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
1310  * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by
1311  * __send_duplicate_bios().
1312  *
1313  * dm_accept_partial_bio informs the dm that the target only wants to process
1314  * additional n_sectors sectors of the bio and the rest of the data should be
1315  * sent in a next bio.
1316  *
1317  * A diagram that explains the arithmetics:
1318  * +--------------------+---------------+-------+
1319  * |         1          |       2       |   3   |
1320  * +--------------------+---------------+-------+
1321  *
1322  * <-------------- *tio->len_ptr --------------->
1323  *                      <----- bio_sectors ----->
1324  *                      <-- n_sectors -->
1325  *
1326  * Region 1 was already iterated over with bio_advance or similar function.
1327  *	(it may be empty if the target doesn't use bio_advance)
1328  * Region 2 is the remaining bio size that the target wants to process.
1329  *	(it may be empty if region 1 is non-empty, although there is no reason
1330  *	 to make it empty)
1331  * The target requires that region 3 is to be sent in the next bio.
1332  *
1333  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1334  * the partially processed part (the sum of regions 1+2) must be the same for all
1335  * copies of the bio.
1336  */
dm_accept_partial_bio(struct bio * bio,unsigned int n_sectors)1337 void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors)
1338 {
1339 	struct dm_target_io *tio = clone_to_tio(bio);
1340 	struct dm_io *io = tio->io;
1341 	unsigned int bio_sectors = bio_sectors(bio);
1342 
1343 	BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
1344 	BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1345 	BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1346 	BUG_ON(bio_sectors > *tio->len_ptr);
1347 	BUG_ON(n_sectors > bio_sectors);
1348 
1349 	*tio->len_ptr -= bio_sectors - n_sectors;
1350 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1351 
1352 	/*
1353 	 * __split_and_process_bio() may have already saved mapped part
1354 	 * for accounting but it is being reduced so update accordingly.
1355 	 */
1356 	dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1357 	io->sectors = n_sectors;
1358 	io->sector_offset = bio_sectors(io->orig_bio);
1359 }
1360 EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1361 
1362 /*
1363  * @clone: clone bio that DM core passed to target's .map function
1364  * @tgt_clone: clone of @clone bio that target needs submitted
1365  *
1366  * Targets should use this interface to submit bios they take
1367  * ownership of when returning DM_MAPIO_SUBMITTED.
1368  *
1369  * Target should also enable ti->accounts_remapped_io
1370  */
dm_submit_bio_remap(struct bio * clone,struct bio * tgt_clone)1371 void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone)
1372 {
1373 	struct dm_target_io *tio = clone_to_tio(clone);
1374 	struct dm_io *io = tio->io;
1375 
1376 	/* establish bio that will get submitted */
1377 	if (!tgt_clone)
1378 		tgt_clone = clone;
1379 
1380 	/*
1381 	 * Account io->origin_bio to DM dev on behalf of target
1382 	 * that took ownership of IO with DM_MAPIO_SUBMITTED.
1383 	 */
1384 	dm_start_io_acct(io, clone);
1385 
1386 	trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk),
1387 			      tio->old_sector);
1388 	submit_bio_noacct(tgt_clone);
1389 }
1390 EXPORT_SYMBOL_GPL(dm_submit_bio_remap);
1391 
__set_swap_bios_limit(struct mapped_device * md,int latch)1392 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1393 {
1394 	mutex_lock(&md->swap_bios_lock);
1395 	while (latch < md->swap_bios) {
1396 		cond_resched();
1397 		down(&md->swap_bios_semaphore);
1398 		md->swap_bios--;
1399 	}
1400 	while (latch > md->swap_bios) {
1401 		cond_resched();
1402 		up(&md->swap_bios_semaphore);
1403 		md->swap_bios++;
1404 	}
1405 	mutex_unlock(&md->swap_bios_lock);
1406 }
1407 
__map_bio(struct bio * clone)1408 static void __map_bio(struct bio *clone)
1409 {
1410 	struct dm_target_io *tio = clone_to_tio(clone);
1411 	struct dm_target *ti = tio->ti;
1412 	struct dm_io *io = tio->io;
1413 	struct mapped_device *md = io->md;
1414 	int r;
1415 
1416 	clone->bi_end_io = clone_endio;
1417 
1418 	/*
1419 	 * Map the clone.
1420 	 */
1421 	tio->old_sector = clone->bi_iter.bi_sector;
1422 
1423 	if (static_branch_unlikely(&swap_bios_enabled) &&
1424 	    unlikely(swap_bios_limit(ti, clone))) {
1425 		int latch = get_swap_bios();
1426 
1427 		if (unlikely(latch != md->swap_bios))
1428 			__set_swap_bios_limit(md, latch);
1429 		down(&md->swap_bios_semaphore);
1430 	}
1431 
1432 	if (likely(ti->type->map == linear_map))
1433 		r = linear_map(ti, clone);
1434 	else if (ti->type->map == stripe_map)
1435 		r = stripe_map(ti, clone);
1436 	else
1437 		r = ti->type->map(ti, clone);
1438 
1439 	switch (r) {
1440 	case DM_MAPIO_SUBMITTED:
1441 		/* target has assumed ownership of this io */
1442 		if (!ti->accounts_remapped_io)
1443 			dm_start_io_acct(io, clone);
1444 		break;
1445 	case DM_MAPIO_REMAPPED:
1446 		dm_submit_bio_remap(clone, NULL);
1447 		break;
1448 	case DM_MAPIO_KILL:
1449 	case DM_MAPIO_REQUEUE:
1450 		if (static_branch_unlikely(&swap_bios_enabled) &&
1451 		    unlikely(swap_bios_limit(ti, clone)))
1452 			up(&md->swap_bios_semaphore);
1453 		free_tio(clone);
1454 		if (r == DM_MAPIO_KILL)
1455 			dm_io_dec_pending(io, BLK_STS_IOERR);
1456 		else
1457 			dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
1458 		break;
1459 	default:
1460 		DMCRIT("unimplemented target map return value: %d", r);
1461 		BUG();
1462 	}
1463 }
1464 
setup_split_accounting(struct clone_info * ci,unsigned int len)1465 static void setup_split_accounting(struct clone_info *ci, unsigned int len)
1466 {
1467 	struct dm_io *io = ci->io;
1468 
1469 	if (ci->sector_count > len) {
1470 		/*
1471 		 * Split needed, save the mapped part for accounting.
1472 		 * NOTE: dm_accept_partial_bio() will update accordingly.
1473 		 */
1474 		dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1475 		io->sectors = len;
1476 		io->sector_offset = bio_sectors(ci->bio);
1477 	}
1478 }
1479 
alloc_multiple_bios(struct bio_list * blist,struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned * len,gfp_t gfp_flag)1480 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1481 				struct dm_target *ti, unsigned int num_bios,
1482 				unsigned *len, gfp_t gfp_flag)
1483 {
1484 	struct bio *bio;
1485 	int try = (gfp_flag & GFP_NOWAIT) ? 0 : 1;
1486 
1487 	for (; try < 2; try++) {
1488 		int bio_nr;
1489 
1490 		if (try && num_bios > 1)
1491 			mutex_lock(&ci->io->md->table_devices_lock);
1492 		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1493 			bio = alloc_tio(ci, ti, bio_nr, len,
1494 					try ? GFP_NOIO : GFP_NOWAIT);
1495 			if (!bio)
1496 				break;
1497 
1498 			bio_list_add(blist, bio);
1499 		}
1500 		if (try && num_bios > 1)
1501 			mutex_unlock(&ci->io->md->table_devices_lock);
1502 		if (bio_nr == num_bios)
1503 			return;
1504 
1505 		while ((bio = bio_list_pop(blist)))
1506 			free_tio(bio);
1507 	}
1508 }
1509 
__send_duplicate_bios(struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned int * len,gfp_t gfp_flag)1510 static unsigned int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1511 					  unsigned int num_bios, unsigned int *len,
1512 					  gfp_t gfp_flag)
1513 {
1514 	struct bio_list blist = BIO_EMPTY_LIST;
1515 	struct bio *clone;
1516 	unsigned int ret = 0;
1517 
1518 	if (WARN_ON_ONCE(num_bios == 0)) /* num_bios = 0 is a bug in caller */
1519 		return 0;
1520 
1521 	/* dm_accept_partial_bio() is not supported with shared tio->len_ptr */
1522 	if (len)
1523 		setup_split_accounting(ci, *len);
1524 
1525 	/*
1526 	 * Using alloc_multiple_bios(), even if num_bios is 1, to consistently
1527 	 * support allocating using GFP_NOWAIT with GFP_NOIO fallback.
1528 	 */
1529 	alloc_multiple_bios(&blist, ci, ti, num_bios, len, gfp_flag);
1530 	while ((clone = bio_list_pop(&blist))) {
1531 		if (num_bios > 1)
1532 			dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
1533 		__map_bio(clone);
1534 		ret += 1;
1535 	}
1536 
1537 	return ret;
1538 }
1539 
__send_empty_flush(struct clone_info * ci)1540 static void __send_empty_flush(struct clone_info *ci)
1541 {
1542 	struct dm_table *t = ci->map;
1543 	struct bio flush_bio;
1544 
1545 	/*
1546 	 * Use an on-stack bio for this, it's safe since we don't
1547 	 * need to reference it after submit. It's just used as
1548 	 * the basis for the clone(s).
1549 	 */
1550 	bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
1551 		 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC);
1552 
1553 	ci->bio = &flush_bio;
1554 	ci->sector_count = 0;
1555 	ci->io->tio.clone.bi_iter.bi_size = 0;
1556 
1557 	if (!t->flush_bypasses_map) {
1558 		for (unsigned int i = 0; i < t->num_targets; i++) {
1559 			unsigned int bios;
1560 			struct dm_target *ti = dm_table_get_target(t, i);
1561 
1562 			if (unlikely(ti->num_flush_bios == 0))
1563 				continue;
1564 
1565 			atomic_add(ti->num_flush_bios, &ci->io->io_count);
1566 			bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios,
1567 						     NULL, GFP_NOWAIT);
1568 			atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count);
1569 		}
1570 	} else {
1571 		/*
1572 		 * Note that there's no need to grab t->devices_lock here
1573 		 * because the targets that support flush optimization don't
1574 		 * modify the list of devices.
1575 		 */
1576 		struct list_head *devices = dm_table_get_devices(t);
1577 		unsigned int len = 0;
1578 		struct dm_dev_internal *dd;
1579 		list_for_each_entry(dd, devices, list) {
1580 			struct bio *clone;
1581 			/*
1582 			 * Note that the structure dm_target_io is not
1583 			 * associated with any target (because the device may be
1584 			 * used by multiple targets), so we set tio->ti = NULL.
1585 			 * We must check for NULL in the I/O processing path, to
1586 			 * avoid NULL pointer dereference.
1587 			 */
1588 			clone = alloc_tio(ci, NULL, 0, &len, GFP_NOIO);
1589 			atomic_add(1, &ci->io->io_count);
1590 			bio_set_dev(clone, dd->dm_dev->bdev);
1591 			clone->bi_end_io = clone_endio;
1592 			dm_submit_bio_remap(clone, NULL);
1593 		}
1594 	}
1595 
1596 	/*
1597 	 * alloc_io() takes one extra reference for submission, so the
1598 	 * reference won't reach 0 without the following subtraction
1599 	 */
1600 	atomic_sub(1, &ci->io->io_count);
1601 
1602 	bio_uninit(ci->bio);
1603 }
1604 
__send_abnormal_io(struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned int max_granularity,unsigned int max_sectors)1605 static void __send_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1606 			       unsigned int num_bios, unsigned int max_granularity,
1607 			       unsigned int max_sectors)
1608 {
1609 	unsigned int len, bios;
1610 
1611 	len = min_t(sector_t, ci->sector_count,
1612 		    __max_io_len(ti, ci->sector, max_granularity, max_sectors));
1613 
1614 	atomic_add(num_bios, &ci->io->io_count);
1615 	bios = __send_duplicate_bios(ci, ti, num_bios, &len, GFP_NOIO);
1616 	/*
1617 	 * alloc_io() takes one extra reference for submission, so the
1618 	 * reference won't reach 0 without the following (+1) subtraction
1619 	 */
1620 	atomic_sub(num_bios - bios + 1, &ci->io->io_count);
1621 
1622 	ci->sector += len;
1623 	ci->sector_count -= len;
1624 }
1625 
is_abnormal_io(struct bio * bio)1626 static bool is_abnormal_io(struct bio *bio)
1627 {
1628 	switch (bio_op(bio)) {
1629 	case REQ_OP_READ:
1630 	case REQ_OP_WRITE:
1631 	case REQ_OP_FLUSH:
1632 		return false;
1633 	case REQ_OP_DISCARD:
1634 	case REQ_OP_SECURE_ERASE:
1635 	case REQ_OP_WRITE_ZEROES:
1636 	case REQ_OP_ZONE_RESET_ALL:
1637 		return true;
1638 	default:
1639 		return false;
1640 	}
1641 }
1642 
__process_abnormal_io(struct clone_info * ci,struct dm_target * ti)1643 static blk_status_t __process_abnormal_io(struct clone_info *ci,
1644 					  struct dm_target *ti)
1645 {
1646 	unsigned int num_bios = 0;
1647 	unsigned int max_granularity = 0;
1648 	unsigned int max_sectors = 0;
1649 	struct queue_limits *limits = dm_get_queue_limits(ti->table->md);
1650 
1651 	switch (bio_op(ci->bio)) {
1652 	case REQ_OP_DISCARD:
1653 		num_bios = ti->num_discard_bios;
1654 		max_sectors = limits->max_discard_sectors;
1655 		if (ti->max_discard_granularity)
1656 			max_granularity = max_sectors;
1657 		break;
1658 	case REQ_OP_SECURE_ERASE:
1659 		num_bios = ti->num_secure_erase_bios;
1660 		max_sectors = limits->max_secure_erase_sectors;
1661 		break;
1662 	case REQ_OP_WRITE_ZEROES:
1663 		num_bios = ti->num_write_zeroes_bios;
1664 		max_sectors = limits->max_write_zeroes_sectors;
1665 		break;
1666 	default:
1667 		break;
1668 	}
1669 
1670 	/*
1671 	 * Even though the device advertised support for this type of
1672 	 * request, that does not mean every target supports it, and
1673 	 * reconfiguration might also have changed that since the
1674 	 * check was performed.
1675 	 */
1676 	if (unlikely(!num_bios))
1677 		return BLK_STS_NOTSUPP;
1678 
1679 	__send_abnormal_io(ci, ti, num_bios, max_granularity, max_sectors);
1680 
1681 	return BLK_STS_OK;
1682 }
1683 
1684 /*
1685  * Reuse ->bi_private as dm_io list head for storing all dm_io instances
1686  * associated with this bio, and this bio's bi_private needs to be
1687  * stored in dm_io->data before the reuse.
1688  *
1689  * bio->bi_private is owned by fs or upper layer, so block layer won't
1690  * touch it after splitting. Meantime it won't be changed by anyone after
1691  * bio is submitted. So this reuse is safe.
1692  */
dm_poll_list_head(struct bio * bio)1693 static inline struct dm_io **dm_poll_list_head(struct bio *bio)
1694 {
1695 	return (struct dm_io **)&bio->bi_private;
1696 }
1697 
dm_queue_poll_io(struct bio * bio,struct dm_io * io)1698 static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
1699 {
1700 	struct dm_io **head = dm_poll_list_head(bio);
1701 
1702 	if (!(bio->bi_opf & REQ_DM_POLL_LIST)) {
1703 		bio->bi_opf |= REQ_DM_POLL_LIST;
1704 		/*
1705 		 * Save .bi_private into dm_io, so that we can reuse
1706 		 * .bi_private as dm_io list head for storing dm_io list
1707 		 */
1708 		io->data = bio->bi_private;
1709 
1710 		/* tell block layer to poll for completion */
1711 		bio->bi_cookie = ~BLK_QC_T_NONE;
1712 
1713 		io->next = NULL;
1714 	} else {
1715 		/*
1716 		 * bio recursed due to split, reuse original poll list,
1717 		 * and save bio->bi_private too.
1718 		 */
1719 		io->data = (*head)->data;
1720 		io->next = *head;
1721 	}
1722 
1723 	*head = io;
1724 }
1725 
1726 /*
1727  * Select the correct strategy for processing a non-flush bio.
1728  */
__split_and_process_bio(struct clone_info * ci)1729 static blk_status_t __split_and_process_bio(struct clone_info *ci)
1730 {
1731 	struct bio *clone;
1732 	struct dm_target *ti;
1733 	unsigned int len;
1734 
1735 	ti = dm_table_find_target(ci->map, ci->sector);
1736 	if (unlikely(!ti))
1737 		return BLK_STS_IOERR;
1738 
1739 	if (unlikely(ci->is_abnormal_io))
1740 		return __process_abnormal_io(ci, ti);
1741 
1742 	/*
1743 	 * Only support bio polling for normal IO, and the target io is
1744 	 * exactly inside the dm_io instance (verified in dm_poll_dm_io)
1745 	 */
1746 	ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED);
1747 
1748 	len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
1749 	setup_split_accounting(ci, len);
1750 
1751 	if (unlikely(ci->bio->bi_opf & REQ_NOWAIT)) {
1752 		if (unlikely(!dm_target_supports_nowait(ti->type)))
1753 			return BLK_STS_NOTSUPP;
1754 
1755 		clone = alloc_tio(ci, ti, 0, &len, GFP_NOWAIT);
1756 		if (unlikely(!clone))
1757 			return BLK_STS_AGAIN;
1758 	} else {
1759 		clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO);
1760 	}
1761 	__map_bio(clone);
1762 
1763 	ci->sector += len;
1764 	ci->sector_count -= len;
1765 
1766 	return BLK_STS_OK;
1767 }
1768 
init_clone_info(struct clone_info * ci,struct dm_io * io,struct dm_table * map,struct bio * bio,bool is_abnormal)1769 static void init_clone_info(struct clone_info *ci, struct dm_io *io,
1770 			    struct dm_table *map, struct bio *bio, bool is_abnormal)
1771 {
1772 	ci->map = map;
1773 	ci->io = io;
1774 	ci->bio = bio;
1775 	ci->is_abnormal_io = is_abnormal;
1776 	ci->submit_as_polled = false;
1777 	ci->sector = bio->bi_iter.bi_sector;
1778 	ci->sector_count = bio_sectors(bio);
1779 
1780 	/* Shouldn't happen but sector_count was being set to 0 so... */
1781 	if (static_branch_unlikely(&zoned_enabled) &&
1782 	    WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count))
1783 		ci->sector_count = 0;
1784 }
1785 
1786 #ifdef CONFIG_BLK_DEV_ZONED
dm_zone_bio_needs_split(struct mapped_device * md,struct bio * bio)1787 static inline bool dm_zone_bio_needs_split(struct mapped_device *md,
1788 					   struct bio *bio)
1789 {
1790 	/*
1791 	 * For mapped device that need zone append emulation, we must
1792 	 * split any large BIO that straddles zone boundaries.
1793 	 */
1794 	return dm_emulate_zone_append(md) && bio_straddles_zones(bio) &&
1795 		!bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING);
1796 }
dm_zone_plug_bio(struct mapped_device * md,struct bio * bio)1797 static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio)
1798 {
1799 	return dm_emulate_zone_append(md) && blk_zone_plug_bio(bio, 0);
1800 }
1801 
__send_zone_reset_all_emulated(struct clone_info * ci,struct dm_target * ti)1802 static blk_status_t __send_zone_reset_all_emulated(struct clone_info *ci,
1803 						   struct dm_target *ti)
1804 {
1805 	struct bio_list blist = BIO_EMPTY_LIST;
1806 	struct mapped_device *md = ci->io->md;
1807 	unsigned int zone_sectors = md->disk->queue->limits.chunk_sectors;
1808 	unsigned long *need_reset;
1809 	unsigned int i, nr_zones, nr_reset;
1810 	unsigned int num_bios = 0;
1811 	blk_status_t sts = BLK_STS_OK;
1812 	sector_t sector = ti->begin;
1813 	struct bio *clone;
1814 	int ret;
1815 
1816 	nr_zones = ti->len >> ilog2(zone_sectors);
1817 	need_reset = bitmap_zalloc(nr_zones, GFP_NOIO);
1818 	if (!need_reset)
1819 		return BLK_STS_RESOURCE;
1820 
1821 	ret = dm_zone_get_reset_bitmap(md, ci->map, ti->begin,
1822 				       nr_zones, need_reset);
1823 	if (ret) {
1824 		sts = BLK_STS_IOERR;
1825 		goto free_bitmap;
1826 	}
1827 
1828 	/* If we have no zone to reset, we are done. */
1829 	nr_reset = bitmap_weight(need_reset, nr_zones);
1830 	if (!nr_reset)
1831 		goto free_bitmap;
1832 
1833 	atomic_add(nr_zones, &ci->io->io_count);
1834 
1835 	for (i = 0; i < nr_zones; i++) {
1836 
1837 		if (!test_bit(i, need_reset)) {
1838 			sector += zone_sectors;
1839 			continue;
1840 		}
1841 
1842 		if (bio_list_empty(&blist)) {
1843 			/* This may take a while, so be nice to others */
1844 			if (num_bios)
1845 				cond_resched();
1846 
1847 			/*
1848 			 * We may need to reset thousands of zones, so let's
1849 			 * not go crazy with the clone allocation.
1850 			 */
1851 			alloc_multiple_bios(&blist, ci, ti, min(nr_reset, 32),
1852 					    NULL, GFP_NOIO);
1853 		}
1854 
1855 		/* Get a clone and change it to a regular reset operation. */
1856 		clone = bio_list_pop(&blist);
1857 		clone->bi_opf &= ~REQ_OP_MASK;
1858 		clone->bi_opf |= REQ_OP_ZONE_RESET | REQ_SYNC;
1859 		clone->bi_iter.bi_sector = sector;
1860 		clone->bi_iter.bi_size = 0;
1861 		__map_bio(clone);
1862 
1863 		sector += zone_sectors;
1864 		num_bios++;
1865 		nr_reset--;
1866 	}
1867 
1868 	WARN_ON_ONCE(!bio_list_empty(&blist));
1869 	atomic_sub(nr_zones - num_bios, &ci->io->io_count);
1870 	ci->sector_count = 0;
1871 
1872 free_bitmap:
1873 	bitmap_free(need_reset);
1874 
1875 	return sts;
1876 }
1877 
__send_zone_reset_all_native(struct clone_info * ci,struct dm_target * ti)1878 static void __send_zone_reset_all_native(struct clone_info *ci,
1879 					 struct dm_target *ti)
1880 {
1881 	unsigned int bios;
1882 
1883 	atomic_add(1, &ci->io->io_count);
1884 	bios = __send_duplicate_bios(ci, ti, 1, NULL, GFP_NOIO);
1885 	atomic_sub(1 - bios, &ci->io->io_count);
1886 
1887 	ci->sector_count = 0;
1888 }
1889 
__send_zone_reset_all(struct clone_info * ci)1890 static blk_status_t __send_zone_reset_all(struct clone_info *ci)
1891 {
1892 	struct dm_table *t = ci->map;
1893 	blk_status_t sts = BLK_STS_OK;
1894 
1895 	for (unsigned int i = 0; i < t->num_targets; i++) {
1896 		struct dm_target *ti = dm_table_get_target(t, i);
1897 
1898 		if (ti->zone_reset_all_supported) {
1899 			__send_zone_reset_all_native(ci, ti);
1900 			continue;
1901 		}
1902 
1903 		sts = __send_zone_reset_all_emulated(ci, ti);
1904 		if (sts != BLK_STS_OK)
1905 			break;
1906 	}
1907 
1908 	/* Release the reference that alloc_io() took for submission. */
1909 	atomic_sub(1, &ci->io->io_count);
1910 
1911 	return sts;
1912 }
1913 
1914 #else
dm_zone_bio_needs_split(struct mapped_device * md,struct bio * bio)1915 static inline bool dm_zone_bio_needs_split(struct mapped_device *md,
1916 					   struct bio *bio)
1917 {
1918 	return false;
1919 }
dm_zone_plug_bio(struct mapped_device * md,struct bio * bio)1920 static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio)
1921 {
1922 	return false;
1923 }
__send_zone_reset_all(struct clone_info * ci)1924 static blk_status_t __send_zone_reset_all(struct clone_info *ci)
1925 {
1926 	return BLK_STS_NOTSUPP;
1927 }
1928 #endif
1929 
1930 /*
1931  * Entry point to split a bio into clones and submit them to the targets.
1932  */
dm_split_and_process_bio(struct mapped_device * md,struct dm_table * map,struct bio * bio)1933 static void dm_split_and_process_bio(struct mapped_device *md,
1934 				     struct dm_table *map, struct bio *bio)
1935 {
1936 	struct clone_info ci;
1937 	struct dm_io *io;
1938 	blk_status_t error = BLK_STS_OK;
1939 	bool is_abnormal, need_split;
1940 
1941 	is_abnormal = is_abnormal_io(bio);
1942 	if (static_branch_unlikely(&zoned_enabled)) {
1943 		/* Special case REQ_OP_ZONE_RESET_ALL as it cannot be split. */
1944 		need_split = (bio_op(bio) != REQ_OP_ZONE_RESET_ALL) &&
1945 			(is_abnormal || dm_zone_bio_needs_split(md, bio));
1946 	} else {
1947 		need_split = is_abnormal;
1948 	}
1949 
1950 	if (unlikely(need_split)) {
1951 		/*
1952 		 * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc)
1953 		 * otherwise associated queue_limits won't be imposed.
1954 		 * Also split the BIO for mapped devices needing zone append
1955 		 * emulation to ensure that the BIO does not cross zone
1956 		 * boundaries.
1957 		 */
1958 		bio = bio_split_to_limits(bio);
1959 		if (!bio)
1960 			return;
1961 	}
1962 
1963 	/*
1964 	 * Use the block layer zone write plugging for mapped devices that
1965 	 * need zone append emulation (e.g. dm-crypt).
1966 	 */
1967 	if (static_branch_unlikely(&zoned_enabled) && dm_zone_plug_bio(md, bio))
1968 		return;
1969 
1970 	/* Only support nowait for normal IO */
1971 	if (unlikely(bio->bi_opf & REQ_NOWAIT) && !is_abnormal) {
1972 		io = alloc_io(md, bio, GFP_NOWAIT);
1973 		if (unlikely(!io)) {
1974 			/* Unable to do anything without dm_io. */
1975 			bio_wouldblock_error(bio);
1976 			return;
1977 		}
1978 	} else {
1979 		io = alloc_io(md, bio, GFP_NOIO);
1980 	}
1981 	init_clone_info(&ci, io, map, bio, is_abnormal);
1982 
1983 	if (bio->bi_opf & REQ_PREFLUSH) {
1984 		__send_empty_flush(&ci);
1985 		/* dm_io_complete submits any data associated with flush */
1986 		goto out;
1987 	}
1988 
1989 	if (static_branch_unlikely(&zoned_enabled) &&
1990 	    (bio_op(bio) == REQ_OP_ZONE_RESET_ALL)) {
1991 		error = __send_zone_reset_all(&ci);
1992 		goto out;
1993 	}
1994 
1995 	error = __split_and_process_bio(&ci);
1996 	if (error || !ci.sector_count)
1997 		goto out;
1998 	/*
1999 	 * Remainder must be passed to submit_bio_noacct() so it gets handled
2000 	 * *after* bios already submitted have been completely processed.
2001 	 */
2002 	bio_trim(bio, io->sectors, ci.sector_count);
2003 	trace_block_split(bio, bio->bi_iter.bi_sector);
2004 	bio_inc_remaining(bio);
2005 	submit_bio_noacct(bio);
2006 out:
2007 	/*
2008 	 * Drop the extra reference count for non-POLLED bio, and hold one
2009 	 * reference for POLLED bio, which will be released in dm_poll_bio
2010 	 *
2011 	 * Add every dm_io instance into the dm_io list head which is stored
2012 	 * in bio->bi_private, so that dm_poll_bio can poll them all.
2013 	 */
2014 	if (error || !ci.submit_as_polled) {
2015 		/*
2016 		 * In case of submission failure, the extra reference for
2017 		 * submitting io isn't consumed yet
2018 		 */
2019 		if (error)
2020 			atomic_dec(&io->io_count);
2021 		dm_io_dec_pending(io, error);
2022 	} else
2023 		dm_queue_poll_io(bio, io);
2024 }
2025 
dm_submit_bio(struct bio * bio)2026 static void dm_submit_bio(struct bio *bio)
2027 {
2028 	struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
2029 	int srcu_idx;
2030 	struct dm_table *map;
2031 
2032 	map = dm_get_live_table(md, &srcu_idx);
2033 
2034 	/* If suspended, or map not yet available, queue this IO for later */
2035 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) ||
2036 	    unlikely(!map)) {
2037 		if (bio->bi_opf & REQ_NOWAIT)
2038 			bio_wouldblock_error(bio);
2039 		else if (bio->bi_opf & REQ_RAHEAD)
2040 			bio_io_error(bio);
2041 		else
2042 			queue_io(md, bio);
2043 		goto out;
2044 	}
2045 
2046 	dm_split_and_process_bio(md, map, bio);
2047 out:
2048 	dm_put_live_table(md, srcu_idx);
2049 }
2050 
dm_poll_dm_io(struct dm_io * io,struct io_comp_batch * iob,unsigned int flags)2051 static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
2052 			  unsigned int flags)
2053 {
2054 	WARN_ON_ONCE(!dm_tio_is_normal(&io->tio));
2055 
2056 	/* don't poll if the mapped io is done */
2057 	if (atomic_read(&io->io_count) > 1)
2058 		bio_poll(&io->tio.clone, iob, flags);
2059 
2060 	/* bio_poll holds the last reference */
2061 	return atomic_read(&io->io_count) == 1;
2062 }
2063 
dm_poll_bio(struct bio * bio,struct io_comp_batch * iob,unsigned int flags)2064 static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
2065 		       unsigned int flags)
2066 {
2067 	struct dm_io **head = dm_poll_list_head(bio);
2068 	struct dm_io *list = *head;
2069 	struct dm_io *tmp = NULL;
2070 	struct dm_io *curr, *next;
2071 
2072 	/* Only poll normal bio which was marked as REQ_DM_POLL_LIST */
2073 	if (!(bio->bi_opf & REQ_DM_POLL_LIST))
2074 		return 0;
2075 
2076 	WARN_ON_ONCE(!list);
2077 
2078 	/*
2079 	 * Restore .bi_private before possibly completing dm_io.
2080 	 *
2081 	 * bio_poll() is only possible once @bio has been completely
2082 	 * submitted via submit_bio_noacct()'s depth-first submission.
2083 	 * So there is no dm_queue_poll_io() race associated with
2084 	 * clearing REQ_DM_POLL_LIST here.
2085 	 */
2086 	bio->bi_opf &= ~REQ_DM_POLL_LIST;
2087 	bio->bi_private = list->data;
2088 
2089 	for (curr = list, next = curr->next; curr; curr = next, next =
2090 			curr ? curr->next : NULL) {
2091 		if (dm_poll_dm_io(curr, iob, flags)) {
2092 			/*
2093 			 * clone_endio() has already occurred, so no
2094 			 * error handling is needed here.
2095 			 */
2096 			__dm_io_dec_pending(curr);
2097 		} else {
2098 			curr->next = tmp;
2099 			tmp = curr;
2100 		}
2101 	}
2102 
2103 	/* Not done? */
2104 	if (tmp) {
2105 		bio->bi_opf |= REQ_DM_POLL_LIST;
2106 		/* Reset bio->bi_private to dm_io list head */
2107 		*head = tmp;
2108 		return 0;
2109 	}
2110 	return 1;
2111 }
2112 
2113 /*
2114  *---------------------------------------------------------------
2115  * An IDR is used to keep track of allocated minor numbers.
2116  *---------------------------------------------------------------
2117  */
free_minor(int minor)2118 static void free_minor(int minor)
2119 {
2120 	spin_lock(&_minor_lock);
2121 	idr_remove(&_minor_idr, minor);
2122 	spin_unlock(&_minor_lock);
2123 }
2124 
2125 /*
2126  * See if the device with a specific minor # is free.
2127  */
specific_minor(int minor)2128 static int specific_minor(int minor)
2129 {
2130 	int r;
2131 
2132 	if (minor >= (1 << MINORBITS))
2133 		return -EINVAL;
2134 
2135 	idr_preload(GFP_KERNEL);
2136 	spin_lock(&_minor_lock);
2137 
2138 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
2139 
2140 	spin_unlock(&_minor_lock);
2141 	idr_preload_end();
2142 	if (r < 0)
2143 		return r == -ENOSPC ? -EBUSY : r;
2144 	return 0;
2145 }
2146 
next_free_minor(int * minor)2147 static int next_free_minor(int *minor)
2148 {
2149 	int r;
2150 
2151 	idr_preload(GFP_KERNEL);
2152 	spin_lock(&_minor_lock);
2153 
2154 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
2155 
2156 	spin_unlock(&_minor_lock);
2157 	idr_preload_end();
2158 	if (r < 0)
2159 		return r;
2160 	*minor = r;
2161 	return 0;
2162 }
2163 
2164 static const struct block_device_operations dm_blk_dops;
2165 static const struct block_device_operations dm_rq_blk_dops;
2166 static const struct dax_operations dm_dax_ops;
2167 
2168 static void dm_wq_work(struct work_struct *work);
2169 
2170 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
dm_queue_destroy_crypto_profile(struct request_queue * q)2171 static void dm_queue_destroy_crypto_profile(struct request_queue *q)
2172 {
2173 	dm_destroy_crypto_profile(q->crypto_profile);
2174 }
2175 
2176 #else /* CONFIG_BLK_INLINE_ENCRYPTION */
2177 
dm_queue_destroy_crypto_profile(struct request_queue * q)2178 static inline void dm_queue_destroy_crypto_profile(struct request_queue *q)
2179 {
2180 }
2181 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
2182 
cleanup_mapped_device(struct mapped_device * md)2183 static void cleanup_mapped_device(struct mapped_device *md)
2184 {
2185 	if (md->wq)
2186 		destroy_workqueue(md->wq);
2187 	dm_free_md_mempools(md->mempools);
2188 
2189 	if (md->dax_dev) {
2190 		dax_remove_host(md->disk);
2191 		kill_dax(md->dax_dev);
2192 		put_dax(md->dax_dev);
2193 		md->dax_dev = NULL;
2194 	}
2195 
2196 	if (md->disk) {
2197 		spin_lock(&_minor_lock);
2198 		md->disk->private_data = NULL;
2199 		spin_unlock(&_minor_lock);
2200 		if (dm_get_md_type(md) != DM_TYPE_NONE) {
2201 			struct table_device *td;
2202 
2203 			dm_sysfs_exit(md);
2204 			list_for_each_entry(td, &md->table_devices, list) {
2205 				bd_unlink_disk_holder(td->dm_dev.bdev,
2206 						      md->disk);
2207 			}
2208 
2209 			/*
2210 			 * Hold lock to make sure del_gendisk() won't concurrent
2211 			 * with open/close_table_device().
2212 			 */
2213 			mutex_lock(&md->table_devices_lock);
2214 			del_gendisk(md->disk);
2215 			mutex_unlock(&md->table_devices_lock);
2216 		}
2217 		dm_queue_destroy_crypto_profile(md->queue);
2218 		put_disk(md->disk);
2219 	}
2220 
2221 	if (md->pending_io) {
2222 		free_percpu(md->pending_io);
2223 		md->pending_io = NULL;
2224 	}
2225 
2226 	cleanup_srcu_struct(&md->io_barrier);
2227 
2228 	mutex_destroy(&md->suspend_lock);
2229 	mutex_destroy(&md->type_lock);
2230 	mutex_destroy(&md->table_devices_lock);
2231 	mutex_destroy(&md->swap_bios_lock);
2232 
2233 	dm_mq_cleanup_mapped_device(md);
2234 }
2235 
2236 /*
2237  * Allocate and initialise a blank device with a given minor.
2238  */
alloc_dev(int minor)2239 static struct mapped_device *alloc_dev(int minor)
2240 {
2241 	int r, numa_node_id = dm_get_numa_node();
2242 	struct dax_device *dax_dev;
2243 	struct mapped_device *md;
2244 	void *old_md;
2245 
2246 	md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
2247 	if (!md) {
2248 		DMERR("unable to allocate device, out of memory.");
2249 		return NULL;
2250 	}
2251 
2252 	if (!try_module_get(THIS_MODULE))
2253 		goto bad_module_get;
2254 
2255 	/* get a minor number for the dev */
2256 	if (minor == DM_ANY_MINOR)
2257 		r = next_free_minor(&minor);
2258 	else
2259 		r = specific_minor(minor);
2260 	if (r < 0)
2261 		goto bad_minor;
2262 
2263 	r = init_srcu_struct(&md->io_barrier);
2264 	if (r < 0)
2265 		goto bad_io_barrier;
2266 
2267 	md->numa_node_id = numa_node_id;
2268 	md->init_tio_pdu = false;
2269 	md->type = DM_TYPE_NONE;
2270 	mutex_init(&md->suspend_lock);
2271 	mutex_init(&md->type_lock);
2272 	mutex_init(&md->table_devices_lock);
2273 	spin_lock_init(&md->deferred_lock);
2274 	atomic_set(&md->holders, 1);
2275 	atomic_set(&md->open_count, 0);
2276 	atomic_set(&md->event_nr, 0);
2277 	atomic_set(&md->uevent_seq, 0);
2278 	INIT_LIST_HEAD(&md->uevent_list);
2279 	INIT_LIST_HEAD(&md->table_devices);
2280 	spin_lock_init(&md->uevent_lock);
2281 
2282 	/*
2283 	 * default to bio-based until DM table is loaded and md->type
2284 	 * established. If request-based table is loaded: blk-mq will
2285 	 * override accordingly.
2286 	 */
2287 	md->disk = blk_alloc_disk(NULL, md->numa_node_id);
2288 	if (IS_ERR(md->disk))
2289 		goto bad;
2290 	md->queue = md->disk->queue;
2291 
2292 	init_waitqueue_head(&md->wait);
2293 	INIT_WORK(&md->work, dm_wq_work);
2294 	INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
2295 	init_waitqueue_head(&md->eventq);
2296 	init_completion(&md->kobj_holder.completion);
2297 
2298 	md->requeue_list = NULL;
2299 	md->swap_bios = get_swap_bios();
2300 	sema_init(&md->swap_bios_semaphore, md->swap_bios);
2301 	mutex_init(&md->swap_bios_lock);
2302 
2303 	md->disk->major = _major;
2304 	md->disk->first_minor = minor;
2305 	md->disk->minors = 1;
2306 	md->disk->flags |= GENHD_FL_NO_PART;
2307 	md->disk->fops = &dm_blk_dops;
2308 	md->disk->private_data = md;
2309 	sprintf(md->disk->disk_name, "dm-%d", minor);
2310 
2311 	dax_dev = alloc_dax(md, &dm_dax_ops);
2312 	if (IS_ERR(dax_dev)) {
2313 		if (PTR_ERR(dax_dev) != -EOPNOTSUPP)
2314 			goto bad;
2315 	} else {
2316 		set_dax_nocache(dax_dev);
2317 		set_dax_nomc(dax_dev);
2318 		md->dax_dev = dax_dev;
2319 		if (dax_add_host(dax_dev, md->disk))
2320 			goto bad;
2321 	}
2322 
2323 	format_dev_t(md->name, MKDEV(_major, minor));
2324 
2325 	md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
2326 	if (!md->wq)
2327 		goto bad;
2328 
2329 	md->pending_io = alloc_percpu(unsigned long);
2330 	if (!md->pending_io)
2331 		goto bad;
2332 
2333 	r = dm_stats_init(&md->stats);
2334 	if (r < 0)
2335 		goto bad;
2336 
2337 	/* Populate the mapping, nobody knows we exist yet */
2338 	spin_lock(&_minor_lock);
2339 	old_md = idr_replace(&_minor_idr, md, minor);
2340 	spin_unlock(&_minor_lock);
2341 
2342 	BUG_ON(old_md != MINOR_ALLOCED);
2343 
2344 	return md;
2345 
2346 bad:
2347 	cleanup_mapped_device(md);
2348 bad_io_barrier:
2349 	free_minor(minor);
2350 bad_minor:
2351 	module_put(THIS_MODULE);
2352 bad_module_get:
2353 	kvfree(md);
2354 	return NULL;
2355 }
2356 
2357 static void unlock_fs(struct mapped_device *md);
2358 
free_dev(struct mapped_device * md)2359 static void free_dev(struct mapped_device *md)
2360 {
2361 	int minor = MINOR(disk_devt(md->disk));
2362 
2363 	unlock_fs(md);
2364 
2365 	cleanup_mapped_device(md);
2366 
2367 	WARN_ON_ONCE(!list_empty(&md->table_devices));
2368 	dm_stats_cleanup(&md->stats);
2369 	free_minor(minor);
2370 
2371 	module_put(THIS_MODULE);
2372 	kvfree(md);
2373 }
2374 
2375 /*
2376  * Bind a table to the device.
2377  */
event_callback(void * context)2378 static void event_callback(void *context)
2379 {
2380 	unsigned long flags;
2381 	LIST_HEAD(uevents);
2382 	struct mapped_device *md = context;
2383 
2384 	spin_lock_irqsave(&md->uevent_lock, flags);
2385 	list_splice_init(&md->uevent_list, &uevents);
2386 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2387 
2388 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2389 
2390 	atomic_inc(&md->event_nr);
2391 	wake_up(&md->eventq);
2392 	dm_issue_global_event();
2393 }
2394 
2395 /*
2396  * Returns old map, which caller must destroy.
2397  */
__bind(struct mapped_device * md,struct dm_table * t,struct queue_limits * limits)2398 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2399 			       struct queue_limits *limits)
2400 {
2401 	struct dm_table *old_map;
2402 	sector_t size;
2403 	int ret;
2404 
2405 	lockdep_assert_held(&md->suspend_lock);
2406 
2407 	size = dm_table_get_size(t);
2408 
2409 	/*
2410 	 * Wipe any geometry if the size of the table changed.
2411 	 */
2412 	if (size != dm_get_size(md))
2413 		memset(&md->geometry, 0, sizeof(md->geometry));
2414 
2415 	set_capacity(md->disk, size);
2416 
2417 	dm_table_event_callback(t, event_callback, md);
2418 
2419 	if (dm_table_request_based(t)) {
2420 		/*
2421 		 * Leverage the fact that request-based DM targets are
2422 		 * immutable singletons - used to optimize dm_mq_queue_rq.
2423 		 */
2424 		md->immutable_target = dm_table_get_immutable_target(t);
2425 
2426 		/*
2427 		 * There is no need to reload with request-based dm because the
2428 		 * size of front_pad doesn't change.
2429 		 *
2430 		 * Note for future: If you are to reload bioset, prep-ed
2431 		 * requests in the queue may refer to bio from the old bioset,
2432 		 * so you must walk through the queue to unprep.
2433 		 */
2434 		if (!md->mempools) {
2435 			md->mempools = t->mempools;
2436 			t->mempools = NULL;
2437 		}
2438 	} else {
2439 		/*
2440 		 * The md may already have mempools that need changing.
2441 		 * If so, reload bioset because front_pad may have changed
2442 		 * because a different table was loaded.
2443 		 */
2444 		dm_free_md_mempools(md->mempools);
2445 		md->mempools = t->mempools;
2446 		t->mempools = NULL;
2447 	}
2448 
2449 	ret = dm_table_set_restrictions(t, md->queue, limits);
2450 	if (ret) {
2451 		old_map = ERR_PTR(ret);
2452 		goto out;
2453 	}
2454 
2455 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2456 	rcu_assign_pointer(md->map, (void *)t);
2457 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
2458 
2459 	if (old_map)
2460 		dm_sync_table(md);
2461 out:
2462 	return old_map;
2463 }
2464 
2465 /*
2466  * Returns unbound table for the caller to free.
2467  */
__unbind(struct mapped_device * md)2468 static struct dm_table *__unbind(struct mapped_device *md)
2469 {
2470 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
2471 
2472 	if (!map)
2473 		return NULL;
2474 
2475 	dm_table_event_callback(map, NULL, NULL);
2476 	RCU_INIT_POINTER(md->map, NULL);
2477 	dm_sync_table(md);
2478 
2479 	return map;
2480 }
2481 
2482 /*
2483  * Constructor for a new device.
2484  */
dm_create(int minor,struct mapped_device ** result)2485 int dm_create(int minor, struct mapped_device **result)
2486 {
2487 	struct mapped_device *md;
2488 
2489 	md = alloc_dev(minor);
2490 	if (!md)
2491 		return -ENXIO;
2492 
2493 	dm_ima_reset_data(md);
2494 
2495 	*result = md;
2496 	return 0;
2497 }
2498 
2499 /*
2500  * Functions to manage md->type.
2501  * All are required to hold md->type_lock.
2502  */
dm_lock_md_type(struct mapped_device * md)2503 void dm_lock_md_type(struct mapped_device *md)
2504 {
2505 	mutex_lock(&md->type_lock);
2506 }
2507 
dm_unlock_md_type(struct mapped_device * md)2508 void dm_unlock_md_type(struct mapped_device *md)
2509 {
2510 	mutex_unlock(&md->type_lock);
2511 }
2512 
dm_set_md_type(struct mapped_device * md,enum dm_queue_mode type)2513 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2514 {
2515 	BUG_ON(!mutex_is_locked(&md->type_lock));
2516 	md->type = type;
2517 }
2518 
dm_get_md_type(struct mapped_device * md)2519 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2520 {
2521 	return md->type;
2522 }
2523 
dm_get_immutable_target_type(struct mapped_device * md)2524 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2525 {
2526 	return md->immutable_target_type;
2527 }
2528 
2529 /*
2530  * Setup the DM device's queue based on md's type
2531  */
dm_setup_md_queue(struct mapped_device * md,struct dm_table * t)2532 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2533 {
2534 	enum dm_queue_mode type = dm_table_get_type(t);
2535 	struct queue_limits limits;
2536 	struct table_device *td;
2537 	int r;
2538 
2539 	WARN_ON_ONCE(type == DM_TYPE_NONE);
2540 
2541 	if (type == DM_TYPE_REQUEST_BASED) {
2542 		md->disk->fops = &dm_rq_blk_dops;
2543 		r = dm_mq_init_request_queue(md, t);
2544 		if (r) {
2545 			DMERR("Cannot initialize queue for request-based dm mapped device");
2546 			return r;
2547 		}
2548 	}
2549 
2550 	r = dm_calculate_queue_limits(t, &limits);
2551 	if (r) {
2552 		DMERR("Cannot calculate initial queue limits");
2553 		return r;
2554 	}
2555 	r = dm_table_set_restrictions(t, md->queue, &limits);
2556 	if (r)
2557 		return r;
2558 
2559 	/*
2560 	 * Hold lock to make sure add_disk() and del_gendisk() won't concurrent
2561 	 * with open_table_device() and close_table_device().
2562 	 */
2563 	mutex_lock(&md->table_devices_lock);
2564 	r = add_disk(md->disk);
2565 	mutex_unlock(&md->table_devices_lock);
2566 	if (r)
2567 		return r;
2568 
2569 	/*
2570 	 * Register the holder relationship for devices added before the disk
2571 	 * was live.
2572 	 */
2573 	list_for_each_entry(td, &md->table_devices, list) {
2574 		r = bd_link_disk_holder(td->dm_dev.bdev, md->disk);
2575 		if (r)
2576 			goto out_undo_holders;
2577 	}
2578 
2579 	r = dm_sysfs_init(md);
2580 	if (r)
2581 		goto out_undo_holders;
2582 
2583 	md->type = type;
2584 	return 0;
2585 
2586 out_undo_holders:
2587 	list_for_each_entry_continue_reverse(td, &md->table_devices, list)
2588 		bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
2589 	mutex_lock(&md->table_devices_lock);
2590 	del_gendisk(md->disk);
2591 	mutex_unlock(&md->table_devices_lock);
2592 	return r;
2593 }
2594 
dm_get_md(dev_t dev)2595 struct mapped_device *dm_get_md(dev_t dev)
2596 {
2597 	struct mapped_device *md;
2598 	unsigned int minor = MINOR(dev);
2599 
2600 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2601 		return NULL;
2602 
2603 	spin_lock(&_minor_lock);
2604 
2605 	md = idr_find(&_minor_idr, minor);
2606 	if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2607 	    test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2608 		md = NULL;
2609 		goto out;
2610 	}
2611 	dm_get(md);
2612 out:
2613 	spin_unlock(&_minor_lock);
2614 
2615 	return md;
2616 }
2617 EXPORT_SYMBOL_GPL(dm_get_md);
2618 
dm_get_mdptr(struct mapped_device * md)2619 void *dm_get_mdptr(struct mapped_device *md)
2620 {
2621 	return md->interface_ptr;
2622 }
2623 
dm_set_mdptr(struct mapped_device * md,void * ptr)2624 void dm_set_mdptr(struct mapped_device *md, void *ptr)
2625 {
2626 	md->interface_ptr = ptr;
2627 }
2628 
dm_get(struct mapped_device * md)2629 void dm_get(struct mapped_device *md)
2630 {
2631 	atomic_inc(&md->holders);
2632 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
2633 }
2634 
dm_hold(struct mapped_device * md)2635 int dm_hold(struct mapped_device *md)
2636 {
2637 	spin_lock(&_minor_lock);
2638 	if (test_bit(DMF_FREEING, &md->flags)) {
2639 		spin_unlock(&_minor_lock);
2640 		return -EBUSY;
2641 	}
2642 	dm_get(md);
2643 	spin_unlock(&_minor_lock);
2644 	return 0;
2645 }
2646 EXPORT_SYMBOL_GPL(dm_hold);
2647 
dm_device_name(struct mapped_device * md)2648 const char *dm_device_name(struct mapped_device *md)
2649 {
2650 	return md->name;
2651 }
2652 EXPORT_SYMBOL_GPL(dm_device_name);
2653 
__dm_destroy(struct mapped_device * md,bool wait)2654 static void __dm_destroy(struct mapped_device *md, bool wait)
2655 {
2656 	struct dm_table *map;
2657 	int srcu_idx;
2658 
2659 	might_sleep();
2660 
2661 	spin_lock(&_minor_lock);
2662 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2663 	set_bit(DMF_FREEING, &md->flags);
2664 	spin_unlock(&_minor_lock);
2665 
2666 	blk_mark_disk_dead(md->disk);
2667 
2668 	/*
2669 	 * Take suspend_lock so that presuspend and postsuspend methods
2670 	 * do not race with internal suspend.
2671 	 */
2672 	mutex_lock(&md->suspend_lock);
2673 	map = dm_get_live_table(md, &srcu_idx);
2674 	if (!dm_suspended_md(md)) {
2675 		dm_table_presuspend_targets(map);
2676 		set_bit(DMF_SUSPENDED, &md->flags);
2677 		set_bit(DMF_POST_SUSPENDING, &md->flags);
2678 		dm_table_postsuspend_targets(map);
2679 	}
2680 	/* dm_put_live_table must be before fsleep, otherwise deadlock is possible */
2681 	dm_put_live_table(md, srcu_idx);
2682 	mutex_unlock(&md->suspend_lock);
2683 
2684 	/*
2685 	 * Rare, but there may be I/O requests still going to complete,
2686 	 * for example.  Wait for all references to disappear.
2687 	 * No one should increment the reference count of the mapped_device,
2688 	 * after the mapped_device state becomes DMF_FREEING.
2689 	 */
2690 	if (wait)
2691 		while (atomic_read(&md->holders))
2692 			fsleep(1000);
2693 	else if (atomic_read(&md->holders))
2694 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2695 		       dm_device_name(md), atomic_read(&md->holders));
2696 
2697 	dm_table_destroy(__unbind(md));
2698 	free_dev(md);
2699 }
2700 
dm_destroy(struct mapped_device * md)2701 void dm_destroy(struct mapped_device *md)
2702 {
2703 	__dm_destroy(md, true);
2704 }
2705 
dm_destroy_immediate(struct mapped_device * md)2706 void dm_destroy_immediate(struct mapped_device *md)
2707 {
2708 	__dm_destroy(md, false);
2709 }
2710 
dm_put(struct mapped_device * md)2711 void dm_put(struct mapped_device *md)
2712 {
2713 	atomic_dec(&md->holders);
2714 }
2715 EXPORT_SYMBOL_GPL(dm_put);
2716 
dm_in_flight_bios(struct mapped_device * md)2717 static bool dm_in_flight_bios(struct mapped_device *md)
2718 {
2719 	int cpu;
2720 	unsigned long sum = 0;
2721 
2722 	for_each_possible_cpu(cpu)
2723 		sum += *per_cpu_ptr(md->pending_io, cpu);
2724 
2725 	return sum != 0;
2726 }
2727 
dm_wait_for_bios_completion(struct mapped_device * md,unsigned int task_state)2728 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
2729 {
2730 	int r = 0;
2731 	DEFINE_WAIT(wait);
2732 
2733 	while (true) {
2734 		prepare_to_wait(&md->wait, &wait, task_state);
2735 
2736 		if (!dm_in_flight_bios(md))
2737 			break;
2738 
2739 		if (signal_pending_state(task_state, current)) {
2740 			r = -ERESTARTSYS;
2741 			break;
2742 		}
2743 
2744 		io_schedule();
2745 	}
2746 	finish_wait(&md->wait, &wait);
2747 
2748 	smp_rmb();
2749 
2750 	return r;
2751 }
2752 
dm_wait_for_completion(struct mapped_device * md,unsigned int task_state)2753 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
2754 {
2755 	int r = 0;
2756 
2757 	if (!queue_is_mq(md->queue))
2758 		return dm_wait_for_bios_completion(md, task_state);
2759 
2760 	while (true) {
2761 		if (!blk_mq_queue_inflight(md->queue))
2762 			break;
2763 
2764 		if (signal_pending_state(task_state, current)) {
2765 			r = -ERESTARTSYS;
2766 			break;
2767 		}
2768 
2769 		fsleep(5000);
2770 	}
2771 
2772 	return r;
2773 }
2774 
2775 /*
2776  * Process the deferred bios
2777  */
dm_wq_work(struct work_struct * work)2778 static void dm_wq_work(struct work_struct *work)
2779 {
2780 	struct mapped_device *md = container_of(work, struct mapped_device, work);
2781 	struct bio *bio;
2782 
2783 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2784 		spin_lock_irq(&md->deferred_lock);
2785 		bio = bio_list_pop(&md->deferred);
2786 		spin_unlock_irq(&md->deferred_lock);
2787 
2788 		if (!bio)
2789 			break;
2790 
2791 		submit_bio_noacct(bio);
2792 		cond_resched();
2793 	}
2794 }
2795 
dm_queue_flush(struct mapped_device * md)2796 static void dm_queue_flush(struct mapped_device *md)
2797 {
2798 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2799 	smp_mb__after_atomic();
2800 	queue_work(md->wq, &md->work);
2801 }
2802 
2803 /*
2804  * Swap in a new table, returning the old one for the caller to destroy.
2805  */
dm_swap_table(struct mapped_device * md,struct dm_table * table)2806 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2807 {
2808 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2809 	struct queue_limits limits;
2810 	int r;
2811 
2812 	mutex_lock(&md->suspend_lock);
2813 
2814 	/* device must be suspended */
2815 	if (!dm_suspended_md(md))
2816 		goto out;
2817 
2818 	/*
2819 	 * If the new table has no data devices, retain the existing limits.
2820 	 * This helps multipath with queue_if_no_path if all paths disappear,
2821 	 * then new I/O is queued based on these limits, and then some paths
2822 	 * reappear.
2823 	 */
2824 	if (dm_table_has_no_data_devices(table)) {
2825 		live_map = dm_get_live_table_fast(md);
2826 		if (live_map)
2827 			limits = md->queue->limits;
2828 		dm_put_live_table_fast(md);
2829 	}
2830 
2831 	if (!live_map) {
2832 		r = dm_calculate_queue_limits(table, &limits);
2833 		if (r) {
2834 			map = ERR_PTR(r);
2835 			goto out;
2836 		}
2837 	}
2838 
2839 	map = __bind(md, table, &limits);
2840 	dm_issue_global_event();
2841 
2842 out:
2843 	mutex_unlock(&md->suspend_lock);
2844 	return map;
2845 }
2846 
2847 /*
2848  * Functions to lock and unlock any filesystem running on the
2849  * device.
2850  */
lock_fs(struct mapped_device * md)2851 static int lock_fs(struct mapped_device *md)
2852 {
2853 	int r;
2854 
2855 	WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2856 
2857 	r = bdev_freeze(md->disk->part0);
2858 	if (!r)
2859 		set_bit(DMF_FROZEN, &md->flags);
2860 	return r;
2861 }
2862 
unlock_fs(struct mapped_device * md)2863 static void unlock_fs(struct mapped_device *md)
2864 {
2865 	if (!test_bit(DMF_FROZEN, &md->flags))
2866 		return;
2867 	bdev_thaw(md->disk->part0);
2868 	clear_bit(DMF_FROZEN, &md->flags);
2869 }
2870 
2871 /*
2872  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2873  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2874  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2875  *
2876  * If __dm_suspend returns 0, the device is completely quiescent
2877  * now. There is no request-processing activity. All new requests
2878  * are being added to md->deferred list.
2879  */
__dm_suspend(struct mapped_device * md,struct dm_table * map,unsigned int suspend_flags,unsigned int task_state,int dmf_suspended_flag)2880 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2881 			unsigned int suspend_flags, unsigned int task_state,
2882 			int dmf_suspended_flag)
2883 {
2884 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2885 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2886 	int r;
2887 
2888 	lockdep_assert_held(&md->suspend_lock);
2889 
2890 	/*
2891 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2892 	 * This flag is cleared before dm_suspend returns.
2893 	 */
2894 	if (noflush)
2895 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2896 	else
2897 		DMDEBUG("%s: suspending with flush", dm_device_name(md));
2898 
2899 	/*
2900 	 * This gets reverted if there's an error later and the targets
2901 	 * provide the .presuspend_undo hook.
2902 	 */
2903 	dm_table_presuspend_targets(map);
2904 
2905 	/*
2906 	 * Flush I/O to the device.
2907 	 * Any I/O submitted after lock_fs() may not be flushed.
2908 	 * noflush takes precedence over do_lockfs.
2909 	 * (lock_fs() flushes I/Os and waits for them to complete.)
2910 	 */
2911 	if (!noflush && do_lockfs) {
2912 		r = lock_fs(md);
2913 		if (r) {
2914 			dm_table_presuspend_undo_targets(map);
2915 			return r;
2916 		}
2917 	}
2918 
2919 	/*
2920 	 * Here we must make sure that no processes are submitting requests
2921 	 * to target drivers i.e. no one may be executing
2922 	 * dm_split_and_process_bio from dm_submit_bio.
2923 	 *
2924 	 * To get all processes out of dm_split_and_process_bio in dm_submit_bio,
2925 	 * we take the write lock. To prevent any process from reentering
2926 	 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread
2927 	 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
2928 	 * flush_workqueue(md->wq).
2929 	 */
2930 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2931 	if (map)
2932 		synchronize_srcu(&md->io_barrier);
2933 
2934 	/*
2935 	 * Stop md->queue before flushing md->wq in case request-based
2936 	 * dm defers requests to md->wq from md->queue.
2937 	 */
2938 	if (dm_request_based(md))
2939 		dm_stop_queue(md->queue);
2940 
2941 	flush_workqueue(md->wq);
2942 
2943 	/*
2944 	 * At this point no more requests are entering target request routines.
2945 	 * We call dm_wait_for_completion to wait for all existing requests
2946 	 * to finish.
2947 	 */
2948 	r = dm_wait_for_completion(md, task_state);
2949 	if (!r)
2950 		set_bit(dmf_suspended_flag, &md->flags);
2951 
2952 	if (noflush)
2953 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2954 	if (map)
2955 		synchronize_srcu(&md->io_barrier);
2956 
2957 	/* were we interrupted ? */
2958 	if (r < 0) {
2959 		dm_queue_flush(md);
2960 
2961 		if (dm_request_based(md))
2962 			dm_start_queue(md->queue);
2963 
2964 		unlock_fs(md);
2965 		dm_table_presuspend_undo_targets(map);
2966 		/* pushback list is already flushed, so skip flush */
2967 	}
2968 
2969 	return r;
2970 }
2971 
2972 /*
2973  * We need to be able to change a mapping table under a mounted
2974  * filesystem.  For example we might want to move some data in
2975  * the background.  Before the table can be swapped with
2976  * dm_bind_table, dm_suspend must be called to flush any in
2977  * flight bios and ensure that any further io gets deferred.
2978  */
2979 /*
2980  * Suspend mechanism in request-based dm.
2981  *
2982  * 1. Flush all I/Os by lock_fs() if needed.
2983  * 2. Stop dispatching any I/O by stopping the request_queue.
2984  * 3. Wait for all in-flight I/Os to be completed or requeued.
2985  *
2986  * To abort suspend, start the request_queue.
2987  */
dm_suspend(struct mapped_device * md,unsigned int suspend_flags)2988 int dm_suspend(struct mapped_device *md, unsigned int suspend_flags)
2989 {
2990 	struct dm_table *map = NULL;
2991 	int r = 0;
2992 
2993 retry:
2994 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2995 
2996 	if (dm_suspended_md(md)) {
2997 		r = -EINVAL;
2998 		goto out_unlock;
2999 	}
3000 
3001 	if (dm_suspended_internally_md(md)) {
3002 		/* already internally suspended, wait for internal resume */
3003 		mutex_unlock(&md->suspend_lock);
3004 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
3005 		if (r)
3006 			return r;
3007 		goto retry;
3008 	}
3009 
3010 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3011 	if (!map) {
3012 		/* avoid deadlock with fs/namespace.c:do_mount() */
3013 		suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
3014 	}
3015 
3016 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
3017 	if (r)
3018 		goto out_unlock;
3019 
3020 	set_bit(DMF_POST_SUSPENDING, &md->flags);
3021 	dm_table_postsuspend_targets(map);
3022 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
3023 
3024 out_unlock:
3025 	mutex_unlock(&md->suspend_lock);
3026 	return r;
3027 }
3028 
__dm_resume(struct mapped_device * md,struct dm_table * map)3029 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
3030 {
3031 	if (map) {
3032 		int r = dm_table_resume_targets(map);
3033 
3034 		if (r)
3035 			return r;
3036 	}
3037 
3038 	dm_queue_flush(md);
3039 
3040 	/*
3041 	 * Flushing deferred I/Os must be done after targets are resumed
3042 	 * so that mapping of targets can work correctly.
3043 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
3044 	 */
3045 	if (dm_request_based(md))
3046 		dm_start_queue(md->queue);
3047 
3048 	unlock_fs(md);
3049 
3050 	return 0;
3051 }
3052 
dm_resume(struct mapped_device * md)3053 int dm_resume(struct mapped_device *md)
3054 {
3055 	int r;
3056 	struct dm_table *map = NULL;
3057 
3058 retry:
3059 	r = -EINVAL;
3060 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
3061 
3062 	if (!dm_suspended_md(md))
3063 		goto out;
3064 
3065 	if (dm_suspended_internally_md(md)) {
3066 		/* already internally suspended, wait for internal resume */
3067 		mutex_unlock(&md->suspend_lock);
3068 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
3069 		if (r)
3070 			return r;
3071 		goto retry;
3072 	}
3073 
3074 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3075 	if (!map || !dm_table_get_size(map))
3076 		goto out;
3077 
3078 	r = __dm_resume(md, map);
3079 	if (r)
3080 		goto out;
3081 
3082 	clear_bit(DMF_SUSPENDED, &md->flags);
3083 out:
3084 	mutex_unlock(&md->suspend_lock);
3085 
3086 	return r;
3087 }
3088 
3089 /*
3090  * Internal suspend/resume works like userspace-driven suspend. It waits
3091  * until all bios finish and prevents issuing new bios to the target drivers.
3092  * It may be used only from the kernel.
3093  */
3094 
__dm_internal_suspend(struct mapped_device * md,unsigned int suspend_flags)3095 static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags)
3096 {
3097 	struct dm_table *map = NULL;
3098 
3099 	lockdep_assert_held(&md->suspend_lock);
3100 
3101 	if (md->internal_suspend_count++)
3102 		return; /* nested internal suspend */
3103 
3104 	if (dm_suspended_md(md)) {
3105 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3106 		return; /* nest suspend */
3107 	}
3108 
3109 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3110 
3111 	/*
3112 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
3113 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
3114 	 * would require changing .presuspend to return an error -- avoid this
3115 	 * until there is a need for more elaborate variants of internal suspend.
3116 	 */
3117 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
3118 			    DMF_SUSPENDED_INTERNALLY);
3119 
3120 	set_bit(DMF_POST_SUSPENDING, &md->flags);
3121 	dm_table_postsuspend_targets(map);
3122 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
3123 }
3124 
__dm_internal_resume(struct mapped_device * md)3125 static void __dm_internal_resume(struct mapped_device *md)
3126 {
3127 	int r;
3128 	struct dm_table *map;
3129 
3130 	BUG_ON(!md->internal_suspend_count);
3131 
3132 	if (--md->internal_suspend_count)
3133 		return; /* resume from nested internal suspend */
3134 
3135 	if (dm_suspended_md(md))
3136 		goto done; /* resume from nested suspend */
3137 
3138 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3139 	r = __dm_resume(md, map);
3140 	if (r) {
3141 		/*
3142 		 * If a preresume method of some target failed, we are in a
3143 		 * tricky situation. We can't return an error to the caller. We
3144 		 * can't fake success because then the "resume" and
3145 		 * "postsuspend" methods would not be paired correctly, and it
3146 		 * would break various targets, for example it would cause list
3147 		 * corruption in the "origin" target.
3148 		 *
3149 		 * So, we fake normal suspend here, to make sure that the
3150 		 * "resume" and "postsuspend" methods will be paired correctly.
3151 		 */
3152 		DMERR("Preresume method failed: %d", r);
3153 		set_bit(DMF_SUSPENDED, &md->flags);
3154 	}
3155 done:
3156 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3157 	smp_mb__after_atomic();
3158 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
3159 }
3160 
dm_internal_suspend_noflush(struct mapped_device * md)3161 void dm_internal_suspend_noflush(struct mapped_device *md)
3162 {
3163 	mutex_lock(&md->suspend_lock);
3164 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
3165 	mutex_unlock(&md->suspend_lock);
3166 }
3167 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
3168 
dm_internal_resume(struct mapped_device * md)3169 void dm_internal_resume(struct mapped_device *md)
3170 {
3171 	mutex_lock(&md->suspend_lock);
3172 	__dm_internal_resume(md);
3173 	mutex_unlock(&md->suspend_lock);
3174 }
3175 EXPORT_SYMBOL_GPL(dm_internal_resume);
3176 
3177 /*
3178  * Fast variants of internal suspend/resume hold md->suspend_lock,
3179  * which prevents interaction with userspace-driven suspend.
3180  */
3181 
dm_internal_suspend_fast(struct mapped_device * md)3182 void dm_internal_suspend_fast(struct mapped_device *md)
3183 {
3184 	mutex_lock(&md->suspend_lock);
3185 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
3186 		return;
3187 
3188 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
3189 	synchronize_srcu(&md->io_barrier);
3190 	flush_workqueue(md->wq);
3191 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
3192 }
3193 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
3194 
dm_internal_resume_fast(struct mapped_device * md)3195 void dm_internal_resume_fast(struct mapped_device *md)
3196 {
3197 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
3198 		goto done;
3199 
3200 	dm_queue_flush(md);
3201 
3202 done:
3203 	mutex_unlock(&md->suspend_lock);
3204 }
3205 EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
3206 
3207 /*
3208  *---------------------------------------------------------------
3209  * Event notification.
3210  *---------------------------------------------------------------
3211  */
dm_kobject_uevent(struct mapped_device * md,enum kobject_action action,unsigned int cookie,bool need_resize_uevent)3212 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
3213 		      unsigned int cookie, bool need_resize_uevent)
3214 {
3215 	int r;
3216 	unsigned int noio_flag;
3217 	char udev_cookie[DM_COOKIE_LENGTH];
3218 	char *envp[3] = { NULL, NULL, NULL };
3219 	char **envpp = envp;
3220 	if (cookie) {
3221 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
3222 			 DM_COOKIE_ENV_VAR_NAME, cookie);
3223 		*envpp++ = udev_cookie;
3224 	}
3225 	if (need_resize_uevent) {
3226 		*envpp++ = "RESIZE=1";
3227 	}
3228 
3229 	noio_flag = memalloc_noio_save();
3230 
3231 	r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp);
3232 
3233 	memalloc_noio_restore(noio_flag);
3234 
3235 	return r;
3236 }
3237 
dm_next_uevent_seq(struct mapped_device * md)3238 uint32_t dm_next_uevent_seq(struct mapped_device *md)
3239 {
3240 	return atomic_add_return(1, &md->uevent_seq);
3241 }
3242 
dm_get_event_nr(struct mapped_device * md)3243 uint32_t dm_get_event_nr(struct mapped_device *md)
3244 {
3245 	return atomic_read(&md->event_nr);
3246 }
3247 
dm_wait_event(struct mapped_device * md,int event_nr)3248 int dm_wait_event(struct mapped_device *md, int event_nr)
3249 {
3250 	return wait_event_interruptible(md->eventq,
3251 			(event_nr != atomic_read(&md->event_nr)));
3252 }
3253 
dm_uevent_add(struct mapped_device * md,struct list_head * elist)3254 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
3255 {
3256 	unsigned long flags;
3257 
3258 	spin_lock_irqsave(&md->uevent_lock, flags);
3259 	list_add(elist, &md->uevent_list);
3260 	spin_unlock_irqrestore(&md->uevent_lock, flags);
3261 }
3262 
3263 /*
3264  * The gendisk is only valid as long as you have a reference
3265  * count on 'md'.
3266  */
dm_disk(struct mapped_device * md)3267 struct gendisk *dm_disk(struct mapped_device *md)
3268 {
3269 	return md->disk;
3270 }
3271 EXPORT_SYMBOL_GPL(dm_disk);
3272 
dm_kobject(struct mapped_device * md)3273 struct kobject *dm_kobject(struct mapped_device *md)
3274 {
3275 	return &md->kobj_holder.kobj;
3276 }
3277 
dm_get_from_kobject(struct kobject * kobj)3278 struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
3279 {
3280 	struct mapped_device *md;
3281 
3282 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
3283 
3284 	spin_lock(&_minor_lock);
3285 	if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
3286 		md = NULL;
3287 		goto out;
3288 	}
3289 	dm_get(md);
3290 out:
3291 	spin_unlock(&_minor_lock);
3292 
3293 	return md;
3294 }
3295 
dm_suspended_md(struct mapped_device * md)3296 int dm_suspended_md(struct mapped_device *md)
3297 {
3298 	return test_bit(DMF_SUSPENDED, &md->flags);
3299 }
3300 
dm_post_suspending_md(struct mapped_device * md)3301 static int dm_post_suspending_md(struct mapped_device *md)
3302 {
3303 	return test_bit(DMF_POST_SUSPENDING, &md->flags);
3304 }
3305 
dm_suspended_internally_md(struct mapped_device * md)3306 int dm_suspended_internally_md(struct mapped_device *md)
3307 {
3308 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3309 }
3310 
dm_test_deferred_remove_flag(struct mapped_device * md)3311 int dm_test_deferred_remove_flag(struct mapped_device *md)
3312 {
3313 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
3314 }
3315 
dm_suspended(struct dm_target * ti)3316 int dm_suspended(struct dm_target *ti)
3317 {
3318 	return dm_suspended_md(ti->table->md);
3319 }
3320 EXPORT_SYMBOL_GPL(dm_suspended);
3321 
dm_post_suspending(struct dm_target * ti)3322 int dm_post_suspending(struct dm_target *ti)
3323 {
3324 	return dm_post_suspending_md(ti->table->md);
3325 }
3326 EXPORT_SYMBOL_GPL(dm_post_suspending);
3327 
dm_noflush_suspending(struct dm_target * ti)3328 int dm_noflush_suspending(struct dm_target *ti)
3329 {
3330 	return __noflush_suspending(ti->table->md);
3331 }
3332 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
3333 
dm_free_md_mempools(struct dm_md_mempools * pools)3334 void dm_free_md_mempools(struct dm_md_mempools *pools)
3335 {
3336 	if (!pools)
3337 		return;
3338 
3339 	bioset_exit(&pools->bs);
3340 	bioset_exit(&pools->io_bs);
3341 
3342 	kfree(pools);
3343 }
3344 
3345 struct dm_pr {
3346 	u64	old_key;
3347 	u64	new_key;
3348 	u32	flags;
3349 	bool	abort;
3350 	bool	fail_early;
3351 	int	ret;
3352 	enum pr_type type;
3353 	struct pr_keys *read_keys;
3354 	struct pr_held_reservation *rsv;
3355 };
3356 
dm_call_pr(struct block_device * bdev,iterate_devices_callout_fn fn,struct dm_pr * pr)3357 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
3358 		      struct dm_pr *pr)
3359 {
3360 	struct mapped_device *md = bdev->bd_disk->private_data;
3361 	struct dm_table *table;
3362 	struct dm_target *ti;
3363 	int ret = -ENOTTY, srcu_idx;
3364 
3365 	table = dm_get_live_table(md, &srcu_idx);
3366 	if (!table || !dm_table_get_size(table))
3367 		goto out;
3368 
3369 	/* We only support devices that have a single target */
3370 	if (table->num_targets != 1)
3371 		goto out;
3372 	ti = dm_table_get_target(table, 0);
3373 
3374 	if (dm_suspended_md(md)) {
3375 		ret = -EAGAIN;
3376 		goto out;
3377 	}
3378 
3379 	ret = -EINVAL;
3380 	if (!ti->type->iterate_devices)
3381 		goto out;
3382 
3383 	ti->type->iterate_devices(ti, fn, pr);
3384 	ret = 0;
3385 out:
3386 	dm_put_live_table(md, srcu_idx);
3387 	return ret;
3388 }
3389 
3390 /*
3391  * For register / unregister we need to manually call out to every path.
3392  */
__dm_pr_register(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3393 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3394 			    sector_t start, sector_t len, void *data)
3395 {
3396 	struct dm_pr *pr = data;
3397 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3398 	int ret;
3399 
3400 	if (!ops || !ops->pr_register) {
3401 		pr->ret = -EOPNOTSUPP;
3402 		return -1;
3403 	}
3404 
3405 	ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3406 	if (!ret)
3407 		return 0;
3408 
3409 	if (!pr->ret)
3410 		pr->ret = ret;
3411 
3412 	if (pr->fail_early)
3413 		return -1;
3414 
3415 	return 0;
3416 }
3417 
dm_pr_register(struct block_device * bdev,u64 old_key,u64 new_key,u32 flags)3418 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
3419 			  u32 flags)
3420 {
3421 	struct dm_pr pr = {
3422 		.old_key	= old_key,
3423 		.new_key	= new_key,
3424 		.flags		= flags,
3425 		.fail_early	= true,
3426 		.ret		= 0,
3427 	};
3428 	int ret;
3429 
3430 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3431 	if (ret) {
3432 		/* Didn't even get to register a path */
3433 		return ret;
3434 	}
3435 
3436 	if (!pr.ret)
3437 		return 0;
3438 	ret = pr.ret;
3439 
3440 	if (!new_key)
3441 		return ret;
3442 
3443 	/* unregister all paths if we failed to register any path */
3444 	pr.old_key = new_key;
3445 	pr.new_key = 0;
3446 	pr.flags = 0;
3447 	pr.fail_early = false;
3448 	(void) dm_call_pr(bdev, __dm_pr_register, &pr);
3449 	return ret;
3450 }
3451 
3452 
__dm_pr_reserve(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3453 static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev,
3454 			   sector_t start, sector_t len, void *data)
3455 {
3456 	struct dm_pr *pr = data;
3457 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3458 
3459 	if (!ops || !ops->pr_reserve) {
3460 		pr->ret = -EOPNOTSUPP;
3461 		return -1;
3462 	}
3463 
3464 	pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags);
3465 	if (!pr->ret)
3466 		return -1;
3467 
3468 	return 0;
3469 }
3470 
dm_pr_reserve(struct block_device * bdev,u64 key,enum pr_type type,u32 flags)3471 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3472 			 u32 flags)
3473 {
3474 	struct dm_pr pr = {
3475 		.old_key	= key,
3476 		.flags		= flags,
3477 		.type		= type,
3478 		.fail_early	= false,
3479 		.ret		= 0,
3480 	};
3481 	int ret;
3482 
3483 	ret = dm_call_pr(bdev, __dm_pr_reserve, &pr);
3484 	if (ret)
3485 		return ret;
3486 
3487 	return pr.ret;
3488 }
3489 
3490 /*
3491  * If there is a non-All Registrants type of reservation, the release must be
3492  * sent down the holding path. For the cases where there is no reservation or
3493  * the path is not the holder the device will also return success, so we must
3494  * try each path to make sure we got the correct path.
3495  */
__dm_pr_release(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3496 static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev,
3497 			   sector_t start, sector_t len, void *data)
3498 {
3499 	struct dm_pr *pr = data;
3500 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3501 
3502 	if (!ops || !ops->pr_release) {
3503 		pr->ret = -EOPNOTSUPP;
3504 		return -1;
3505 	}
3506 
3507 	pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type);
3508 	if (pr->ret)
3509 		return -1;
3510 
3511 	return 0;
3512 }
3513 
dm_pr_release(struct block_device * bdev,u64 key,enum pr_type type)3514 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3515 {
3516 	struct dm_pr pr = {
3517 		.old_key	= key,
3518 		.type		= type,
3519 		.fail_early	= false,
3520 	};
3521 	int ret;
3522 
3523 	ret = dm_call_pr(bdev, __dm_pr_release, &pr);
3524 	if (ret)
3525 		return ret;
3526 
3527 	return pr.ret;
3528 }
3529 
__dm_pr_preempt(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3530 static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev,
3531 			   sector_t start, sector_t len, void *data)
3532 {
3533 	struct dm_pr *pr = data;
3534 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3535 
3536 	if (!ops || !ops->pr_preempt) {
3537 		pr->ret = -EOPNOTSUPP;
3538 		return -1;
3539 	}
3540 
3541 	pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type,
3542 				  pr->abort);
3543 	if (!pr->ret)
3544 		return -1;
3545 
3546 	return 0;
3547 }
3548 
dm_pr_preempt(struct block_device * bdev,u64 old_key,u64 new_key,enum pr_type type,bool abort)3549 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3550 			 enum pr_type type, bool abort)
3551 {
3552 	struct dm_pr pr = {
3553 		.new_key	= new_key,
3554 		.old_key	= old_key,
3555 		.type		= type,
3556 		.fail_early	= false,
3557 	};
3558 	int ret;
3559 
3560 	ret = dm_call_pr(bdev, __dm_pr_preempt, &pr);
3561 	if (ret)
3562 		return ret;
3563 
3564 	return pr.ret;
3565 }
3566 
dm_pr_clear(struct block_device * bdev,u64 key)3567 static int dm_pr_clear(struct block_device *bdev, u64 key)
3568 {
3569 	struct mapped_device *md = bdev->bd_disk->private_data;
3570 	const struct pr_ops *ops;
3571 	int r, srcu_idx;
3572 
3573 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3574 	if (r < 0)
3575 		goto out;
3576 
3577 	ops = bdev->bd_disk->fops->pr_ops;
3578 	if (ops && ops->pr_clear)
3579 		r = ops->pr_clear(bdev, key);
3580 	else
3581 		r = -EOPNOTSUPP;
3582 out:
3583 	dm_unprepare_ioctl(md, srcu_idx);
3584 	return r;
3585 }
3586 
__dm_pr_read_keys(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3587 static int __dm_pr_read_keys(struct dm_target *ti, struct dm_dev *dev,
3588 			     sector_t start, sector_t len, void *data)
3589 {
3590 	struct dm_pr *pr = data;
3591 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3592 
3593 	if (!ops || !ops->pr_read_keys) {
3594 		pr->ret = -EOPNOTSUPP;
3595 		return -1;
3596 	}
3597 
3598 	pr->ret = ops->pr_read_keys(dev->bdev, pr->read_keys);
3599 	if (!pr->ret)
3600 		return -1;
3601 
3602 	return 0;
3603 }
3604 
dm_pr_read_keys(struct block_device * bdev,struct pr_keys * keys)3605 static int dm_pr_read_keys(struct block_device *bdev, struct pr_keys *keys)
3606 {
3607 	struct dm_pr pr = {
3608 		.read_keys = keys,
3609 	};
3610 	int ret;
3611 
3612 	ret = dm_call_pr(bdev, __dm_pr_read_keys, &pr);
3613 	if (ret)
3614 		return ret;
3615 
3616 	return pr.ret;
3617 }
3618 
__dm_pr_read_reservation(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3619 static int __dm_pr_read_reservation(struct dm_target *ti, struct dm_dev *dev,
3620 				    sector_t start, sector_t len, void *data)
3621 {
3622 	struct dm_pr *pr = data;
3623 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3624 
3625 	if (!ops || !ops->pr_read_reservation) {
3626 		pr->ret = -EOPNOTSUPP;
3627 		return -1;
3628 	}
3629 
3630 	pr->ret = ops->pr_read_reservation(dev->bdev, pr->rsv);
3631 	if (!pr->ret)
3632 		return -1;
3633 
3634 	return 0;
3635 }
3636 
dm_pr_read_reservation(struct block_device * bdev,struct pr_held_reservation * rsv)3637 static int dm_pr_read_reservation(struct block_device *bdev,
3638 				  struct pr_held_reservation *rsv)
3639 {
3640 	struct dm_pr pr = {
3641 		.rsv = rsv,
3642 	};
3643 	int ret;
3644 
3645 	ret = dm_call_pr(bdev, __dm_pr_read_reservation, &pr);
3646 	if (ret)
3647 		return ret;
3648 
3649 	return pr.ret;
3650 }
3651 
3652 static const struct pr_ops dm_pr_ops = {
3653 	.pr_register	= dm_pr_register,
3654 	.pr_reserve	= dm_pr_reserve,
3655 	.pr_release	= dm_pr_release,
3656 	.pr_preempt	= dm_pr_preempt,
3657 	.pr_clear	= dm_pr_clear,
3658 	.pr_read_keys	= dm_pr_read_keys,
3659 	.pr_read_reservation = dm_pr_read_reservation,
3660 };
3661 
3662 static const struct block_device_operations dm_blk_dops = {
3663 	.submit_bio = dm_submit_bio,
3664 	.poll_bio = dm_poll_bio,
3665 	.open = dm_blk_open,
3666 	.release = dm_blk_close,
3667 	.ioctl = dm_blk_ioctl,
3668 	.getgeo = dm_blk_getgeo,
3669 	.report_zones = dm_blk_report_zones,
3670 	.pr_ops = &dm_pr_ops,
3671 	.owner = THIS_MODULE
3672 };
3673 
3674 static const struct block_device_operations dm_rq_blk_dops = {
3675 	.open = dm_blk_open,
3676 	.release = dm_blk_close,
3677 	.ioctl = dm_blk_ioctl,
3678 	.getgeo = dm_blk_getgeo,
3679 	.pr_ops = &dm_pr_ops,
3680 	.owner = THIS_MODULE
3681 };
3682 
3683 static const struct dax_operations dm_dax_ops = {
3684 	.direct_access = dm_dax_direct_access,
3685 	.zero_page_range = dm_dax_zero_page_range,
3686 	.recovery_write = dm_dax_recovery_write,
3687 };
3688 
3689 /*
3690  * module hooks
3691  */
3692 module_init(dm_init);
3693 module_exit(dm_exit);
3694 
3695 module_param(major, uint, 0);
3696 MODULE_PARM_DESC(major, "The major number of the device mapper");
3697 
3698 module_param(reserved_bio_based_ios, uint, 0644);
3699 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3700 
3701 module_param(dm_numa_node, int, 0644);
3702 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3703 
3704 module_param(swap_bios, int, 0644);
3705 MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
3706 
3707 MODULE_DESCRIPTION(DM_NAME " driver");
3708 MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>");
3709 MODULE_LICENSE("GPL");
3710