1 // SPDX-License-Identifier: GPL-2.0
14 #include <linux/backing-dev.h>
30 #include "blk-throttle.h"
32 #include "blk-mq-sched.h"
33 #include "blk-rq-qos.h"
34 #include "blk-cgroup.h"
63 				disk->disk_name, sectors,  in set_capacity()
68 	bdev_set_nr_sectors(disk->part0, sectors);  in set_capacity()
90 	    (disk->flags & GENHD_FL_HIDDEN))  in set_capacity_and_notify()
94 		disk->disk_name, capacity, size);  in set_capacity_and_notify()
102 	kobject_uevent_env(&disk_to_dev(disk)->kobj, KOBJ_CHANGE, envp);  in set_capacity_and_notify()
114 		struct disk_stats *ptr = per_cpu_ptr(part->bd_stats, cpu);  in part_stat_read_all()
115 		int group;  in part_stat_read_all()  local
117 		for (group = 0; group < NR_STAT_GROUPS; group++) {  in part_stat_read_all()
118 			stat->nsecs[group] += ptr->nsecs[group];  in part_stat_read_all()
119 			stat->sectors[group] += ptr->sectors[group];  in part_stat_read_all()
120 			stat->ios[group] += ptr->ios[group];  in part_stat_read_all()
121 			stat->merges[group] += ptr->merges[group];  in part_stat_read_all()
124 		stat->io_ticks += ptr->io_ticks;  in part_stat_read_all()
155  * bdev_count_inflight - get the number of inflight IOs for a block device.
160  * bio-based block device, and from blk_account_io_start() for rq-based block
189 /* index in the above - for now: assume no multimajor ranges */
201 	for (dp = major_names[major_to_index(offset)]; dp; dp = dp->next)  in blkdev_show()
202 		if (dp->major == offset)  in blkdev_show()
203 			seq_printf(seqf, "%3d %s\n", dp->major, dp->name);  in blkdev_show()
209  * __register_blkdev - register a new block device
211  * @major: the requested major device number [1..BLKDEV_MAJOR_MAX-1]. If
214  * @probe: pre-devtmpfs / pre-udev callback used to create disks when their
215  *	   pre-created device node is accessed. When a probe call uses
223  *  - if a major device number was requested in range [1..BLKDEV_MAJOR_MAX-1]
225  *  - if any unused major number was requested with @major = 0 parameter
227  *    [1..BLKDEV_MAJOR_MAX-1] or a negative error code otherwise
229  * See Documentation/admin-guide/devices.txt for the list of allocated
244 		for (index = ARRAY_SIZE(major_names)-1; index > 0; index--) {  in __register_blkdev()
252 			ret = -EBUSY;  in __register_blkdev()
261 		       __func__, major, BLKDEV_MAJOR_MAX-1, name);  in __register_blkdev()
263 		ret = -EINVAL;  in __register_blkdev()
269 		ret = -ENOMEM;  in __register_blkdev()
273 	p->major = major;  in __register_blkdev()
275 	p->probe = probe;  in __register_blkdev()
277 	strscpy(p->name, name, sizeof(p->name));  in __register_blkdev()
278 	p->next = NULL;  in __register_blkdev()
282 	for (n = &major_names[index]; *n; n = &(*n)->next) {  in __register_blkdev()
283 		if ((*n)->major == major)  in __register_blkdev()
289 		ret = -EBUSY;  in __register_blkdev()
311 	for (n = &major_names[index]; *n; n = &(*n)->next)  in unregister_blkdev()
312 		if ((*n)->major == major)  in unregister_blkdev()
314 	if (!*n || strcmp((*n)->name, name)) {  in unregister_blkdev()
318 		*n = p->next;  in unregister_blkdev()
331 	idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT - 1, GFP_KERNEL);  in blk_alloc_ext_minor()
332 	if (idx == -ENOSPC)  in blk_alloc_ext_minor()
333 		return -EBUSY;  in blk_alloc_ext_minor()
348 	xa_for_each(&disk->part_tbl, idx, part) {  in disk_uevent()
351 		if (!kobject_get_unless_zero(&part->bd_device.kobj))  in disk_uevent()
356 		put_device(&part->bd_device);  in disk_uevent()
369 		return -EINVAL;  in disk_scan_partitions()
370 	if (disk->open_partitions)  in disk_scan_partitions()
371 		return -EBUSY;  in disk_scan_partitions()
380 		ret = bd_prepare_to_claim(disk->part0, disk_scan_partitions,  in disk_scan_partitions()
386 	set_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
396 	 * and this will cause that re-assemble partitioned raid device will  in disk_scan_partitions()
399 	clear_bit(GD_NEED_PART_SCAN, &disk->state);  in disk_scan_partitions()
401 		bd_abort_claiming(disk->part0, disk_scan_partitions);  in disk_scan_partitions()
409 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in add_disk_final()
410 		bdev_add(disk->part0, ddev->devt);  in add_disk_final()
422 	blk_apply_bdi_limits(disk->bdi, &disk->queue->limits);  in add_disk_final()
424 	set_bit(GD_ADDED, &disk->state);  in add_disk_final()
435 	if (WARN_ON_ONCE(bdev_nr_sectors(disk->part0) > BLK_DEV_MAX_SECTORS))  in __add_disk()
436 		return -EINVAL;  in __add_disk()
438 	if (queue_is_mq(disk->queue)) {  in __add_disk()
440 		 * ->submit_bio and ->poll_bio are bypassed for blk-mq drivers.  in __add_disk()
442 		if (disk->fops->submit_bio || disk->fops->poll_bio)  in __add_disk()
443 			return -EINVAL;  in __add_disk()
445 		if (!disk->fops->submit_bio)  in __add_disk()
446 			return -EINVAL;  in __add_disk()
447 		bdev_set_flag(disk->part0, BD_HAS_SUBMIT_BIO);  in __add_disk()
454 	if (blk_queue_is_zoned(disk->queue))  in __add_disk()
455 		disk->flags |= GENHD_FL_NO_PART;  in __add_disk()
464 	ret = -EINVAL;  in __add_disk()
465 	if (disk->major) {  in __add_disk()
466 		if (WARN_ON(!disk->minors))  in __add_disk()
469 		if (disk->minors > DISK_MAX_PARTS) {  in __add_disk()
472 			disk->minors = DISK_MAX_PARTS;  in __add_disk()
474 		if (disk->first_minor > MINORMASK ||  in __add_disk()
475 		    disk->minors > MINORMASK + 1 ||  in __add_disk()
476 		    disk->first_minor + disk->minors > MINORMASK + 1)  in __add_disk()
479 		if (WARN_ON(disk->minors))  in __add_disk()
485 		disk->major = BLOCK_EXT_MAJOR;  in __add_disk()
486 		disk->first_minor = ret;  in __add_disk()
492 	ddev->parent = parent;  in __add_disk()
493 	ddev->groups = groups;  in __add_disk()
494 	dev_set_name(ddev, "%s", disk->disk_name);  in __add_disk()
497 	if (!(disk->flags & GENHD_FL_HIDDEN))  in __add_disk()
498 		ddev->devt = MKDEV(disk->major, disk->first_minor);  in __add_disk()
507 	ret = sysfs_create_link(block_depr, &ddev->kobj,  in __add_disk()
508 				kobject_name(&ddev->kobj));  in __add_disk()
519 	disk->part0->bd_holder_dir =  in __add_disk()
520 		kobject_create_and_add("holders", &ddev->kobj);  in __add_disk()
521 	if (!disk->part0->bd_holder_dir) {  in __add_disk()
522 		ret = -ENOMEM;  in __add_disk()
525 	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);  in __add_disk()
526 	if (!disk->slave_dir) {  in __add_disk()
527 		ret = -ENOMEM;  in __add_disk()
535 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in __add_disk()
536 		ret = bdi_register(disk->bdi, "%u:%u",  in __add_disk()
537 				   disk->major, disk->first_minor);  in __add_disk()
540 		bdi_set_owner(disk->bdi, ddev);  in __add_disk()
541 		ret = sysfs_create_link(&ddev->kobj,  in __add_disk()
542 					&disk->bdi->dev->kobj, "bdi");  in __add_disk()
551 		disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);  in __add_disk()
556 	if (!(disk->flags & GENHD_FL_HIDDEN))  in __add_disk()
557 		bdi_unregister(disk->bdi);  in __add_disk()
560 	rq_qos_exit(disk->queue);  in __add_disk()
562 	kobject_put(disk->slave_dir);  in __add_disk()
563 	disk->slave_dir = NULL;  in __add_disk()
565 	kobject_put(disk->part0->bd_holder_dir);  in __add_disk()
572 	if (disk->major == BLOCK_EXT_MAJOR)  in __add_disk()
573 		blk_free_ext_minor(disk->first_minor);  in __add_disk()
579  * add_disk_fwnode - add disk information to kernel list with fwnode
581  * @disk: per-device partitioning information
582  * @groups: Additional per-device sysfs groups
596 	if (queue_is_mq(disk->queue)) {  in add_disk_fwnode()
597 		set = disk->queue->tag_set;  in add_disk_fwnode()
599 		down_read(&set->update_nr_hwq_lock);  in add_disk_fwnode()
601 		up_read(&set->update_nr_hwq_lock);  in add_disk_fwnode()
609 	 * of read lock `set->update_nr_hwq_lock` for avoiding unnecessary  in add_disk_fwnode()
610 	 * lock dependency on `disk->open_mutex` from scanning partition.  in add_disk_fwnode()
619  * device_add_disk - add disk information to kernel list
621  * @disk: per-device partitioning information
622  * @groups: Additional per-device sysfs groups
642 	 * disk->open_mutex.  in blk_report_disk_dead()
644 	lockdep_assert_not_held(&disk->open_mutex);  in blk_report_disk_dead()
647 	xa_for_each(&disk->part_tbl, idx, bdev) {  in blk_report_disk_dead()
648 		if (!kobject_get_unless_zero(&bdev->bd_device.kobj))  in blk_report_disk_dead()
654 		put_device(&bdev->bd_device);  in blk_report_disk_dead()
665 	if (test_and_set_bit(GD_DEAD, &disk->state))  in __blk_mark_disk_dead()
668 	if (test_bit(GD_OWNS_QUEUE, &disk->state))  in __blk_mark_disk_dead()
669 		blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue);  in __blk_mark_disk_dead()
679 	return blk_queue_start_drain(disk->queue);  in __blk_mark_disk_dead()
683  * blk_mark_disk_dead - mark a disk as dead
691 	blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue);  in blk_mark_disk_dead()
699 	struct request_queue *q = disk->queue;  in __del_gendisk()
706 	if (WARN_ON_ONCE(!disk_live(disk) && !(disk->flags & GENHD_FL_HIDDEN)))  in __del_gendisk()
714 	mutex_lock(&disk->open_mutex);  in __del_gendisk()
715 	xa_for_each(&disk->part_tbl, idx, part)  in __del_gendisk()
717 	mutex_unlock(&disk->open_mutex);  in __del_gendisk()
723 	if (!test_bit(GD_DEAD, &disk->state))  in __del_gendisk()
729 	mutex_lock(&disk->open_mutex);  in __del_gendisk()
733 	xa_for_each_start(&disk->part_tbl, idx, part, 1)  in __del_gendisk()
735 	mutex_unlock(&disk->open_mutex);  in __del_gendisk()
737 	if (!(disk->flags & GENHD_FL_HIDDEN)) {  in __del_gendisk()
738 		sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");  in __del_gendisk()
744 		bdi_unregister(disk->bdi);  in __del_gendisk()
749 	kobject_put(disk->part0->bd_holder_dir);  in __del_gendisk()
750 	kobject_put(disk->slave_dir);  in __del_gendisk()
751 	disk->slave_dir = NULL;  in __del_gendisk()
753 	part_stat_set_all(disk->part0, 0);  in __del_gendisk()
754 	disk->part0->bd_stamp = 0;  in __del_gendisk()
772 	 * If the disk does not own the queue, allow using passthrough requests  in __del_gendisk()
773 	 * again.  Else leave the queue frozen to fail all I/O.  in __del_gendisk()
775 	if (!test_bit(GD_OWNS_QUEUE, &disk->state))  in __del_gendisk()
786 	struct blk_mq_tag_set *set = q->tag_set;  in disable_elv_switch()
789 	down_write(&set->update_nr_hwq_lock);  in disable_elv_switch()
791 	up_write(&set->update_nr_hwq_lock);  in disable_elv_switch()
795  * del_gendisk - remove the gendisk
818 	if (!queue_is_mq(disk->queue)) {  in del_gendisk()
821 		set = disk->queue->tag_set;  in del_gendisk()
823 		disable_elv_switch(disk->queue);  in del_gendisk()
826 		down_read(&set->update_nr_hwq_lock);  in del_gendisk()
828 		up_read(&set->update_nr_hwq_lock);  in del_gendisk()
835  * invalidate_disk - invalidate the disk
846 	struct block_device *bdev = disk->part0;  in invalidate_disk()
849 	bdev->bd_mapping->wb_err = 0;  in invalidate_disk()
854 /* sysfs access to bad-blocks list. */
861 	if (!disk->bb)  in disk_badblocks_show()
864 	return badblocks_show(disk->bb, page, 0);  in disk_badblocks_show()
873 	if (!disk->bb)  in disk_badblocks_store()
874 		return -ENXIO;  in disk_badblocks_store()
876 	return badblocks_store(disk->bb, page, len, 0);  in disk_badblocks_store()
886 	for (n = &major_names[major_to_index(major)]; *n; n = &(*n)->next) {  in blk_probe_dev()
887 		if ((*n)->major == major && (*n)->probe) {  in blk_probe_dev()
888 			(*n)->probe(devt);  in blk_probe_dev()
904 	error = request_module("block-major-%d-%d", MAJOR(devt), MINOR(devt));  in blk_request_module()
905 	/* Make old-style 2.4 aliases work */  in blk_request_module()
907 		error = request_module("block-major-%d", MAJOR(devt));  in blk_request_module()
923 		return ERR_PTR(-ENOMEM);  in disk_seqf_start()
925 	seqf->private = iter;  in disk_seqf_start()
931 	} while (skip--);  in disk_seqf_start()
941 	dev = class_dev_iter_next(seqf->private);  in disk_seqf_next()
950 	struct class_dev_iter *iter = seqf->private;  in disk_seqf_stop()
952 	/* stop is called even after start failed :-( */  in disk_seqf_stop()
956 		seqf->private = NULL;  in disk_seqf_stop()
976 	if (!get_capacity(sgp) || (sgp->flags & GENHD_FL_HIDDEN))  in show_partition()
980 	xa_for_each(&sgp->part_tbl, idx, part) {  in show_partition()
984 			   MAJOR(part->bd_dev), MINOR(part->bd_dev),  in show_partition()
1010 	/* create top-level block dir */  in genhd_device_init()
1022 	return sysfs_emit(buf, "%d\n", disk->minors);  in disk_range_show()
1031 		(disk->flags & GENHD_FL_NO_PART) ? 1 : DISK_MAX_PARTS);  in disk_ext_range_show()
1040 		       (disk->flags & GENHD_FL_REMOVABLE ? 1 : 0));  in disk_removable_show()
1049 		       (disk->flags & GENHD_FL_HIDDEN ? 1 : 0));  in disk_hidden_show()
1112  * For bio-based device, started from bdev_start_io_acct();
1113  * For rq-based device, started from blk_mq_start_request();
1140 	return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_alignment_offset_show()
1149 	return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0));  in disk_discard_alignment_show()
1157 	return sysfs_emit(buf, "%llu\n", disk->diskseq);  in diskseq_show()
1205 	__ATTR(make-it-fail, 0644, part_fail_show, part_fail_store);
1210 	__ATTR(io-timeout-fail, 0644, part_timeout_show, part_timeout_store);
1245 	if (a == &dev_attr_badblocks.attr && !disk->bb)  in disk_visible()
1247 	return a->mode;  in disk_visible()
1267  * disk_release - releases all allocated resources of the gendisk
1287 	blk_trace_remove(disk->queue);  in disk_release()
1297 	if (queue_is_mq(disk->queue) &&  in disk_release()
1298 	    test_bit(GD_OWNS_QUEUE, &disk->state) &&  in disk_release()
1299 	    !test_bit(GD_ADDED, &disk->state)) {  in disk_release()
1300 		blk_sync_queue(disk->queue);  in disk_release()
1301 		blk_mq_cancel_work_sync(disk->queue);  in disk_release()
1302 		blk_mq_exit_queue(disk->queue);  in disk_release()
1307 	bioset_exit(&disk->bio_split);  in disk_release()
1310 	kfree(disk->random);  in disk_release()
1312 	xa_destroy(&disk->part_tbl);  in disk_release()
1314 	kobject_put(&disk->queue_kobj);  in disk_release()
1315 	disk->queue->disk = NULL;  in disk_release()
1316 	blk_put_queue(disk->queue);  in disk_release()
1318 	if (test_bit(GD_ADDED, &disk->state) && disk->fops->free_disk)  in disk_release()
1319 		disk->fops->free_disk(disk);  in disk_release()
1321 	bdev_drop(disk->part0);	/* frees the disk */  in disk_release()
1328 	return add_uevent_var(env, "DISKSEQ=%llu", disk->diskseq);  in block_uevent()
1341 	if (disk->fops->devnode)  in block_devnode()
1342 		return disk->fops->devnode(disk, mode);  in block_devnode()
1370 	if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next)  in diskstats_show()
1378 	xa_for_each(&gp->part_tbl, idx, hd) {  in diskstats_show()
1389 		seq_put_decimal_ull_width(seqf, "",  MAJOR(hd->bd_dev), 4);  in diskstats_show()
1390 		seq_put_decimal_ull_width(seqf, " ", MINOR(hd->bd_dev), 7);  in diskstats_show()
1446 	part = xa_load(&disk->part_tbl, partno);  in part_devt()
1448 		devt = part->bd_dev;  in part_devt()
1463 	if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))  in __alloc_disk_node()
1466 	disk->bdi = bdi_alloc(node_id);  in __alloc_disk_node()
1467 	if (!disk->bdi)  in __alloc_disk_node()
1470 	/* bdev_alloc() might need the queue, set before the first call */  in __alloc_disk_node()
1471 	disk->queue = q;  in __alloc_disk_node()
1473 	disk->part0 = bdev_alloc(disk, 0);  in __alloc_disk_node()
1474 	if (!disk->part0)  in __alloc_disk_node()
1477 	disk->node_id = node_id;  in __alloc_disk_node()
1478 	mutex_init(&disk->open_mutex);  in __alloc_disk_node()
1479 	xa_init(&disk->part_tbl);  in __alloc_disk_node()
1480 	if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))  in __alloc_disk_node()
1488 	disk_to_dev(disk)->class = &block_class;  in __alloc_disk_node()
1489 	disk_to_dev(disk)->type = &disk_type;  in __alloc_disk_node()
1492 	q->disk = disk;  in __alloc_disk_node()
1493 	lockdep_init_map(&disk->lockdep_map, "(bio completion)", lkclass, 0);  in __alloc_disk_node()
1495 	INIT_LIST_HEAD(&disk->slave_bdevs);  in __alloc_disk_node()
1498 	mutex_init(&disk->error_injection_lock);  in __alloc_disk_node()
1499 	INIT_LIST_HEAD(&disk->error_injection_list);  in __alloc_disk_node()
1501 	mutex_init(&disk->rqos_state_mutex);  in __alloc_disk_node()
1502 	kobject_init(&disk->queue_kobj, &blk_queue_ktype);  in __alloc_disk_node()
1506 	xa_erase(&disk->part_tbl, 0);  in __alloc_disk_node()
1508 	xa_destroy(&disk->part_tbl);  in __alloc_disk_node()
1509 	disk->part0->bd_disk = NULL;  in __alloc_disk_node()
1510 	bdev_drop(disk->part0);  in __alloc_disk_node()
1512 	bdi_put(disk->bdi);  in __alloc_disk_node()
1514 	bioset_exit(&disk->bio_split);  in __alloc_disk_node()
1534 		return ERR_PTR(-ENOMEM);  in __blk_alloc_disk()
1536 	set_bit(GD_OWNS_QUEUE, &disk->state);  in __blk_alloc_disk()
1542  * put_disk - decrements the gendisk refcount
1548  * Note: for blk-mq disk put_disk must be called before freeing the tag_set
1568 	kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);  in set_disk_ro_uevent()
1572  * set_disk_ro - set a gendisk read-only
1574  * @read_only:	%true to set the disk read-only, %false set the disk read/write
1577  * read-only flag set. set_disk_ro() is typically used by device drivers to
1578  * indicate whether the underlying physical device is write-protected.
1583 		if (test_and_set_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1586 		if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))  in set_disk_ro()
1595 	disk->diskseq = atomic64_inc_return(&diskseq);  in inc_diskseq()