105 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
107 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
121 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
122 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
126 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
132 	struct gendisk *disk = ns->head->disk;  in nvme_mpath_start_request()
134 	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {  in nvme_mpath_start_request()
157 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),  in nvme_mpath_end_request()
170 		if (!ns->head->disk)  in nvme_kick_requeue_lists()
172 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
174 			disk_uevent(ns->head->disk, KOBJ_CHANGE);  in nvme_kick_requeue_lists()
190 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()  local
194 	if (!head)  in nvme_mpath_clear_current_path()
198 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
199 			rcu_assign_pointer(head->current_path[node], NULL);  in nvme_mpath_clear_current_path()
216 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
223 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()  local
224 	sector_t capacity = get_capacity(head->disk);  in nvme_mpath_revalidate_paths()
228 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_mpath_revalidate_paths()
229 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_mpath_revalidate_paths()
230 				 srcu_read_lock_held(&head->srcu)) {  in nvme_mpath_revalidate_paths()
234 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_mpath_revalidate_paths()
237 		rcu_assign_pointer(head->current_path[node], NULL);  in nvme_mpath_revalidate_paths()
238 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_revalidate_paths()
258 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)  in __nvme_find_path()  argument
263 	list_for_each_entry_srcu(ns, &head->list, siblings,  in __nvme_find_path()
264 				 srcu_read_lock_held(&head->srcu)) {  in __nvme_find_path()
269 		    READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)  in __nvme_find_path()
295 		rcu_assign_pointer(head->current_path[node], found);  in __nvme_find_path()
299 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,  in nvme_next_ns()  argument
302 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
306 	return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);  in nvme_next_ns()
309 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)  in nvme_round_robin_path()  argument
313 	struct nvme_ns *old = srcu_dereference(head->current_path[node],  in nvme_round_robin_path()
314 					       &head->srcu);  in nvme_round_robin_path()
317 		return __nvme_find_path(head, node);  in nvme_round_robin_path()
319 	if (list_is_singular(&head->list)) {  in nvme_round_robin_path()
325 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
327 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
353 	rcu_assign_pointer(head->current_path[node], found);  in nvme_round_robin_path()
357 static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)  in nvme_queue_depth_path()  argument
363 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_queue_depth_path()
364 				 srcu_read_lock_held(&head->srcu)) {  in nvme_queue_depth_path()
400 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)  in nvme_numa_path()  argument
405 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_numa_path()
407 		return __nvme_find_path(head, node);  in nvme_numa_path()
409 		return __nvme_find_path(head, node);  in nvme_numa_path()
413 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)  in nvme_find_path()  argument
415 	switch (READ_ONCE(head->subsys->iopolicy)) {  in nvme_find_path()
417 		return nvme_queue_depth_path(head);  in nvme_find_path()
419 		return nvme_round_robin_path(head);  in nvme_find_path()
421 		return nvme_numa_path(head);  in nvme_find_path()
425 static bool nvme_available_path(struct nvme_ns_head *head)  in nvme_available_path()  argument
429 	if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags))  in nvme_available_path()
432 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_available_path()
433 				 srcu_read_lock_held(&head->srcu)) {  in nvme_available_path()
450 	struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;  in nvme_ns_head_submit_bio()  local
451 	struct device *dev = disk_to_dev(head->disk);  in nvme_ns_head_submit_bio()
464 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_submit_bio()
465 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
469 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
472 	} else if (nvme_available_path(head)) {  in nvme_ns_head_submit_bio()
475 		spin_lock_irq(&head->requeue_lock);  in nvme_ns_head_submit_bio()
476 		bio_list_add(&head->requeue_list, bio);  in nvme_ns_head_submit_bio()
477 		spin_unlock_irq(&head->requeue_lock);  in nvme_ns_head_submit_bio()
484 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_submit_bio()
502 	struct nvme_ns_head *head = disk->private_data;  in nvme_ns_head_get_unique_id()  local
506 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_get_unique_id()
507 	ns = nvme_find_path(head);  in nvme_ns_head_get_unique_id()
510 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_get_unique_id()
518 	struct nvme_ns_head *head = disk->private_data;  in nvme_ns_head_report_zones()  local
522 	srcu_idx = srcu_read_lock(&head->srcu);  in nvme_ns_head_report_zones()
523 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
526 	srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_ns_head_report_zones()
574 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)  in nvme_add_ns_head_cdev()  argument
578 	head->cdev_device.parent = &head->subsys->dev;  in nvme_add_ns_head_cdev()
579 	ret = dev_set_name(&head->cdev_device, "ng%dn%d",  in nvme_add_ns_head_cdev()
580 			   head->subsys->instance, head->instance);  in nvme_add_ns_head_cdev()
583 	ret = nvme_cdev_add(&head->cdev, &head->cdev_device,  in nvme_add_ns_head_cdev()
590 	struct nvme_ns_head *head =  in nvme_partition_scan_work()  local
594 					     &head->disk->state)))  in nvme_partition_scan_work()
597 	mutex_lock(&head->disk->open_mutex);  in nvme_partition_scan_work()
598 	bdev_disk_changed(head->disk, false);  in nvme_partition_scan_work()
599 	mutex_unlock(&head->disk->open_mutex);  in nvme_partition_scan_work()
604 	struct nvme_ns_head *head =  in nvme_requeue_work()  local
608 	spin_lock_irq(&head->requeue_lock);  in nvme_requeue_work()
609 	next = bio_list_get(&head->requeue_list);  in nvme_requeue_work()
610 	spin_unlock_irq(&head->requeue_lock);  in nvme_requeue_work()
620 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)  in nvme_mpath_alloc_disk()  argument
624 	mutex_init(&head->lock);  in nvme_mpath_alloc_disk()
625 	bio_list_init(&head->requeue_list);  in nvme_mpath_alloc_disk()
626 	spin_lock_init(&head->requeue_lock);  in nvme_mpath_alloc_disk()
627 	INIT_WORK(&head->requeue_work, nvme_requeue_work);  in nvme_mpath_alloc_disk()
628 	INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work);  in nvme_mpath_alloc_disk()
636 	    !nvme_is_unique_nsid(ctrl, head) || !multipath)  in nvme_mpath_alloc_disk()
642 	if (head->ids.csi == NVME_CSI_ZNS)  in nvme_mpath_alloc_disk()
645 	head->disk = blk_alloc_disk(&lim, ctrl->numa_node);  in nvme_mpath_alloc_disk()
646 	if (IS_ERR(head->disk))  in nvme_mpath_alloc_disk()
647 		return PTR_ERR(head->disk);  in nvme_mpath_alloc_disk()
648 	head->disk->fops = &nvme_ns_head_ops;  in nvme_mpath_alloc_disk()
649 	head->disk->private_data = head;  in nvme_mpath_alloc_disk()
659 	set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state);  in nvme_mpath_alloc_disk()
660 	sprintf(head->disk->disk_name, "nvme%dn%d",  in nvme_mpath_alloc_disk()
661 			ctrl->subsys->instance, head->instance);  in nvme_mpath_alloc_disk()
667 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()  local
670 	if (!head->disk)  in nvme_mpath_set_live()
678 	if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {  in nvme_mpath_set_live()
679 		rc = device_add_disk(&head->subsys->dev, head->disk,  in nvme_mpath_set_live()
682 			clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags);  in nvme_mpath_set_live()
685 		nvme_add_ns_head_cdev(head);  in nvme_mpath_set_live()
686 		kblockd_schedule_work(&head->partition_scan_work);  in nvme_mpath_set_live()
689 	mutex_lock(&head->lock);  in nvme_mpath_set_live()
693 		srcu_idx = srcu_read_lock(&head->srcu);  in nvme_mpath_set_live()
695 			__nvme_find_path(head, node);  in nvme_mpath_set_live()
696 		srcu_read_unlock(&head->srcu, srcu_idx);  in nvme_mpath_set_live()
698 	mutex_unlock(&head->lock);  in nvme_mpath_set_live()
700 	synchronize_srcu(&head->srcu);  in nvme_mpath_set_live()
701 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_set_live()
797 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
799 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
803 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
996 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
997 		ns->head->disk->nr_zones = ns->disk->nr_zones;  in nvme_mpath_add_disk()
1001 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)  in nvme_mpath_shutdown_disk()  argument
1003 	if (!head->disk)  in nvme_mpath_shutdown_disk()
1005 	if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {  in nvme_mpath_shutdown_disk()
1006 		nvme_cdev_del(&head->cdev, &head->cdev_device);  in nvme_mpath_shutdown_disk()
1011 		synchronize_srcu(&head->srcu);  in nvme_mpath_shutdown_disk()
1012 		kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_shutdown_disk()
1013 		del_gendisk(head->disk);  in nvme_mpath_shutdown_disk()
1017 void nvme_mpath_remove_disk(struct nvme_ns_head *head)  in nvme_mpath_remove_disk()  argument
1019 	if (!head->disk)  in nvme_mpath_remove_disk()
1022 	kblockd_schedule_work(&head->requeue_work);  in nvme_mpath_remove_disk()
1023 	flush_work(&head->requeue_work);  in nvme_mpath_remove_disk()
1024 	flush_work(&head->partition_scan_work);  in nvme_mpath_remove_disk()
1025 	put_disk(head->disk);  in nvme_mpath_remove_disk()