Lines Matching +full:dev +full:- +full:ctrl
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2017-2018 Christoph Hellwig.
6 #include <linux/backing-dev.h>
18 bool *arg = kp->arg; in multipath_param_set()
27 return -EINVAL; in multipath_param_set()
46 bool *arg = kp->arg; in multipath_always_on_set()
66 …"create multipath node always except for private namespace with non-unique nsid; note that this al…
70 [NVME_IOPOLICY_RR] = "round-robin",
71 [NVME_IOPOLICY_QD] = "queue-depth",
84 return -EINVAL; in nvme_iopolicy_parse()
92 return -EINVAL; in nvme_set_iopolicy()
110 "Default multipath I/O policy; 'numa' (default), 'round-robin' or 'queue-depth'");
114 subsys->iopolicy = iopolicy; in nvme_mpath_default_iopolicy()
121 lockdep_assert_held(&subsys->lock); in nvme_mpath_unfreeze()
122 list_for_each_entry(h, &subsys->nsheads, entry) in nvme_mpath_unfreeze()
123 if (h->disk) in nvme_mpath_unfreeze()
124 blk_mq_unfreeze_queue_nomemrestore(h->disk->queue); in nvme_mpath_unfreeze()
131 lockdep_assert_held(&subsys->lock); in nvme_mpath_wait_freeze()
132 list_for_each_entry(h, &subsys->nsheads, entry) in nvme_mpath_wait_freeze()
133 if (h->disk) in nvme_mpath_wait_freeze()
134 blk_mq_freeze_queue_wait(h->disk->queue); in nvme_mpath_wait_freeze()
141 lockdep_assert_held(&subsys->lock); in nvme_mpath_start_freeze()
142 list_for_each_entry(h, &subsys->nsheads, entry) in nvme_mpath_start_freeze()
143 if (h->disk) in nvme_mpath_start_freeze()
144 blk_freeze_queue_start(h->disk->queue); in nvme_mpath_start_freeze()
149 struct nvme_ns *ns = req->q->queuedata; in nvme_failover_req()
150 u16 status = nvme_req(req)->status & NVME_SCT_SC_MASK; in nvme_failover_req()
155 atomic_long_inc(&ns->failover); in nvme_failover_req()
159 * ready to serve this namespace. Kick of a re-read of the ANA in nvme_failover_req()
162 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { in nvme_failover_req()
163 set_bit(NVME_NS_ANA_PENDING, &ns->flags); in nvme_failover_req()
164 queue_work(nvme_wq, &ns->ctrl->ana_work); in nvme_failover_req()
167 spin_lock_irqsave(&ns->head->requeue_lock, flags); in nvme_failover_req()
168 for (bio = req->bio; bio; bio = bio->bi_next) in nvme_failover_req()
169 bio_set_dev(bio, ns->head->disk->part0); in nvme_failover_req()
170 blk_steal_bios(&ns->head->requeue_list, req); in nvme_failover_req()
171 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); in nvme_failover_req()
173 nvme_req(req)->status = 0; in nvme_failover_req()
175 kblockd_schedule_work(&ns->head->requeue_work); in nvme_failover_req()
180 struct nvme_ns *ns = rq->q->queuedata; in nvme_mpath_start_request()
181 struct gendisk *disk = ns->head->disk; in nvme_mpath_start_request()
183 if ((READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) && in nvme_mpath_start_request()
184 !(nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE)) { in nvme_mpath_start_request()
185 atomic_inc(&ns->ctrl->nr_active); in nvme_mpath_start_request()
186 nvme_req(rq)->flags |= NVME_MPATH_CNT_ACTIVE; in nvme_mpath_start_request()
189 if (!blk_queue_io_stat(disk->queue) || in nvme_mpath_start_request()
190 (nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) in nvme_mpath_start_request()
193 !blk_rq_passthrough_stats(rq, disk->queue)) in nvme_mpath_start_request()
196 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; in nvme_mpath_start_request()
197 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), in nvme_mpath_start_request()
204 struct nvme_ns *ns = rq->q->queuedata; in nvme_mpath_end_request()
206 if (nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE) in nvme_mpath_end_request()
207 atomic_dec_if_positive(&ns->ctrl->nr_active); in nvme_mpath_end_request()
209 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) in nvme_mpath_end_request()
211 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), in nvme_mpath_end_request()
213 nvme_req(rq)->start_time); in nvme_mpath_end_request()
216 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) in nvme_kick_requeue_lists() argument
221 srcu_idx = srcu_read_lock(&ctrl->srcu); in nvme_kick_requeue_lists()
222 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, in nvme_kick_requeue_lists()
223 srcu_read_lock_held(&ctrl->srcu)) { in nvme_kick_requeue_lists()
224 if (!ns->head->disk) in nvme_kick_requeue_lists()
226 kblockd_schedule_work(&ns->head->requeue_work); in nvme_kick_requeue_lists()
227 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) in nvme_kick_requeue_lists()
228 disk_uevent(ns->head->disk, KOBJ_CHANGE); in nvme_kick_requeue_lists()
230 srcu_read_unlock(&ctrl->srcu, srcu_idx); in nvme_kick_requeue_lists()
236 [NVME_ANA_NONOPTIMIZED] = "non-optimized",
238 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss",
244 struct nvme_ns_head *head = ns->head; in nvme_mpath_clear_current_path()
249 if (ns == rcu_access_pointer(head->current_path[node])) { in nvme_mpath_clear_current_path()
250 rcu_assign_pointer(head->current_path[node], NULL); in nvme_mpath_clear_current_path()
257 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) in nvme_mpath_clear_ctrl_paths() argument
262 srcu_idx = srcu_read_lock(&ctrl->srcu); in nvme_mpath_clear_ctrl_paths()
263 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, in nvme_mpath_clear_ctrl_paths()
264 srcu_read_lock_held(&ctrl->srcu)) { in nvme_mpath_clear_ctrl_paths()
266 kblockd_schedule_work(&ns->head->requeue_work); in nvme_mpath_clear_ctrl_paths()
268 srcu_read_unlock(&ctrl->srcu, srcu_idx); in nvme_mpath_clear_ctrl_paths()
273 sector_t capacity = get_capacity(head->disk); in nvme_mpath_revalidate_paths()
278 srcu_idx = srcu_read_lock(&head->srcu); in nvme_mpath_revalidate_paths()
279 list_for_each_entry_srcu(ns, &head->list, siblings, in nvme_mpath_revalidate_paths()
280 srcu_read_lock_held(&head->srcu)) { in nvme_mpath_revalidate_paths()
281 if (capacity != get_capacity(ns->disk)) in nvme_mpath_revalidate_paths()
282 clear_bit(NVME_NS_READY, &ns->flags); in nvme_mpath_revalidate_paths()
284 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_mpath_revalidate_paths()
287 rcu_assign_pointer(head->current_path[node], NULL); in nvme_mpath_revalidate_paths()
288 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_revalidate_paths()
294 struct gendisk *disk = head->disk; in nvme_mpath_revalidate_zones()
297 if (!disk || !blk_queue_is_zoned(disk->queue) || in nvme_mpath_revalidate_zones()
298 !test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) in nvme_mpath_revalidate_zones()
312 enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); in nvme_path_is_disabled()
321 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || in nvme_path_is_disabled()
322 !test_bit(NVME_NS_READY, &ns->flags)) in nvme_path_is_disabled()
328 __must_hold_shared(&head->srcu) in __nvme_find_path()
333 list_for_each_entry_srcu(ns, &head->list, siblings, in __nvme_find_path()
334 srcu_read_lock_held(&head->srcu)) { in __nvme_find_path()
338 if (ns->ctrl->numa_node != NUMA_NO_NODE && in __nvme_find_path()
339 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) in __nvme_find_path()
340 distance = node_distance(node, ns->ctrl->numa_node); in __nvme_find_path()
344 switch (ns->ana_state) { in __nvme_find_path()
365 rcu_assign_pointer(head->current_path[node], found); in __nvme_find_path()
371 __must_hold_shared(&head->srcu) in nvme_next_ns()
373 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, in nvme_next_ns()
377 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); in nvme_next_ns()
381 __must_hold_shared(&head->srcu) in nvme_round_robin_path()
385 struct nvme_ns *old = srcu_dereference(head->current_path[node], in nvme_round_robin_path()
386 &head->srcu); in nvme_round_robin_path()
391 if (list_is_singular(&head->list)) { in nvme_round_robin_path()
403 if (ns->ana_state == NVME_ANA_OPTIMIZED) { in nvme_round_robin_path()
407 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) in nvme_round_robin_path()
412 * The loop above skips the current path for round-robin semantics. in nvme_round_robin_path()
414 * - no other optimized path found and current is optimized, in nvme_round_robin_path()
415 * - no other usable path found and current is usable. in nvme_round_robin_path()
418 (old->ana_state == NVME_ANA_OPTIMIZED || in nvme_round_robin_path()
419 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) in nvme_round_robin_path()
425 rcu_assign_pointer(head->current_path[node], found); in nvme_round_robin_path()
430 __must_hold_shared(&head->srcu) in nvme_queue_depth_path()
436 list_for_each_entry_srcu(ns, &head->list, siblings, in nvme_queue_depth_path()
437 srcu_read_lock_held(&head->srcu)) { in nvme_queue_depth_path()
441 depth = atomic_read(&ns->ctrl->nr_active); in nvme_queue_depth_path()
443 switch (ns->ana_state) { in nvme_queue_depth_path()
469 return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE && in nvme_path_is_optimized()
470 ns->ana_state == NVME_ANA_OPTIMIZED; in nvme_path_is_optimized()
474 __must_hold_shared(&head->srcu) in nvme_numa_path()
479 ns = srcu_dereference(head->current_path[node], &head->srcu); in nvme_numa_path()
489 switch (READ_ONCE(head->subsys->iopolicy)) { in nvme_find_path()
500 __must_hold_shared(&head->srcu) in nvme_available_path()
504 if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) in nvme_available_path()
507 list_for_each_entry_srcu(ns, &head->list, siblings, in nvme_available_path()
508 srcu_read_lock_held(&head->srcu)) { in nvme_available_path()
509 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) in nvme_available_path()
511 switch (nvme_ctrl_state(ns->ctrl)) { in nvme_available_path()
522 * If "head->delayed_removal_secs" is configured (i.e., non-zero), do in nvme_available_path()
528 * non-zero, this flag is set to true. When zero, the flag is cleared. in nvme_available_path()
535 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; in nvme_ns_head_submit_bio()
536 struct device *dev = disk_to_dev(head->disk); in nvme_ns_head_submit_bio() local
549 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_submit_bio()
552 bio_set_dev(bio, ns->disk->part0); in nvme_ns_head_submit_bio()
555 * enters submit_bio_noacct() for the per-path device. The EOD in nvme_ns_head_submit_bio()
559 bio->bi_opf |= REQ_NVME_MPATH; in nvme_ns_head_submit_bio()
560 trace_block_bio_remap(bio, disk_devt(ns->head->disk), in nvme_ns_head_submit_bio()
561 bio->bi_iter.bi_sector); in nvme_ns_head_submit_bio()
564 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); in nvme_ns_head_submit_bio()
566 spin_lock_irq(&head->requeue_lock); in nvme_ns_head_submit_bio()
567 bio_list_add(&head->requeue_list, bio); in nvme_ns_head_submit_bio()
568 spin_unlock_irq(&head->requeue_lock); in nvme_ns_head_submit_bio()
569 atomic_long_inc(&head->io_requeue_no_usable_path_count); in nvme_ns_head_submit_bio()
571 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); in nvme_ns_head_submit_bio()
574 atomic_long_inc(&head->io_fail_no_available_path_count); in nvme_ns_head_submit_bio()
577 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_submit_bio()
582 if (!nvme_tryget_ns_head(disk->private_data)) in nvme_ns_head_open()
583 return -ENXIO; in nvme_ns_head_open()
589 nvme_put_ns_head(disk->private_data); in nvme_ns_head_release()
595 struct nvme_ns_head *head = disk->private_data; in nvme_ns_head_get_unique_id()
597 int srcu_idx, ret = -EWOULDBLOCK; in nvme_ns_head_get_unique_id()
599 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_get_unique_id()
603 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_get_unique_id()
611 struct nvme_ns_head *head = disk->private_data; in nvme_ns_head_report_zones()
613 int srcu_idx, ret = -EWOULDBLOCK; in nvme_ns_head_report_zones()
615 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_report_zones()
619 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_report_zones()
651 head->cdev_device.parent = &head->subsys->dev; in nvme_add_ns_head_cdev()
652 snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, in nvme_add_ns_head_cdev()
653 head->instance); in nvme_add_ns_head_cdev()
656 if (nvme_cdev_add(name, &head->cdev, &head->cdev_device, in nvme_add_ns_head_cdev()
658 dev_err(disk_to_dev(head->disk), in nvme_add_ns_head_cdev()
663 set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); in nvme_add_ns_head_cdev()
672 &head->disk->state))) in nvme_partition_scan_work()
675 mutex_lock(&head->disk->open_mutex); in nvme_partition_scan_work()
676 bdev_disk_changed(head->disk, false); in nvme_partition_scan_work()
677 mutex_unlock(&head->disk->open_mutex); in nvme_partition_scan_work()
686 spin_lock_irq(&head->requeue_lock); in nvme_requeue_work()
687 next = bio_list_get(&head->requeue_list); in nvme_requeue_work()
688 spin_unlock_irq(&head->requeue_lock); in nvme_requeue_work()
691 next = bio->bi_next; in nvme_requeue_work()
692 bio->bi_next = NULL; in nvme_requeue_work()
700 if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { in nvme_remove_head()
706 synchronize_srcu(&head->srcu); in nvme_remove_head()
707 kblockd_schedule_work(&head->requeue_work); in nvme_remove_head()
709 if (test_and_clear_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags)) in nvme_remove_head()
710 nvme_cdev_del(&head->cdev, &head->cdev_device); in nvme_remove_head()
711 del_gendisk(head->disk); in nvme_remove_head()
722 mutex_lock(&head->subsys->lock); in nvme_remove_head_work()
723 if (list_empty(&head->list)) { in nvme_remove_head_work()
724 list_del_init(&head->entry); in nvme_remove_head_work()
727 mutex_unlock(&head->subsys->lock); in nvme_remove_head_work()
734 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) in nvme_mpath_alloc_disk() argument
738 mutex_init(&head->lock); in nvme_mpath_alloc_disk()
739 spin_lock_init(&head->requeue_lock); in nvme_mpath_alloc_disk()
740 INIT_WORK(&head->requeue_work, nvme_requeue_work); in nvme_mpath_alloc_disk()
741 INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); in nvme_mpath_alloc_disk()
742 INIT_DELAYED_WORK(&head->remove_work, nvme_remove_head_work); in nvme_mpath_alloc_disk()
754 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || in nvme_mpath_alloc_disk()
759 if (!nvme_is_unique_nsid(ctrl, head)) in nvme_mpath_alloc_disk()
766 if (head->ids.csi == NVME_CSI_ZNS) in nvme_mpath_alloc_disk()
769 head->disk = blk_alloc_disk(&lim, ctrl->numa_node); in nvme_mpath_alloc_disk()
770 if (IS_ERR(head->disk)) in nvme_mpath_alloc_disk()
771 return PTR_ERR(head->disk); in nvme_mpath_alloc_disk()
772 head->disk->fops = &nvme_ns_head_ops; in nvme_mpath_alloc_disk()
773 head->disk->private_data = head; in nvme_mpath_alloc_disk()
783 set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); in nvme_mpath_alloc_disk()
784 sprintf(head->disk->disk_name, "nvme%dn%d", in nvme_mpath_alloc_disk()
785 ctrl->subsys->instance, head->instance); in nvme_mpath_alloc_disk()
792 struct nvme_ns_head *head = ns->head; in nvme_mpath_set_live()
795 if (!head->disk) in nvme_mpath_set_live()
803 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { in nvme_mpath_set_live()
804 rc = device_add_disk(&head->subsys->dev, head->disk, in nvme_mpath_set_live()
807 clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags); in nvme_mpath_set_live()
811 queue_work(nvme_wq, &head->partition_scan_work); in nvme_mpath_set_live()
814 nvme_mpath_add_sysfs_link(ns->head); in nvme_mpath_set_live()
816 mutex_lock(&head->lock); in nvme_mpath_set_live()
820 srcu_idx = srcu_read_lock(&head->srcu); in nvme_mpath_set_live()
823 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_mpath_set_live()
825 mutex_unlock(&head->lock); in nvme_mpath_set_live()
827 synchronize_srcu(&head->srcu); in nvme_mpath_set_live()
829 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_set_live()
832 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, in nvme_parse_ana_log() argument
833 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, in nvme_parse_ana_log() argument
835 __must_hold(&ctrl->ana_lock) in nvme_parse_ana_log()
837 void *base = ctrl->ana_log_buf; in nvme_parse_ana_log()
841 lockdep_assert_held(&ctrl->ana_lock); in nvme_parse_ana_log()
843 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { in nvme_parse_ana_log()
848 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) in nvme_parse_ana_log()
849 return -EINVAL; in nvme_parse_ana_log()
851 nr_nsids = le32_to_cpu(desc->nnsids); in nvme_parse_ana_log()
854 if (WARN_ON_ONCE(desc->grpid == 0)) in nvme_parse_ana_log()
855 return -EINVAL; in nvme_parse_ana_log()
856 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) in nvme_parse_ana_log()
857 return -EINVAL; in nvme_parse_ana_log()
858 if (WARN_ON_ONCE(desc->state == 0)) in nvme_parse_ana_log()
859 return -EINVAL; in nvme_parse_ana_log()
860 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) in nvme_parse_ana_log()
861 return -EINVAL; in nvme_parse_ana_log()
864 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) in nvme_parse_ana_log()
865 return -EINVAL; in nvme_parse_ana_log()
867 error = cb(ctrl, desc, data); in nvme_parse_ana_log()
885 ns->ana_grpid = le32_to_cpu(desc->grpid); in nvme_update_ns_ana_state()
886 ns->ana_state = desc->state; in nvme_update_ns_ana_state()
887 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); in nvme_update_ns_ana_state()
892 * nvme_mpath_init_identify() and the ctrl will never complete in nvme_update_ns_ana_state()
897 if (nvme_state_is_live(ns->ana_state) && in nvme_update_ns_ana_state()
898 nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) in nvme_update_ns_ana_state()
905 * or non-optimized) while we alloc the ns then sysfs link would in nvme_update_ns_ana_state()
916 if (test_bit(NVME_NSHEAD_DISK_LIVE, &ns->head->flags)) in nvme_update_ns_ana_state()
917 nvme_mpath_add_sysfs_link(ns->head); in nvme_update_ns_ana_state()
921 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, in nvme_update_ana_state() argument
924 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; in nvme_update_ana_state()
929 dev_dbg(ctrl->device, "ANA group %d: %s.\n", in nvme_update_ana_state()
930 le32_to_cpu(desc->grpid), in nvme_update_ana_state()
931 nvme_ana_state_names[desc->state]); in nvme_update_ana_state()
933 if (desc->state == NVME_ANA_CHANGE) in nvme_update_ana_state()
939 srcu_idx = srcu_read_lock(&ctrl->srcu); in nvme_update_ana_state()
940 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, in nvme_update_ana_state()
941 srcu_read_lock_held(&ctrl->srcu)) { in nvme_update_ana_state()
944 nsid = le32_to_cpu(desc->nsids[n]); in nvme_update_ana_state()
945 if (ns->head->ns_id < nsid) in nvme_update_ana_state()
947 if (ns->head->ns_id == nsid) in nvme_update_ana_state()
951 if (ns->head->ns_id > nsid) in nvme_update_ana_state()
954 srcu_read_unlock(&ctrl->srcu, srcu_idx); in nvme_update_ana_state()
958 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) in nvme_read_ana_log() argument
963 mutex_lock(&ctrl->ana_lock); in nvme_read_ana_log()
964 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, in nvme_read_ana_log()
965 ctrl->ana_log_buf, ctrl->ana_log_size, 0); in nvme_read_ana_log()
967 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); in nvme_read_ana_log()
971 error = nvme_parse_ana_log(ctrl, &nr_change_groups, in nvme_read_ana_log()
988 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); in nvme_read_ana_log()
990 timer_delete_sync(&ctrl->anatt_timer); in nvme_read_ana_log()
992 mutex_unlock(&ctrl->ana_lock); in nvme_read_ana_log()
998 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); in nvme_ana_work() local
1000 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) in nvme_ana_work()
1003 nvme_read_ana_log(ctrl); in nvme_ana_work()
1006 void nvme_mpath_update(struct nvme_ctrl *ctrl) in nvme_mpath_update() argument
1010 if (!ctrl->ana_log_buf) in nvme_mpath_update()
1013 mutex_lock(&ctrl->ana_lock); in nvme_mpath_update()
1014 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); in nvme_mpath_update()
1015 mutex_unlock(&ctrl->ana_lock); in nvme_mpath_update()
1020 struct nvme_ctrl *ctrl = timer_container_of(ctrl, t, anatt_timer); in nvme_anatt_timeout() local
1022 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); in nvme_anatt_timeout()
1023 nvme_reset_ctrl(ctrl); in nvme_anatt_timeout()
1026 void nvme_mpath_stop(struct nvme_ctrl *ctrl) in nvme_mpath_stop() argument
1028 if (!nvme_ctrl_use_ana(ctrl)) in nvme_mpath_stop()
1030 timer_delete_sync(&ctrl->anatt_timer); in nvme_mpath_stop()
1031 cancel_work_sync(&ctrl->ana_work); in nvme_mpath_stop()
1038 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, in nvme_subsys_iopolicy_show() argument
1042 container_of(dev, struct nvme_subsystem, dev); in nvme_subsys_iopolicy_show()
1045 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); in nvme_subsys_iopolicy_show()
1051 struct nvme_ctrl *ctrl; in nvme_subsys_iopolicy_update() local
1052 int old_iopolicy = READ_ONCE(subsys->iopolicy); in nvme_subsys_iopolicy_update()
1057 WRITE_ONCE(subsys->iopolicy, iopolicy); in nvme_subsys_iopolicy_update()
1061 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) in nvme_subsys_iopolicy_update()
1062 nvme_mpath_clear_ctrl_paths(ctrl); in nvme_subsys_iopolicy_update()
1066 subsys->subnqn, in nvme_subsys_iopolicy_update()
1071 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, in nvme_subsys_iopolicy_store() argument
1075 container_of(dev, struct nvme_subsystem, dev); in nvme_subsys_iopolicy_store()
1088 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, in ana_grpid_show() argument
1091 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); in ana_grpid_show()
1095 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, in ana_state_show() argument
1098 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); in ana_state_show()
1100 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); in ana_state_show()
1104 static ssize_t queue_depth_show(struct device *dev, in queue_depth_show() argument
1107 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); in queue_depth_show()
1109 if (ns->head->subsys->iopolicy != NVME_IOPOLICY_QD) in queue_depth_show()
1112 return sysfs_emit(buf, "%d\n", atomic_read(&ns->ctrl->nr_active)); in queue_depth_show()
1116 static ssize_t numa_nodes_show(struct device *dev, struct device_attribute *attr, in numa_nodes_show() argument
1122 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); in numa_nodes_show()
1123 struct nvme_ns_head *head = ns->head; in numa_nodes_show()
1125 if (head->subsys->iopolicy != NVME_IOPOLICY_NUMA) in numa_nodes_show()
1130 srcu_idx = srcu_read_lock(&head->srcu); in numa_nodes_show()
1132 current_ns = srcu_dereference(head->current_path[node], in numa_nodes_show()
1133 &head->srcu); in numa_nodes_show()
1137 srcu_read_unlock(&head->srcu, srcu_idx); in numa_nodes_show()
1143 static ssize_t delayed_removal_secs_show(struct device *dev, in delayed_removal_secs_show() argument
1146 struct gendisk *disk = dev_to_disk(dev); in delayed_removal_secs_show()
1147 struct nvme_ns_head *head = disk->private_data; in delayed_removal_secs_show()
1150 mutex_lock(&head->subsys->lock); in delayed_removal_secs_show()
1151 ret = sysfs_emit(buf, "%u\n", head->delayed_removal_secs); in delayed_removal_secs_show()
1152 mutex_unlock(&head->subsys->lock); in delayed_removal_secs_show()
1156 static ssize_t delayed_removal_secs_store(struct device *dev, in delayed_removal_secs_store() argument
1159 struct gendisk *disk = dev_to_disk(dev); in delayed_removal_secs_store()
1160 struct nvme_ns_head *head = disk->private_data; in delayed_removal_secs_store()
1168 mutex_lock(&head->subsys->lock); in delayed_removal_secs_store()
1169 head->delayed_removal_secs = sec; in delayed_removal_secs_store()
1171 set_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags); in delayed_removal_secs_store()
1173 clear_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags); in delayed_removal_secs_store()
1174 mutex_unlock(&head->subsys->lock); in delayed_removal_secs_store()
1179 synchronize_srcu(&head->srcu); in delayed_removal_secs_store()
1186 static ssize_t multipath_failover_count_show(struct device *dev, in multipath_failover_count_show() argument
1189 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); in multipath_failover_count_show()
1191 return sysfs_emit(buf, "%lu\n", atomic_long_read(&ns->failover)); in multipath_failover_count_show()
1194 static ssize_t multipath_failover_count_store(struct device *dev, in multipath_failover_count_store() argument
1199 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); in multipath_failover_count_store()
1203 return -EINVAL; in multipath_failover_count_store()
1205 atomic_long_set(&ns->failover, failover); in multipath_failover_count_store()
1212 static ssize_t io_requeue_no_usable_path_count_show(struct device *dev, in io_requeue_no_usable_path_count_show() argument
1215 struct gendisk *disk = dev_to_disk(dev); in io_requeue_no_usable_path_count_show()
1216 struct nvme_ns_head *head = disk->private_data; in io_requeue_no_usable_path_count_show()
1219 atomic_long_read(&head->io_requeue_no_usable_path_count)); in io_requeue_no_usable_path_count_show()
1222 static ssize_t io_requeue_no_usable_path_count_store(struct device *dev, in io_requeue_no_usable_path_count_store() argument
1227 struct gendisk *disk = dev_to_disk(dev); in io_requeue_no_usable_path_count_store()
1228 struct nvme_ns_head *head = disk->private_data; in io_requeue_no_usable_path_count_store()
1232 return -EINVAL; in io_requeue_no_usable_path_count_store()
1234 atomic_long_set(&head->io_requeue_no_usable_path_count, requeue_cnt); in io_requeue_no_usable_path_count_store()
1241 static ssize_t io_fail_no_available_path_count_show(struct device *dev, in io_fail_no_available_path_count_show() argument
1244 struct gendisk *disk = dev_to_disk(dev); in io_fail_no_available_path_count_show()
1245 struct nvme_ns_head *head = disk->private_data; in io_fail_no_available_path_count_show()
1248 atomic_long_read(&head->io_fail_no_available_path_count)); in io_fail_no_available_path_count_show()
1251 static ssize_t io_fail_no_available_path_count_store(struct device *dev, in io_fail_no_available_path_count_store() argument
1256 struct gendisk *disk = dev_to_disk(dev); in io_fail_no_available_path_count_store()
1257 struct nvme_ns_head *head = disk->private_data; in io_fail_no_available_path_count_store()
1261 return -EINVAL; in io_fail_no_available_path_count_store()
1263 atomic_long_set(&head->io_fail_no_available_path_count, fail_cnt); in io_fail_no_available_path_count_store()
1270 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, in nvme_lookup_ana_group_desc() argument
1275 if (desc->grpid != dst->grpid) in nvme_lookup_ana_group_desc()
1279 return -ENXIO; /* just break out of the loop */ in nvme_lookup_ana_group_desc()
1293 if (!test_bit(GD_ADDED, &head->disk->state)) in nvme_mpath_add_sysfs_link()
1296 kobj = &disk_to_dev(head->disk)->kobj; in nvme_mpath_add_sysfs_link()
1299 * loop through each ns chained through the head->list and create the in nvme_mpath_add_sysfs_link()
1302 srcu_idx = srcu_read_lock(&head->srcu); in nvme_mpath_add_sysfs_link()
1304 list_for_each_entry_srcu(ns, &head->list, siblings, in nvme_mpath_add_sysfs_link()
1305 srcu_read_lock_held(&head->srcu)) { in nvme_mpath_add_sysfs_link()
1310 if (!test_bit(GD_ADDED, &ns->disk->state)) in nvme_mpath_add_sysfs_link()
1315 * When path ana state transitions from optimized to non- in nvme_mpath_add_sysfs_link()
1316 * optimized or vice-versa, the nvme_mpath_set_live() is in nvme_mpath_add_sysfs_link()
1318 * link already exists for the given path and we attempt to re- in nvme_mpath_add_sysfs_link()
1325 if (test_and_set_bit(NVME_NS_SYSFS_ATTR_LINK, &ns->flags)) in nvme_mpath_add_sysfs_link()
1328 target = disk_to_dev(ns->disk); in nvme_mpath_add_sysfs_link()
1331 * ns path gendisk kobject @target->kobj. in nvme_mpath_add_sysfs_link()
1334 &target->kobj, dev_name(target)); in nvme_mpath_add_sysfs_link()
1336 dev_err(disk_to_dev(ns->head->disk), in nvme_mpath_add_sysfs_link()
1339 clear_bit(NVME_NS_SYSFS_ATTR_LINK, &ns->flags); in nvme_mpath_add_sysfs_link()
1343 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_mpath_add_sysfs_link()
1351 if (!test_bit(NVME_NS_SYSFS_ATTR_LINK, &ns->flags)) in nvme_mpath_remove_sysfs_link()
1354 target = disk_to_dev(ns->disk); in nvme_mpath_remove_sysfs_link()
1355 kobj = &disk_to_dev(ns->head->disk)->kobj; in nvme_mpath_remove_sysfs_link()
1358 clear_bit(NVME_NS_SYSFS_ATTR_LINK, &ns->flags); in nvme_mpath_remove_sysfs_link()
1363 if (nvme_ctrl_use_ana(ns->ctrl)) { in nvme_mpath_add_disk()
1369 mutex_lock(&ns->ctrl->ana_lock); in nvme_mpath_add_disk()
1370 ns->ana_grpid = le32_to_cpu(anagrpid); in nvme_mpath_add_disk()
1371 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); in nvme_mpath_add_disk()
1372 mutex_unlock(&ns->ctrl->ana_lock); in nvme_mpath_add_disk()
1377 /* group desc not found: trigger a re-read */ in nvme_mpath_add_disk()
1378 set_bit(NVME_NS_ANA_PENDING, &ns->flags); in nvme_mpath_add_disk()
1379 queue_work(nvme_wq, &ns->ctrl->ana_work); in nvme_mpath_add_disk()
1382 ns->ana_state = NVME_ANA_OPTIMIZED; in nvme_mpath_add_disk()
1392 if (!head->disk) in nvme_mpath_remove_disk()
1395 mutex_lock(&head->subsys->lock); in nvme_mpath_remove_disk()
1398 * head->list is expected to be empty. However, nvme_ns_remove() and in nvme_mpath_remove_disk()
1399 * nvme_init_ns_head() can run concurrently and so if head->delayed_ in nvme_mpath_remove_disk()
1401 * this point, head->list may no longer be empty. Therefore, we recheck in nvme_mpath_remove_disk()
1402 * head->list here. If it is no longer empty then we skip enqueuing the in nvme_mpath_remove_disk()
1405 if (!list_empty(&head->list)) in nvme_mpath_remove_disk()
1412 if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) { in nvme_mpath_remove_disk()
1413 mod_delayed_work(nvme_wq, &head->remove_work, in nvme_mpath_remove_disk()
1414 head->delayed_removal_secs * HZ); in nvme_mpath_remove_disk()
1416 list_del_init(&head->entry); in nvme_mpath_remove_disk()
1420 mutex_unlock(&head->subsys->lock); in nvme_mpath_remove_disk()
1427 if (!head->disk) in nvme_mpath_put_disk()
1430 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_put_disk()
1431 flush_work(&head->requeue_work); in nvme_mpath_put_disk()
1432 flush_work(&head->partition_scan_work); in nvme_mpath_put_disk()
1433 put_disk(head->disk); in nvme_mpath_put_disk()
1436 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) in nvme_mpath_init_ctrl() argument
1438 mutex_init(&ctrl->ana_lock); in nvme_mpath_init_ctrl()
1439 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); in nvme_mpath_init_ctrl()
1440 INIT_WORK(&ctrl->ana_work, nvme_ana_work); in nvme_mpath_init_ctrl()
1443 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) in nvme_mpath_init_identify() argument
1445 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; in nvme_mpath_init_identify()
1450 if (!multipath || !ctrl->subsys || in nvme_mpath_init_identify()
1451 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) in nvme_mpath_init_identify()
1455 atomic_set(&ctrl->nr_active, 0); in nvme_mpath_init_identify()
1457 if (!ctrl->max_namespaces || in nvme_mpath_init_identify()
1458 ctrl->max_namespaces > le32_to_cpu(id->nn)) { in nvme_mpath_init_identify()
1459 dev_err(ctrl->device, in nvme_mpath_init_identify()
1460 "Invalid MNAN value %u\n", ctrl->max_namespaces); in nvme_mpath_init_identify()
1461 return -EINVAL; in nvme_mpath_init_identify()
1464 ctrl->anacap = id->anacap; in nvme_mpath_init_identify()
1465 ctrl->anatt = id->anatt; in nvme_mpath_init_identify()
1466 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); in nvme_mpath_init_identify()
1467 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); in nvme_mpath_init_identify()
1470 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + in nvme_mpath_init_identify()
1471 ctrl->max_namespaces * sizeof(__le32); in nvme_mpath_init_identify()
1473 dev_err(ctrl->device, in nvme_mpath_init_identify()
1476 dev_err(ctrl->device, "disabling ANA support.\n"); in nvme_mpath_init_identify()
1479 if (ana_log_size > ctrl->ana_log_size) { in nvme_mpath_init_identify()
1480 nvme_mpath_stop(ctrl); in nvme_mpath_init_identify()
1481 nvme_mpath_uninit(ctrl); in nvme_mpath_init_identify()
1482 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); in nvme_mpath_init_identify()
1483 if (!ctrl->ana_log_buf) in nvme_mpath_init_identify()
1484 return -ENOMEM; in nvme_mpath_init_identify()
1486 ctrl->ana_log_size = ana_log_size; in nvme_mpath_init_identify()
1487 error = nvme_read_ana_log(ctrl); in nvme_mpath_init_identify()
1493 nvme_mpath_uninit(ctrl); in nvme_mpath_init_identify()
1497 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) in nvme_mpath_uninit() argument
1499 kvfree(ctrl->ana_log_buf); in nvme_mpath_uninit()
1500 ctrl->ana_log_buf = NULL; in nvme_mpath_uninit()
1501 ctrl->ana_log_size = 0; in nvme_mpath_uninit()