1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2017-2018 Christoph Hellwig.
4 */
5
6 #include <linux/backing-dev.h>
7 #include <linux/moduleparam.h>
8 #include <linux/vmalloc.h>
9 #include <trace/events/block.h>
10 #include "nvme.h"
11
12 bool multipath = true;
13 module_param(multipath, bool, 0444);
14 MODULE_PARM_DESC(multipath,
15 "turn on native support for multiple controllers per subsystem");
16
17 static const char *nvme_iopolicy_names[] = {
18 [NVME_IOPOLICY_NUMA] = "numa",
19 [NVME_IOPOLICY_RR] = "round-robin",
20 [NVME_IOPOLICY_QD] = "queue-depth",
21 };
22
23 static int iopolicy = NVME_IOPOLICY_NUMA;
24
nvme_set_iopolicy(const char * val,const struct kernel_param * kp)25 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
26 {
27 if (!val)
28 return -EINVAL;
29 if (!strncmp(val, "numa", 4))
30 iopolicy = NVME_IOPOLICY_NUMA;
31 else if (!strncmp(val, "round-robin", 11))
32 iopolicy = NVME_IOPOLICY_RR;
33 else if (!strncmp(val, "queue-depth", 11))
34 iopolicy = NVME_IOPOLICY_QD;
35 else
36 return -EINVAL;
37
38 return 0;
39 }
40
nvme_get_iopolicy(char * buf,const struct kernel_param * kp)41 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
42 {
43 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
44 }
45
46 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
47 &iopolicy, 0644);
48 MODULE_PARM_DESC(iopolicy,
49 "Default multipath I/O policy; 'numa' (default), 'round-robin' or 'queue-depth'");
50
nvme_mpath_default_iopolicy(struct nvme_subsystem * subsys)51 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
52 {
53 subsys->iopolicy = iopolicy;
54 }
55
nvme_mpath_unfreeze(struct nvme_subsystem * subsys)56 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
57 {
58 struct nvme_ns_head *h;
59
60 lockdep_assert_held(&subsys->lock);
61 list_for_each_entry(h, &subsys->nsheads, entry)
62 if (h->disk)
63 blk_mq_unfreeze_queue(h->disk->queue);
64 }
65
nvme_mpath_wait_freeze(struct nvme_subsystem * subsys)66 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
67 {
68 struct nvme_ns_head *h;
69
70 lockdep_assert_held(&subsys->lock);
71 list_for_each_entry(h, &subsys->nsheads, entry)
72 if (h->disk)
73 blk_mq_freeze_queue_wait(h->disk->queue);
74 }
75
nvme_mpath_start_freeze(struct nvme_subsystem * subsys)76 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
77 {
78 struct nvme_ns_head *h;
79
80 lockdep_assert_held(&subsys->lock);
81 list_for_each_entry(h, &subsys->nsheads, entry)
82 if (h->disk)
83 blk_freeze_queue_start(h->disk->queue);
84 }
85
nvme_failover_req(struct request * req)86 void nvme_failover_req(struct request *req)
87 {
88 struct nvme_ns *ns = req->q->queuedata;
89 u16 status = nvme_req(req)->status & NVME_SCT_SC_MASK;
90 unsigned long flags;
91 struct bio *bio;
92
93 nvme_mpath_clear_current_path(ns);
94
95 /*
96 * If we got back an ANA error, we know the controller is alive but not
97 * ready to serve this namespace. Kick of a re-read of the ANA
98 * information page, and just try any other available path for now.
99 */
100 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {
101 set_bit(NVME_NS_ANA_PENDING, &ns->flags);
102 queue_work(nvme_wq, &ns->ctrl->ana_work);
103 }
104
105 spin_lock_irqsave(&ns->head->requeue_lock, flags);
106 for (bio = req->bio; bio; bio = bio->bi_next) {
107 bio_set_dev(bio, ns->head->disk->part0);
108 if (bio->bi_opf & REQ_POLLED) {
109 bio->bi_opf &= ~REQ_POLLED;
110 bio->bi_cookie = BLK_QC_T_NONE;
111 }
112 /*
113 * The alternate request queue that we may end up submitting
114 * the bio to may be frozen temporarily, in this case REQ_NOWAIT
115 * will fail the I/O immediately with EAGAIN to the issuer.
116 * We are not in the issuer context which cannot block. Clear
117 * the flag to avoid spurious EAGAIN I/O failures.
118 */
119 bio->bi_opf &= ~REQ_NOWAIT;
120 }
121 blk_steal_bios(&ns->head->requeue_list, req);
122 spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
123
124 nvme_req(req)->status = 0;
125 nvme_end_req(req);
126 kblockd_schedule_work(&ns->head->requeue_work);
127 }
128
nvme_mpath_start_request(struct request * rq)129 void nvme_mpath_start_request(struct request *rq)
130 {
131 struct nvme_ns *ns = rq->q->queuedata;
132 struct gendisk *disk = ns->head->disk;
133
134 if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {
135 atomic_inc(&ns->ctrl->nr_active);
136 nvme_req(rq)->flags |= NVME_MPATH_CNT_ACTIVE;
137 }
138
139 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
140 return;
141
142 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
143 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
144 jiffies);
145 }
146 EXPORT_SYMBOL_GPL(nvme_mpath_start_request);
147
nvme_mpath_end_request(struct request * rq)148 void nvme_mpath_end_request(struct request *rq)
149 {
150 struct nvme_ns *ns = rq->q->queuedata;
151
152 if (nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE)
153 atomic_dec_if_positive(&ns->ctrl->nr_active);
154
155 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
156 return;
157 bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
158 blk_rq_bytes(rq) >> SECTOR_SHIFT,
159 nvme_req(rq)->start_time);
160 }
161
nvme_kick_requeue_lists(struct nvme_ctrl * ctrl)162 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
163 {
164 struct nvme_ns *ns;
165 int srcu_idx;
166
167 srcu_idx = srcu_read_lock(&ctrl->srcu);
168 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
169 if (!ns->head->disk)
170 continue;
171 kblockd_schedule_work(&ns->head->requeue_work);
172 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)
173 disk_uevent(ns->head->disk, KOBJ_CHANGE);
174 }
175 srcu_read_unlock(&ctrl->srcu, srcu_idx);
176 }
177
178 static const char *nvme_ana_state_names[] = {
179 [0] = "invalid state",
180 [NVME_ANA_OPTIMIZED] = "optimized",
181 [NVME_ANA_NONOPTIMIZED] = "non-optimized",
182 [NVME_ANA_INACCESSIBLE] = "inaccessible",
183 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss",
184 [NVME_ANA_CHANGE] = "change",
185 };
186
nvme_mpath_clear_current_path(struct nvme_ns * ns)187 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
188 {
189 struct nvme_ns_head *head = ns->head;
190 bool changed = false;
191 int node;
192
193 if (!head)
194 goto out;
195
196 for_each_node(node) {
197 if (ns == rcu_access_pointer(head->current_path[node])) {
198 rcu_assign_pointer(head->current_path[node], NULL);
199 changed = true;
200 }
201 }
202 out:
203 return changed;
204 }
205
nvme_mpath_clear_ctrl_paths(struct nvme_ctrl * ctrl)206 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
207 {
208 struct nvme_ns *ns;
209 int srcu_idx;
210
211 srcu_idx = srcu_read_lock(&ctrl->srcu);
212 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
213 nvme_mpath_clear_current_path(ns);
214 kblockd_schedule_work(&ns->head->requeue_work);
215 }
216 srcu_read_unlock(&ctrl->srcu, srcu_idx);
217 }
218
nvme_mpath_revalidate_paths(struct nvme_ns * ns)219 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
220 {
221 struct nvme_ns_head *head = ns->head;
222 sector_t capacity = get_capacity(head->disk);
223 int node;
224 int srcu_idx;
225
226 srcu_idx = srcu_read_lock(&head->srcu);
227 list_for_each_entry_rcu(ns, &head->list, siblings) {
228 if (capacity != get_capacity(ns->disk))
229 clear_bit(NVME_NS_READY, &ns->flags);
230 }
231 srcu_read_unlock(&head->srcu, srcu_idx);
232
233 for_each_node(node)
234 rcu_assign_pointer(head->current_path[node], NULL);
235 kblockd_schedule_work(&head->requeue_work);
236 }
237
nvme_path_is_disabled(struct nvme_ns * ns)238 static bool nvme_path_is_disabled(struct nvme_ns *ns)
239 {
240 enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);
241
242 /*
243 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should
244 * still be able to complete assuming that the controller is connected.
245 * Otherwise it will fail immediately and return to the requeue list.
246 */
247 if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING)
248 return true;
249 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
250 !test_bit(NVME_NS_READY, &ns->flags))
251 return true;
252 return false;
253 }
254
__nvme_find_path(struct nvme_ns_head * head,int node)255 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
256 {
257 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
258 struct nvme_ns *found = NULL, *fallback = NULL, *ns;
259
260 list_for_each_entry_rcu(ns, &head->list, siblings) {
261 if (nvme_path_is_disabled(ns))
262 continue;
263
264 if (ns->ctrl->numa_node != NUMA_NO_NODE &&
265 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
266 distance = node_distance(node, ns->ctrl->numa_node);
267 else
268 distance = LOCAL_DISTANCE;
269
270 switch (ns->ana_state) {
271 case NVME_ANA_OPTIMIZED:
272 if (distance < found_distance) {
273 found_distance = distance;
274 found = ns;
275 }
276 break;
277 case NVME_ANA_NONOPTIMIZED:
278 if (distance < fallback_distance) {
279 fallback_distance = distance;
280 fallback = ns;
281 }
282 break;
283 default:
284 break;
285 }
286 }
287
288 if (!found)
289 found = fallback;
290 if (found)
291 rcu_assign_pointer(head->current_path[node], found);
292 return found;
293 }
294
nvme_next_ns(struct nvme_ns_head * head,struct nvme_ns * ns)295 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
296 struct nvme_ns *ns)
297 {
298 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
299 siblings);
300 if (ns)
301 return ns;
302 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
303 }
304
nvme_round_robin_path(struct nvme_ns_head * head)305 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)
306 {
307 struct nvme_ns *ns, *found = NULL;
308 int node = numa_node_id();
309 struct nvme_ns *old = srcu_dereference(head->current_path[node],
310 &head->srcu);
311
312 if (unlikely(!old))
313 return __nvme_find_path(head, node);
314
315 if (list_is_singular(&head->list)) {
316 if (nvme_path_is_disabled(old))
317 return NULL;
318 return old;
319 }
320
321 for (ns = nvme_next_ns(head, old);
322 ns && ns != old;
323 ns = nvme_next_ns(head, ns)) {
324 if (nvme_path_is_disabled(ns))
325 continue;
326
327 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
328 found = ns;
329 goto out;
330 }
331 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
332 found = ns;
333 }
334
335 /*
336 * The loop above skips the current path for round-robin semantics.
337 * Fall back to the current path if either:
338 * - no other optimized path found and current is optimized,
339 * - no other usable path found and current is usable.
340 */
341 if (!nvme_path_is_disabled(old) &&
342 (old->ana_state == NVME_ANA_OPTIMIZED ||
343 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED)))
344 return old;
345
346 if (!found)
347 return NULL;
348 out:
349 rcu_assign_pointer(head->current_path[node], found);
350 return found;
351 }
352
nvme_queue_depth_path(struct nvme_ns_head * head)353 static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head)
354 {
355 struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns;
356 unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX;
357 unsigned int depth;
358
359 list_for_each_entry_rcu(ns, &head->list, siblings) {
360 if (nvme_path_is_disabled(ns))
361 continue;
362
363 depth = atomic_read(&ns->ctrl->nr_active);
364
365 switch (ns->ana_state) {
366 case NVME_ANA_OPTIMIZED:
367 if (depth < min_depth_opt) {
368 min_depth_opt = depth;
369 best_opt = ns;
370 }
371 break;
372 case NVME_ANA_NONOPTIMIZED:
373 if (depth < min_depth_nonopt) {
374 min_depth_nonopt = depth;
375 best_nonopt = ns;
376 }
377 break;
378 default:
379 break;
380 }
381
382 if (min_depth_opt == 0)
383 return best_opt;
384 }
385
386 return best_opt ? best_opt : best_nonopt;
387 }
388
nvme_path_is_optimized(struct nvme_ns * ns)389 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
390 {
391 return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
392 ns->ana_state == NVME_ANA_OPTIMIZED;
393 }
394
nvme_numa_path(struct nvme_ns_head * head)395 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
396 {
397 int node = numa_node_id();
398 struct nvme_ns *ns;
399
400 ns = srcu_dereference(head->current_path[node], &head->srcu);
401 if (unlikely(!ns))
402 return __nvme_find_path(head, node);
403 if (unlikely(!nvme_path_is_optimized(ns)))
404 return __nvme_find_path(head, node);
405 return ns;
406 }
407
nvme_find_path(struct nvme_ns_head * head)408 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
409 {
410 switch (READ_ONCE(head->subsys->iopolicy)) {
411 case NVME_IOPOLICY_QD:
412 return nvme_queue_depth_path(head);
413 case NVME_IOPOLICY_RR:
414 return nvme_round_robin_path(head);
415 default:
416 return nvme_numa_path(head);
417 }
418 }
419
nvme_available_path(struct nvme_ns_head * head)420 static bool nvme_available_path(struct nvme_ns_head *head)
421 {
422 struct nvme_ns *ns;
423
424 if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags))
425 return NULL;
426
427 list_for_each_entry_rcu(ns, &head->list, siblings) {
428 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
429 continue;
430 switch (nvme_ctrl_state(ns->ctrl)) {
431 case NVME_CTRL_LIVE:
432 case NVME_CTRL_RESETTING:
433 case NVME_CTRL_CONNECTING:
434 /* fallthru */
435 return true;
436 default:
437 break;
438 }
439 }
440 return false;
441 }
442
nvme_ns_head_submit_bio(struct bio * bio)443 static void nvme_ns_head_submit_bio(struct bio *bio)
444 {
445 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
446 struct device *dev = disk_to_dev(head->disk);
447 struct nvme_ns *ns;
448 int srcu_idx;
449
450 /*
451 * The namespace might be going away and the bio might be moved to a
452 * different queue via blk_steal_bios(), so we need to use the bio_split
453 * pool from the original queue to allocate the bvecs from.
454 */
455 bio = bio_split_to_limits(bio);
456 if (!bio)
457 return;
458
459 srcu_idx = srcu_read_lock(&head->srcu);
460 ns = nvme_find_path(head);
461 if (likely(ns)) {
462 bio_set_dev(bio, ns->disk->part0);
463 bio->bi_opf |= REQ_NVME_MPATH;
464 trace_block_bio_remap(bio, disk_devt(ns->head->disk),
465 bio->bi_iter.bi_sector);
466 submit_bio_noacct(bio);
467 } else if (nvme_available_path(head)) {
468 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
469
470 spin_lock_irq(&head->requeue_lock);
471 bio_list_add(&head->requeue_list, bio);
472 spin_unlock_irq(&head->requeue_lock);
473 } else {
474 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
475
476 bio_io_error(bio);
477 }
478
479 srcu_read_unlock(&head->srcu, srcu_idx);
480 }
481
nvme_ns_head_open(struct gendisk * disk,blk_mode_t mode)482 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode)
483 {
484 if (!nvme_tryget_ns_head(disk->private_data))
485 return -ENXIO;
486 return 0;
487 }
488
nvme_ns_head_release(struct gendisk * disk)489 static void nvme_ns_head_release(struct gendisk *disk)
490 {
491 nvme_put_ns_head(disk->private_data);
492 }
493
nvme_ns_head_get_unique_id(struct gendisk * disk,u8 id[16],enum blk_unique_id type)494 static int nvme_ns_head_get_unique_id(struct gendisk *disk, u8 id[16],
495 enum blk_unique_id type)
496 {
497 struct nvme_ns_head *head = disk->private_data;
498 struct nvme_ns *ns;
499 int srcu_idx, ret = -EWOULDBLOCK;
500
501 srcu_idx = srcu_read_lock(&head->srcu);
502 ns = nvme_find_path(head);
503 if (ns)
504 ret = nvme_ns_get_unique_id(ns, id, type);
505 srcu_read_unlock(&head->srcu, srcu_idx);
506 return ret;
507 }
508
509 #ifdef CONFIG_BLK_DEV_ZONED
nvme_ns_head_report_zones(struct gendisk * disk,sector_t sector,unsigned int nr_zones,report_zones_cb cb,void * data)510 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
511 unsigned int nr_zones, report_zones_cb cb, void *data)
512 {
513 struct nvme_ns_head *head = disk->private_data;
514 struct nvme_ns *ns;
515 int srcu_idx, ret = -EWOULDBLOCK;
516
517 srcu_idx = srcu_read_lock(&head->srcu);
518 ns = nvme_find_path(head);
519 if (ns)
520 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);
521 srcu_read_unlock(&head->srcu, srcu_idx);
522 return ret;
523 }
524 #else
525 #define nvme_ns_head_report_zones NULL
526 #endif /* CONFIG_BLK_DEV_ZONED */
527
528 const struct block_device_operations nvme_ns_head_ops = {
529 .owner = THIS_MODULE,
530 .submit_bio = nvme_ns_head_submit_bio,
531 .open = nvme_ns_head_open,
532 .release = nvme_ns_head_release,
533 .ioctl = nvme_ns_head_ioctl,
534 .compat_ioctl = blkdev_compat_ptr_ioctl,
535 .getgeo = nvme_getgeo,
536 .get_unique_id = nvme_ns_head_get_unique_id,
537 .report_zones = nvme_ns_head_report_zones,
538 .pr_ops = &nvme_pr_ops,
539 };
540
cdev_to_ns_head(struct cdev * cdev)541 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev)
542 {
543 return container_of(cdev, struct nvme_ns_head, cdev);
544 }
545
nvme_ns_head_chr_open(struct inode * inode,struct file * file)546 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file)
547 {
548 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev)))
549 return -ENXIO;
550 return 0;
551 }
552
nvme_ns_head_chr_release(struct inode * inode,struct file * file)553 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file)
554 {
555 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev));
556 return 0;
557 }
558
559 static const struct file_operations nvme_ns_head_chr_fops = {
560 .owner = THIS_MODULE,
561 .open = nvme_ns_head_chr_open,
562 .release = nvme_ns_head_chr_release,
563 .unlocked_ioctl = nvme_ns_head_chr_ioctl,
564 .compat_ioctl = compat_ptr_ioctl,
565 .uring_cmd = nvme_ns_head_chr_uring_cmd,
566 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
567 };
568
nvme_add_ns_head_cdev(struct nvme_ns_head * head)569 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
570 {
571 int ret;
572
573 head->cdev_device.parent = &head->subsys->dev;
574 ret = dev_set_name(&head->cdev_device, "ng%dn%d",
575 head->subsys->instance, head->instance);
576 if (ret)
577 return ret;
578 ret = nvme_cdev_add(&head->cdev, &head->cdev_device,
579 &nvme_ns_head_chr_fops, THIS_MODULE);
580 return ret;
581 }
582
nvme_requeue_work(struct work_struct * work)583 static void nvme_requeue_work(struct work_struct *work)
584 {
585 struct nvme_ns_head *head =
586 container_of(work, struct nvme_ns_head, requeue_work);
587 struct bio *bio, *next;
588
589 spin_lock_irq(&head->requeue_lock);
590 next = bio_list_get(&head->requeue_list);
591 spin_unlock_irq(&head->requeue_lock);
592
593 while ((bio = next) != NULL) {
594 next = bio->bi_next;
595 bio->bi_next = NULL;
596
597 submit_bio_noacct(bio);
598 }
599 }
600
nvme_mpath_alloc_disk(struct nvme_ctrl * ctrl,struct nvme_ns_head * head)601 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
602 {
603 struct queue_limits lim;
604
605 mutex_init(&head->lock);
606 bio_list_init(&head->requeue_list);
607 spin_lock_init(&head->requeue_lock);
608 INIT_WORK(&head->requeue_work, nvme_requeue_work);
609
610 /*
611 * Add a multipath node if the subsystems supports multiple controllers.
612 * We also do this for private namespaces as the namespace sharing flag
613 * could change after a rescan.
614 */
615 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
616 !nvme_is_unique_nsid(ctrl, head) || !multipath)
617 return 0;
618
619 blk_set_stacking_limits(&lim);
620 lim.dma_alignment = 3;
621 lim.features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL;
622 if (head->ids.csi == NVME_CSI_ZNS)
623 lim.features |= BLK_FEAT_ZONED;
624 else
625 lim.max_zone_append_sectors = 0;
626
627 head->disk = blk_alloc_disk(&lim, ctrl->numa_node);
628 if (IS_ERR(head->disk))
629 return PTR_ERR(head->disk);
630 head->disk->fops = &nvme_ns_head_ops;
631 head->disk->private_data = head;
632 sprintf(head->disk->disk_name, "nvme%dn%d",
633 ctrl->subsys->instance, head->instance);
634 return 0;
635 }
636
nvme_mpath_set_live(struct nvme_ns * ns)637 static void nvme_mpath_set_live(struct nvme_ns *ns)
638 {
639 struct nvme_ns_head *head = ns->head;
640 int rc;
641
642 if (!head->disk)
643 return;
644
645 /*
646 * test_and_set_bit() is used because it is protecting against two nvme
647 * paths simultaneously calling device_add_disk() on the same namespace
648 * head.
649 */
650 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
651 rc = device_add_disk(&head->subsys->dev, head->disk,
652 nvme_ns_attr_groups);
653 if (rc) {
654 clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags);
655 return;
656 }
657 nvme_add_ns_head_cdev(head);
658 }
659
660 mutex_lock(&head->lock);
661 if (nvme_path_is_optimized(ns)) {
662 int node, srcu_idx;
663
664 srcu_idx = srcu_read_lock(&head->srcu);
665 for_each_online_node(node)
666 __nvme_find_path(head, node);
667 srcu_read_unlock(&head->srcu, srcu_idx);
668 }
669 mutex_unlock(&head->lock);
670
671 synchronize_srcu(&head->srcu);
672 kblockd_schedule_work(&head->requeue_work);
673 }
674
nvme_parse_ana_log(struct nvme_ctrl * ctrl,void * data,int (* cb)(struct nvme_ctrl * ctrl,struct nvme_ana_group_desc *,void *))675 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
676 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
677 void *))
678 {
679 void *base = ctrl->ana_log_buf;
680 size_t offset = sizeof(struct nvme_ana_rsp_hdr);
681 int error, i;
682
683 lockdep_assert_held(&ctrl->ana_lock);
684
685 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
686 struct nvme_ana_group_desc *desc = base + offset;
687 u32 nr_nsids;
688 size_t nsid_buf_size;
689
690 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
691 return -EINVAL;
692
693 nr_nsids = le32_to_cpu(desc->nnsids);
694 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids);
695
696 if (WARN_ON_ONCE(desc->grpid == 0))
697 return -EINVAL;
698 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
699 return -EINVAL;
700 if (WARN_ON_ONCE(desc->state == 0))
701 return -EINVAL;
702 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
703 return -EINVAL;
704
705 offset += sizeof(*desc);
706 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
707 return -EINVAL;
708
709 error = cb(ctrl, desc, data);
710 if (error)
711 return error;
712
713 offset += nsid_buf_size;
714 }
715
716 return 0;
717 }
718
nvme_state_is_live(enum nvme_ana_state state)719 static inline bool nvme_state_is_live(enum nvme_ana_state state)
720 {
721 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
722 }
723
nvme_update_ns_ana_state(struct nvme_ana_group_desc * desc,struct nvme_ns * ns)724 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
725 struct nvme_ns *ns)
726 {
727 ns->ana_grpid = le32_to_cpu(desc->grpid);
728 ns->ana_state = desc->state;
729 clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
730 /*
731 * nvme_mpath_set_live() will trigger I/O to the multipath path device
732 * and in turn to this path device. However we cannot accept this I/O
733 * if the controller is not live. This may deadlock if called from
734 * nvme_mpath_init_identify() and the ctrl will never complete
735 * initialization, preventing I/O from completing. For this case we
736 * will reprocess the ANA log page in nvme_mpath_update() once the
737 * controller is ready.
738 */
739 if (nvme_state_is_live(ns->ana_state) &&
740 nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)
741 nvme_mpath_set_live(ns);
742 }
743
nvme_update_ana_state(struct nvme_ctrl * ctrl,struct nvme_ana_group_desc * desc,void * data)744 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
745 struct nvme_ana_group_desc *desc, void *data)
746 {
747 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
748 unsigned *nr_change_groups = data;
749 struct nvme_ns *ns;
750 int srcu_idx;
751
752 dev_dbg(ctrl->device, "ANA group %d: %s.\n",
753 le32_to_cpu(desc->grpid),
754 nvme_ana_state_names[desc->state]);
755
756 if (desc->state == NVME_ANA_CHANGE)
757 (*nr_change_groups)++;
758
759 if (!nr_nsids)
760 return 0;
761
762 srcu_idx = srcu_read_lock(&ctrl->srcu);
763 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
764 unsigned nsid;
765 again:
766 nsid = le32_to_cpu(desc->nsids[n]);
767 if (ns->head->ns_id < nsid)
768 continue;
769 if (ns->head->ns_id == nsid)
770 nvme_update_ns_ana_state(desc, ns);
771 if (++n == nr_nsids)
772 break;
773 if (ns->head->ns_id > nsid)
774 goto again;
775 }
776 srcu_read_unlock(&ctrl->srcu, srcu_idx);
777 return 0;
778 }
779
nvme_read_ana_log(struct nvme_ctrl * ctrl)780 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)
781 {
782 u32 nr_change_groups = 0;
783 int error;
784
785 mutex_lock(&ctrl->ana_lock);
786 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM,
787 ctrl->ana_log_buf, ctrl->ana_log_size, 0);
788 if (error) {
789 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
790 goto out_unlock;
791 }
792
793 error = nvme_parse_ana_log(ctrl, &nr_change_groups,
794 nvme_update_ana_state);
795 if (error)
796 goto out_unlock;
797
798 /*
799 * In theory we should have an ANATT timer per group as they might enter
800 * the change state at different times. But that is a lot of overhead
801 * just to protect against a target that keeps entering new changes
802 * states while never finishing previous ones. But we'll still
803 * eventually time out once all groups are in change state, so this
804 * isn't a big deal.
805 *
806 * We also double the ANATT value to provide some slack for transports
807 * or AEN processing overhead.
808 */
809 if (nr_change_groups)
810 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
811 else
812 del_timer_sync(&ctrl->anatt_timer);
813 out_unlock:
814 mutex_unlock(&ctrl->ana_lock);
815 return error;
816 }
817
nvme_ana_work(struct work_struct * work)818 static void nvme_ana_work(struct work_struct *work)
819 {
820 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
821
822 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
823 return;
824
825 nvme_read_ana_log(ctrl);
826 }
827
nvme_mpath_update(struct nvme_ctrl * ctrl)828 void nvme_mpath_update(struct nvme_ctrl *ctrl)
829 {
830 u32 nr_change_groups = 0;
831
832 if (!ctrl->ana_log_buf)
833 return;
834
835 mutex_lock(&ctrl->ana_lock);
836 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);
837 mutex_unlock(&ctrl->ana_lock);
838 }
839
nvme_anatt_timeout(struct timer_list * t)840 static void nvme_anatt_timeout(struct timer_list *t)
841 {
842 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
843
844 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
845 nvme_reset_ctrl(ctrl);
846 }
847
nvme_mpath_stop(struct nvme_ctrl * ctrl)848 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
849 {
850 if (!nvme_ctrl_use_ana(ctrl))
851 return;
852 del_timer_sync(&ctrl->anatt_timer);
853 cancel_work_sync(&ctrl->ana_work);
854 }
855
856 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \
857 struct device_attribute subsys_attr_##_name = \
858 __ATTR(_name, _mode, _show, _store)
859
nvme_subsys_iopolicy_show(struct device * dev,struct device_attribute * attr,char * buf)860 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
861 struct device_attribute *attr, char *buf)
862 {
863 struct nvme_subsystem *subsys =
864 container_of(dev, struct nvme_subsystem, dev);
865
866 return sysfs_emit(buf, "%s\n",
867 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
868 }
869
nvme_subsys_iopolicy_update(struct nvme_subsystem * subsys,int iopolicy)870 static void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys,
871 int iopolicy)
872 {
873 struct nvme_ctrl *ctrl;
874 int old_iopolicy = READ_ONCE(subsys->iopolicy);
875
876 if (old_iopolicy == iopolicy)
877 return;
878
879 WRITE_ONCE(subsys->iopolicy, iopolicy);
880
881 /* iopolicy changes clear the mpath by design */
882 mutex_lock(&nvme_subsystems_lock);
883 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
884 nvme_mpath_clear_ctrl_paths(ctrl);
885 mutex_unlock(&nvme_subsystems_lock);
886
887 pr_notice("subsysnqn %s iopolicy changed from %s to %s\n",
888 subsys->subnqn,
889 nvme_iopolicy_names[old_iopolicy],
890 nvme_iopolicy_names[iopolicy]);
891 }
892
nvme_subsys_iopolicy_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)893 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
894 struct device_attribute *attr, const char *buf, size_t count)
895 {
896 struct nvme_subsystem *subsys =
897 container_of(dev, struct nvme_subsystem, dev);
898 int i;
899
900 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
901 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
902 nvme_subsys_iopolicy_update(subsys, i);
903 return count;
904 }
905 }
906
907 return -EINVAL;
908 }
909 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
910 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
911
ana_grpid_show(struct device * dev,struct device_attribute * attr,char * buf)912 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
913 char *buf)
914 {
915 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
916 }
917 DEVICE_ATTR_RO(ana_grpid);
918
ana_state_show(struct device * dev,struct device_attribute * attr,char * buf)919 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
920 char *buf)
921 {
922 struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
923
924 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
925 }
926 DEVICE_ATTR_RO(ana_state);
927
nvme_lookup_ana_group_desc(struct nvme_ctrl * ctrl,struct nvme_ana_group_desc * desc,void * data)928 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
929 struct nvme_ana_group_desc *desc, void *data)
930 {
931 struct nvme_ana_group_desc *dst = data;
932
933 if (desc->grpid != dst->grpid)
934 return 0;
935
936 *dst = *desc;
937 return -ENXIO; /* just break out of the loop */
938 }
939
nvme_mpath_add_disk(struct nvme_ns * ns,__le32 anagrpid)940 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)
941 {
942 if (nvme_ctrl_use_ana(ns->ctrl)) {
943 struct nvme_ana_group_desc desc = {
944 .grpid = anagrpid,
945 .state = 0,
946 };
947
948 mutex_lock(&ns->ctrl->ana_lock);
949 ns->ana_grpid = le32_to_cpu(anagrpid);
950 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);
951 mutex_unlock(&ns->ctrl->ana_lock);
952 if (desc.state) {
953 /* found the group desc: update */
954 nvme_update_ns_ana_state(&desc, ns);
955 } else {
956 /* group desc not found: trigger a re-read */
957 set_bit(NVME_NS_ANA_PENDING, &ns->flags);
958 queue_work(nvme_wq, &ns->ctrl->ana_work);
959 }
960 } else {
961 ns->ana_state = NVME_ANA_OPTIMIZED;
962 nvme_mpath_set_live(ns);
963 }
964
965 #ifdef CONFIG_BLK_DEV_ZONED
966 if (blk_queue_is_zoned(ns->queue) && ns->head->disk)
967 ns->head->disk->nr_zones = ns->disk->nr_zones;
968 #endif
969 }
970
nvme_mpath_shutdown_disk(struct nvme_ns_head * head)971 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)
972 {
973 if (!head->disk)
974 return;
975 if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
976 nvme_cdev_del(&head->cdev, &head->cdev_device);
977 del_gendisk(head->disk);
978 }
979 /*
980 * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared
981 * to allow multipath to fail all I/O.
982 */
983 synchronize_srcu(&head->srcu);
984 kblockd_schedule_work(&head->requeue_work);
985 }
986
nvme_mpath_remove_disk(struct nvme_ns_head * head)987 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
988 {
989 if (!head->disk)
990 return;
991 /* make sure all pending bios are cleaned up */
992 kblockd_schedule_work(&head->requeue_work);
993 flush_work(&head->requeue_work);
994 put_disk(head->disk);
995 }
996
nvme_mpath_init_ctrl(struct nvme_ctrl * ctrl)997 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
998 {
999 mutex_init(&ctrl->ana_lock);
1000 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
1001 INIT_WORK(&ctrl->ana_work, nvme_ana_work);
1002 }
1003
nvme_mpath_init_identify(struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)1004 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
1005 {
1006 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT;
1007 size_t ana_log_size;
1008 int error = 0;
1009
1010 /* check if multipath is enabled and we have the capability */
1011 if (!multipath || !ctrl->subsys ||
1012 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA))
1013 return 0;
1014
1015 /* initialize this in the identify path to cover controller resets */
1016 atomic_set(&ctrl->nr_active, 0);
1017
1018 if (!ctrl->max_namespaces ||
1019 ctrl->max_namespaces > le32_to_cpu(id->nn)) {
1020 dev_err(ctrl->device,
1021 "Invalid MNAN value %u\n", ctrl->max_namespaces);
1022 return -EINVAL;
1023 }
1024
1025 ctrl->anacap = id->anacap;
1026 ctrl->anatt = id->anatt;
1027 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
1028 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
1029
1030 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
1031 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) +
1032 ctrl->max_namespaces * sizeof(__le32);
1033 if (ana_log_size > max_transfer_size) {
1034 dev_err(ctrl->device,
1035 "ANA log page size (%zd) larger than MDTS (%zd).\n",
1036 ana_log_size, max_transfer_size);
1037 dev_err(ctrl->device, "disabling ANA support.\n");
1038 goto out_uninit;
1039 }
1040 if (ana_log_size > ctrl->ana_log_size) {
1041 nvme_mpath_stop(ctrl);
1042 nvme_mpath_uninit(ctrl);
1043 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL);
1044 if (!ctrl->ana_log_buf)
1045 return -ENOMEM;
1046 }
1047 ctrl->ana_log_size = ana_log_size;
1048 error = nvme_read_ana_log(ctrl);
1049 if (error)
1050 goto out_uninit;
1051 return 0;
1052
1053 out_uninit:
1054 nvme_mpath_uninit(ctrl);
1055 return error;
1056 }
1057
nvme_mpath_uninit(struct nvme_ctrl * ctrl)1058 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
1059 {
1060 kvfree(ctrl->ana_log_buf);
1061 ctrl->ana_log_buf = NULL;
1062 ctrl->ana_log_size = 0;
1063 }
1064