1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 [NVME_IOPOLICY_QD] = "queue-depth", 21 }; 22 23 static int iopolicy = NVME_IOPOLICY_NUMA; 24 25 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 26 { 27 if (!val) 28 return -EINVAL; 29 if (!strncmp(val, "numa", 4)) 30 iopolicy = NVME_IOPOLICY_NUMA; 31 else if (!strncmp(val, "round-robin", 11)) 32 iopolicy = NVME_IOPOLICY_RR; 33 else if (!strncmp(val, "queue-depth", 11)) 34 iopolicy = NVME_IOPOLICY_QD; 35 else 36 return -EINVAL; 37 38 return 0; 39 } 40 41 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 42 { 43 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 44 } 45 46 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 47 &iopolicy, 0644); 48 MODULE_PARM_DESC(iopolicy, 49 "Default multipath I/O policy; 'numa' (default), 'round-robin' or 'queue-depth'"); 50 51 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 52 { 53 subsys->iopolicy = iopolicy; 54 } 55 56 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 57 { 58 struct nvme_ns_head *h; 59 60 lockdep_assert_held(&subsys->lock); 61 list_for_each_entry(h, &subsys->nsheads, entry) 62 if (h->disk) 63 blk_mq_unfreeze_queue(h->disk->queue); 64 } 65 66 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 67 { 68 struct nvme_ns_head *h; 69 70 lockdep_assert_held(&subsys->lock); 71 list_for_each_entry(h, &subsys->nsheads, entry) 72 if (h->disk) 73 blk_mq_freeze_queue_wait(h->disk->queue); 74 } 75 76 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 77 { 78 struct nvme_ns_head *h; 79 80 lockdep_assert_held(&subsys->lock); 81 list_for_each_entry(h, &subsys->nsheads, entry) 82 if (h->disk) 83 blk_freeze_queue_start(h->disk->queue); 84 } 85 86 void nvme_failover_req(struct request *req) 87 { 88 struct nvme_ns *ns = req->q->queuedata; 89 u16 status = nvme_req(req)->status & NVME_SCT_SC_MASK; 90 unsigned long flags; 91 struct bio *bio; 92 93 nvme_mpath_clear_current_path(ns); 94 95 /* 96 * If we got back an ANA error, we know the controller is alive but not 97 * ready to serve this namespace. Kick of a re-read of the ANA 98 * information page, and just try any other available path for now. 99 */ 100 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 101 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 102 queue_work(nvme_wq, &ns->ctrl->ana_work); 103 } 104 105 spin_lock_irqsave(&ns->head->requeue_lock, flags); 106 for (bio = req->bio; bio; bio = bio->bi_next) { 107 bio_set_dev(bio, ns->head->disk->part0); 108 if (bio->bi_opf & REQ_POLLED) { 109 bio->bi_opf &= ~REQ_POLLED; 110 bio->bi_cookie = BLK_QC_T_NONE; 111 } 112 /* 113 * The alternate request queue that we may end up submitting 114 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 115 * will fail the I/O immediately with EAGAIN to the issuer. 116 * We are not in the issuer context which cannot block. Clear 117 * the flag to avoid spurious EAGAIN I/O failures. 118 */ 119 bio->bi_opf &= ~REQ_NOWAIT; 120 } 121 blk_steal_bios(&ns->head->requeue_list, req); 122 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 123 124 nvme_req(req)->status = 0; 125 nvme_end_req(req); 126 kblockd_schedule_work(&ns->head->requeue_work); 127 } 128 129 void nvme_mpath_start_request(struct request *rq) 130 { 131 struct nvme_ns *ns = rq->q->queuedata; 132 struct gendisk *disk = ns->head->disk; 133 134 if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) { 135 atomic_inc(&ns->ctrl->nr_active); 136 nvme_req(rq)->flags |= NVME_MPATH_CNT_ACTIVE; 137 } 138 139 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) 140 return; 141 142 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; 143 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), 144 jiffies); 145 } 146 EXPORT_SYMBOL_GPL(nvme_mpath_start_request); 147 148 void nvme_mpath_end_request(struct request *rq) 149 { 150 struct nvme_ns *ns = rq->q->queuedata; 151 152 if (nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE) 153 atomic_dec_if_positive(&ns->ctrl->nr_active); 154 155 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) 156 return; 157 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), 158 blk_rq_bytes(rq) >> SECTOR_SHIFT, 159 nvme_req(rq)->start_time); 160 } 161 162 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 163 { 164 struct nvme_ns *ns; 165 int srcu_idx; 166 167 srcu_idx = srcu_read_lock(&ctrl->srcu); 168 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 169 if (!ns->head->disk) 170 continue; 171 kblockd_schedule_work(&ns->head->requeue_work); 172 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 173 disk_uevent(ns->head->disk, KOBJ_CHANGE); 174 } 175 srcu_read_unlock(&ctrl->srcu, srcu_idx); 176 } 177 178 static const char *nvme_ana_state_names[] = { 179 [0] = "invalid state", 180 [NVME_ANA_OPTIMIZED] = "optimized", 181 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 182 [NVME_ANA_INACCESSIBLE] = "inaccessible", 183 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 184 [NVME_ANA_CHANGE] = "change", 185 }; 186 187 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 188 { 189 struct nvme_ns_head *head = ns->head; 190 bool changed = false; 191 int node; 192 193 if (!head) 194 goto out; 195 196 for_each_node(node) { 197 if (ns == rcu_access_pointer(head->current_path[node])) { 198 rcu_assign_pointer(head->current_path[node], NULL); 199 changed = true; 200 } 201 } 202 out: 203 return changed; 204 } 205 206 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 207 { 208 struct nvme_ns *ns; 209 int srcu_idx; 210 211 srcu_idx = srcu_read_lock(&ctrl->srcu); 212 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 213 nvme_mpath_clear_current_path(ns); 214 kblockd_schedule_work(&ns->head->requeue_work); 215 } 216 srcu_read_unlock(&ctrl->srcu, srcu_idx); 217 } 218 219 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 220 { 221 struct nvme_ns_head *head = ns->head; 222 sector_t capacity = get_capacity(head->disk); 223 int node; 224 int srcu_idx; 225 226 srcu_idx = srcu_read_lock(&head->srcu); 227 list_for_each_entry_rcu(ns, &head->list, siblings) { 228 if (capacity != get_capacity(ns->disk)) 229 clear_bit(NVME_NS_READY, &ns->flags); 230 } 231 srcu_read_unlock(&head->srcu, srcu_idx); 232 233 for_each_node(node) 234 rcu_assign_pointer(head->current_path[node], NULL); 235 kblockd_schedule_work(&head->requeue_work); 236 } 237 238 static bool nvme_path_is_disabled(struct nvme_ns *ns) 239 { 240 enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); 241 242 /* 243 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 244 * still be able to complete assuming that the controller is connected. 245 * Otherwise it will fail immediately and return to the requeue list. 246 */ 247 if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING) 248 return true; 249 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 250 !test_bit(NVME_NS_READY, &ns->flags)) 251 return true; 252 return false; 253 } 254 255 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 256 { 257 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 258 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 259 260 list_for_each_entry_rcu(ns, &head->list, siblings) { 261 if (nvme_path_is_disabled(ns)) 262 continue; 263 264 if (ns->ctrl->numa_node != NUMA_NO_NODE && 265 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 266 distance = node_distance(node, ns->ctrl->numa_node); 267 else 268 distance = LOCAL_DISTANCE; 269 270 switch (ns->ana_state) { 271 case NVME_ANA_OPTIMIZED: 272 if (distance < found_distance) { 273 found_distance = distance; 274 found = ns; 275 } 276 break; 277 case NVME_ANA_NONOPTIMIZED: 278 if (distance < fallback_distance) { 279 fallback_distance = distance; 280 fallback = ns; 281 } 282 break; 283 default: 284 break; 285 } 286 } 287 288 if (!found) 289 found = fallback; 290 if (found) 291 rcu_assign_pointer(head->current_path[node], found); 292 return found; 293 } 294 295 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 296 struct nvme_ns *ns) 297 { 298 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 299 siblings); 300 if (ns) 301 return ns; 302 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 303 } 304 305 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head) 306 { 307 struct nvme_ns *ns, *found = NULL; 308 int node = numa_node_id(); 309 struct nvme_ns *old = srcu_dereference(head->current_path[node], 310 &head->srcu); 311 312 if (unlikely(!old)) 313 return __nvme_find_path(head, node); 314 315 if (list_is_singular(&head->list)) { 316 if (nvme_path_is_disabled(old)) 317 return NULL; 318 return old; 319 } 320 321 for (ns = nvme_next_ns(head, old); 322 ns && ns != old; 323 ns = nvme_next_ns(head, ns)) { 324 if (nvme_path_is_disabled(ns)) 325 continue; 326 327 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 328 found = ns; 329 goto out; 330 } 331 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 332 found = ns; 333 } 334 335 /* 336 * The loop above skips the current path for round-robin semantics. 337 * Fall back to the current path if either: 338 * - no other optimized path found and current is optimized, 339 * - no other usable path found and current is usable. 340 */ 341 if (!nvme_path_is_disabled(old) && 342 (old->ana_state == NVME_ANA_OPTIMIZED || 343 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 344 return old; 345 346 if (!found) 347 return NULL; 348 out: 349 rcu_assign_pointer(head->current_path[node], found); 350 return found; 351 } 352 353 static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head) 354 { 355 struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns; 356 unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX; 357 unsigned int depth; 358 359 list_for_each_entry_rcu(ns, &head->list, siblings) { 360 if (nvme_path_is_disabled(ns)) 361 continue; 362 363 depth = atomic_read(&ns->ctrl->nr_active); 364 365 switch (ns->ana_state) { 366 case NVME_ANA_OPTIMIZED: 367 if (depth < min_depth_opt) { 368 min_depth_opt = depth; 369 best_opt = ns; 370 } 371 break; 372 case NVME_ANA_NONOPTIMIZED: 373 if (depth < min_depth_nonopt) { 374 min_depth_nonopt = depth; 375 best_nonopt = ns; 376 } 377 break; 378 default: 379 break; 380 } 381 382 if (min_depth_opt == 0) 383 return best_opt; 384 } 385 386 return best_opt ? best_opt : best_nonopt; 387 } 388 389 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 390 { 391 return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE && 392 ns->ana_state == NVME_ANA_OPTIMIZED; 393 } 394 395 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head) 396 { 397 int node = numa_node_id(); 398 struct nvme_ns *ns; 399 400 ns = srcu_dereference(head->current_path[node], &head->srcu); 401 if (unlikely(!ns)) 402 return __nvme_find_path(head, node); 403 if (unlikely(!nvme_path_is_optimized(ns))) 404 return __nvme_find_path(head, node); 405 return ns; 406 } 407 408 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 409 { 410 switch (READ_ONCE(head->subsys->iopolicy)) { 411 case NVME_IOPOLICY_QD: 412 return nvme_queue_depth_path(head); 413 case NVME_IOPOLICY_RR: 414 return nvme_round_robin_path(head); 415 default: 416 return nvme_numa_path(head); 417 } 418 } 419 420 static bool nvme_available_path(struct nvme_ns_head *head) 421 { 422 struct nvme_ns *ns; 423 424 if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) 425 return NULL; 426 427 list_for_each_entry_rcu(ns, &head->list, siblings) { 428 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 429 continue; 430 switch (nvme_ctrl_state(ns->ctrl)) { 431 case NVME_CTRL_LIVE: 432 case NVME_CTRL_RESETTING: 433 case NVME_CTRL_CONNECTING: 434 return true; 435 default: 436 break; 437 } 438 } 439 return false; 440 } 441 442 static void nvme_ns_head_submit_bio(struct bio *bio) 443 { 444 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 445 struct device *dev = disk_to_dev(head->disk); 446 struct nvme_ns *ns; 447 int srcu_idx; 448 449 /* 450 * The namespace might be going away and the bio might be moved to a 451 * different queue via blk_steal_bios(), so we need to use the bio_split 452 * pool from the original queue to allocate the bvecs from. 453 */ 454 bio = bio_split_to_limits(bio); 455 if (!bio) 456 return; 457 458 srcu_idx = srcu_read_lock(&head->srcu); 459 ns = nvme_find_path(head); 460 if (likely(ns)) { 461 bio_set_dev(bio, ns->disk->part0); 462 bio->bi_opf |= REQ_NVME_MPATH; 463 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 464 bio->bi_iter.bi_sector); 465 submit_bio_noacct(bio); 466 } else if (nvme_available_path(head)) { 467 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 468 469 spin_lock_irq(&head->requeue_lock); 470 bio_list_add(&head->requeue_list, bio); 471 spin_unlock_irq(&head->requeue_lock); 472 } else { 473 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 474 475 bio_io_error(bio); 476 } 477 478 srcu_read_unlock(&head->srcu, srcu_idx); 479 } 480 481 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode) 482 { 483 if (!nvme_tryget_ns_head(disk->private_data)) 484 return -ENXIO; 485 return 0; 486 } 487 488 static void nvme_ns_head_release(struct gendisk *disk) 489 { 490 nvme_put_ns_head(disk->private_data); 491 } 492 493 static int nvme_ns_head_get_unique_id(struct gendisk *disk, u8 id[16], 494 enum blk_unique_id type) 495 { 496 struct nvme_ns_head *head = disk->private_data; 497 struct nvme_ns *ns; 498 int srcu_idx, ret = -EWOULDBLOCK; 499 500 srcu_idx = srcu_read_lock(&head->srcu); 501 ns = nvme_find_path(head); 502 if (ns) 503 ret = nvme_ns_get_unique_id(ns, id, type); 504 srcu_read_unlock(&head->srcu, srcu_idx); 505 return ret; 506 } 507 508 #ifdef CONFIG_BLK_DEV_ZONED 509 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 510 unsigned int nr_zones, report_zones_cb cb, void *data) 511 { 512 struct nvme_ns_head *head = disk->private_data; 513 struct nvme_ns *ns; 514 int srcu_idx, ret = -EWOULDBLOCK; 515 516 srcu_idx = srcu_read_lock(&head->srcu); 517 ns = nvme_find_path(head); 518 if (ns) 519 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 520 srcu_read_unlock(&head->srcu, srcu_idx); 521 return ret; 522 } 523 #else 524 #define nvme_ns_head_report_zones NULL 525 #endif /* CONFIG_BLK_DEV_ZONED */ 526 527 const struct block_device_operations nvme_ns_head_ops = { 528 .owner = THIS_MODULE, 529 .submit_bio = nvme_ns_head_submit_bio, 530 .open = nvme_ns_head_open, 531 .release = nvme_ns_head_release, 532 .ioctl = nvme_ns_head_ioctl, 533 .compat_ioctl = blkdev_compat_ptr_ioctl, 534 .getgeo = nvme_getgeo, 535 .get_unique_id = nvme_ns_head_get_unique_id, 536 .report_zones = nvme_ns_head_report_zones, 537 .pr_ops = &nvme_pr_ops, 538 }; 539 540 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 541 { 542 return container_of(cdev, struct nvme_ns_head, cdev); 543 } 544 545 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 546 { 547 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 548 return -ENXIO; 549 return 0; 550 } 551 552 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 553 { 554 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 555 return 0; 556 } 557 558 static const struct file_operations nvme_ns_head_chr_fops = { 559 .owner = THIS_MODULE, 560 .open = nvme_ns_head_chr_open, 561 .release = nvme_ns_head_chr_release, 562 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 563 .compat_ioctl = compat_ptr_ioctl, 564 .uring_cmd = nvme_ns_head_chr_uring_cmd, 565 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, 566 }; 567 568 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 569 { 570 int ret; 571 572 head->cdev_device.parent = &head->subsys->dev; 573 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 574 head->subsys->instance, head->instance); 575 if (ret) 576 return ret; 577 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 578 &nvme_ns_head_chr_fops, THIS_MODULE); 579 return ret; 580 } 581 582 static void nvme_partition_scan_work(struct work_struct *work) 583 { 584 struct nvme_ns_head *head = 585 container_of(work, struct nvme_ns_head, partition_scan_work); 586 587 if (WARN_ON_ONCE(!test_and_clear_bit(GD_SUPPRESS_PART_SCAN, 588 &head->disk->state))) 589 return; 590 591 mutex_lock(&head->disk->open_mutex); 592 bdev_disk_changed(head->disk, false); 593 mutex_unlock(&head->disk->open_mutex); 594 } 595 596 static void nvme_requeue_work(struct work_struct *work) 597 { 598 struct nvme_ns_head *head = 599 container_of(work, struct nvme_ns_head, requeue_work); 600 struct bio *bio, *next; 601 602 spin_lock_irq(&head->requeue_lock); 603 next = bio_list_get(&head->requeue_list); 604 spin_unlock_irq(&head->requeue_lock); 605 606 while ((bio = next) != NULL) { 607 next = bio->bi_next; 608 bio->bi_next = NULL; 609 610 submit_bio_noacct(bio); 611 } 612 } 613 614 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 615 { 616 struct queue_limits lim; 617 618 mutex_init(&head->lock); 619 bio_list_init(&head->requeue_list); 620 spin_lock_init(&head->requeue_lock); 621 INIT_WORK(&head->requeue_work, nvme_requeue_work); 622 INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); 623 624 /* 625 * Add a multipath node if the subsystems supports multiple controllers. 626 * We also do this for private namespaces as the namespace sharing flag 627 * could change after a rescan. 628 */ 629 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 630 !nvme_is_unique_nsid(ctrl, head) || !multipath) 631 return 0; 632 633 blk_set_stacking_limits(&lim); 634 lim.dma_alignment = 3; 635 lim.features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL; 636 if (head->ids.csi == NVME_CSI_ZNS) 637 lim.features |= BLK_FEAT_ZONED; 638 else 639 lim.max_zone_append_sectors = 0; 640 641 head->disk = blk_alloc_disk(&lim, ctrl->numa_node); 642 if (IS_ERR(head->disk)) 643 return PTR_ERR(head->disk); 644 head->disk->fops = &nvme_ns_head_ops; 645 head->disk->private_data = head; 646 647 /* 648 * We need to suppress the partition scan from occuring within the 649 * controller's scan_work context. If a path error occurs here, the IO 650 * will wait until a path becomes available or all paths are torn down, 651 * but that action also occurs within scan_work, so it would deadlock. 652 * Defer the partion scan to a different context that does not block 653 * scan_work. 654 */ 655 set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); 656 sprintf(head->disk->disk_name, "nvme%dn%d", 657 ctrl->subsys->instance, head->instance); 658 return 0; 659 } 660 661 static void nvme_mpath_set_live(struct nvme_ns *ns) 662 { 663 struct nvme_ns_head *head = ns->head; 664 int rc; 665 666 if (!head->disk) 667 return; 668 669 /* 670 * test_and_set_bit() is used because it is protecting against two nvme 671 * paths simultaneously calling device_add_disk() on the same namespace 672 * head. 673 */ 674 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 675 rc = device_add_disk(&head->subsys->dev, head->disk, 676 nvme_ns_attr_groups); 677 if (rc) { 678 clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags); 679 return; 680 } 681 nvme_add_ns_head_cdev(head); 682 kblockd_schedule_work(&head->partition_scan_work); 683 } 684 685 mutex_lock(&head->lock); 686 if (nvme_path_is_optimized(ns)) { 687 int node, srcu_idx; 688 689 srcu_idx = srcu_read_lock(&head->srcu); 690 for_each_online_node(node) 691 __nvme_find_path(head, node); 692 srcu_read_unlock(&head->srcu, srcu_idx); 693 } 694 mutex_unlock(&head->lock); 695 696 synchronize_srcu(&head->srcu); 697 kblockd_schedule_work(&head->requeue_work); 698 } 699 700 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 701 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 702 void *)) 703 { 704 void *base = ctrl->ana_log_buf; 705 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 706 int error, i; 707 708 lockdep_assert_held(&ctrl->ana_lock); 709 710 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 711 struct nvme_ana_group_desc *desc = base + offset; 712 u32 nr_nsids; 713 size_t nsid_buf_size; 714 715 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 716 return -EINVAL; 717 718 nr_nsids = le32_to_cpu(desc->nnsids); 719 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 720 721 if (WARN_ON_ONCE(desc->grpid == 0)) 722 return -EINVAL; 723 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 724 return -EINVAL; 725 if (WARN_ON_ONCE(desc->state == 0)) 726 return -EINVAL; 727 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 728 return -EINVAL; 729 730 offset += sizeof(*desc); 731 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 732 return -EINVAL; 733 734 error = cb(ctrl, desc, data); 735 if (error) 736 return error; 737 738 offset += nsid_buf_size; 739 } 740 741 return 0; 742 } 743 744 static inline bool nvme_state_is_live(enum nvme_ana_state state) 745 { 746 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 747 } 748 749 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 750 struct nvme_ns *ns) 751 { 752 ns->ana_grpid = le32_to_cpu(desc->grpid); 753 ns->ana_state = desc->state; 754 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 755 /* 756 * nvme_mpath_set_live() will trigger I/O to the multipath path device 757 * and in turn to this path device. However we cannot accept this I/O 758 * if the controller is not live. This may deadlock if called from 759 * nvme_mpath_init_identify() and the ctrl will never complete 760 * initialization, preventing I/O from completing. For this case we 761 * will reprocess the ANA log page in nvme_mpath_update() once the 762 * controller is ready. 763 */ 764 if (nvme_state_is_live(ns->ana_state) && 765 nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 766 nvme_mpath_set_live(ns); 767 } 768 769 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 770 struct nvme_ana_group_desc *desc, void *data) 771 { 772 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 773 unsigned *nr_change_groups = data; 774 struct nvme_ns *ns; 775 int srcu_idx; 776 777 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 778 le32_to_cpu(desc->grpid), 779 nvme_ana_state_names[desc->state]); 780 781 if (desc->state == NVME_ANA_CHANGE) 782 (*nr_change_groups)++; 783 784 if (!nr_nsids) 785 return 0; 786 787 srcu_idx = srcu_read_lock(&ctrl->srcu); 788 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 789 unsigned nsid; 790 again: 791 nsid = le32_to_cpu(desc->nsids[n]); 792 if (ns->head->ns_id < nsid) 793 continue; 794 if (ns->head->ns_id == nsid) 795 nvme_update_ns_ana_state(desc, ns); 796 if (++n == nr_nsids) 797 break; 798 if (ns->head->ns_id > nsid) 799 goto again; 800 } 801 srcu_read_unlock(&ctrl->srcu, srcu_idx); 802 return 0; 803 } 804 805 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 806 { 807 u32 nr_change_groups = 0; 808 int error; 809 810 mutex_lock(&ctrl->ana_lock); 811 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 812 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 813 if (error) { 814 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 815 goto out_unlock; 816 } 817 818 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 819 nvme_update_ana_state); 820 if (error) 821 goto out_unlock; 822 823 /* 824 * In theory we should have an ANATT timer per group as they might enter 825 * the change state at different times. But that is a lot of overhead 826 * just to protect against a target that keeps entering new changes 827 * states while never finishing previous ones. But we'll still 828 * eventually time out once all groups are in change state, so this 829 * isn't a big deal. 830 * 831 * We also double the ANATT value to provide some slack for transports 832 * or AEN processing overhead. 833 */ 834 if (nr_change_groups) 835 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 836 else 837 del_timer_sync(&ctrl->anatt_timer); 838 out_unlock: 839 mutex_unlock(&ctrl->ana_lock); 840 return error; 841 } 842 843 static void nvme_ana_work(struct work_struct *work) 844 { 845 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 846 847 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 848 return; 849 850 nvme_read_ana_log(ctrl); 851 } 852 853 void nvme_mpath_update(struct nvme_ctrl *ctrl) 854 { 855 u32 nr_change_groups = 0; 856 857 if (!ctrl->ana_log_buf) 858 return; 859 860 mutex_lock(&ctrl->ana_lock); 861 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 862 mutex_unlock(&ctrl->ana_lock); 863 } 864 865 static void nvme_anatt_timeout(struct timer_list *t) 866 { 867 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 868 869 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 870 nvme_reset_ctrl(ctrl); 871 } 872 873 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 874 { 875 if (!nvme_ctrl_use_ana(ctrl)) 876 return; 877 del_timer_sync(&ctrl->anatt_timer); 878 cancel_work_sync(&ctrl->ana_work); 879 } 880 881 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 882 struct device_attribute subsys_attr_##_name = \ 883 __ATTR(_name, _mode, _show, _store) 884 885 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 886 struct device_attribute *attr, char *buf) 887 { 888 struct nvme_subsystem *subsys = 889 container_of(dev, struct nvme_subsystem, dev); 890 891 return sysfs_emit(buf, "%s\n", 892 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 893 } 894 895 static void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, 896 int iopolicy) 897 { 898 struct nvme_ctrl *ctrl; 899 int old_iopolicy = READ_ONCE(subsys->iopolicy); 900 901 if (old_iopolicy == iopolicy) 902 return; 903 904 WRITE_ONCE(subsys->iopolicy, iopolicy); 905 906 /* iopolicy changes clear the mpath by design */ 907 mutex_lock(&nvme_subsystems_lock); 908 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 909 nvme_mpath_clear_ctrl_paths(ctrl); 910 mutex_unlock(&nvme_subsystems_lock); 911 912 pr_notice("subsysnqn %s iopolicy changed from %s to %s\n", 913 subsys->subnqn, 914 nvme_iopolicy_names[old_iopolicy], 915 nvme_iopolicy_names[iopolicy]); 916 } 917 918 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 919 struct device_attribute *attr, const char *buf, size_t count) 920 { 921 struct nvme_subsystem *subsys = 922 container_of(dev, struct nvme_subsystem, dev); 923 int i; 924 925 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 926 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 927 nvme_subsys_iopolicy_update(subsys, i); 928 return count; 929 } 930 } 931 932 return -EINVAL; 933 } 934 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 935 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 936 937 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 938 char *buf) 939 { 940 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 941 } 942 DEVICE_ATTR_RO(ana_grpid); 943 944 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 945 char *buf) 946 { 947 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 948 949 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 950 } 951 DEVICE_ATTR_RO(ana_state); 952 953 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 954 struct nvme_ana_group_desc *desc, void *data) 955 { 956 struct nvme_ana_group_desc *dst = data; 957 958 if (desc->grpid != dst->grpid) 959 return 0; 960 961 *dst = *desc; 962 return -ENXIO; /* just break out of the loop */ 963 } 964 965 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 966 { 967 if (nvme_ctrl_use_ana(ns->ctrl)) { 968 struct nvme_ana_group_desc desc = { 969 .grpid = anagrpid, 970 .state = 0, 971 }; 972 973 mutex_lock(&ns->ctrl->ana_lock); 974 ns->ana_grpid = le32_to_cpu(anagrpid); 975 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 976 mutex_unlock(&ns->ctrl->ana_lock); 977 if (desc.state) { 978 /* found the group desc: update */ 979 nvme_update_ns_ana_state(&desc, ns); 980 } else { 981 /* group desc not found: trigger a re-read */ 982 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 983 queue_work(nvme_wq, &ns->ctrl->ana_work); 984 } 985 } else { 986 ns->ana_state = NVME_ANA_OPTIMIZED; 987 nvme_mpath_set_live(ns); 988 } 989 990 #ifdef CONFIG_BLK_DEV_ZONED 991 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 992 ns->head->disk->nr_zones = ns->disk->nr_zones; 993 #endif 994 } 995 996 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 997 { 998 if (!head->disk) 999 return; 1000 if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 1001 nvme_cdev_del(&head->cdev, &head->cdev_device); 1002 /* 1003 * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared 1004 * to allow multipath to fail all I/O. 1005 */ 1006 synchronize_srcu(&head->srcu); 1007 kblockd_schedule_work(&head->requeue_work); 1008 del_gendisk(head->disk); 1009 } 1010 } 1011 1012 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 1013 { 1014 if (!head->disk) 1015 return; 1016 /* make sure all pending bios are cleaned up */ 1017 kblockd_schedule_work(&head->requeue_work); 1018 flush_work(&head->requeue_work); 1019 flush_work(&head->partition_scan_work); 1020 put_disk(head->disk); 1021 } 1022 1023 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 1024 { 1025 mutex_init(&ctrl->ana_lock); 1026 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 1027 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 1028 } 1029 1030 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 1031 { 1032 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 1033 size_t ana_log_size; 1034 int error = 0; 1035 1036 /* check if multipath is enabled and we have the capability */ 1037 if (!multipath || !ctrl->subsys || 1038 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 1039 return 0; 1040 1041 /* initialize this in the identify path to cover controller resets */ 1042 atomic_set(&ctrl->nr_active, 0); 1043 1044 if (!ctrl->max_namespaces || 1045 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 1046 dev_err(ctrl->device, 1047 "Invalid MNAN value %u\n", ctrl->max_namespaces); 1048 return -EINVAL; 1049 } 1050 1051 ctrl->anacap = id->anacap; 1052 ctrl->anatt = id->anatt; 1053 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 1054 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 1055 1056 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 1057 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 1058 ctrl->max_namespaces * sizeof(__le32); 1059 if (ana_log_size > max_transfer_size) { 1060 dev_err(ctrl->device, 1061 "ANA log page size (%zd) larger than MDTS (%zd).\n", 1062 ana_log_size, max_transfer_size); 1063 dev_err(ctrl->device, "disabling ANA support.\n"); 1064 goto out_uninit; 1065 } 1066 if (ana_log_size > ctrl->ana_log_size) { 1067 nvme_mpath_stop(ctrl); 1068 nvme_mpath_uninit(ctrl); 1069 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 1070 if (!ctrl->ana_log_buf) 1071 return -ENOMEM; 1072 } 1073 ctrl->ana_log_size = ana_log_size; 1074 error = nvme_read_ana_log(ctrl); 1075 if (error) 1076 goto out_uninit; 1077 return 0; 1078 1079 out_uninit: 1080 nvme_mpath_uninit(ctrl); 1081 return error; 1082 } 1083 1084 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 1085 { 1086 kvfree(ctrl->ana_log_buf); 1087 ctrl->ana_log_buf = NULL; 1088 ctrl->ana_log_size = 0; 1089 } 1090