1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 }; 21 22 static int iopolicy = NVME_IOPOLICY_NUMA; 23 24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 25 { 26 if (!val) 27 return -EINVAL; 28 if (!strncmp(val, "numa", 4)) 29 iopolicy = NVME_IOPOLICY_NUMA; 30 else if (!strncmp(val, "round-robin", 11)) 31 iopolicy = NVME_IOPOLICY_RR; 32 else 33 return -EINVAL; 34 35 return 0; 36 } 37 38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 39 { 40 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 41 } 42 43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 44 &iopolicy, 0644); 45 MODULE_PARM_DESC(iopolicy, 46 "Default multipath I/O policy; 'numa' (default) or 'round-robin'"); 47 48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 49 { 50 subsys->iopolicy = iopolicy; 51 } 52 53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 54 { 55 struct nvme_ns_head *h; 56 57 lockdep_assert_held(&subsys->lock); 58 list_for_each_entry(h, &subsys->nsheads, entry) 59 if (h->disk) 60 blk_mq_unfreeze_queue(h->disk->queue); 61 } 62 63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 64 { 65 struct nvme_ns_head *h; 66 67 lockdep_assert_held(&subsys->lock); 68 list_for_each_entry(h, &subsys->nsheads, entry) 69 if (h->disk) 70 blk_mq_freeze_queue_wait(h->disk->queue); 71 } 72 73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 74 { 75 struct nvme_ns_head *h; 76 77 lockdep_assert_held(&subsys->lock); 78 list_for_each_entry(h, &subsys->nsheads, entry) 79 if (h->disk) 80 blk_freeze_queue_start(h->disk->queue); 81 } 82 83 void nvme_failover_req(struct request *req) 84 { 85 struct nvme_ns *ns = req->q->queuedata; 86 u16 status = nvme_req(req)->status & 0x7ff; 87 unsigned long flags; 88 struct bio *bio; 89 90 nvme_mpath_clear_current_path(ns); 91 92 /* 93 * If we got back an ANA error, we know the controller is alive but not 94 * ready to serve this namespace. Kick of a re-read of the ANA 95 * information page, and just try any other available path for now. 96 */ 97 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 98 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 99 queue_work(nvme_wq, &ns->ctrl->ana_work); 100 } 101 102 spin_lock_irqsave(&ns->head->requeue_lock, flags); 103 for (bio = req->bio; bio; bio = bio->bi_next) { 104 bio_set_dev(bio, ns->head->disk->part0); 105 if (bio->bi_opf & REQ_POLLED) { 106 bio->bi_opf &= ~REQ_POLLED; 107 bio->bi_cookie = BLK_QC_T_NONE; 108 } 109 /* 110 * The alternate request queue that we may end up submitting 111 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 112 * will fail the I/O immediately with EAGAIN to the issuer. 113 * We are not in the issuer context which cannot block. Clear 114 * the flag to avoid spurious EAGAIN I/O failures. 115 */ 116 bio->bi_opf &= ~REQ_NOWAIT; 117 } 118 blk_steal_bios(&ns->head->requeue_list, req); 119 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 120 121 blk_mq_end_request(req, 0); 122 kblockd_schedule_work(&ns->head->requeue_work); 123 } 124 125 void nvme_mpath_start_request(struct request *rq) 126 { 127 struct nvme_ns *ns = rq->q->queuedata; 128 struct gendisk *disk = ns->head->disk; 129 130 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) 131 return; 132 133 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; 134 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), 135 jiffies); 136 } 137 EXPORT_SYMBOL_GPL(nvme_mpath_start_request); 138 139 void nvme_mpath_end_request(struct request *rq) 140 { 141 struct nvme_ns *ns = rq->q->queuedata; 142 143 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) 144 return; 145 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), 146 blk_rq_bytes(rq) >> SECTOR_SHIFT, 147 nvme_req(rq)->start_time); 148 } 149 150 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 151 { 152 struct nvme_ns *ns; 153 154 down_read(&ctrl->namespaces_rwsem); 155 list_for_each_entry(ns, &ctrl->namespaces, list) { 156 if (!ns->head->disk) 157 continue; 158 kblockd_schedule_work(&ns->head->requeue_work); 159 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 160 disk_uevent(ns->head->disk, KOBJ_CHANGE); 161 } 162 up_read(&ctrl->namespaces_rwsem); 163 } 164 165 static const char *nvme_ana_state_names[] = { 166 [0] = "invalid state", 167 [NVME_ANA_OPTIMIZED] = "optimized", 168 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 169 [NVME_ANA_INACCESSIBLE] = "inaccessible", 170 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 171 [NVME_ANA_CHANGE] = "change", 172 }; 173 174 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 175 { 176 struct nvme_ns_head *head = ns->head; 177 bool changed = false; 178 int node; 179 180 if (!head) 181 goto out; 182 183 for_each_node(node) { 184 if (ns == rcu_access_pointer(head->current_path[node])) { 185 rcu_assign_pointer(head->current_path[node], NULL); 186 changed = true; 187 } 188 } 189 out: 190 return changed; 191 } 192 193 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 194 { 195 struct nvme_ns *ns; 196 197 down_read(&ctrl->namespaces_rwsem); 198 list_for_each_entry(ns, &ctrl->namespaces, list) { 199 nvme_mpath_clear_current_path(ns); 200 kblockd_schedule_work(&ns->head->requeue_work); 201 } 202 up_read(&ctrl->namespaces_rwsem); 203 } 204 205 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 206 { 207 struct nvme_ns_head *head = ns->head; 208 sector_t capacity = get_capacity(head->disk); 209 int node; 210 int srcu_idx; 211 212 srcu_idx = srcu_read_lock(&head->srcu); 213 list_for_each_entry_rcu(ns, &head->list, siblings) { 214 if (capacity != get_capacity(ns->disk)) 215 clear_bit(NVME_NS_READY, &ns->flags); 216 } 217 srcu_read_unlock(&head->srcu, srcu_idx); 218 219 for_each_node(node) 220 rcu_assign_pointer(head->current_path[node], NULL); 221 kblockd_schedule_work(&head->requeue_work); 222 } 223 224 static bool nvme_path_is_disabled(struct nvme_ns *ns) 225 { 226 enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); 227 228 /* 229 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 230 * still be able to complete assuming that the controller is connected. 231 * Otherwise it will fail immediately and return to the requeue list. 232 */ 233 if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING) 234 return true; 235 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 236 !test_bit(NVME_NS_READY, &ns->flags)) 237 return true; 238 return false; 239 } 240 241 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 242 { 243 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 244 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 245 246 list_for_each_entry_rcu(ns, &head->list, siblings) { 247 if (nvme_path_is_disabled(ns)) 248 continue; 249 250 if (ns->ctrl->numa_node != NUMA_NO_NODE && 251 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 252 distance = node_distance(node, ns->ctrl->numa_node); 253 else 254 distance = LOCAL_DISTANCE; 255 256 switch (ns->ana_state) { 257 case NVME_ANA_OPTIMIZED: 258 if (distance < found_distance) { 259 found_distance = distance; 260 found = ns; 261 } 262 break; 263 case NVME_ANA_NONOPTIMIZED: 264 if (distance < fallback_distance) { 265 fallback_distance = distance; 266 fallback = ns; 267 } 268 break; 269 default: 270 break; 271 } 272 } 273 274 if (!found) 275 found = fallback; 276 if (found) 277 rcu_assign_pointer(head->current_path[node], found); 278 return found; 279 } 280 281 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 282 struct nvme_ns *ns) 283 { 284 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 285 siblings); 286 if (ns) 287 return ns; 288 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 289 } 290 291 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, 292 int node, struct nvme_ns *old) 293 { 294 struct nvme_ns *ns, *found = NULL; 295 296 if (list_is_singular(&head->list)) { 297 if (nvme_path_is_disabled(old)) 298 return NULL; 299 return old; 300 } 301 302 for (ns = nvme_next_ns(head, old); 303 ns && ns != old; 304 ns = nvme_next_ns(head, ns)) { 305 if (nvme_path_is_disabled(ns)) 306 continue; 307 308 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 309 found = ns; 310 goto out; 311 } 312 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 313 found = ns; 314 } 315 316 /* 317 * The loop above skips the current path for round-robin semantics. 318 * Fall back to the current path if either: 319 * - no other optimized path found and current is optimized, 320 * - no other usable path found and current is usable. 321 */ 322 if (!nvme_path_is_disabled(old) && 323 (old->ana_state == NVME_ANA_OPTIMIZED || 324 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 325 return old; 326 327 if (!found) 328 return NULL; 329 out: 330 rcu_assign_pointer(head->current_path[node], found); 331 return found; 332 } 333 334 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 335 { 336 return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE && 337 ns->ana_state == NVME_ANA_OPTIMIZED; 338 } 339 340 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 341 { 342 int node = numa_node_id(); 343 struct nvme_ns *ns; 344 345 ns = srcu_dereference(head->current_path[node], &head->srcu); 346 if (unlikely(!ns)) 347 return __nvme_find_path(head, node); 348 349 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR) 350 return nvme_round_robin_path(head, node, ns); 351 if (unlikely(!nvme_path_is_optimized(ns))) 352 return __nvme_find_path(head, node); 353 return ns; 354 } 355 356 static bool nvme_available_path(struct nvme_ns_head *head) 357 { 358 struct nvme_ns *ns; 359 360 list_for_each_entry_rcu(ns, &head->list, siblings) { 361 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 362 continue; 363 switch (nvme_ctrl_state(ns->ctrl)) { 364 case NVME_CTRL_LIVE: 365 case NVME_CTRL_RESETTING: 366 case NVME_CTRL_CONNECTING: 367 /* fallthru */ 368 return true; 369 default: 370 break; 371 } 372 } 373 return false; 374 } 375 376 static void nvme_ns_head_submit_bio(struct bio *bio) 377 { 378 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 379 struct device *dev = disk_to_dev(head->disk); 380 struct nvme_ns *ns; 381 int srcu_idx; 382 383 /* 384 * The namespace might be going away and the bio might be moved to a 385 * different queue via blk_steal_bios(), so we need to use the bio_split 386 * pool from the original queue to allocate the bvecs from. 387 */ 388 bio = bio_split_to_limits(bio); 389 if (!bio) 390 return; 391 392 srcu_idx = srcu_read_lock(&head->srcu); 393 ns = nvme_find_path(head); 394 if (likely(ns)) { 395 bio_set_dev(bio, ns->disk->part0); 396 bio->bi_opf |= REQ_NVME_MPATH; 397 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 398 bio->bi_iter.bi_sector); 399 submit_bio_noacct(bio); 400 } else if (nvme_available_path(head)) { 401 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 402 403 spin_lock_irq(&head->requeue_lock); 404 bio_list_add(&head->requeue_list, bio); 405 spin_unlock_irq(&head->requeue_lock); 406 } else { 407 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 408 409 bio_io_error(bio); 410 } 411 412 srcu_read_unlock(&head->srcu, srcu_idx); 413 } 414 415 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode) 416 { 417 if (!nvme_tryget_ns_head(disk->private_data)) 418 return -ENXIO; 419 return 0; 420 } 421 422 static void nvme_ns_head_release(struct gendisk *disk) 423 { 424 nvme_put_ns_head(disk->private_data); 425 } 426 427 #ifdef CONFIG_BLK_DEV_ZONED 428 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 429 unsigned int nr_zones, report_zones_cb cb, void *data) 430 { 431 struct nvme_ns_head *head = disk->private_data; 432 struct nvme_ns *ns; 433 int srcu_idx, ret = -EWOULDBLOCK; 434 435 srcu_idx = srcu_read_lock(&head->srcu); 436 ns = nvme_find_path(head); 437 if (ns) 438 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 439 srcu_read_unlock(&head->srcu, srcu_idx); 440 return ret; 441 } 442 #else 443 #define nvme_ns_head_report_zones NULL 444 #endif /* CONFIG_BLK_DEV_ZONED */ 445 446 const struct block_device_operations nvme_ns_head_ops = { 447 .owner = THIS_MODULE, 448 .submit_bio = nvme_ns_head_submit_bio, 449 .open = nvme_ns_head_open, 450 .release = nvme_ns_head_release, 451 .ioctl = nvme_ns_head_ioctl, 452 .compat_ioctl = blkdev_compat_ptr_ioctl, 453 .getgeo = nvme_getgeo, 454 .report_zones = nvme_ns_head_report_zones, 455 .pr_ops = &nvme_pr_ops, 456 }; 457 458 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 459 { 460 return container_of(cdev, struct nvme_ns_head, cdev); 461 } 462 463 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 464 { 465 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 466 return -ENXIO; 467 return 0; 468 } 469 470 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 471 { 472 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 473 return 0; 474 } 475 476 static const struct file_operations nvme_ns_head_chr_fops = { 477 .owner = THIS_MODULE, 478 .open = nvme_ns_head_chr_open, 479 .release = nvme_ns_head_chr_release, 480 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 481 .compat_ioctl = compat_ptr_ioctl, 482 .uring_cmd = nvme_ns_head_chr_uring_cmd, 483 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, 484 }; 485 486 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 487 { 488 int ret; 489 490 head->cdev_device.parent = &head->subsys->dev; 491 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 492 head->subsys->instance, head->instance); 493 if (ret) 494 return ret; 495 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 496 &nvme_ns_head_chr_fops, THIS_MODULE); 497 return ret; 498 } 499 500 static void nvme_requeue_work(struct work_struct *work) 501 { 502 struct nvme_ns_head *head = 503 container_of(work, struct nvme_ns_head, requeue_work); 504 struct bio *bio, *next; 505 506 spin_lock_irq(&head->requeue_lock); 507 next = bio_list_get(&head->requeue_list); 508 spin_unlock_irq(&head->requeue_lock); 509 510 while ((bio = next) != NULL) { 511 next = bio->bi_next; 512 bio->bi_next = NULL; 513 514 submit_bio_noacct(bio); 515 } 516 } 517 518 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 519 { 520 struct queue_limits lim; 521 bool vwc = false; 522 523 mutex_init(&head->lock); 524 bio_list_init(&head->requeue_list); 525 spin_lock_init(&head->requeue_lock); 526 INIT_WORK(&head->requeue_work, nvme_requeue_work); 527 528 /* 529 * Add a multipath node if the subsystems supports multiple controllers. 530 * We also do this for private namespaces as the namespace sharing flag 531 * could change after a rescan. 532 */ 533 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 534 !nvme_is_unique_nsid(ctrl, head) || !multipath) 535 return 0; 536 537 blk_set_stacking_limits(&lim); 538 lim.dma_alignment = 3; 539 if (head->ids.csi != NVME_CSI_ZNS) 540 lim.max_zone_append_sectors = 0; 541 542 head->disk = blk_alloc_disk(&lim, ctrl->numa_node); 543 if (IS_ERR(head->disk)) 544 return PTR_ERR(head->disk); 545 head->disk->fops = &nvme_ns_head_ops; 546 head->disk->private_data = head; 547 sprintf(head->disk->disk_name, "nvme%dn%d", 548 ctrl->subsys->instance, head->instance); 549 550 blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue); 551 blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue); 552 blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue); 553 /* 554 * This assumes all controllers that refer to a namespace either 555 * support poll queues or not. That is not a strict guarantee, 556 * but if the assumption is wrong the effect is only suboptimal 557 * performance but not correctness problem. 558 */ 559 if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL && 560 ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues) 561 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue); 562 563 /* we need to propagate up the VMC settings */ 564 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 565 vwc = true; 566 blk_queue_write_cache(head->disk->queue, vwc, vwc); 567 return 0; 568 } 569 570 static void nvme_mpath_set_live(struct nvme_ns *ns) 571 { 572 struct nvme_ns_head *head = ns->head; 573 int rc; 574 575 if (!head->disk) 576 return; 577 578 /* 579 * test_and_set_bit() is used because it is protecting against two nvme 580 * paths simultaneously calling device_add_disk() on the same namespace 581 * head. 582 */ 583 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 584 rc = device_add_disk(&head->subsys->dev, head->disk, 585 nvme_ns_attr_groups); 586 if (rc) { 587 clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); 588 return; 589 } 590 nvme_add_ns_head_cdev(head); 591 } 592 593 mutex_lock(&head->lock); 594 if (nvme_path_is_optimized(ns)) { 595 int node, srcu_idx; 596 597 srcu_idx = srcu_read_lock(&head->srcu); 598 for_each_node(node) 599 __nvme_find_path(head, node); 600 srcu_read_unlock(&head->srcu, srcu_idx); 601 } 602 mutex_unlock(&head->lock); 603 604 synchronize_srcu(&head->srcu); 605 kblockd_schedule_work(&head->requeue_work); 606 } 607 608 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 609 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 610 void *)) 611 { 612 void *base = ctrl->ana_log_buf; 613 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 614 int error, i; 615 616 lockdep_assert_held(&ctrl->ana_lock); 617 618 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 619 struct nvme_ana_group_desc *desc = base + offset; 620 u32 nr_nsids; 621 size_t nsid_buf_size; 622 623 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 624 return -EINVAL; 625 626 nr_nsids = le32_to_cpu(desc->nnsids); 627 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 628 629 if (WARN_ON_ONCE(desc->grpid == 0)) 630 return -EINVAL; 631 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 632 return -EINVAL; 633 if (WARN_ON_ONCE(desc->state == 0)) 634 return -EINVAL; 635 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 636 return -EINVAL; 637 638 offset += sizeof(*desc); 639 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 640 return -EINVAL; 641 642 error = cb(ctrl, desc, data); 643 if (error) 644 return error; 645 646 offset += nsid_buf_size; 647 } 648 649 return 0; 650 } 651 652 static inline bool nvme_state_is_live(enum nvme_ana_state state) 653 { 654 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 655 } 656 657 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 658 struct nvme_ns *ns) 659 { 660 ns->ana_grpid = le32_to_cpu(desc->grpid); 661 ns->ana_state = desc->state; 662 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 663 /* 664 * nvme_mpath_set_live() will trigger I/O to the multipath path device 665 * and in turn to this path device. However we cannot accept this I/O 666 * if the controller is not live. This may deadlock if called from 667 * nvme_mpath_init_identify() and the ctrl will never complete 668 * initialization, preventing I/O from completing. For this case we 669 * will reprocess the ANA log page in nvme_mpath_update() once the 670 * controller is ready. 671 */ 672 if (nvme_state_is_live(ns->ana_state) && 673 nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 674 nvme_mpath_set_live(ns); 675 } 676 677 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 678 struct nvme_ana_group_desc *desc, void *data) 679 { 680 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 681 unsigned *nr_change_groups = data; 682 struct nvme_ns *ns; 683 684 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 685 le32_to_cpu(desc->grpid), 686 nvme_ana_state_names[desc->state]); 687 688 if (desc->state == NVME_ANA_CHANGE) 689 (*nr_change_groups)++; 690 691 if (!nr_nsids) 692 return 0; 693 694 down_read(&ctrl->namespaces_rwsem); 695 list_for_each_entry(ns, &ctrl->namespaces, list) { 696 unsigned nsid; 697 again: 698 nsid = le32_to_cpu(desc->nsids[n]); 699 if (ns->head->ns_id < nsid) 700 continue; 701 if (ns->head->ns_id == nsid) 702 nvme_update_ns_ana_state(desc, ns); 703 if (++n == nr_nsids) 704 break; 705 if (ns->head->ns_id > nsid) 706 goto again; 707 } 708 up_read(&ctrl->namespaces_rwsem); 709 return 0; 710 } 711 712 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 713 { 714 u32 nr_change_groups = 0; 715 int error; 716 717 mutex_lock(&ctrl->ana_lock); 718 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 719 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 720 if (error) { 721 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 722 goto out_unlock; 723 } 724 725 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 726 nvme_update_ana_state); 727 if (error) 728 goto out_unlock; 729 730 /* 731 * In theory we should have an ANATT timer per group as they might enter 732 * the change state at different times. But that is a lot of overhead 733 * just to protect against a target that keeps entering new changes 734 * states while never finishing previous ones. But we'll still 735 * eventually time out once all groups are in change state, so this 736 * isn't a big deal. 737 * 738 * We also double the ANATT value to provide some slack for transports 739 * or AEN processing overhead. 740 */ 741 if (nr_change_groups) 742 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 743 else 744 del_timer_sync(&ctrl->anatt_timer); 745 out_unlock: 746 mutex_unlock(&ctrl->ana_lock); 747 return error; 748 } 749 750 static void nvme_ana_work(struct work_struct *work) 751 { 752 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 753 754 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 755 return; 756 757 nvme_read_ana_log(ctrl); 758 } 759 760 void nvme_mpath_update(struct nvme_ctrl *ctrl) 761 { 762 u32 nr_change_groups = 0; 763 764 if (!ctrl->ana_log_buf) 765 return; 766 767 mutex_lock(&ctrl->ana_lock); 768 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 769 mutex_unlock(&ctrl->ana_lock); 770 } 771 772 static void nvme_anatt_timeout(struct timer_list *t) 773 { 774 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 775 776 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 777 nvme_reset_ctrl(ctrl); 778 } 779 780 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 781 { 782 if (!nvme_ctrl_use_ana(ctrl)) 783 return; 784 del_timer_sync(&ctrl->anatt_timer); 785 cancel_work_sync(&ctrl->ana_work); 786 } 787 788 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 789 struct device_attribute subsys_attr_##_name = \ 790 __ATTR(_name, _mode, _show, _store) 791 792 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 793 struct device_attribute *attr, char *buf) 794 { 795 struct nvme_subsystem *subsys = 796 container_of(dev, struct nvme_subsystem, dev); 797 798 return sysfs_emit(buf, "%s\n", 799 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 800 } 801 802 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 803 struct device_attribute *attr, const char *buf, size_t count) 804 { 805 struct nvme_subsystem *subsys = 806 container_of(dev, struct nvme_subsystem, dev); 807 int i; 808 809 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 810 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 811 WRITE_ONCE(subsys->iopolicy, i); 812 return count; 813 } 814 } 815 816 return -EINVAL; 817 } 818 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 819 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 820 821 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 822 char *buf) 823 { 824 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 825 } 826 DEVICE_ATTR_RO(ana_grpid); 827 828 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 829 char *buf) 830 { 831 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 832 833 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 834 } 835 DEVICE_ATTR_RO(ana_state); 836 837 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 838 struct nvme_ana_group_desc *desc, void *data) 839 { 840 struct nvme_ana_group_desc *dst = data; 841 842 if (desc->grpid != dst->grpid) 843 return 0; 844 845 *dst = *desc; 846 return -ENXIO; /* just break out of the loop */ 847 } 848 849 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 850 { 851 if (nvme_ctrl_use_ana(ns->ctrl)) { 852 struct nvme_ana_group_desc desc = { 853 .grpid = anagrpid, 854 .state = 0, 855 }; 856 857 mutex_lock(&ns->ctrl->ana_lock); 858 ns->ana_grpid = le32_to_cpu(anagrpid); 859 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 860 mutex_unlock(&ns->ctrl->ana_lock); 861 if (desc.state) { 862 /* found the group desc: update */ 863 nvme_update_ns_ana_state(&desc, ns); 864 } else { 865 /* group desc not found: trigger a re-read */ 866 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 867 queue_work(nvme_wq, &ns->ctrl->ana_work); 868 } 869 } else { 870 ns->ana_state = NVME_ANA_OPTIMIZED; 871 nvme_mpath_set_live(ns); 872 } 873 874 if (blk_queue_stable_writes(ns->queue) && ns->head->disk) 875 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 876 ns->head->disk->queue); 877 #ifdef CONFIG_BLK_DEV_ZONED 878 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 879 ns->head->disk->nr_zones = ns->disk->nr_zones; 880 #endif 881 } 882 883 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 884 { 885 if (!head->disk) 886 return; 887 kblockd_schedule_work(&head->requeue_work); 888 if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 889 nvme_cdev_del(&head->cdev, &head->cdev_device); 890 del_gendisk(head->disk); 891 } 892 } 893 894 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 895 { 896 if (!head->disk) 897 return; 898 /* make sure all pending bios are cleaned up */ 899 kblockd_schedule_work(&head->requeue_work); 900 flush_work(&head->requeue_work); 901 put_disk(head->disk); 902 } 903 904 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 905 { 906 mutex_init(&ctrl->ana_lock); 907 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 908 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 909 } 910 911 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 912 { 913 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 914 size_t ana_log_size; 915 int error = 0; 916 917 /* check if multipath is enabled and we have the capability */ 918 if (!multipath || !ctrl->subsys || 919 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 920 return 0; 921 922 if (!ctrl->max_namespaces || 923 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 924 dev_err(ctrl->device, 925 "Invalid MNAN value %u\n", ctrl->max_namespaces); 926 return -EINVAL; 927 } 928 929 ctrl->anacap = id->anacap; 930 ctrl->anatt = id->anatt; 931 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 932 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 933 934 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 935 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 936 ctrl->max_namespaces * sizeof(__le32); 937 if (ana_log_size > max_transfer_size) { 938 dev_err(ctrl->device, 939 "ANA log page size (%zd) larger than MDTS (%zd).\n", 940 ana_log_size, max_transfer_size); 941 dev_err(ctrl->device, "disabling ANA support.\n"); 942 goto out_uninit; 943 } 944 if (ana_log_size > ctrl->ana_log_size) { 945 nvme_mpath_stop(ctrl); 946 nvme_mpath_uninit(ctrl); 947 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 948 if (!ctrl->ana_log_buf) 949 return -ENOMEM; 950 } 951 ctrl->ana_log_size = ana_log_size; 952 error = nvme_read_ana_log(ctrl); 953 if (error) 954 goto out_uninit; 955 return 0; 956 957 out_uninit: 958 nvme_mpath_uninit(ctrl); 959 return error; 960 } 961 962 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 963 { 964 kvfree(ctrl->ana_log_buf); 965 ctrl->ana_log_buf = NULL; 966 ctrl->ana_log_size = 0; 967 } 968