1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 }; 21 22 static int iopolicy = NVME_IOPOLICY_NUMA; 23 24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 25 { 26 if (!val) 27 return -EINVAL; 28 if (!strncmp(val, "numa", 4)) 29 iopolicy = NVME_IOPOLICY_NUMA; 30 else if (!strncmp(val, "round-robin", 11)) 31 iopolicy = NVME_IOPOLICY_RR; 32 else 33 return -EINVAL; 34 35 return 0; 36 } 37 38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 39 { 40 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 41 } 42 43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 44 &iopolicy, 0644); 45 MODULE_PARM_DESC(iopolicy, 46 "Default multipath I/O policy; 'numa' (default) or 'round-robin'"); 47 48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 49 { 50 subsys->iopolicy = iopolicy; 51 } 52 53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 54 { 55 struct nvme_ns_head *h; 56 57 lockdep_assert_held(&subsys->lock); 58 list_for_each_entry(h, &subsys->nsheads, entry) 59 if (h->disk) 60 blk_mq_unfreeze_queue(h->disk->queue); 61 } 62 63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 64 { 65 struct nvme_ns_head *h; 66 67 lockdep_assert_held(&subsys->lock); 68 list_for_each_entry(h, &subsys->nsheads, entry) 69 if (h->disk) 70 blk_mq_freeze_queue_wait(h->disk->queue); 71 } 72 73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 74 { 75 struct nvme_ns_head *h; 76 77 lockdep_assert_held(&subsys->lock); 78 list_for_each_entry(h, &subsys->nsheads, entry) 79 if (h->disk) 80 blk_freeze_queue_start(h->disk->queue); 81 } 82 83 void nvme_failover_req(struct request *req) 84 { 85 struct nvme_ns *ns = req->q->queuedata; 86 u16 status = nvme_req(req)->status & 0x7ff; 87 unsigned long flags; 88 struct bio *bio; 89 90 nvme_mpath_clear_current_path(ns); 91 92 /* 93 * If we got back an ANA error, we know the controller is alive but not 94 * ready to serve this namespace. Kick of a re-read of the ANA 95 * information page, and just try any other available path for now. 96 */ 97 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 98 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 99 queue_work(nvme_wq, &ns->ctrl->ana_work); 100 } 101 102 spin_lock_irqsave(&ns->head->requeue_lock, flags); 103 for (bio = req->bio; bio; bio = bio->bi_next) { 104 bio_set_dev(bio, ns->head->disk->part0); 105 if (bio->bi_opf & REQ_POLLED) { 106 bio->bi_opf &= ~REQ_POLLED; 107 bio->bi_cookie = BLK_QC_T_NONE; 108 } 109 /* 110 * The alternate request queue that we may end up submitting 111 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 112 * will fail the I/O immediately with EAGAIN to the issuer. 113 * We are not in the issuer context which cannot block. Clear 114 * the flag to avoid spurious EAGAIN I/O failures. 115 */ 116 bio->bi_opf &= ~REQ_NOWAIT; 117 } 118 blk_steal_bios(&ns->head->requeue_list, req); 119 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 120 121 nvme_req(req)->status = 0; 122 nvme_end_req(req); 123 kblockd_schedule_work(&ns->head->requeue_work); 124 } 125 126 void nvme_mpath_start_request(struct request *rq) 127 { 128 struct nvme_ns *ns = rq->q->queuedata; 129 struct gendisk *disk = ns->head->disk; 130 131 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) 132 return; 133 134 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; 135 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), 136 jiffies); 137 } 138 EXPORT_SYMBOL_GPL(nvme_mpath_start_request); 139 140 void nvme_mpath_end_request(struct request *rq) 141 { 142 struct nvme_ns *ns = rq->q->queuedata; 143 144 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) 145 return; 146 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), 147 blk_rq_bytes(rq) >> SECTOR_SHIFT, 148 nvme_req(rq)->start_time); 149 } 150 151 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 152 { 153 struct nvme_ns *ns; 154 int srcu_idx; 155 156 srcu_idx = srcu_read_lock(&ctrl->srcu); 157 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 158 if (!ns->head->disk) 159 continue; 160 kblockd_schedule_work(&ns->head->requeue_work); 161 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 162 disk_uevent(ns->head->disk, KOBJ_CHANGE); 163 } 164 srcu_read_unlock(&ctrl->srcu, srcu_idx); 165 } 166 167 static const char *nvme_ana_state_names[] = { 168 [0] = "invalid state", 169 [NVME_ANA_OPTIMIZED] = "optimized", 170 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 171 [NVME_ANA_INACCESSIBLE] = "inaccessible", 172 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 173 [NVME_ANA_CHANGE] = "change", 174 }; 175 176 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 177 { 178 struct nvme_ns_head *head = ns->head; 179 bool changed = false; 180 int node; 181 182 if (!head) 183 goto out; 184 185 for_each_node(node) { 186 if (ns == rcu_access_pointer(head->current_path[node])) { 187 rcu_assign_pointer(head->current_path[node], NULL); 188 changed = true; 189 } 190 } 191 out: 192 return changed; 193 } 194 195 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 196 { 197 struct nvme_ns *ns; 198 int srcu_idx; 199 200 srcu_idx = srcu_read_lock(&ctrl->srcu); 201 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 202 nvme_mpath_clear_current_path(ns); 203 kblockd_schedule_work(&ns->head->requeue_work); 204 } 205 srcu_read_unlock(&ctrl->srcu, srcu_idx); 206 } 207 208 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 209 { 210 struct nvme_ns_head *head = ns->head; 211 sector_t capacity = get_capacity(head->disk); 212 int node; 213 int srcu_idx; 214 215 srcu_idx = srcu_read_lock(&head->srcu); 216 list_for_each_entry_rcu(ns, &head->list, siblings) { 217 if (capacity != get_capacity(ns->disk)) 218 clear_bit(NVME_NS_READY, &ns->flags); 219 } 220 srcu_read_unlock(&head->srcu, srcu_idx); 221 222 for_each_node(node) 223 rcu_assign_pointer(head->current_path[node], NULL); 224 kblockd_schedule_work(&head->requeue_work); 225 } 226 227 static bool nvme_path_is_disabled(struct nvme_ns *ns) 228 { 229 enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); 230 231 /* 232 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 233 * still be able to complete assuming that the controller is connected. 234 * Otherwise it will fail immediately and return to the requeue list. 235 */ 236 if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING) 237 return true; 238 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 239 !test_bit(NVME_NS_READY, &ns->flags)) 240 return true; 241 return false; 242 } 243 244 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 245 { 246 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 247 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 248 249 list_for_each_entry_rcu(ns, &head->list, siblings) { 250 if (nvme_path_is_disabled(ns)) 251 continue; 252 253 if (ns->ctrl->numa_node != NUMA_NO_NODE && 254 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 255 distance = node_distance(node, ns->ctrl->numa_node); 256 else 257 distance = LOCAL_DISTANCE; 258 259 switch (ns->ana_state) { 260 case NVME_ANA_OPTIMIZED: 261 if (distance < found_distance) { 262 found_distance = distance; 263 found = ns; 264 } 265 break; 266 case NVME_ANA_NONOPTIMIZED: 267 if (distance < fallback_distance) { 268 fallback_distance = distance; 269 fallback = ns; 270 } 271 break; 272 default: 273 break; 274 } 275 } 276 277 if (!found) 278 found = fallback; 279 if (found) 280 rcu_assign_pointer(head->current_path[node], found); 281 return found; 282 } 283 284 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 285 struct nvme_ns *ns) 286 { 287 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 288 siblings); 289 if (ns) 290 return ns; 291 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 292 } 293 294 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, 295 int node, struct nvme_ns *old) 296 { 297 struct nvme_ns *ns, *found = NULL; 298 299 if (list_is_singular(&head->list)) { 300 if (nvme_path_is_disabled(old)) 301 return NULL; 302 return old; 303 } 304 305 for (ns = nvme_next_ns(head, old); 306 ns && ns != old; 307 ns = nvme_next_ns(head, ns)) { 308 if (nvme_path_is_disabled(ns)) 309 continue; 310 311 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 312 found = ns; 313 goto out; 314 } 315 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 316 found = ns; 317 } 318 319 /* 320 * The loop above skips the current path for round-robin semantics. 321 * Fall back to the current path if either: 322 * - no other optimized path found and current is optimized, 323 * - no other usable path found and current is usable. 324 */ 325 if (!nvme_path_is_disabled(old) && 326 (old->ana_state == NVME_ANA_OPTIMIZED || 327 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 328 return old; 329 330 if (!found) 331 return NULL; 332 out: 333 rcu_assign_pointer(head->current_path[node], found); 334 return found; 335 } 336 337 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 338 { 339 return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE && 340 ns->ana_state == NVME_ANA_OPTIMIZED; 341 } 342 343 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 344 { 345 int node = numa_node_id(); 346 struct nvme_ns *ns; 347 348 ns = srcu_dereference(head->current_path[node], &head->srcu); 349 if (unlikely(!ns)) 350 return __nvme_find_path(head, node); 351 352 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR) 353 return nvme_round_robin_path(head, node, ns); 354 if (unlikely(!nvme_path_is_optimized(ns))) 355 return __nvme_find_path(head, node); 356 return ns; 357 } 358 359 static bool nvme_available_path(struct nvme_ns_head *head) 360 { 361 struct nvme_ns *ns; 362 363 list_for_each_entry_rcu(ns, &head->list, siblings) { 364 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 365 continue; 366 switch (nvme_ctrl_state(ns->ctrl)) { 367 case NVME_CTRL_LIVE: 368 case NVME_CTRL_RESETTING: 369 case NVME_CTRL_CONNECTING: 370 /* fallthru */ 371 return true; 372 default: 373 break; 374 } 375 } 376 return false; 377 } 378 379 static void nvme_ns_head_submit_bio(struct bio *bio) 380 { 381 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 382 struct device *dev = disk_to_dev(head->disk); 383 struct nvme_ns *ns; 384 int srcu_idx; 385 386 /* 387 * The namespace might be going away and the bio might be moved to a 388 * different queue via blk_steal_bios(), so we need to use the bio_split 389 * pool from the original queue to allocate the bvecs from. 390 */ 391 bio = bio_split_to_limits(bio); 392 if (!bio) 393 return; 394 395 srcu_idx = srcu_read_lock(&head->srcu); 396 ns = nvme_find_path(head); 397 if (likely(ns)) { 398 bio_set_dev(bio, ns->disk->part0); 399 bio->bi_opf |= REQ_NVME_MPATH; 400 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 401 bio->bi_iter.bi_sector); 402 submit_bio_noacct(bio); 403 } else if (nvme_available_path(head)) { 404 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 405 406 spin_lock_irq(&head->requeue_lock); 407 bio_list_add(&head->requeue_list, bio); 408 spin_unlock_irq(&head->requeue_lock); 409 } else { 410 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 411 412 bio_io_error(bio); 413 } 414 415 srcu_read_unlock(&head->srcu, srcu_idx); 416 } 417 418 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode) 419 { 420 if (!nvme_tryget_ns_head(disk->private_data)) 421 return -ENXIO; 422 return 0; 423 } 424 425 static void nvme_ns_head_release(struct gendisk *disk) 426 { 427 nvme_put_ns_head(disk->private_data); 428 } 429 430 #ifdef CONFIG_BLK_DEV_ZONED 431 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 432 unsigned int nr_zones, report_zones_cb cb, void *data) 433 { 434 struct nvme_ns_head *head = disk->private_data; 435 struct nvme_ns *ns; 436 int srcu_idx, ret = -EWOULDBLOCK; 437 438 srcu_idx = srcu_read_lock(&head->srcu); 439 ns = nvme_find_path(head); 440 if (ns) 441 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 442 srcu_read_unlock(&head->srcu, srcu_idx); 443 return ret; 444 } 445 #else 446 #define nvme_ns_head_report_zones NULL 447 #endif /* CONFIG_BLK_DEV_ZONED */ 448 449 const struct block_device_operations nvme_ns_head_ops = { 450 .owner = THIS_MODULE, 451 .submit_bio = nvme_ns_head_submit_bio, 452 .open = nvme_ns_head_open, 453 .release = nvme_ns_head_release, 454 .ioctl = nvme_ns_head_ioctl, 455 .compat_ioctl = blkdev_compat_ptr_ioctl, 456 .getgeo = nvme_getgeo, 457 .report_zones = nvme_ns_head_report_zones, 458 .pr_ops = &nvme_pr_ops, 459 }; 460 461 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 462 { 463 return container_of(cdev, struct nvme_ns_head, cdev); 464 } 465 466 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 467 { 468 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 469 return -ENXIO; 470 return 0; 471 } 472 473 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 474 { 475 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 476 return 0; 477 } 478 479 static const struct file_operations nvme_ns_head_chr_fops = { 480 .owner = THIS_MODULE, 481 .open = nvme_ns_head_chr_open, 482 .release = nvme_ns_head_chr_release, 483 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 484 .compat_ioctl = compat_ptr_ioctl, 485 .uring_cmd = nvme_ns_head_chr_uring_cmd, 486 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, 487 }; 488 489 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 490 { 491 int ret; 492 493 head->cdev_device.parent = &head->subsys->dev; 494 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 495 head->subsys->instance, head->instance); 496 if (ret) 497 return ret; 498 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 499 &nvme_ns_head_chr_fops, THIS_MODULE); 500 return ret; 501 } 502 503 static void nvme_requeue_work(struct work_struct *work) 504 { 505 struct nvme_ns_head *head = 506 container_of(work, struct nvme_ns_head, requeue_work); 507 struct bio *bio, *next; 508 509 spin_lock_irq(&head->requeue_lock); 510 next = bio_list_get(&head->requeue_list); 511 spin_unlock_irq(&head->requeue_lock); 512 513 while ((bio = next) != NULL) { 514 next = bio->bi_next; 515 bio->bi_next = NULL; 516 517 submit_bio_noacct(bio); 518 } 519 } 520 521 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 522 { 523 struct queue_limits lim; 524 525 mutex_init(&head->lock); 526 bio_list_init(&head->requeue_list); 527 spin_lock_init(&head->requeue_lock); 528 INIT_WORK(&head->requeue_work, nvme_requeue_work); 529 530 /* 531 * Add a multipath node if the subsystems supports multiple controllers. 532 * We also do this for private namespaces as the namespace sharing flag 533 * could change after a rescan. 534 */ 535 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 536 !nvme_is_unique_nsid(ctrl, head) || !multipath) 537 return 0; 538 539 blk_set_stacking_limits(&lim); 540 lim.dma_alignment = 3; 541 lim.features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL; 542 if (head->ids.csi != NVME_CSI_ZNS) 543 lim.max_zone_append_sectors = 0; 544 545 head->disk = blk_alloc_disk(&lim, ctrl->numa_node); 546 if (IS_ERR(head->disk)) 547 return PTR_ERR(head->disk); 548 head->disk->fops = &nvme_ns_head_ops; 549 head->disk->private_data = head; 550 sprintf(head->disk->disk_name, "nvme%dn%d", 551 ctrl->subsys->instance, head->instance); 552 return 0; 553 } 554 555 static void nvme_mpath_set_live(struct nvme_ns *ns) 556 { 557 struct nvme_ns_head *head = ns->head; 558 int rc; 559 560 if (!head->disk) 561 return; 562 563 /* 564 * test_and_set_bit() is used because it is protecting against two nvme 565 * paths simultaneously calling device_add_disk() on the same namespace 566 * head. 567 */ 568 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 569 rc = device_add_disk(&head->subsys->dev, head->disk, 570 nvme_ns_attr_groups); 571 if (rc) { 572 clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); 573 return; 574 } 575 nvme_add_ns_head_cdev(head); 576 } 577 578 mutex_lock(&head->lock); 579 if (nvme_path_is_optimized(ns)) { 580 int node, srcu_idx; 581 582 srcu_idx = srcu_read_lock(&head->srcu); 583 for_each_online_node(node) 584 __nvme_find_path(head, node); 585 srcu_read_unlock(&head->srcu, srcu_idx); 586 } 587 mutex_unlock(&head->lock); 588 589 synchronize_srcu(&head->srcu); 590 kblockd_schedule_work(&head->requeue_work); 591 } 592 593 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 594 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 595 void *)) 596 { 597 void *base = ctrl->ana_log_buf; 598 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 599 int error, i; 600 601 lockdep_assert_held(&ctrl->ana_lock); 602 603 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 604 struct nvme_ana_group_desc *desc = base + offset; 605 u32 nr_nsids; 606 size_t nsid_buf_size; 607 608 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 609 return -EINVAL; 610 611 nr_nsids = le32_to_cpu(desc->nnsids); 612 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 613 614 if (WARN_ON_ONCE(desc->grpid == 0)) 615 return -EINVAL; 616 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 617 return -EINVAL; 618 if (WARN_ON_ONCE(desc->state == 0)) 619 return -EINVAL; 620 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 621 return -EINVAL; 622 623 offset += sizeof(*desc); 624 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 625 return -EINVAL; 626 627 error = cb(ctrl, desc, data); 628 if (error) 629 return error; 630 631 offset += nsid_buf_size; 632 } 633 634 return 0; 635 } 636 637 static inline bool nvme_state_is_live(enum nvme_ana_state state) 638 { 639 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 640 } 641 642 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 643 struct nvme_ns *ns) 644 { 645 ns->ana_grpid = le32_to_cpu(desc->grpid); 646 ns->ana_state = desc->state; 647 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 648 /* 649 * nvme_mpath_set_live() will trigger I/O to the multipath path device 650 * and in turn to this path device. However we cannot accept this I/O 651 * if the controller is not live. This may deadlock if called from 652 * nvme_mpath_init_identify() and the ctrl will never complete 653 * initialization, preventing I/O from completing. For this case we 654 * will reprocess the ANA log page in nvme_mpath_update() once the 655 * controller is ready. 656 */ 657 if (nvme_state_is_live(ns->ana_state) && 658 nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE) 659 nvme_mpath_set_live(ns); 660 } 661 662 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 663 struct nvme_ana_group_desc *desc, void *data) 664 { 665 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 666 unsigned *nr_change_groups = data; 667 struct nvme_ns *ns; 668 int srcu_idx; 669 670 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 671 le32_to_cpu(desc->grpid), 672 nvme_ana_state_names[desc->state]); 673 674 if (desc->state == NVME_ANA_CHANGE) 675 (*nr_change_groups)++; 676 677 if (!nr_nsids) 678 return 0; 679 680 srcu_idx = srcu_read_lock(&ctrl->srcu); 681 list_for_each_entry_rcu(ns, &ctrl->namespaces, list) { 682 unsigned nsid; 683 again: 684 nsid = le32_to_cpu(desc->nsids[n]); 685 if (ns->head->ns_id < nsid) 686 continue; 687 if (ns->head->ns_id == nsid) 688 nvme_update_ns_ana_state(desc, ns); 689 if (++n == nr_nsids) 690 break; 691 if (ns->head->ns_id > nsid) 692 goto again; 693 } 694 srcu_read_unlock(&ctrl->srcu, srcu_idx); 695 return 0; 696 } 697 698 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 699 { 700 u32 nr_change_groups = 0; 701 int error; 702 703 mutex_lock(&ctrl->ana_lock); 704 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 705 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 706 if (error) { 707 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 708 goto out_unlock; 709 } 710 711 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 712 nvme_update_ana_state); 713 if (error) 714 goto out_unlock; 715 716 /* 717 * In theory we should have an ANATT timer per group as they might enter 718 * the change state at different times. But that is a lot of overhead 719 * just to protect against a target that keeps entering new changes 720 * states while never finishing previous ones. But we'll still 721 * eventually time out once all groups are in change state, so this 722 * isn't a big deal. 723 * 724 * We also double the ANATT value to provide some slack for transports 725 * or AEN processing overhead. 726 */ 727 if (nr_change_groups) 728 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 729 else 730 del_timer_sync(&ctrl->anatt_timer); 731 out_unlock: 732 mutex_unlock(&ctrl->ana_lock); 733 return error; 734 } 735 736 static void nvme_ana_work(struct work_struct *work) 737 { 738 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 739 740 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 741 return; 742 743 nvme_read_ana_log(ctrl); 744 } 745 746 void nvme_mpath_update(struct nvme_ctrl *ctrl) 747 { 748 u32 nr_change_groups = 0; 749 750 if (!ctrl->ana_log_buf) 751 return; 752 753 mutex_lock(&ctrl->ana_lock); 754 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 755 mutex_unlock(&ctrl->ana_lock); 756 } 757 758 static void nvme_anatt_timeout(struct timer_list *t) 759 { 760 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 761 762 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 763 nvme_reset_ctrl(ctrl); 764 } 765 766 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 767 { 768 if (!nvme_ctrl_use_ana(ctrl)) 769 return; 770 del_timer_sync(&ctrl->anatt_timer); 771 cancel_work_sync(&ctrl->ana_work); 772 } 773 774 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 775 struct device_attribute subsys_attr_##_name = \ 776 __ATTR(_name, _mode, _show, _store) 777 778 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 779 struct device_attribute *attr, char *buf) 780 { 781 struct nvme_subsystem *subsys = 782 container_of(dev, struct nvme_subsystem, dev); 783 784 return sysfs_emit(buf, "%s\n", 785 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 786 } 787 788 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 789 struct device_attribute *attr, const char *buf, size_t count) 790 { 791 struct nvme_subsystem *subsys = 792 container_of(dev, struct nvme_subsystem, dev); 793 int i; 794 795 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 796 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 797 WRITE_ONCE(subsys->iopolicy, i); 798 return count; 799 } 800 } 801 802 return -EINVAL; 803 } 804 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 805 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 806 807 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 808 char *buf) 809 { 810 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 811 } 812 DEVICE_ATTR_RO(ana_grpid); 813 814 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 815 char *buf) 816 { 817 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 818 819 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 820 } 821 DEVICE_ATTR_RO(ana_state); 822 823 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 824 struct nvme_ana_group_desc *desc, void *data) 825 { 826 struct nvme_ana_group_desc *dst = data; 827 828 if (desc->grpid != dst->grpid) 829 return 0; 830 831 *dst = *desc; 832 return -ENXIO; /* just break out of the loop */ 833 } 834 835 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 836 { 837 if (nvme_ctrl_use_ana(ns->ctrl)) { 838 struct nvme_ana_group_desc desc = { 839 .grpid = anagrpid, 840 .state = 0, 841 }; 842 843 mutex_lock(&ns->ctrl->ana_lock); 844 ns->ana_grpid = le32_to_cpu(anagrpid); 845 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 846 mutex_unlock(&ns->ctrl->ana_lock); 847 if (desc.state) { 848 /* found the group desc: update */ 849 nvme_update_ns_ana_state(&desc, ns); 850 } else { 851 /* group desc not found: trigger a re-read */ 852 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 853 queue_work(nvme_wq, &ns->ctrl->ana_work); 854 } 855 } else { 856 ns->ana_state = NVME_ANA_OPTIMIZED; 857 nvme_mpath_set_live(ns); 858 } 859 860 #ifdef CONFIG_BLK_DEV_ZONED 861 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 862 ns->head->disk->nr_zones = ns->disk->nr_zones; 863 #endif 864 } 865 866 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 867 { 868 if (!head->disk) 869 return; 870 kblockd_schedule_work(&head->requeue_work); 871 if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 872 nvme_cdev_del(&head->cdev, &head->cdev_device); 873 del_gendisk(head->disk); 874 } 875 } 876 877 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 878 { 879 if (!head->disk) 880 return; 881 /* make sure all pending bios are cleaned up */ 882 kblockd_schedule_work(&head->requeue_work); 883 flush_work(&head->requeue_work); 884 put_disk(head->disk); 885 } 886 887 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 888 { 889 mutex_init(&ctrl->ana_lock); 890 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 891 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 892 } 893 894 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 895 { 896 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 897 size_t ana_log_size; 898 int error = 0; 899 900 /* check if multipath is enabled and we have the capability */ 901 if (!multipath || !ctrl->subsys || 902 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 903 return 0; 904 905 if (!ctrl->max_namespaces || 906 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 907 dev_err(ctrl->device, 908 "Invalid MNAN value %u\n", ctrl->max_namespaces); 909 return -EINVAL; 910 } 911 912 ctrl->anacap = id->anacap; 913 ctrl->anatt = id->anatt; 914 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 915 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 916 917 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 918 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 919 ctrl->max_namespaces * sizeof(__le32); 920 if (ana_log_size > max_transfer_size) { 921 dev_err(ctrl->device, 922 "ANA log page size (%zd) larger than MDTS (%zd).\n", 923 ana_log_size, max_transfer_size); 924 dev_err(ctrl->device, "disabling ANA support.\n"); 925 goto out_uninit; 926 } 927 if (ana_log_size > ctrl->ana_log_size) { 928 nvme_mpath_stop(ctrl); 929 nvme_mpath_uninit(ctrl); 930 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 931 if (!ctrl->ana_log_buf) 932 return -ENOMEM; 933 } 934 ctrl->ana_log_size = ana_log_size; 935 error = nvme_read_ana_log(ctrl); 936 if (error) 937 goto out_uninit; 938 return 0; 939 940 out_uninit: 941 nvme_mpath_uninit(ctrl); 942 return error; 943 } 944 945 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 946 { 947 kvfree(ctrl->ana_log_buf); 948 ctrl->ana_log_buf = NULL; 949 ctrl->ana_log_size = 0; 950 } 951