1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common code for the NVMe target. 4 * Copyright (c) 2015-2016 HGST, a Western Digital Company. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/hex.h> 8 #include <linux/module.h> 9 #include <linux/random.h> 10 #include <linux/rculist.h> 11 #include <linux/pci-p2pdma.h> 12 #include <linux/scatterlist.h> 13 14 #include <generated/utsrelease.h> 15 16 #define CREATE_TRACE_POINTS 17 #include "trace.h" 18 19 #include "nvmet.h" 20 #include "debugfs.h" 21 22 struct kmem_cache *nvmet_bvec_cache; 23 struct workqueue_struct *buffered_io_wq; 24 struct workqueue_struct *zbd_wq; 25 static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX]; 26 static DEFINE_IDA(cntlid_ida); 27 28 struct workqueue_struct *nvmet_wq; 29 EXPORT_SYMBOL_GPL(nvmet_wq); 30 struct workqueue_struct *nvmet_aen_wq; 31 EXPORT_SYMBOL_GPL(nvmet_aen_wq); 32 33 /* 34 * This read/write semaphore is used to synchronize access to configuration 35 * information on a target system that will result in discovery log page 36 * information change for at least one host. 37 * The full list of resources to protected by this semaphore is: 38 * 39 * - subsystems list 40 * - per-subsystem allowed hosts list 41 * - allow_any_host subsystem attribute 42 * - nvmet_genctr 43 * - the nvmet_transports array 44 * 45 * When updating any of those lists/structures write lock should be obtained, 46 * while when reading (populating discovery log page or checking host-subsystem 47 * link) read lock is obtained to allow concurrent reads. 48 */ 49 DECLARE_RWSEM(nvmet_config_sem); 50 51 u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1]; 52 u64 nvmet_ana_chgcnt; 53 DECLARE_RWSEM(nvmet_ana_sem); 54 55 inline u16 errno_to_nvme_status(struct nvmet_req *req, int errno) 56 { 57 switch (errno) { 58 case 0: 59 return NVME_SC_SUCCESS; 60 case -ENOSPC: 61 req->error_loc = offsetof(struct nvme_rw_command, length); 62 return NVME_SC_CAP_EXCEEDED | NVME_STATUS_DNR; 63 case -EREMOTEIO: 64 req->error_loc = offsetof(struct nvme_rw_command, slba); 65 return NVME_SC_LBA_RANGE | NVME_STATUS_DNR; 66 case -EOPNOTSUPP: 67 req->error_loc = offsetof(struct nvme_common_command, opcode); 68 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 69 case -ENODATA: 70 req->error_loc = offsetof(struct nvme_rw_command, nsid); 71 return NVME_SC_ACCESS_DENIED; 72 case -EIO: 73 fallthrough; 74 default: 75 req->error_loc = offsetof(struct nvme_common_command, opcode); 76 return NVME_SC_INTERNAL | NVME_STATUS_DNR; 77 } 78 } 79 80 u16 nvmet_report_invalid_opcode(struct nvmet_req *req) 81 { 82 pr_debug("unhandled cmd %d on qid %d\n", req->cmd->common.opcode, 83 req->sq->qid); 84 85 req->error_loc = offsetof(struct nvme_common_command, opcode); 86 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 87 } 88 89 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, 90 const char *subsysnqn); 91 92 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf, 93 size_t len) 94 { 95 if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len) { 96 req->error_loc = offsetof(struct nvme_common_command, dptr); 97 return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; 98 } 99 return 0; 100 } 101 102 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len) 103 { 104 if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len) { 105 req->error_loc = offsetof(struct nvme_common_command, dptr); 106 return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; 107 } 108 return 0; 109 } 110 111 u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len) 112 { 113 if (sg_zero_buffer(req->sg, req->sg_cnt, len, off) != len) { 114 req->error_loc = offsetof(struct nvme_common_command, dptr); 115 return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; 116 } 117 return 0; 118 } 119 120 static u32 nvmet_max_nsid(struct nvmet_subsys *subsys) 121 { 122 struct nvmet_ns *cur; 123 unsigned long idx; 124 u32 nsid = 0; 125 126 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, cur) 127 nsid = cur->nsid; 128 129 return nsid; 130 } 131 132 static u32 nvmet_async_event_result(struct nvmet_async_event *aen) 133 { 134 return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); 135 } 136 137 static void nvmet_async_events_failall(struct nvmet_ctrl *ctrl) 138 { 139 struct nvmet_req *req; 140 141 mutex_lock(&ctrl->lock); 142 while (ctrl->nr_async_event_cmds) { 143 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; 144 mutex_unlock(&ctrl->lock); 145 nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_STATUS_DNR); 146 mutex_lock(&ctrl->lock); 147 } 148 mutex_unlock(&ctrl->lock); 149 } 150 151 static void nvmet_async_events_process(struct nvmet_ctrl *ctrl) 152 { 153 struct nvmet_async_event *aen; 154 struct nvmet_req *req; 155 156 mutex_lock(&ctrl->lock); 157 while (ctrl->nr_async_event_cmds && !list_empty(&ctrl->async_events)) { 158 aen = list_first_entry(&ctrl->async_events, 159 struct nvmet_async_event, entry); 160 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; 161 nvmet_set_result(req, nvmet_async_event_result(aen)); 162 163 list_del(&aen->entry); 164 kfree(aen); 165 166 mutex_unlock(&ctrl->lock); 167 trace_nvmet_async_event(ctrl, req->cqe->result.u32); 168 nvmet_req_complete(req, 0); 169 mutex_lock(&ctrl->lock); 170 } 171 mutex_unlock(&ctrl->lock); 172 } 173 174 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) 175 { 176 struct nvmet_async_event *aen, *tmp; 177 178 mutex_lock(&ctrl->lock); 179 list_for_each_entry_safe(aen, tmp, &ctrl->async_events, entry) { 180 list_del(&aen->entry); 181 kfree(aen); 182 } 183 mutex_unlock(&ctrl->lock); 184 } 185 186 static void nvmet_async_event_work(struct work_struct *work) 187 { 188 struct nvmet_ctrl *ctrl = 189 container_of(work, struct nvmet_ctrl, async_event_work); 190 191 nvmet_async_events_process(ctrl); 192 } 193 194 void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, 195 u8 event_info, u8 log_page) 196 { 197 struct nvmet_async_event *aen; 198 199 aen = kmalloc_obj(*aen); 200 if (!aen) 201 return; 202 203 aen->event_type = event_type; 204 aen->event_info = event_info; 205 aen->log_page = log_page; 206 207 mutex_lock(&ctrl->lock); 208 list_add_tail(&aen->entry, &ctrl->async_events); 209 mutex_unlock(&ctrl->lock); 210 211 queue_work(nvmet_aen_wq, &ctrl->async_event_work); 212 } 213 214 static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) 215 { 216 u32 i; 217 218 mutex_lock(&ctrl->lock); 219 if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES) 220 goto out_unlock; 221 222 for (i = 0; i < ctrl->nr_changed_ns; i++) { 223 if (ctrl->changed_ns_list[i] == nsid) 224 goto out_unlock; 225 } 226 227 if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) { 228 ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff); 229 ctrl->nr_changed_ns = U32_MAX; 230 goto out_unlock; 231 } 232 233 ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid; 234 out_unlock: 235 mutex_unlock(&ctrl->lock); 236 } 237 238 void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid) 239 { 240 struct nvmet_ctrl *ctrl; 241 242 lockdep_assert_held(&subsys->lock); 243 244 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 245 nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid)); 246 if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_NS_ATTR)) 247 continue; 248 nvmet_add_async_event(ctrl, NVME_AER_NOTICE, 249 NVME_AER_NOTICE_NS_CHANGED, 250 NVME_LOG_CHANGED_NS); 251 } 252 } 253 254 void nvmet_send_ana_event(struct nvmet_subsys *subsys, 255 struct nvmet_port *port) 256 { 257 struct nvmet_ctrl *ctrl; 258 259 mutex_lock(&subsys->lock); 260 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 261 if (port && ctrl->port != port) 262 continue; 263 if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_ANA_CHANGE)) 264 continue; 265 nvmet_add_async_event(ctrl, NVME_AER_NOTICE, 266 NVME_AER_NOTICE_ANA, NVME_LOG_ANA); 267 } 268 mutex_unlock(&subsys->lock); 269 } 270 271 void nvmet_port_send_ana_event(struct nvmet_port *port) 272 { 273 struct nvmet_subsys_link *p; 274 275 down_read(&nvmet_config_sem); 276 list_for_each_entry(p, &port->subsystems, entry) 277 nvmet_send_ana_event(p->subsys, port); 278 up_read(&nvmet_config_sem); 279 } 280 281 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops) 282 { 283 int ret = 0; 284 285 down_write(&nvmet_config_sem); 286 if (nvmet_transports[ops->type]) 287 ret = -EINVAL; 288 else 289 nvmet_transports[ops->type] = ops; 290 up_write(&nvmet_config_sem); 291 292 return ret; 293 } 294 EXPORT_SYMBOL_GPL(nvmet_register_transport); 295 296 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops) 297 { 298 down_write(&nvmet_config_sem); 299 nvmet_transports[ops->type] = NULL; 300 up_write(&nvmet_config_sem); 301 } 302 EXPORT_SYMBOL_GPL(nvmet_unregister_transport); 303 304 void nvmet_port_del_ctrls(struct nvmet_port *port, struct nvmet_subsys *subsys) 305 { 306 struct nvmet_ctrl *ctrl; 307 308 mutex_lock(&subsys->lock); 309 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 310 if (ctrl->port == port) 311 ctrl->ops->delete_ctrl(ctrl); 312 } 313 mutex_unlock(&subsys->lock); 314 } 315 316 int nvmet_enable_port(struct nvmet_port *port) 317 { 318 const struct nvmet_fabrics_ops *ops; 319 int ret; 320 321 lockdep_assert_held(&nvmet_config_sem); 322 323 if (port->disc_addr.trtype == NVMF_TRTYPE_MAX) 324 return -EINVAL; 325 326 ops = nvmet_transports[port->disc_addr.trtype]; 327 if (!ops) { 328 up_write(&nvmet_config_sem); 329 request_module("nvmet-transport-%d", port->disc_addr.trtype); 330 down_write(&nvmet_config_sem); 331 ops = nvmet_transports[port->disc_addr.trtype]; 332 if (!ops) { 333 pr_err("transport type %d not supported\n", 334 port->disc_addr.trtype); 335 return -EINVAL; 336 } 337 } 338 339 if (!try_module_get(ops->owner)) 340 return -EINVAL; 341 342 /* 343 * If the user requested PI support and the transport isn't pi capable, 344 * don't enable the port. 345 */ 346 if (port->pi_enable && !(ops->flags & NVMF_METADATA_SUPPORTED)) { 347 pr_err("T10-PI is not supported by transport type %d\n", 348 port->disc_addr.trtype); 349 ret = -EINVAL; 350 goto out_put; 351 } 352 353 ret = ops->add_port(port); 354 if (ret) 355 goto out_put; 356 357 /* If the transport didn't set inline_data_size, then disable it. */ 358 if (port->inline_data_size < 0) 359 port->inline_data_size = 0; 360 361 /* 362 * If the transport didn't set the max_queue_size properly, then clamp 363 * it to the target limits. Also set default values in case the 364 * transport didn't set it at all. 365 */ 366 if (port->max_queue_size < 0) 367 port->max_queue_size = NVMET_MAX_QUEUE_SIZE; 368 else 369 port->max_queue_size = clamp_t(int, port->max_queue_size, 370 NVMET_MIN_QUEUE_SIZE, 371 NVMET_MAX_QUEUE_SIZE); 372 373 /* 374 * If the transport didn't set the mdts properly, then clamp it to the 375 * target limits. Also set default values in case the transport didn't 376 * set it at all. 377 */ 378 if (port->mdts < 0 || port->mdts > NVMET_MAX_MDTS) 379 port->mdts = 0; 380 381 port->enabled = true; 382 port->tr_ops = ops; 383 return 0; 384 385 out_put: 386 module_put(ops->owner); 387 return ret; 388 } 389 390 void nvmet_disable_port(struct nvmet_port *port) 391 { 392 const struct nvmet_fabrics_ops *ops; 393 394 lockdep_assert_held(&nvmet_config_sem); 395 396 port->enabled = false; 397 port->tr_ops = NULL; 398 399 ops = nvmet_transports[port->disc_addr.trtype]; 400 ops->remove_port(port); 401 module_put(ops->owner); 402 } 403 404 static void nvmet_keep_alive_timer(struct work_struct *work) 405 { 406 struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work), 407 struct nvmet_ctrl, ka_work); 408 bool reset_tbkas = ctrl->reset_tbkas; 409 410 ctrl->reset_tbkas = false; 411 if (reset_tbkas) { 412 pr_debug("ctrl %d reschedule traffic based keep-alive timer\n", 413 ctrl->cntlid); 414 queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); 415 return; 416 } 417 418 pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n", 419 ctrl->cntlid, ctrl->kato); 420 421 nvmet_ctrl_fatal_error(ctrl); 422 } 423 424 void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl) 425 { 426 if (unlikely(ctrl->kato == 0)) 427 return; 428 429 pr_debug("ctrl %d start keep-alive timer for %d secs\n", 430 ctrl->cntlid, ctrl->kato); 431 432 queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); 433 } 434 435 void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl) 436 { 437 if (unlikely(ctrl->kato == 0)) 438 return; 439 440 pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid); 441 442 cancel_delayed_work_sync(&ctrl->ka_work); 443 } 444 445 u16 nvmet_req_find_ns(struct nvmet_req *req) 446 { 447 u32 nsid = le32_to_cpu(req->cmd->common.nsid); 448 struct nvmet_subsys *subsys = nvmet_req_subsys(req); 449 450 req->ns = xa_load(&subsys->namespaces, nsid); 451 if (unlikely(!req->ns || !req->ns->enabled)) { 452 req->error_loc = offsetof(struct nvme_common_command, nsid); 453 if (!req->ns) /* ns doesn't exist! */ 454 return NVME_SC_INVALID_NS | NVME_STATUS_DNR; 455 456 /* ns exists but it's disabled */ 457 req->ns = NULL; 458 return NVME_SC_INTERNAL_PATH_ERROR; 459 } 460 461 percpu_ref_get(&req->ns->ref); 462 return NVME_SC_SUCCESS; 463 } 464 465 static void nvmet_destroy_namespace(struct percpu_ref *ref) 466 { 467 struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref); 468 469 complete(&ns->disable_done); 470 } 471 472 void nvmet_put_namespace(struct nvmet_ns *ns) 473 { 474 percpu_ref_put(&ns->ref); 475 } 476 477 static void nvmet_ns_dev_disable(struct nvmet_ns *ns) 478 { 479 nvmet_bdev_ns_disable(ns); 480 nvmet_file_ns_disable(ns); 481 } 482 483 static int nvmet_p2pmem_ns_enable(struct nvmet_ns *ns) 484 { 485 int ret; 486 struct pci_dev *p2p_dev; 487 488 if (!ns->use_p2pmem) 489 return 0; 490 491 if (!ns->bdev) { 492 pr_err("peer-to-peer DMA is not supported by non-block device namespaces\n"); 493 return -EINVAL; 494 } 495 496 if (!blk_queue_pci_p2pdma(ns->bdev->bd_disk->queue)) { 497 pr_err("peer-to-peer DMA is not supported by the driver of %s\n", 498 ns->device_path); 499 return -EINVAL; 500 } 501 502 if (ns->p2p_dev) { 503 ret = pci_p2pdma_distance(ns->p2p_dev, nvmet_ns_dev(ns), true); 504 if (ret < 0) 505 return -EINVAL; 506 } else { 507 /* 508 * Right now we just check that there is p2pmem available so 509 * we can report an error to the user right away if there 510 * is not. We'll find the actual device to use once we 511 * setup the controller when the port's device is available. 512 */ 513 514 p2p_dev = pci_p2pmem_find(nvmet_ns_dev(ns)); 515 if (!p2p_dev) { 516 pr_err("no peer-to-peer memory is available for %s\n", 517 ns->device_path); 518 return -EINVAL; 519 } 520 521 pci_dev_put(p2p_dev); 522 } 523 524 return 0; 525 } 526 527 static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl, 528 struct nvmet_ns *ns) 529 { 530 struct device *clients[2]; 531 struct pci_dev *p2p_dev; 532 int ret; 533 534 lockdep_assert_held(&ctrl->subsys->lock); 535 536 if (!ctrl->p2p_client || !ns->use_p2pmem) 537 return; 538 539 if (ns->p2p_dev) { 540 ret = pci_p2pdma_distance(ns->p2p_dev, ctrl->p2p_client, true); 541 if (ret < 0) 542 return; 543 544 p2p_dev = pci_dev_get(ns->p2p_dev); 545 } else { 546 clients[0] = ctrl->p2p_client; 547 clients[1] = nvmet_ns_dev(ns); 548 549 p2p_dev = pci_p2pmem_find_many(clients, ARRAY_SIZE(clients)); 550 if (!p2p_dev) { 551 pr_err("no peer-to-peer memory is available that's supported by %s and %s\n", 552 dev_name(ctrl->p2p_client), ns->device_path); 553 return; 554 } 555 } 556 557 ret = radix_tree_insert(&ctrl->p2p_ns_map, ns->nsid, p2p_dev); 558 if (ret < 0) 559 pci_dev_put(p2p_dev); 560 561 pr_info("using p2pmem on %s for nsid %d\n", pci_name(p2p_dev), 562 ns->nsid); 563 } 564 565 bool nvmet_ns_revalidate(struct nvmet_ns *ns) 566 { 567 loff_t oldsize = ns->size; 568 569 if (ns->bdev) 570 nvmet_bdev_ns_revalidate(ns); 571 else 572 nvmet_file_ns_revalidate(ns); 573 574 return oldsize != ns->size; 575 } 576 577 int nvmet_ns_enable(struct nvmet_ns *ns) 578 { 579 struct nvmet_subsys *subsys = ns->subsys; 580 struct nvmet_ctrl *ctrl; 581 int ret; 582 583 mutex_lock(&subsys->lock); 584 ret = 0; 585 586 if (nvmet_is_passthru_subsys(subsys)) { 587 pr_info("cannot enable both passthru and regular namespaces for a single subsystem"); 588 goto out_unlock; 589 } 590 591 if (ns->enabled) 592 goto out_unlock; 593 594 ret = nvmet_bdev_ns_enable(ns); 595 if (ret == -ENOTBLK) 596 ret = nvmet_file_ns_enable(ns); 597 if (ret) 598 goto out_unlock; 599 600 ret = nvmet_p2pmem_ns_enable(ns); 601 if (ret) 602 goto out_dev_disable; 603 604 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 605 nvmet_p2pmem_ns_add_p2p(ctrl, ns); 606 607 if (ns->pr.enable) { 608 ret = nvmet_pr_init_ns(ns); 609 if (ret) 610 goto out_dev_put; 611 } 612 613 ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); 614 if (ret) 615 goto out_pr_exit; 616 617 nvmet_ns_changed(subsys, ns->nsid); 618 ns->enabled = true; 619 xa_set_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); 620 nvmet_debugfs_ns_setup(ns); 621 ret = 0; 622 out_unlock: 623 mutex_unlock(&subsys->lock); 624 return ret; 625 out_pr_exit: 626 if (ns->pr.enable) 627 nvmet_pr_exit_ns(ns); 628 out_dev_put: 629 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 630 pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); 631 out_dev_disable: 632 nvmet_ns_dev_disable(ns); 633 goto out_unlock; 634 } 635 636 void nvmet_ns_disable(struct nvmet_ns *ns) 637 { 638 struct nvmet_subsys *subsys = ns->subsys; 639 struct nvmet_ctrl *ctrl; 640 641 mutex_lock(&subsys->lock); 642 if (!ns->enabled) 643 goto out_unlock; 644 645 ns->enabled = false; 646 xa_clear_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); 647 nvmet_debugfs_ns_free(ns); 648 649 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 650 pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); 651 652 mutex_unlock(&subsys->lock); 653 654 /* 655 * Now that we removed the namespaces from the lookup list, we 656 * can kill the per_cpu ref and wait for any remaining references 657 * to be dropped, as well as a RCU grace period for anyone only 658 * using the namespace under rcu_read_lock(). Note that we can't 659 * use call_rcu here as we need to ensure the namespaces have 660 * been fully destroyed before unloading the module. 661 */ 662 percpu_ref_kill(&ns->ref); 663 synchronize_rcu(); 664 wait_for_completion(&ns->disable_done); 665 percpu_ref_exit(&ns->ref); 666 667 if (ns->pr.enable) 668 nvmet_pr_exit_ns(ns); 669 670 mutex_lock(&subsys->lock); 671 nvmet_ns_changed(subsys, ns->nsid); 672 nvmet_ns_dev_disable(ns); 673 out_unlock: 674 mutex_unlock(&subsys->lock); 675 } 676 677 void nvmet_ns_free(struct nvmet_ns *ns) 678 { 679 struct nvmet_subsys *subsys = ns->subsys; 680 681 nvmet_ns_disable(ns); 682 683 mutex_lock(&subsys->lock); 684 685 xa_erase(&subsys->namespaces, ns->nsid); 686 if (ns->nsid == subsys->max_nsid) 687 subsys->max_nsid = nvmet_max_nsid(subsys); 688 689 subsys->nr_namespaces--; 690 mutex_unlock(&subsys->lock); 691 692 down_write(&nvmet_ana_sem); 693 nvmet_ana_group_enabled[ns->anagrpid]--; 694 up_write(&nvmet_ana_sem); 695 696 kfree(ns->device_path); 697 kfree(ns); 698 } 699 700 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid) 701 { 702 struct nvmet_ns *ns; 703 704 mutex_lock(&subsys->lock); 705 706 if (subsys->nr_namespaces == NVMET_MAX_NAMESPACES) 707 goto out_unlock; 708 709 ns = kzalloc_obj(*ns); 710 if (!ns) 711 goto out_unlock; 712 713 init_completion(&ns->disable_done); 714 715 ns->nsid = nsid; 716 ns->subsys = subsys; 717 718 if (ns->nsid > subsys->max_nsid) 719 subsys->max_nsid = nsid; 720 721 if (xa_insert(&subsys->namespaces, ns->nsid, ns, GFP_KERNEL)) 722 goto out_exit; 723 724 subsys->nr_namespaces++; 725 726 mutex_unlock(&subsys->lock); 727 728 down_write(&nvmet_ana_sem); 729 ns->anagrpid = NVMET_DEFAULT_ANA_GRPID; 730 nvmet_ana_group_enabled[ns->anagrpid]++; 731 up_write(&nvmet_ana_sem); 732 733 uuid_gen(&ns->uuid); 734 ns->buffered_io = false; 735 ns->csi = NVME_CSI_NVM; 736 737 return ns; 738 out_exit: 739 subsys->max_nsid = nvmet_max_nsid(subsys); 740 kfree(ns); 741 out_unlock: 742 mutex_unlock(&subsys->lock); 743 return NULL; 744 } 745 746 static void nvmet_update_sq_head(struct nvmet_req *req) 747 { 748 if (req->sq->size) { 749 u32 old_sqhd, new_sqhd; 750 751 old_sqhd = READ_ONCE(req->sq->sqhd); 752 do { 753 new_sqhd = (old_sqhd + 1) % req->sq->size; 754 } while (!try_cmpxchg(&req->sq->sqhd, &old_sqhd, new_sqhd)); 755 } 756 req->cqe->sq_head = cpu_to_le16(req->sq->sqhd & 0x0000FFFF); 757 } 758 759 static void nvmet_set_error(struct nvmet_req *req, u16 status) 760 { 761 struct nvmet_ctrl *ctrl = req->sq->ctrl; 762 struct nvme_error_slot *new_error_slot; 763 unsigned long flags; 764 765 req->cqe->status = cpu_to_le16(status << 1); 766 767 if (!ctrl || req->error_loc == NVMET_NO_ERROR_LOC) 768 return; 769 770 spin_lock_irqsave(&ctrl->error_lock, flags); 771 ctrl->err_counter++; 772 new_error_slot = 773 &ctrl->slots[ctrl->err_counter % NVMET_ERROR_LOG_SLOTS]; 774 775 new_error_slot->error_count = cpu_to_le64(ctrl->err_counter); 776 new_error_slot->sqid = cpu_to_le16(req->sq->qid); 777 new_error_slot->cmdid = cpu_to_le16(req->cmd->common.command_id); 778 new_error_slot->status_field = cpu_to_le16(status << 1); 779 new_error_slot->param_error_location = cpu_to_le16(req->error_loc); 780 new_error_slot->lba = cpu_to_le64(req->error_slba); 781 new_error_slot->nsid = req->cmd->common.nsid; 782 spin_unlock_irqrestore(&ctrl->error_lock, flags); 783 784 /* set the more bit for this request */ 785 req->cqe->status |= cpu_to_le16(1 << 14); 786 } 787 788 static void __nvmet_req_complete(struct nvmet_req *req, u16 status) 789 { 790 struct nvmet_ns *ns = req->ns; 791 struct nvmet_pr_per_ctrl_ref *pc_ref = req->pc_ref; 792 793 if (!req->sq->sqhd_disabled) 794 nvmet_update_sq_head(req); 795 req->cqe->sq_id = cpu_to_le16(req->sq->qid); 796 req->cqe->command_id = req->cmd->common.command_id; 797 798 if (unlikely(status)) 799 nvmet_set_error(req, status); 800 801 trace_nvmet_req_complete(req); 802 803 req->ops->queue_response(req); 804 805 if (pc_ref) 806 nvmet_pr_put_ns_pc_ref(pc_ref); 807 if (ns) 808 nvmet_put_namespace(ns); 809 } 810 811 void nvmet_req_complete(struct nvmet_req *req, u16 status) 812 { 813 struct nvmet_sq *sq = req->sq; 814 815 __nvmet_req_complete(req, status); 816 percpu_ref_put(&sq->ref); 817 } 818 EXPORT_SYMBOL_GPL(nvmet_req_complete); 819 820 void nvmet_cq_init(struct nvmet_cq *cq) 821 { 822 refcount_set(&cq->ref, 1); 823 } 824 EXPORT_SYMBOL_GPL(nvmet_cq_init); 825 826 bool nvmet_cq_get(struct nvmet_cq *cq) 827 { 828 return refcount_inc_not_zero(&cq->ref); 829 } 830 EXPORT_SYMBOL_GPL(nvmet_cq_get); 831 832 void nvmet_cq_put(struct nvmet_cq *cq) 833 { 834 if (refcount_dec_and_test(&cq->ref)) 835 nvmet_cq_destroy(cq); 836 } 837 EXPORT_SYMBOL_GPL(nvmet_cq_put); 838 839 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, 840 u16 qid, u16 size) 841 { 842 cq->qid = qid; 843 cq->size = size; 844 845 ctrl->cqs[qid] = cq; 846 } 847 848 void nvmet_cq_destroy(struct nvmet_cq *cq) 849 { 850 struct nvmet_ctrl *ctrl = cq->ctrl; 851 852 if (ctrl) { 853 ctrl->cqs[cq->qid] = NULL; 854 nvmet_ctrl_put(cq->ctrl); 855 cq->ctrl = NULL; 856 } 857 } 858 859 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, 860 u16 qid, u16 size) 861 { 862 sq->sqhd = 0; 863 sq->qid = qid; 864 sq->size = size; 865 866 ctrl->sqs[qid] = sq; 867 } 868 869 static void nvmet_confirm_sq(struct percpu_ref *ref) 870 { 871 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); 872 873 complete(&sq->confirm_done); 874 } 875 876 u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) 877 { 878 if (!ctrl->cqs) 879 return NVME_SC_INTERNAL | NVME_STATUS_DNR; 880 881 if (cqid > ctrl->max_qid) 882 return NVME_SC_QID_INVALID | NVME_STATUS_DNR; 883 884 if ((create && ctrl->cqs[cqid]) || (!create && !ctrl->cqs[cqid])) 885 return NVME_SC_QID_INVALID | NVME_STATUS_DNR; 886 887 return NVME_SC_SUCCESS; 888 } 889 890 u16 nvmet_check_io_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) 891 { 892 if (!cqid) 893 return NVME_SC_QID_INVALID | NVME_STATUS_DNR; 894 return nvmet_check_cqid(ctrl, cqid, create); 895 } 896 897 bool nvmet_cq_in_use(struct nvmet_cq *cq) 898 { 899 return refcount_read(&cq->ref) > 1; 900 } 901 EXPORT_SYMBOL_GPL(nvmet_cq_in_use); 902 903 u16 nvmet_cq_create(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, 904 u16 qid, u16 size) 905 { 906 u16 status; 907 908 status = nvmet_check_cqid(ctrl, qid, true); 909 if (status != NVME_SC_SUCCESS) 910 return status; 911 912 if (!kref_get_unless_zero(&ctrl->ref)) 913 return NVME_SC_INTERNAL | NVME_STATUS_DNR; 914 cq->ctrl = ctrl; 915 916 nvmet_cq_init(cq); 917 nvmet_cq_setup(ctrl, cq, qid, size); 918 919 return NVME_SC_SUCCESS; 920 } 921 EXPORT_SYMBOL_GPL(nvmet_cq_create); 922 923 u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, 924 bool create) 925 { 926 if (!ctrl->sqs) 927 return NVME_SC_INTERNAL | NVME_STATUS_DNR; 928 929 if (sqid > ctrl->max_qid) 930 return NVME_SC_QID_INVALID | NVME_STATUS_DNR; 931 932 if ((create && ctrl->sqs[sqid]) || 933 (!create && !ctrl->sqs[sqid])) 934 return NVME_SC_QID_INVALID | NVME_STATUS_DNR; 935 936 return NVME_SC_SUCCESS; 937 } 938 939 u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, 940 struct nvmet_cq *cq, u16 sqid, u16 size) 941 { 942 u16 status; 943 int ret; 944 945 if (!kref_get_unless_zero(&ctrl->ref)) 946 return NVME_SC_INTERNAL | NVME_STATUS_DNR; 947 948 status = nvmet_check_sqid(ctrl, sqid, true); 949 if (status != NVME_SC_SUCCESS) 950 goto ctrl_put; 951 952 ret = nvmet_sq_init(sq, cq); 953 if (ret) { 954 status = NVME_SC_INTERNAL | NVME_STATUS_DNR; 955 goto ctrl_put; 956 } 957 958 nvmet_sq_setup(ctrl, sq, sqid, size); 959 sq->ctrl = ctrl; 960 961 return NVME_SC_SUCCESS; 962 963 ctrl_put: 964 nvmet_ctrl_put(ctrl); 965 return status; 966 } 967 EXPORT_SYMBOL_GPL(nvmet_sq_create); 968 969 void nvmet_sq_destroy(struct nvmet_sq *sq) 970 { 971 struct nvmet_ctrl *ctrl = sq->ctrl; 972 973 /* 974 * If this is the admin queue, complete all AERs so that our 975 * queue doesn't have outstanding requests on it. 976 */ 977 if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) 978 nvmet_async_events_failall(ctrl); 979 percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); 980 wait_for_completion(&sq->confirm_done); 981 wait_for_completion(&sq->free_done); 982 percpu_ref_exit(&sq->ref); 983 nvmet_auth_sq_free(sq); 984 nvmet_cq_put(sq->cq); 985 986 /* 987 * we must reference the ctrl again after waiting for inflight IO 988 * to complete. Because admin connect may have sneaked in after we 989 * store sq->ctrl locally, but before we killed the percpu_ref. the 990 * admin connect allocates and assigns sq->ctrl, which now needs a 991 * final ref put, as this ctrl is going away. 992 */ 993 ctrl = sq->ctrl; 994 995 if (ctrl) { 996 /* 997 * The teardown flow may take some time, and the host may not 998 * send us keep-alive during this period, hence reset the 999 * traffic based keep-alive timer so we don't trigger a 1000 * controller teardown as a result of a keep-alive expiration. 1001 */ 1002 ctrl->reset_tbkas = true; 1003 sq->ctrl->sqs[sq->qid] = NULL; 1004 nvmet_ctrl_put(ctrl); 1005 sq->ctrl = NULL; /* allows reusing the queue later */ 1006 } 1007 } 1008 EXPORT_SYMBOL_GPL(nvmet_sq_destroy); 1009 1010 static void nvmet_sq_free(struct percpu_ref *ref) 1011 { 1012 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); 1013 1014 complete(&sq->free_done); 1015 } 1016 1017 int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq) 1018 { 1019 int ret; 1020 1021 if (!nvmet_cq_get(cq)) 1022 return -EINVAL; 1023 1024 ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL); 1025 if (ret) { 1026 pr_err("percpu_ref init failed!\n"); 1027 nvmet_cq_put(cq); 1028 return ret; 1029 } 1030 init_completion(&sq->free_done); 1031 init_completion(&sq->confirm_done); 1032 nvmet_auth_sq_init(sq); 1033 sq->cq = cq; 1034 1035 return 0; 1036 } 1037 EXPORT_SYMBOL_GPL(nvmet_sq_init); 1038 1039 static inline u16 nvmet_check_ana_state(struct nvmet_port *port, 1040 struct nvmet_ns *ns) 1041 { 1042 enum nvme_ana_state state = port->ana_state[ns->anagrpid]; 1043 1044 if (unlikely(state == NVME_ANA_INACCESSIBLE)) 1045 return NVME_SC_ANA_INACCESSIBLE; 1046 if (unlikely(state == NVME_ANA_PERSISTENT_LOSS)) 1047 return NVME_SC_ANA_PERSISTENT_LOSS; 1048 if (unlikely(state == NVME_ANA_CHANGE)) 1049 return NVME_SC_ANA_TRANSITION; 1050 return 0; 1051 } 1052 1053 static inline u16 nvmet_io_cmd_check_access(struct nvmet_req *req) 1054 { 1055 if (unlikely(req->ns->readonly)) { 1056 switch (req->cmd->common.opcode) { 1057 case nvme_cmd_read: 1058 case nvme_cmd_flush: 1059 break; 1060 default: 1061 return NVME_SC_NS_WRITE_PROTECTED; 1062 } 1063 } 1064 1065 return 0; 1066 } 1067 1068 static u32 nvmet_io_cmd_transfer_len(struct nvmet_req *req) 1069 { 1070 struct nvme_command *cmd = req->cmd; 1071 u32 metadata_len = 0; 1072 1073 if (nvme_is_fabrics(cmd)) 1074 return nvmet_fabrics_io_cmd_data_len(req); 1075 1076 if (!req->ns) 1077 return 0; 1078 1079 switch (req->cmd->common.opcode) { 1080 case nvme_cmd_read: 1081 case nvme_cmd_write: 1082 case nvme_cmd_zone_append: 1083 if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns)) 1084 metadata_len = nvmet_rw_metadata_len(req); 1085 return nvmet_rw_data_len(req) + metadata_len; 1086 case nvme_cmd_dsm: 1087 return nvmet_dsm_len(req); 1088 case nvme_cmd_zone_mgmt_recv: 1089 return (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2; 1090 default: 1091 return 0; 1092 } 1093 } 1094 1095 static u16 nvmet_parse_io_cmd(struct nvmet_req *req) 1096 { 1097 struct nvme_command *cmd = req->cmd; 1098 u16 ret; 1099 1100 if (nvme_is_fabrics(cmd)) 1101 return nvmet_parse_fabrics_io_cmd(req); 1102 1103 if (unlikely(!nvmet_check_auth_status(req))) 1104 return NVME_SC_AUTH_REQUIRED | NVME_STATUS_DNR; 1105 1106 ret = nvmet_check_ctrl_status(req); 1107 if (unlikely(ret)) 1108 return ret; 1109 1110 if (nvmet_is_passthru_req(req)) 1111 return nvmet_parse_passthru_io_cmd(req); 1112 1113 ret = nvmet_req_find_ns(req); 1114 if (unlikely(ret)) 1115 return ret; 1116 1117 ret = nvmet_check_ana_state(req->port, req->ns); 1118 if (unlikely(ret)) { 1119 req->error_loc = offsetof(struct nvme_common_command, nsid); 1120 return ret; 1121 } 1122 ret = nvmet_io_cmd_check_access(req); 1123 if (unlikely(ret)) { 1124 req->error_loc = offsetof(struct nvme_common_command, nsid); 1125 return ret; 1126 } 1127 1128 if (req->ns->pr.enable) { 1129 ret = nvmet_parse_pr_cmd(req); 1130 if (!ret) 1131 return ret; 1132 } 1133 1134 switch (req->ns->csi) { 1135 case NVME_CSI_NVM: 1136 if (req->ns->file) 1137 ret = nvmet_file_parse_io_cmd(req); 1138 else 1139 ret = nvmet_bdev_parse_io_cmd(req); 1140 break; 1141 case NVME_CSI_ZNS: 1142 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) 1143 ret = nvmet_bdev_zns_parse_io_cmd(req); 1144 else 1145 ret = NVME_SC_INVALID_IO_CMD_SET; 1146 break; 1147 default: 1148 ret = NVME_SC_INVALID_IO_CMD_SET; 1149 } 1150 if (ret) 1151 return ret; 1152 1153 if (req->ns->pr.enable) { 1154 ret = nvmet_pr_check_cmd_access(req); 1155 if (ret) 1156 return ret; 1157 1158 ret = nvmet_pr_get_ns_pc_ref(req); 1159 } 1160 return ret; 1161 } 1162 1163 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_sq *sq, 1164 const struct nvmet_fabrics_ops *ops) 1165 { 1166 u8 flags = req->cmd->common.flags; 1167 u16 status; 1168 1169 req->cq = sq->cq; 1170 req->sq = sq; 1171 req->ops = ops; 1172 req->sg = NULL; 1173 req->metadata_sg = NULL; 1174 req->sg_cnt = 0; 1175 req->metadata_sg_cnt = 0; 1176 req->transfer_len = 0; 1177 req->metadata_len = 0; 1178 req->cqe->result.u64 = 0; 1179 req->cqe->status = 0; 1180 req->cqe->sq_head = 0; 1181 req->ns = NULL; 1182 req->error_loc = NVMET_NO_ERROR_LOC; 1183 req->error_slba = 0; 1184 req->pc_ref = NULL; 1185 1186 /* no support for fused commands yet */ 1187 if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) { 1188 req->error_loc = offsetof(struct nvme_common_command, flags); 1189 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 1190 goto fail; 1191 } 1192 1193 /* 1194 * For fabrics, PSDT field shall describe metadata pointer (MPTR) that 1195 * contains an address of a single contiguous physical buffer that is 1196 * byte aligned. For PCI controllers, this is optional so not enforced. 1197 */ 1198 if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF)) { 1199 if (!req->sq->ctrl || !nvmet_is_pci_ctrl(req->sq->ctrl)) { 1200 req->error_loc = 1201 offsetof(struct nvme_common_command, flags); 1202 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 1203 goto fail; 1204 } 1205 } 1206 1207 if (unlikely(!req->sq->ctrl)) 1208 /* will return an error for any non-connect command: */ 1209 status = nvmet_parse_connect_cmd(req); 1210 else if (likely(req->sq->qid != 0)) 1211 status = nvmet_parse_io_cmd(req); 1212 else 1213 status = nvmet_parse_admin_cmd(req); 1214 1215 if (status) 1216 goto fail; 1217 1218 trace_nvmet_req_init(req, req->cmd); 1219 1220 if (unlikely(!percpu_ref_tryget_live(&sq->ref))) { 1221 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 1222 goto fail; 1223 } 1224 1225 if (sq->ctrl) 1226 sq->ctrl->reset_tbkas = true; 1227 1228 return true; 1229 1230 fail: 1231 __nvmet_req_complete(req, status); 1232 return false; 1233 } 1234 EXPORT_SYMBOL_GPL(nvmet_req_init); 1235 1236 void nvmet_req_uninit(struct nvmet_req *req) 1237 { 1238 percpu_ref_put(&req->sq->ref); 1239 if (req->pc_ref) 1240 nvmet_pr_put_ns_pc_ref(req->pc_ref); 1241 if (req->ns) 1242 nvmet_put_namespace(req->ns); 1243 } 1244 EXPORT_SYMBOL_GPL(nvmet_req_uninit); 1245 1246 size_t nvmet_req_transfer_len(struct nvmet_req *req) 1247 { 1248 if (likely(req->sq->qid != 0)) 1249 return nvmet_io_cmd_transfer_len(req); 1250 if (unlikely(!req->sq->ctrl)) 1251 return nvmet_connect_cmd_data_len(req); 1252 return nvmet_admin_cmd_data_len(req); 1253 } 1254 EXPORT_SYMBOL_GPL(nvmet_req_transfer_len); 1255 1256 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len) 1257 { 1258 if (unlikely(len != req->transfer_len)) { 1259 u16 status; 1260 1261 req->error_loc = offsetof(struct nvme_common_command, dptr); 1262 if (req->cmd->common.flags & NVME_CMD_SGL_ALL) 1263 status = NVME_SC_SGL_INVALID_DATA; 1264 else 1265 status = NVME_SC_INVALID_FIELD; 1266 nvmet_req_complete(req, status | NVME_STATUS_DNR); 1267 return false; 1268 } 1269 1270 return true; 1271 } 1272 EXPORT_SYMBOL_GPL(nvmet_check_transfer_len); 1273 1274 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len) 1275 { 1276 if (unlikely(data_len > req->transfer_len)) { 1277 u16 status; 1278 1279 req->error_loc = offsetof(struct nvme_common_command, dptr); 1280 if (req->cmd->common.flags & NVME_CMD_SGL_ALL) 1281 status = NVME_SC_SGL_INVALID_DATA; 1282 else 1283 status = NVME_SC_INVALID_FIELD; 1284 nvmet_req_complete(req, status | NVME_STATUS_DNR); 1285 return false; 1286 } 1287 1288 return true; 1289 } 1290 1291 static unsigned int nvmet_data_transfer_len(struct nvmet_req *req) 1292 { 1293 return req->transfer_len - req->metadata_len; 1294 } 1295 1296 static int nvmet_req_alloc_p2pmem_sgls(struct pci_dev *p2p_dev, 1297 struct nvmet_req *req) 1298 { 1299 req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt, 1300 nvmet_data_transfer_len(req)); 1301 if (!req->sg) 1302 goto out_err; 1303 1304 if (req->metadata_len) { 1305 req->metadata_sg = pci_p2pmem_alloc_sgl(p2p_dev, 1306 &req->metadata_sg_cnt, req->metadata_len); 1307 if (!req->metadata_sg) 1308 goto out_free_sg; 1309 } 1310 1311 req->p2p_dev = p2p_dev; 1312 1313 return 0; 1314 out_free_sg: 1315 pci_p2pmem_free_sgl(req->p2p_dev, req->sg); 1316 out_err: 1317 return -ENOMEM; 1318 } 1319 1320 static struct pci_dev *nvmet_req_find_p2p_dev(struct nvmet_req *req) 1321 { 1322 if (!IS_ENABLED(CONFIG_PCI_P2PDMA) || 1323 !req->sq->ctrl || !req->sq->qid || !req->ns) 1324 return NULL; 1325 return radix_tree_lookup(&req->sq->ctrl->p2p_ns_map, req->ns->nsid); 1326 } 1327 1328 int nvmet_req_alloc_sgls(struct nvmet_req *req) 1329 { 1330 struct pci_dev *p2p_dev = nvmet_req_find_p2p_dev(req); 1331 1332 if (p2p_dev && !nvmet_req_alloc_p2pmem_sgls(p2p_dev, req)) 1333 return 0; 1334 1335 req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL, 1336 &req->sg_cnt); 1337 if (unlikely(!req->sg)) 1338 goto out; 1339 1340 if (req->metadata_len) { 1341 req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL, 1342 &req->metadata_sg_cnt); 1343 if (unlikely(!req->metadata_sg)) 1344 goto out_free; 1345 } 1346 1347 return 0; 1348 out_free: 1349 sgl_free(req->sg); 1350 out: 1351 return -ENOMEM; 1352 } 1353 EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls); 1354 1355 void nvmet_req_free_sgls(struct nvmet_req *req) 1356 { 1357 if (req->p2p_dev) { 1358 pci_p2pmem_free_sgl(req->p2p_dev, req->sg); 1359 if (req->metadata_sg) 1360 pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg); 1361 req->p2p_dev = NULL; 1362 } else { 1363 sgl_free(req->sg); 1364 if (req->metadata_sg) 1365 sgl_free(req->metadata_sg); 1366 } 1367 1368 req->sg = NULL; 1369 req->metadata_sg = NULL; 1370 req->sg_cnt = 0; 1371 req->metadata_sg_cnt = 0; 1372 } 1373 EXPORT_SYMBOL_GPL(nvmet_req_free_sgls); 1374 1375 static inline bool nvmet_css_supported(u8 cc_css) 1376 { 1377 switch (cc_css << NVME_CC_CSS_SHIFT) { 1378 case NVME_CC_CSS_NVM: 1379 case NVME_CC_CSS_CSI: 1380 return true; 1381 default: 1382 return false; 1383 } 1384 } 1385 1386 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl) 1387 { 1388 lockdep_assert_held(&ctrl->lock); 1389 1390 /* 1391 * Only I/O controllers should verify iosqes,iocqes. 1392 * Strictly speaking, the spec says a discovery controller 1393 * should verify iosqes,iocqes are zeroed, however that 1394 * would break backwards compatibility, so don't enforce it. 1395 */ 1396 if (!nvmet_is_disc_subsys(ctrl->subsys) && 1397 (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES || 1398 nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES)) { 1399 ctrl->csts = NVME_CSTS_CFS; 1400 return; 1401 } 1402 1403 if (nvmet_cc_mps(ctrl->cc) != 0 || 1404 nvmet_cc_ams(ctrl->cc) != 0 || 1405 !nvmet_css_supported(nvmet_cc_css(ctrl->cc))) { 1406 ctrl->csts = NVME_CSTS_CFS; 1407 return; 1408 } 1409 1410 ctrl->csts = NVME_CSTS_RDY; 1411 1412 /* 1413 * Controllers that are not yet enabled should not really enforce the 1414 * keep alive timeout, but we still want to track a timeout and cleanup 1415 * in case a host died before it enabled the controller. Hence, simply 1416 * reset the keep alive timer when the controller is enabled. 1417 */ 1418 if (ctrl->kato) 1419 mod_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); 1420 } 1421 1422 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl) 1423 { 1424 lockdep_assert_held(&ctrl->lock); 1425 1426 /* XXX: tear down queues? */ 1427 ctrl->csts &= ~NVME_CSTS_RDY; 1428 ctrl->cc = 0; 1429 } 1430 1431 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new) 1432 { 1433 u32 old; 1434 1435 mutex_lock(&ctrl->lock); 1436 old = ctrl->cc; 1437 ctrl->cc = new; 1438 1439 if (nvmet_cc_en(new) && !nvmet_cc_en(old)) 1440 nvmet_start_ctrl(ctrl); 1441 if (!nvmet_cc_en(new) && nvmet_cc_en(old)) 1442 nvmet_clear_ctrl(ctrl); 1443 if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) { 1444 nvmet_clear_ctrl(ctrl); 1445 ctrl->csts |= NVME_CSTS_SHST_CMPLT; 1446 } 1447 if (!nvmet_cc_shn(new) && nvmet_cc_shn(old)) 1448 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT; 1449 mutex_unlock(&ctrl->lock); 1450 } 1451 EXPORT_SYMBOL_GPL(nvmet_update_cc); 1452 1453 static void nvmet_init_cap(struct nvmet_ctrl *ctrl) 1454 { 1455 /* command sets supported: NVMe command set: */ 1456 ctrl->cap = (1ULL << 37); 1457 /* Controller supports one or more I/O Command Sets */ 1458 ctrl->cap |= (1ULL << 43); 1459 /* CC.EN timeout in 500msec units: */ 1460 ctrl->cap |= (15ULL << 24); 1461 /* maximum queue entries supported: */ 1462 if (ctrl->ops->get_max_queue_size) 1463 ctrl->cap |= min_t(u16, ctrl->ops->get_max_queue_size(ctrl), 1464 ctrl->port->max_queue_size) - 1; 1465 else 1466 ctrl->cap |= ctrl->port->max_queue_size - 1; 1467 1468 if (nvmet_is_passthru_subsys(ctrl->subsys)) 1469 nvmet_passthrough_override_cap(ctrl); 1470 } 1471 1472 struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, 1473 const char *hostnqn, u16 cntlid, 1474 struct nvmet_req *req) 1475 { 1476 struct nvmet_ctrl *ctrl = NULL; 1477 struct nvmet_subsys *subsys; 1478 1479 subsys = nvmet_find_get_subsys(req->port, subsysnqn); 1480 if (!subsys) { 1481 pr_warn("connect request for invalid subsystem %s!\n", 1482 subsysnqn); 1483 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn); 1484 goto out; 1485 } 1486 1487 mutex_lock(&subsys->lock); 1488 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 1489 if (ctrl->cntlid == cntlid) { 1490 if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) { 1491 pr_warn("hostnqn mismatch.\n"); 1492 continue; 1493 } 1494 if (!kref_get_unless_zero(&ctrl->ref)) 1495 continue; 1496 1497 /* ctrl found */ 1498 goto found; 1499 } 1500 } 1501 1502 ctrl = NULL; /* ctrl not found */ 1503 pr_warn("could not find controller %d for subsys %s / host %s\n", 1504 cntlid, subsysnqn, hostnqn); 1505 req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid); 1506 1507 found: 1508 mutex_unlock(&subsys->lock); 1509 nvmet_subsys_put(subsys); 1510 out: 1511 return ctrl; 1512 } 1513 1514 u16 nvmet_check_ctrl_status(struct nvmet_req *req) 1515 { 1516 if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) { 1517 pr_err("got cmd %d while CC.EN == 0 on qid = %d\n", 1518 req->cmd->common.opcode, req->sq->qid); 1519 return NVME_SC_CMD_SEQ_ERROR | NVME_STATUS_DNR; 1520 } 1521 1522 if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) { 1523 pr_err("got cmd %d while CSTS.RDY == 0 on qid = %d\n", 1524 req->cmd->common.opcode, req->sq->qid); 1525 return NVME_SC_CMD_SEQ_ERROR | NVME_STATUS_DNR; 1526 } 1527 1528 if (unlikely(!nvmet_check_auth_status(req))) { 1529 pr_warn("qid %d not authenticated\n", req->sq->qid); 1530 return NVME_SC_AUTH_REQUIRED | NVME_STATUS_DNR; 1531 } 1532 return 0; 1533 } 1534 1535 bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn) 1536 { 1537 struct nvmet_host_link *p; 1538 1539 lockdep_assert_held(&nvmet_config_sem); 1540 1541 if (subsys->allow_any_host) 1542 return true; 1543 1544 if (nvmet_is_disc_subsys(subsys)) /* allow all access to disc subsys */ 1545 return true; 1546 1547 list_for_each_entry(p, &subsys->hosts, entry) { 1548 if (!strcmp(nvmet_host_name(p->host), hostnqn)) 1549 return true; 1550 } 1551 1552 return false; 1553 } 1554 1555 static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl, 1556 struct device *p2p_client) 1557 { 1558 struct nvmet_ns *ns; 1559 unsigned long idx; 1560 1561 lockdep_assert_held(&ctrl->subsys->lock); 1562 1563 if (!p2p_client) 1564 return; 1565 1566 ctrl->p2p_client = get_device(p2p_client); 1567 1568 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) 1569 nvmet_p2pmem_ns_add_p2p(ctrl, ns); 1570 } 1571 1572 static void nvmet_release_p2p_ns_map(struct nvmet_ctrl *ctrl) 1573 { 1574 struct radix_tree_iter iter; 1575 void __rcu **slot; 1576 1577 lockdep_assert_held(&ctrl->subsys->lock); 1578 1579 radix_tree_for_each_slot(slot, &ctrl->p2p_ns_map, &iter, 0) 1580 pci_dev_put(radix_tree_deref_slot(slot)); 1581 1582 put_device(ctrl->p2p_client); 1583 } 1584 1585 static void nvmet_fatal_error_handler(struct work_struct *work) 1586 { 1587 struct nvmet_ctrl *ctrl = 1588 container_of(work, struct nvmet_ctrl, fatal_err_work); 1589 1590 pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid); 1591 ctrl->ops->delete_ctrl(ctrl); 1592 } 1593 1594 struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) 1595 { 1596 struct nvmet_subsys *subsys; 1597 struct nvmet_ctrl *ctrl; 1598 u32 kato = args->kato; 1599 u8 dhchap_status; 1600 int ret; 1601 1602 args->status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; 1603 subsys = nvmet_find_get_subsys(args->port, args->subsysnqn); 1604 if (!subsys) { 1605 pr_warn("connect request for invalid subsystem %s!\n", 1606 args->subsysnqn); 1607 args->result = IPO_IATTR_CONNECT_DATA(subsysnqn); 1608 args->error_loc = offsetof(struct nvme_common_command, dptr); 1609 return NULL; 1610 } 1611 1612 down_read(&nvmet_config_sem); 1613 if (!nvmet_host_allowed(subsys, args->hostnqn)) { 1614 pr_info("connect by host %s for subsystem %s not allowed\n", 1615 args->hostnqn, args->subsysnqn); 1616 args->result = IPO_IATTR_CONNECT_DATA(hostnqn); 1617 up_read(&nvmet_config_sem); 1618 args->status = NVME_SC_CONNECT_INVALID_HOST | NVME_STATUS_DNR; 1619 args->error_loc = offsetof(struct nvme_common_command, dptr); 1620 goto out_put_subsystem; 1621 } 1622 up_read(&nvmet_config_sem); 1623 1624 args->status = NVME_SC_INTERNAL; 1625 ctrl = kzalloc_obj(*ctrl); 1626 if (!ctrl) 1627 goto out_put_subsystem; 1628 mutex_init(&ctrl->lock); 1629 1630 ctrl->port = args->port; 1631 ctrl->ops = args->ops; 1632 1633 #ifdef CONFIG_NVME_TARGET_PASSTHRU 1634 /* By default, set loop targets to clear IDS by default */ 1635 if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP) 1636 subsys->clear_ids = 1; 1637 #endif 1638 1639 INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); 1640 INIT_LIST_HEAD(&ctrl->async_events); 1641 INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL); 1642 INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler); 1643 INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer); 1644 1645 memcpy(ctrl->hostnqn, args->hostnqn, NVMF_NQN_SIZE); 1646 1647 kref_init(&ctrl->ref); 1648 ctrl->subsys = subsys; 1649 ctrl->pi_support = ctrl->port->pi_enable && ctrl->subsys->pi_support; 1650 nvmet_init_cap(ctrl); 1651 WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL); 1652 1653 ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES, 1654 sizeof(__le32), GFP_KERNEL); 1655 if (!ctrl->changed_ns_list) 1656 goto out_free_ctrl; 1657 1658 /* 1659 * Discovery controllers may use some arbitrary high value 1660 * in order to cleanup stale discovery sessions 1661 */ 1662 if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) 1663 kato = NVMET_DISC_KATO_MS; 1664 1665 /* keep-alive timeout in seconds */ 1666 ctrl->kato = DIV_ROUND_UP(kato, 1000); 1667 1668 ctrl->err_counter = 0; 1669 spin_lock_init(&ctrl->error_lock); 1670 1671 down_read(&nvmet_config_sem); 1672 mutex_lock(&subsys->lock); 1673 1674 ctrl->max_qid = subsys->max_qid; 1675 1676 ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1); 1677 if (!ctrl->sqs) 1678 goto out_free_changed_ns_list; 1679 1680 ctrl->cqs = kzalloc_objs(struct nvmet_cq *, ctrl->max_qid + 1); 1681 if (!ctrl->cqs) 1682 goto out_free_sqs; 1683 1684 ret = ida_alloc_range(&cntlid_ida, 1685 subsys->cntlid_min, subsys->cntlid_max, 1686 GFP_KERNEL); 1687 if (ret < 0) { 1688 args->status = NVME_SC_CONNECT_CTRL_BUSY | NVME_STATUS_DNR; 1689 goto out_free_cqs; 1690 } 1691 ctrl->cntlid = ret; 1692 1693 ret = nvmet_ctrl_init_pr(ctrl); 1694 if (ret) 1695 goto init_pr_fail; 1696 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls); 1697 nvmet_setup_p2p_ns_map(ctrl, args->p2p_client); 1698 nvmet_debugfs_ctrl_setup(ctrl); 1699 mutex_unlock(&subsys->lock); 1700 up_read(&nvmet_config_sem); 1701 1702 nvmet_start_keep_alive_timer(ctrl); 1703 1704 if (args->hostid) 1705 uuid_copy(&ctrl->hostid, args->hostid); 1706 1707 dhchap_status = nvmet_setup_auth(ctrl, args->sq, false); 1708 if (dhchap_status) { 1709 pr_err("Failed to setup authentication, dhchap status %u\n", 1710 dhchap_status); 1711 nvmet_ctrl_put(ctrl); 1712 if (dhchap_status == NVME_AUTH_DHCHAP_FAILURE_FAILED) 1713 args->status = 1714 NVME_SC_CONNECT_INVALID_HOST | NVME_STATUS_DNR; 1715 else 1716 args->status = NVME_SC_INTERNAL; 1717 return NULL; 1718 } 1719 1720 args->status = NVME_SC_SUCCESS; 1721 1722 pr_info("Created %s controller %d for subsystem %s for NQN %s%s%s%s.\n", 1723 nvmet_is_disc_subsys(ctrl->subsys) ? "discovery" : "nvm", 1724 ctrl->cntlid, ctrl->subsys->subsysnqn, ctrl->hostnqn, 1725 ctrl->pi_support ? " T10-PI is enabled" : "", 1726 nvmet_has_auth(ctrl, args->sq) ? " with DH-HMAC-CHAP" : "", 1727 nvmet_queue_tls_keyid(args->sq) ? ", TLS" : ""); 1728 1729 return ctrl; 1730 1731 init_pr_fail: 1732 ida_free(&cntlid_ida, ctrl->cntlid); 1733 out_free_cqs: 1734 kfree(ctrl->cqs); 1735 out_free_sqs: 1736 kfree(ctrl->sqs); 1737 out_free_changed_ns_list: 1738 mutex_unlock(&subsys->lock); 1739 up_read(&nvmet_config_sem); 1740 kfree(ctrl->changed_ns_list); 1741 out_free_ctrl: 1742 kfree(ctrl); 1743 out_put_subsystem: 1744 nvmet_subsys_put(subsys); 1745 return NULL; 1746 } 1747 EXPORT_SYMBOL_GPL(nvmet_alloc_ctrl); 1748 1749 static void nvmet_ctrl_free(struct kref *ref) 1750 { 1751 struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref); 1752 struct nvmet_subsys *subsys = ctrl->subsys; 1753 1754 mutex_lock(&subsys->lock); 1755 nvmet_ctrl_destroy_pr(ctrl); 1756 nvmet_release_p2p_ns_map(ctrl); 1757 list_del(&ctrl->subsys_entry); 1758 mutex_unlock(&subsys->lock); 1759 1760 nvmet_stop_keep_alive_timer(ctrl); 1761 1762 cancel_work_sync(&ctrl->async_event_work); 1763 cancel_work_sync(&ctrl->fatal_err_work); 1764 1765 nvmet_destroy_auth(ctrl); 1766 1767 nvmet_debugfs_ctrl_free(ctrl); 1768 1769 ida_free(&cntlid_ida, ctrl->cntlid); 1770 1771 nvmet_async_events_free(ctrl); 1772 kfree(ctrl->sqs); 1773 kfree(ctrl->cqs); 1774 kfree(ctrl->changed_ns_list); 1775 kfree(ctrl); 1776 1777 nvmet_subsys_put(subsys); 1778 } 1779 1780 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl) 1781 { 1782 kref_put(&ctrl->ref, nvmet_ctrl_free); 1783 } 1784 EXPORT_SYMBOL_GPL(nvmet_ctrl_put); 1785 1786 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl) 1787 { 1788 mutex_lock(&ctrl->lock); 1789 if (!(ctrl->csts & NVME_CSTS_CFS)) { 1790 ctrl->csts |= NVME_CSTS_CFS; 1791 queue_work(nvmet_wq, &ctrl->fatal_err_work); 1792 } 1793 mutex_unlock(&ctrl->lock); 1794 } 1795 EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error); 1796 1797 ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl, 1798 char *traddr, size_t traddr_len) 1799 { 1800 if (!ctrl->ops->host_traddr) 1801 return -EOPNOTSUPP; 1802 return ctrl->ops->host_traddr(ctrl, traddr, traddr_len); 1803 } 1804 1805 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, 1806 const char *subsysnqn) 1807 { 1808 struct nvmet_subsys_link *p; 1809 1810 if (!port) 1811 return NULL; 1812 1813 if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn)) { 1814 if (!kref_get_unless_zero(&nvmet_disc_subsys->ref)) 1815 return NULL; 1816 return nvmet_disc_subsys; 1817 } 1818 1819 down_read(&nvmet_config_sem); 1820 if (!strncmp(nvmet_disc_subsys->subsysnqn, subsysnqn, 1821 NVMF_NQN_SIZE)) { 1822 if (kref_get_unless_zero(&nvmet_disc_subsys->ref)) { 1823 up_read(&nvmet_config_sem); 1824 return nvmet_disc_subsys; 1825 } 1826 } 1827 list_for_each_entry(p, &port->subsystems, entry) { 1828 if (!strncmp(p->subsys->subsysnqn, subsysnqn, 1829 NVMF_NQN_SIZE)) { 1830 if (!kref_get_unless_zero(&p->subsys->ref)) 1831 break; 1832 up_read(&nvmet_config_sem); 1833 return p->subsys; 1834 } 1835 } 1836 up_read(&nvmet_config_sem); 1837 return NULL; 1838 } 1839 1840 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, 1841 enum nvme_subsys_type type) 1842 { 1843 struct nvmet_subsys *subsys; 1844 char serial[NVMET_SN_MAX_SIZE / 2]; 1845 int ret; 1846 1847 subsys = kzalloc_obj(*subsys); 1848 if (!subsys) 1849 return ERR_PTR(-ENOMEM); 1850 1851 subsys->ver = NVMET_DEFAULT_VS; 1852 /* generate a random serial number as our controllers are ephemeral: */ 1853 get_random_bytes(&serial, sizeof(serial)); 1854 bin2hex(subsys->serial, &serial, sizeof(serial)); 1855 1856 subsys->model_number = kstrdup(NVMET_DEFAULT_CTRL_MODEL, GFP_KERNEL); 1857 if (!subsys->model_number) { 1858 ret = -ENOMEM; 1859 goto free_subsys; 1860 } 1861 1862 subsys->ieee_oui = 0; 1863 1864 subsys->firmware_rev = kstrndup(UTS_RELEASE, NVMET_FR_MAX_SIZE, GFP_KERNEL); 1865 if (!subsys->firmware_rev) { 1866 ret = -ENOMEM; 1867 goto free_mn; 1868 } 1869 1870 switch (type) { 1871 case NVME_NQN_NVME: 1872 subsys->max_qid = NVMET_NR_QUEUES; 1873 break; 1874 case NVME_NQN_DISC: 1875 case NVME_NQN_CURR: 1876 subsys->max_qid = 0; 1877 break; 1878 default: 1879 pr_err("%s: Unknown Subsystem type - %d\n", __func__, type); 1880 ret = -EINVAL; 1881 goto free_fr; 1882 } 1883 subsys->type = type; 1884 subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE, 1885 GFP_KERNEL); 1886 if (!subsys->subsysnqn) { 1887 ret = -ENOMEM; 1888 goto free_fr; 1889 } 1890 subsys->cntlid_min = NVME_CNTLID_MIN; 1891 subsys->cntlid_max = NVME_CNTLID_MAX; 1892 kref_init(&subsys->ref); 1893 1894 mutex_init(&subsys->lock); 1895 xa_init(&subsys->namespaces); 1896 INIT_LIST_HEAD(&subsys->ctrls); 1897 INIT_LIST_HEAD(&subsys->hosts); 1898 1899 ret = nvmet_debugfs_subsys_setup(subsys); 1900 if (ret) 1901 goto free_subsysnqn; 1902 1903 return subsys; 1904 1905 free_subsysnqn: 1906 kfree(subsys->subsysnqn); 1907 free_fr: 1908 kfree(subsys->firmware_rev); 1909 free_mn: 1910 kfree(subsys->model_number); 1911 free_subsys: 1912 kfree(subsys); 1913 return ERR_PTR(ret); 1914 } 1915 1916 static void nvmet_subsys_free(struct kref *ref) 1917 { 1918 struct nvmet_subsys *subsys = 1919 container_of(ref, struct nvmet_subsys, ref); 1920 1921 WARN_ON_ONCE(!list_empty(&subsys->ctrls)); 1922 WARN_ON_ONCE(!list_empty(&subsys->hosts)); 1923 WARN_ON_ONCE(!xa_empty(&subsys->namespaces)); 1924 1925 nvmet_debugfs_subsys_free(subsys); 1926 1927 xa_destroy(&subsys->namespaces); 1928 nvmet_passthru_subsys_free(subsys); 1929 1930 kfree(subsys->subsysnqn); 1931 kfree(subsys->model_number); 1932 kfree(subsys->firmware_rev); 1933 kfree(subsys); 1934 } 1935 1936 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys) 1937 { 1938 struct nvmet_ctrl *ctrl; 1939 1940 mutex_lock(&subsys->lock); 1941 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 1942 ctrl->ops->delete_ctrl(ctrl); 1943 mutex_unlock(&subsys->lock); 1944 } 1945 1946 void nvmet_subsys_put(struct nvmet_subsys *subsys) 1947 { 1948 kref_put(&subsys->ref, nvmet_subsys_free); 1949 } 1950 1951 static int __init nvmet_init(void) 1952 { 1953 int error = -ENOMEM; 1954 1955 nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID] = 1; 1956 1957 nvmet_bvec_cache = kmem_cache_create("nvmet-bvec", 1958 NVMET_MAX_MPOOL_BVEC * sizeof(struct bio_vec), 0, 1959 SLAB_HWCACHE_ALIGN, NULL); 1960 if (!nvmet_bvec_cache) 1961 return -ENOMEM; 1962 1963 zbd_wq = alloc_workqueue("nvmet-zbd-wq", WQ_MEM_RECLAIM | WQ_PERCPU, 1964 0); 1965 if (!zbd_wq) 1966 goto out_destroy_bvec_cache; 1967 1968 buffered_io_wq = alloc_workqueue("nvmet-buffered-io-wq", 1969 WQ_MEM_RECLAIM | WQ_PERCPU, 0); 1970 if (!buffered_io_wq) 1971 goto out_free_zbd_work_queue; 1972 1973 nvmet_wq = alloc_workqueue("nvmet-wq", 1974 WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 0); 1975 if (!nvmet_wq) 1976 goto out_free_buffered_work_queue; 1977 1978 nvmet_aen_wq = alloc_workqueue("nvmet-aen-wq", 1979 WQ_MEM_RECLAIM | WQ_UNBOUND, 0); 1980 if (!nvmet_aen_wq) 1981 goto out_free_nvmet_work_queue; 1982 1983 error = nvmet_init_debugfs(); 1984 if (error) 1985 goto out_free_nvmet_aen_work_queue; 1986 1987 error = nvmet_init_discovery(); 1988 if (error) 1989 goto out_exit_debugfs; 1990 1991 error = nvmet_init_configfs(); 1992 if (error) 1993 goto out_exit_discovery; 1994 1995 return 0; 1996 1997 out_exit_discovery: 1998 nvmet_exit_discovery(); 1999 out_exit_debugfs: 2000 nvmet_exit_debugfs(); 2001 out_free_nvmet_aen_work_queue: 2002 destroy_workqueue(nvmet_aen_wq); 2003 out_free_nvmet_work_queue: 2004 destroy_workqueue(nvmet_wq); 2005 out_free_buffered_work_queue: 2006 destroy_workqueue(buffered_io_wq); 2007 out_free_zbd_work_queue: 2008 destroy_workqueue(zbd_wq); 2009 out_destroy_bvec_cache: 2010 kmem_cache_destroy(nvmet_bvec_cache); 2011 return error; 2012 } 2013 2014 static void __exit nvmet_exit(void) 2015 { 2016 nvmet_exit_configfs(); 2017 nvmet_exit_discovery(); 2018 nvmet_exit_debugfs(); 2019 ida_destroy(&cntlid_ida); 2020 destroy_workqueue(nvmet_aen_wq); 2021 destroy_workqueue(nvmet_wq); 2022 destroy_workqueue(buffered_io_wq); 2023 destroy_workqueue(zbd_wq); 2024 kmem_cache_destroy(nvmet_bvec_cache); 2025 2026 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024); 2027 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024); 2028 } 2029 2030 module_init(nvmet_init); 2031 module_exit(nvmet_exit); 2032 2033 MODULE_DESCRIPTION("NVMe target core framework"); 2034 MODULE_LICENSE("GPL v2"); 2035