1 /* 2 * Copyright (c) 2016 Avago Technologies. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful. 9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, 10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A 11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO 12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID. 13 * See the GNU General Public License for more details, a copy of which 14 * can be found in the file COPYING included with this package 15 * 16 */ 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 #include <linux/module.h> 19 #include <linux/parser.h> 20 #include <uapi/scsi/fc/fc_fs.h> 21 #include <uapi/scsi/fc/fc_els.h> 22 #include <linux/delay.h> 23 24 #include "nvme.h" 25 #include "fabrics.h" 26 #include <linux/nvme-fc-driver.h> 27 #include <linux/nvme-fc.h> 28 29 30 /* *************************** Data Structures/Defines ****************** */ 31 32 33 enum nvme_fc_queue_flags { 34 NVME_FC_Q_CONNECTED = 0, 35 NVME_FC_Q_LIVE, 36 }; 37 38 #define NVME_FC_DEFAULT_DEV_LOSS_TMO 60 /* seconds */ 39 40 struct nvme_fc_queue { 41 struct nvme_fc_ctrl *ctrl; 42 struct device *dev; 43 struct blk_mq_hw_ctx *hctx; 44 void *lldd_handle; 45 size_t cmnd_capsule_len; 46 u32 qnum; 47 u32 rqcnt; 48 u32 seqno; 49 50 u64 connection_id; 51 atomic_t csn; 52 53 unsigned long flags; 54 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 55 56 enum nvme_fcop_flags { 57 FCOP_FLAGS_TERMIO = (1 << 0), 58 FCOP_FLAGS_AEN = (1 << 1), 59 }; 60 61 struct nvmefc_ls_req_op { 62 struct nvmefc_ls_req ls_req; 63 64 struct nvme_fc_rport *rport; 65 struct nvme_fc_queue *queue; 66 struct request *rq; 67 u32 flags; 68 69 int ls_error; 70 struct completion ls_done; 71 struct list_head lsreq_list; /* rport->ls_req_list */ 72 bool req_queued; 73 }; 74 75 enum nvme_fcpop_state { 76 FCPOP_STATE_UNINIT = 0, 77 FCPOP_STATE_IDLE = 1, 78 FCPOP_STATE_ACTIVE = 2, 79 FCPOP_STATE_ABORTED = 3, 80 FCPOP_STATE_COMPLETE = 4, 81 }; 82 83 struct nvme_fc_fcp_op { 84 struct nvme_request nreq; /* 85 * nvme/host/core.c 86 * requires this to be 87 * the 1st element in the 88 * private structure 89 * associated with the 90 * request. 91 */ 92 struct nvmefc_fcp_req fcp_req; 93 94 struct nvme_fc_ctrl *ctrl; 95 struct nvme_fc_queue *queue; 96 struct request *rq; 97 98 atomic_t state; 99 u32 flags; 100 u32 rqno; 101 u32 nents; 102 103 struct nvme_fc_cmd_iu cmd_iu; 104 struct nvme_fc_ersp_iu rsp_iu; 105 }; 106 107 struct nvme_fc_lport { 108 struct nvme_fc_local_port localport; 109 110 struct ida endp_cnt; 111 struct list_head port_list; /* nvme_fc_port_list */ 112 struct list_head endp_list; 113 struct device *dev; /* physical device for dma */ 114 struct nvme_fc_port_template *ops; 115 struct kref ref; 116 atomic_t act_rport_cnt; 117 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 118 119 struct nvme_fc_rport { 120 struct nvme_fc_remote_port remoteport; 121 122 struct list_head endp_list; /* for lport->endp_list */ 123 struct list_head ctrl_list; 124 struct list_head ls_req_list; 125 struct device *dev; /* physical device for dma */ 126 struct nvme_fc_lport *lport; 127 spinlock_t lock; 128 struct kref ref; 129 atomic_t act_ctrl_cnt; 130 unsigned long dev_loss_end; 131 } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ 132 133 enum nvme_fcctrl_flags { 134 FCCTRL_TERMIO = (1 << 0), 135 }; 136 137 struct nvme_fc_ctrl { 138 spinlock_t lock; 139 struct nvme_fc_queue *queues; 140 struct device *dev; 141 struct nvme_fc_lport *lport; 142 struct nvme_fc_rport *rport; 143 u32 cnum; 144 145 bool assoc_active; 146 u64 association_id; 147 148 struct list_head ctrl_list; /* rport->ctrl_list */ 149 150 struct blk_mq_tag_set admin_tag_set; 151 struct blk_mq_tag_set tag_set; 152 153 struct delayed_work connect_work; 154 155 struct kref ref; 156 u32 flags; 157 u32 iocnt; 158 wait_queue_head_t ioabort_wait; 159 160 struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS]; 161 162 struct nvme_ctrl ctrl; 163 }; 164 165 static inline struct nvme_fc_ctrl * 166 to_fc_ctrl(struct nvme_ctrl *ctrl) 167 { 168 return container_of(ctrl, struct nvme_fc_ctrl, ctrl); 169 } 170 171 static inline struct nvme_fc_lport * 172 localport_to_lport(struct nvme_fc_local_port *portptr) 173 { 174 return container_of(portptr, struct nvme_fc_lport, localport); 175 } 176 177 static inline struct nvme_fc_rport * 178 remoteport_to_rport(struct nvme_fc_remote_port *portptr) 179 { 180 return container_of(portptr, struct nvme_fc_rport, remoteport); 181 } 182 183 static inline struct nvmefc_ls_req_op * 184 ls_req_to_lsop(struct nvmefc_ls_req *lsreq) 185 { 186 return container_of(lsreq, struct nvmefc_ls_req_op, ls_req); 187 } 188 189 static inline struct nvme_fc_fcp_op * 190 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq) 191 { 192 return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req); 193 } 194 195 196 197 /* *************************** Globals **************************** */ 198 199 200 static DEFINE_SPINLOCK(nvme_fc_lock); 201 202 static LIST_HEAD(nvme_fc_lport_list); 203 static DEFINE_IDA(nvme_fc_local_port_cnt); 204 static DEFINE_IDA(nvme_fc_ctrl_cnt); 205 206 207 208 /* 209 * These items are short-term. They will eventually be moved into 210 * a generic FC class. See comments in module init. 211 */ 212 static struct class *fc_class; 213 static struct device *fc_udev_device; 214 215 216 /* *********************** FC-NVME Port Management ************************ */ 217 218 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *, 219 struct nvme_fc_queue *, unsigned int); 220 221 static void 222 nvme_fc_free_lport(struct kref *ref) 223 { 224 struct nvme_fc_lport *lport = 225 container_of(ref, struct nvme_fc_lport, ref); 226 unsigned long flags; 227 228 WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED); 229 WARN_ON(!list_empty(&lport->endp_list)); 230 231 /* remove from transport list */ 232 spin_lock_irqsave(&nvme_fc_lock, flags); 233 list_del(&lport->port_list); 234 spin_unlock_irqrestore(&nvme_fc_lock, flags); 235 236 ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num); 237 ida_destroy(&lport->endp_cnt); 238 239 put_device(lport->dev); 240 241 kfree(lport); 242 } 243 244 static void 245 nvme_fc_lport_put(struct nvme_fc_lport *lport) 246 { 247 kref_put(&lport->ref, nvme_fc_free_lport); 248 } 249 250 static int 251 nvme_fc_lport_get(struct nvme_fc_lport *lport) 252 { 253 return kref_get_unless_zero(&lport->ref); 254 } 255 256 257 static struct nvme_fc_lport * 258 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo, 259 struct nvme_fc_port_template *ops, 260 struct device *dev) 261 { 262 struct nvme_fc_lport *lport; 263 unsigned long flags; 264 265 spin_lock_irqsave(&nvme_fc_lock, flags); 266 267 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { 268 if (lport->localport.node_name != pinfo->node_name || 269 lport->localport.port_name != pinfo->port_name) 270 continue; 271 272 if (lport->dev != dev) { 273 lport = ERR_PTR(-EXDEV); 274 goto out_done; 275 } 276 277 if (lport->localport.port_state != FC_OBJSTATE_DELETED) { 278 lport = ERR_PTR(-EEXIST); 279 goto out_done; 280 } 281 282 if (!nvme_fc_lport_get(lport)) { 283 /* 284 * fails if ref cnt already 0. If so, 285 * act as if lport already deleted 286 */ 287 lport = NULL; 288 goto out_done; 289 } 290 291 /* resume the lport */ 292 293 lport->ops = ops; 294 lport->localport.port_role = pinfo->port_role; 295 lport->localport.port_id = pinfo->port_id; 296 lport->localport.port_state = FC_OBJSTATE_ONLINE; 297 298 spin_unlock_irqrestore(&nvme_fc_lock, flags); 299 300 return lport; 301 } 302 303 lport = NULL; 304 305 out_done: 306 spin_unlock_irqrestore(&nvme_fc_lock, flags); 307 308 return lport; 309 } 310 311 /** 312 * nvme_fc_register_localport - transport entry point called by an 313 * LLDD to register the existence of a NVME 314 * host FC port. 315 * @pinfo: pointer to information about the port to be registered 316 * @template: LLDD entrypoints and operational parameters for the port 317 * @dev: physical hardware device node port corresponds to. Will be 318 * used for DMA mappings 319 * @lport_p: pointer to a local port pointer. Upon success, the routine 320 * will allocate a nvme_fc_local_port structure and place its 321 * address in the local port pointer. Upon failure, local port 322 * pointer will be set to 0. 323 * 324 * Returns: 325 * a completion status. Must be 0 upon success; a negative errno 326 * (ex: -ENXIO) upon failure. 327 */ 328 int 329 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo, 330 struct nvme_fc_port_template *template, 331 struct device *dev, 332 struct nvme_fc_local_port **portptr) 333 { 334 struct nvme_fc_lport *newrec; 335 unsigned long flags; 336 int ret, idx; 337 338 if (!template->localport_delete || !template->remoteport_delete || 339 !template->ls_req || !template->fcp_io || 340 !template->ls_abort || !template->fcp_abort || 341 !template->max_hw_queues || !template->max_sgl_segments || 342 !template->max_dif_sgl_segments || !template->dma_boundary) { 343 ret = -EINVAL; 344 goto out_reghost_failed; 345 } 346 347 /* 348 * look to see if there is already a localport that had been 349 * deregistered and in the process of waiting for all the 350 * references to fully be removed. If the references haven't 351 * expired, we can simply re-enable the localport. Remoteports 352 * and controller reconnections should resume naturally. 353 */ 354 newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev); 355 356 /* found an lport, but something about its state is bad */ 357 if (IS_ERR(newrec)) { 358 ret = PTR_ERR(newrec); 359 goto out_reghost_failed; 360 361 /* found existing lport, which was resumed */ 362 } else if (newrec) { 363 *portptr = &newrec->localport; 364 return 0; 365 } 366 367 /* nothing found - allocate a new localport struct */ 368 369 newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz), 370 GFP_KERNEL); 371 if (!newrec) { 372 ret = -ENOMEM; 373 goto out_reghost_failed; 374 } 375 376 idx = ida_simple_get(&nvme_fc_local_port_cnt, 0, 0, GFP_KERNEL); 377 if (idx < 0) { 378 ret = -ENOSPC; 379 goto out_fail_kfree; 380 } 381 382 if (!get_device(dev) && dev) { 383 ret = -ENODEV; 384 goto out_ida_put; 385 } 386 387 INIT_LIST_HEAD(&newrec->port_list); 388 INIT_LIST_HEAD(&newrec->endp_list); 389 kref_init(&newrec->ref); 390 atomic_set(&newrec->act_rport_cnt, 0); 391 newrec->ops = template; 392 newrec->dev = dev; 393 ida_init(&newrec->endp_cnt); 394 newrec->localport.private = &newrec[1]; 395 newrec->localport.node_name = pinfo->node_name; 396 newrec->localport.port_name = pinfo->port_name; 397 newrec->localport.port_role = pinfo->port_role; 398 newrec->localport.port_id = pinfo->port_id; 399 newrec->localport.port_state = FC_OBJSTATE_ONLINE; 400 newrec->localport.port_num = idx; 401 402 spin_lock_irqsave(&nvme_fc_lock, flags); 403 list_add_tail(&newrec->port_list, &nvme_fc_lport_list); 404 spin_unlock_irqrestore(&nvme_fc_lock, flags); 405 406 if (dev) 407 dma_set_seg_boundary(dev, template->dma_boundary); 408 409 *portptr = &newrec->localport; 410 return 0; 411 412 out_ida_put: 413 ida_simple_remove(&nvme_fc_local_port_cnt, idx); 414 out_fail_kfree: 415 kfree(newrec); 416 out_reghost_failed: 417 *portptr = NULL; 418 419 return ret; 420 } 421 EXPORT_SYMBOL_GPL(nvme_fc_register_localport); 422 423 /** 424 * nvme_fc_unregister_localport - transport entry point called by an 425 * LLDD to deregister/remove a previously 426 * registered a NVME host FC port. 427 * @localport: pointer to the (registered) local port that is to be 428 * deregistered. 429 * 430 * Returns: 431 * a completion status. Must be 0 upon success; a negative errno 432 * (ex: -ENXIO) upon failure. 433 */ 434 int 435 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr) 436 { 437 struct nvme_fc_lport *lport = localport_to_lport(portptr); 438 unsigned long flags; 439 440 if (!portptr) 441 return -EINVAL; 442 443 spin_lock_irqsave(&nvme_fc_lock, flags); 444 445 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 446 spin_unlock_irqrestore(&nvme_fc_lock, flags); 447 return -EINVAL; 448 } 449 portptr->port_state = FC_OBJSTATE_DELETED; 450 451 spin_unlock_irqrestore(&nvme_fc_lock, flags); 452 453 if (atomic_read(&lport->act_rport_cnt) == 0) 454 lport->ops->localport_delete(&lport->localport); 455 456 nvme_fc_lport_put(lport); 457 458 return 0; 459 } 460 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport); 461 462 /* 463 * TRADDR strings, per FC-NVME are fixed format: 464 * "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters 465 * udev event will only differ by prefix of what field is 466 * being specified: 467 * "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters 468 * 19 + 43 + null_fudge = 64 characters 469 */ 470 #define FCNVME_TRADDR_LENGTH 64 471 472 static void 473 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport, 474 struct nvme_fc_rport *rport) 475 { 476 char hostaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_HOST_TRADDR=...*/ 477 char tgtaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_TRADDR=...*/ 478 char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL }; 479 480 if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY)) 481 return; 482 483 snprintf(hostaddr, sizeof(hostaddr), 484 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx", 485 lport->localport.node_name, lport->localport.port_name); 486 snprintf(tgtaddr, sizeof(tgtaddr), 487 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx", 488 rport->remoteport.node_name, rport->remoteport.port_name); 489 kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp); 490 } 491 492 static void 493 nvme_fc_free_rport(struct kref *ref) 494 { 495 struct nvme_fc_rport *rport = 496 container_of(ref, struct nvme_fc_rport, ref); 497 struct nvme_fc_lport *lport = 498 localport_to_lport(rport->remoteport.localport); 499 unsigned long flags; 500 501 WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED); 502 WARN_ON(!list_empty(&rport->ctrl_list)); 503 504 /* remove from lport list */ 505 spin_lock_irqsave(&nvme_fc_lock, flags); 506 list_del(&rport->endp_list); 507 spin_unlock_irqrestore(&nvme_fc_lock, flags); 508 509 ida_simple_remove(&lport->endp_cnt, rport->remoteport.port_num); 510 511 kfree(rport); 512 513 nvme_fc_lport_put(lport); 514 } 515 516 static void 517 nvme_fc_rport_put(struct nvme_fc_rport *rport) 518 { 519 kref_put(&rport->ref, nvme_fc_free_rport); 520 } 521 522 static int 523 nvme_fc_rport_get(struct nvme_fc_rport *rport) 524 { 525 return kref_get_unless_zero(&rport->ref); 526 } 527 528 static void 529 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl) 530 { 531 switch (ctrl->ctrl.state) { 532 case NVME_CTRL_NEW: 533 case NVME_CTRL_CONNECTING: 534 /* 535 * As all reconnects were suppressed, schedule a 536 * connect. 537 */ 538 dev_info(ctrl->ctrl.device, 539 "NVME-FC{%d}: connectivity re-established. " 540 "Attempting reconnect\n", ctrl->cnum); 541 542 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0); 543 break; 544 545 case NVME_CTRL_RESETTING: 546 /* 547 * Controller is already in the process of terminating the 548 * association. No need to do anything further. The reconnect 549 * step will naturally occur after the reset completes. 550 */ 551 break; 552 553 default: 554 /* no action to take - let it delete */ 555 break; 556 } 557 } 558 559 static struct nvme_fc_rport * 560 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport, 561 struct nvme_fc_port_info *pinfo) 562 { 563 struct nvme_fc_rport *rport; 564 struct nvme_fc_ctrl *ctrl; 565 unsigned long flags; 566 567 spin_lock_irqsave(&nvme_fc_lock, flags); 568 569 list_for_each_entry(rport, &lport->endp_list, endp_list) { 570 if (rport->remoteport.node_name != pinfo->node_name || 571 rport->remoteport.port_name != pinfo->port_name) 572 continue; 573 574 if (!nvme_fc_rport_get(rport)) { 575 rport = ERR_PTR(-ENOLCK); 576 goto out_done; 577 } 578 579 spin_unlock_irqrestore(&nvme_fc_lock, flags); 580 581 spin_lock_irqsave(&rport->lock, flags); 582 583 /* has it been unregistered */ 584 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) { 585 /* means lldd called us twice */ 586 spin_unlock_irqrestore(&rport->lock, flags); 587 nvme_fc_rport_put(rport); 588 return ERR_PTR(-ESTALE); 589 } 590 591 rport->remoteport.port_role = pinfo->port_role; 592 rport->remoteport.port_id = pinfo->port_id; 593 rport->remoteport.port_state = FC_OBJSTATE_ONLINE; 594 rport->dev_loss_end = 0; 595 596 /* 597 * kick off a reconnect attempt on all associations to the 598 * remote port. A successful reconnects will resume i/o. 599 */ 600 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) 601 nvme_fc_resume_controller(ctrl); 602 603 spin_unlock_irqrestore(&rport->lock, flags); 604 605 return rport; 606 } 607 608 rport = NULL; 609 610 out_done: 611 spin_unlock_irqrestore(&nvme_fc_lock, flags); 612 613 return rport; 614 } 615 616 static inline void 617 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport, 618 struct nvme_fc_port_info *pinfo) 619 { 620 if (pinfo->dev_loss_tmo) 621 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo; 622 else 623 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO; 624 } 625 626 /** 627 * nvme_fc_register_remoteport - transport entry point called by an 628 * LLDD to register the existence of a NVME 629 * subsystem FC port on its fabric. 630 * @localport: pointer to the (registered) local port that the remote 631 * subsystem port is connected to. 632 * @pinfo: pointer to information about the port to be registered 633 * @rport_p: pointer to a remote port pointer. Upon success, the routine 634 * will allocate a nvme_fc_remote_port structure and place its 635 * address in the remote port pointer. Upon failure, remote port 636 * pointer will be set to 0. 637 * 638 * Returns: 639 * a completion status. Must be 0 upon success; a negative errno 640 * (ex: -ENXIO) upon failure. 641 */ 642 int 643 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, 644 struct nvme_fc_port_info *pinfo, 645 struct nvme_fc_remote_port **portptr) 646 { 647 struct nvme_fc_lport *lport = localport_to_lport(localport); 648 struct nvme_fc_rport *newrec; 649 unsigned long flags; 650 int ret, idx; 651 652 if (!nvme_fc_lport_get(lport)) { 653 ret = -ESHUTDOWN; 654 goto out_reghost_failed; 655 } 656 657 /* 658 * look to see if there is already a remoteport that is waiting 659 * for a reconnect (within dev_loss_tmo) with the same WWN's. 660 * If so, transition to it and reconnect. 661 */ 662 newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo); 663 664 /* found an rport, but something about its state is bad */ 665 if (IS_ERR(newrec)) { 666 ret = PTR_ERR(newrec); 667 goto out_lport_put; 668 669 /* found existing rport, which was resumed */ 670 } else if (newrec) { 671 nvme_fc_lport_put(lport); 672 __nvme_fc_set_dev_loss_tmo(newrec, pinfo); 673 nvme_fc_signal_discovery_scan(lport, newrec); 674 *portptr = &newrec->remoteport; 675 return 0; 676 } 677 678 /* nothing found - allocate a new remoteport struct */ 679 680 newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz), 681 GFP_KERNEL); 682 if (!newrec) { 683 ret = -ENOMEM; 684 goto out_lport_put; 685 } 686 687 idx = ida_simple_get(&lport->endp_cnt, 0, 0, GFP_KERNEL); 688 if (idx < 0) { 689 ret = -ENOSPC; 690 goto out_kfree_rport; 691 } 692 693 INIT_LIST_HEAD(&newrec->endp_list); 694 INIT_LIST_HEAD(&newrec->ctrl_list); 695 INIT_LIST_HEAD(&newrec->ls_req_list); 696 kref_init(&newrec->ref); 697 atomic_set(&newrec->act_ctrl_cnt, 0); 698 spin_lock_init(&newrec->lock); 699 newrec->remoteport.localport = &lport->localport; 700 newrec->dev = lport->dev; 701 newrec->lport = lport; 702 newrec->remoteport.private = &newrec[1]; 703 newrec->remoteport.port_role = pinfo->port_role; 704 newrec->remoteport.node_name = pinfo->node_name; 705 newrec->remoteport.port_name = pinfo->port_name; 706 newrec->remoteport.port_id = pinfo->port_id; 707 newrec->remoteport.port_state = FC_OBJSTATE_ONLINE; 708 newrec->remoteport.port_num = idx; 709 __nvme_fc_set_dev_loss_tmo(newrec, pinfo); 710 711 spin_lock_irqsave(&nvme_fc_lock, flags); 712 list_add_tail(&newrec->endp_list, &lport->endp_list); 713 spin_unlock_irqrestore(&nvme_fc_lock, flags); 714 715 nvme_fc_signal_discovery_scan(lport, newrec); 716 717 *portptr = &newrec->remoteport; 718 return 0; 719 720 out_kfree_rport: 721 kfree(newrec); 722 out_lport_put: 723 nvme_fc_lport_put(lport); 724 out_reghost_failed: 725 *portptr = NULL; 726 return ret; 727 } 728 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport); 729 730 static int 731 nvme_fc_abort_lsops(struct nvme_fc_rport *rport) 732 { 733 struct nvmefc_ls_req_op *lsop; 734 unsigned long flags; 735 736 restart: 737 spin_lock_irqsave(&rport->lock, flags); 738 739 list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) { 740 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) { 741 lsop->flags |= FCOP_FLAGS_TERMIO; 742 spin_unlock_irqrestore(&rport->lock, flags); 743 rport->lport->ops->ls_abort(&rport->lport->localport, 744 &rport->remoteport, 745 &lsop->ls_req); 746 goto restart; 747 } 748 } 749 spin_unlock_irqrestore(&rport->lock, flags); 750 751 return 0; 752 } 753 754 static void 755 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) 756 { 757 dev_info(ctrl->ctrl.device, 758 "NVME-FC{%d}: controller connectivity lost. Awaiting " 759 "Reconnect", ctrl->cnum); 760 761 switch (ctrl->ctrl.state) { 762 case NVME_CTRL_NEW: 763 case NVME_CTRL_LIVE: 764 /* 765 * Schedule a controller reset. The reset will terminate the 766 * association and schedule the reconnect timer. Reconnects 767 * will be attempted until either the ctlr_loss_tmo 768 * (max_retries * connect_delay) expires or the remoteport's 769 * dev_loss_tmo expires. 770 */ 771 if (nvme_reset_ctrl(&ctrl->ctrl)) { 772 dev_warn(ctrl->ctrl.device, 773 "NVME-FC{%d}: Couldn't schedule reset.\n", 774 ctrl->cnum); 775 nvme_delete_ctrl(&ctrl->ctrl); 776 } 777 break; 778 779 case NVME_CTRL_CONNECTING: 780 /* 781 * The association has already been terminated and the 782 * controller is attempting reconnects. No need to do anything 783 * futher. Reconnects will be attempted until either the 784 * ctlr_loss_tmo (max_retries * connect_delay) expires or the 785 * remoteport's dev_loss_tmo expires. 786 */ 787 break; 788 789 case NVME_CTRL_RESETTING: 790 /* 791 * Controller is already in the process of terminating the 792 * association. No need to do anything further. The reconnect 793 * step will kick in naturally after the association is 794 * terminated. 795 */ 796 break; 797 798 case NVME_CTRL_DELETING: 799 default: 800 /* no action to take - let it delete */ 801 break; 802 } 803 } 804 805 /** 806 * nvme_fc_unregister_remoteport - transport entry point called by an 807 * LLDD to deregister/remove a previously 808 * registered a NVME subsystem FC port. 809 * @remoteport: pointer to the (registered) remote port that is to be 810 * deregistered. 811 * 812 * Returns: 813 * a completion status. Must be 0 upon success; a negative errno 814 * (ex: -ENXIO) upon failure. 815 */ 816 int 817 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr) 818 { 819 struct nvme_fc_rport *rport = remoteport_to_rport(portptr); 820 struct nvme_fc_ctrl *ctrl; 821 unsigned long flags; 822 823 if (!portptr) 824 return -EINVAL; 825 826 spin_lock_irqsave(&rport->lock, flags); 827 828 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 829 spin_unlock_irqrestore(&rport->lock, flags); 830 return -EINVAL; 831 } 832 portptr->port_state = FC_OBJSTATE_DELETED; 833 834 rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ); 835 836 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 837 /* if dev_loss_tmo==0, dev loss is immediate */ 838 if (!portptr->dev_loss_tmo) { 839 dev_warn(ctrl->ctrl.device, 840 "NVME-FC{%d}: controller connectivity lost.\n", 841 ctrl->cnum); 842 nvme_delete_ctrl(&ctrl->ctrl); 843 } else 844 nvme_fc_ctrl_connectivity_loss(ctrl); 845 } 846 847 spin_unlock_irqrestore(&rport->lock, flags); 848 849 nvme_fc_abort_lsops(rport); 850 851 if (atomic_read(&rport->act_ctrl_cnt) == 0) 852 rport->lport->ops->remoteport_delete(portptr); 853 854 /* 855 * release the reference, which will allow, if all controllers 856 * go away, which should only occur after dev_loss_tmo occurs, 857 * for the rport to be torn down. 858 */ 859 nvme_fc_rport_put(rport); 860 861 return 0; 862 } 863 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport); 864 865 /** 866 * nvme_fc_rescan_remoteport - transport entry point called by an 867 * LLDD to request a nvme device rescan. 868 * @remoteport: pointer to the (registered) remote port that is to be 869 * rescanned. 870 * 871 * Returns: N/A 872 */ 873 void 874 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport) 875 { 876 struct nvme_fc_rport *rport = remoteport_to_rport(remoteport); 877 878 nvme_fc_signal_discovery_scan(rport->lport, rport); 879 } 880 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport); 881 882 int 883 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr, 884 u32 dev_loss_tmo) 885 { 886 struct nvme_fc_rport *rport = remoteport_to_rport(portptr); 887 unsigned long flags; 888 889 spin_lock_irqsave(&rport->lock, flags); 890 891 if (portptr->port_state != FC_OBJSTATE_ONLINE) { 892 spin_unlock_irqrestore(&rport->lock, flags); 893 return -EINVAL; 894 } 895 896 /* a dev_loss_tmo of 0 (immediate) is allowed to be set */ 897 rport->remoteport.dev_loss_tmo = dev_loss_tmo; 898 899 spin_unlock_irqrestore(&rport->lock, flags); 900 901 return 0; 902 } 903 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss); 904 905 906 /* *********************** FC-NVME DMA Handling **************************** */ 907 908 /* 909 * The fcloop device passes in a NULL device pointer. Real LLD's will 910 * pass in a valid device pointer. If NULL is passed to the dma mapping 911 * routines, depending on the platform, it may or may not succeed, and 912 * may crash. 913 * 914 * As such: 915 * Wrapper all the dma routines and check the dev pointer. 916 * 917 * If simple mappings (return just a dma address, we'll noop them, 918 * returning a dma address of 0. 919 * 920 * On more complex mappings (dma_map_sg), a pseudo routine fills 921 * in the scatter list, setting all dma addresses to 0. 922 */ 923 924 static inline dma_addr_t 925 fc_dma_map_single(struct device *dev, void *ptr, size_t size, 926 enum dma_data_direction dir) 927 { 928 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; 929 } 930 931 static inline int 932 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 933 { 934 return dev ? dma_mapping_error(dev, dma_addr) : 0; 935 } 936 937 static inline void 938 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, 939 enum dma_data_direction dir) 940 { 941 if (dev) 942 dma_unmap_single(dev, addr, size, dir); 943 } 944 945 static inline void 946 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 947 enum dma_data_direction dir) 948 { 949 if (dev) 950 dma_sync_single_for_cpu(dev, addr, size, dir); 951 } 952 953 static inline void 954 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, 955 enum dma_data_direction dir) 956 { 957 if (dev) 958 dma_sync_single_for_device(dev, addr, size, dir); 959 } 960 961 /* pseudo dma_map_sg call */ 962 static int 963 fc_map_sg(struct scatterlist *sg, int nents) 964 { 965 struct scatterlist *s; 966 int i; 967 968 WARN_ON(nents == 0 || sg[0].length == 0); 969 970 for_each_sg(sg, s, nents, i) { 971 s->dma_address = 0L; 972 #ifdef CONFIG_NEED_SG_DMA_LENGTH 973 s->dma_length = s->length; 974 #endif 975 } 976 return nents; 977 } 978 979 static inline int 980 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 981 enum dma_data_direction dir) 982 { 983 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); 984 } 985 986 static inline void 987 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 988 enum dma_data_direction dir) 989 { 990 if (dev) 991 dma_unmap_sg(dev, sg, nents, dir); 992 } 993 994 /* *********************** FC-NVME LS Handling **************************** */ 995 996 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *); 997 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *); 998 999 1000 static void 1001 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop) 1002 { 1003 struct nvme_fc_rport *rport = lsop->rport; 1004 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1005 unsigned long flags; 1006 1007 spin_lock_irqsave(&rport->lock, flags); 1008 1009 if (!lsop->req_queued) { 1010 spin_unlock_irqrestore(&rport->lock, flags); 1011 return; 1012 } 1013 1014 list_del(&lsop->lsreq_list); 1015 1016 lsop->req_queued = false; 1017 1018 spin_unlock_irqrestore(&rport->lock, flags); 1019 1020 fc_dma_unmap_single(rport->dev, lsreq->rqstdma, 1021 (lsreq->rqstlen + lsreq->rsplen), 1022 DMA_BIDIRECTIONAL); 1023 1024 nvme_fc_rport_put(rport); 1025 } 1026 1027 static int 1028 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport, 1029 struct nvmefc_ls_req_op *lsop, 1030 void (*done)(struct nvmefc_ls_req *req, int status)) 1031 { 1032 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1033 unsigned long flags; 1034 int ret = 0; 1035 1036 if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 1037 return -ECONNREFUSED; 1038 1039 if (!nvme_fc_rport_get(rport)) 1040 return -ESHUTDOWN; 1041 1042 lsreq->done = done; 1043 lsop->rport = rport; 1044 lsop->req_queued = false; 1045 INIT_LIST_HEAD(&lsop->lsreq_list); 1046 init_completion(&lsop->ls_done); 1047 1048 lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr, 1049 lsreq->rqstlen + lsreq->rsplen, 1050 DMA_BIDIRECTIONAL); 1051 if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) { 1052 ret = -EFAULT; 1053 goto out_putrport; 1054 } 1055 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; 1056 1057 spin_lock_irqsave(&rport->lock, flags); 1058 1059 list_add_tail(&lsop->lsreq_list, &rport->ls_req_list); 1060 1061 lsop->req_queued = true; 1062 1063 spin_unlock_irqrestore(&rport->lock, flags); 1064 1065 ret = rport->lport->ops->ls_req(&rport->lport->localport, 1066 &rport->remoteport, lsreq); 1067 if (ret) 1068 goto out_unlink; 1069 1070 return 0; 1071 1072 out_unlink: 1073 lsop->ls_error = ret; 1074 spin_lock_irqsave(&rport->lock, flags); 1075 lsop->req_queued = false; 1076 list_del(&lsop->lsreq_list); 1077 spin_unlock_irqrestore(&rport->lock, flags); 1078 fc_dma_unmap_single(rport->dev, lsreq->rqstdma, 1079 (lsreq->rqstlen + lsreq->rsplen), 1080 DMA_BIDIRECTIONAL); 1081 out_putrport: 1082 nvme_fc_rport_put(rport); 1083 1084 return ret; 1085 } 1086 1087 static void 1088 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status) 1089 { 1090 struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq); 1091 1092 lsop->ls_error = status; 1093 complete(&lsop->ls_done); 1094 } 1095 1096 static int 1097 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop) 1098 { 1099 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 1100 struct fcnvme_ls_rjt *rjt = lsreq->rspaddr; 1101 int ret; 1102 1103 ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done); 1104 1105 if (!ret) { 1106 /* 1107 * No timeout/not interruptible as we need the struct 1108 * to exist until the lldd calls us back. Thus mandate 1109 * wait until driver calls back. lldd responsible for 1110 * the timeout action 1111 */ 1112 wait_for_completion(&lsop->ls_done); 1113 1114 __nvme_fc_finish_ls_req(lsop); 1115 1116 ret = lsop->ls_error; 1117 } 1118 1119 if (ret) 1120 return ret; 1121 1122 /* ACC or RJT payload ? */ 1123 if (rjt->w0.ls_cmd == FCNVME_LS_RJT) 1124 return -ENXIO; 1125 1126 return 0; 1127 } 1128 1129 static int 1130 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport, 1131 struct nvmefc_ls_req_op *lsop, 1132 void (*done)(struct nvmefc_ls_req *req, int status)) 1133 { 1134 /* don't wait for completion */ 1135 1136 return __nvme_fc_send_ls_req(rport, lsop, done); 1137 } 1138 1139 /* Validation Error indexes into the string table below */ 1140 enum { 1141 VERR_NO_ERROR = 0, 1142 VERR_LSACC = 1, 1143 VERR_LSDESC_RQST = 2, 1144 VERR_LSDESC_RQST_LEN = 3, 1145 VERR_ASSOC_ID = 4, 1146 VERR_ASSOC_ID_LEN = 5, 1147 VERR_CONN_ID = 6, 1148 VERR_CONN_ID_LEN = 7, 1149 VERR_CR_ASSOC = 8, 1150 VERR_CR_ASSOC_ACC_LEN = 9, 1151 VERR_CR_CONN = 10, 1152 VERR_CR_CONN_ACC_LEN = 11, 1153 VERR_DISCONN = 12, 1154 VERR_DISCONN_ACC_LEN = 13, 1155 }; 1156 1157 static char *validation_errors[] = { 1158 "OK", 1159 "Not LS_ACC", 1160 "Not LSDESC_RQST", 1161 "Bad LSDESC_RQST Length", 1162 "Not Association ID", 1163 "Bad Association ID Length", 1164 "Not Connection ID", 1165 "Bad Connection ID Length", 1166 "Not CR_ASSOC Rqst", 1167 "Bad CR_ASSOC ACC Length", 1168 "Not CR_CONN Rqst", 1169 "Bad CR_CONN ACC Length", 1170 "Not Disconnect Rqst", 1171 "Bad Disconnect ACC Length", 1172 }; 1173 1174 static int 1175 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, 1176 struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio) 1177 { 1178 struct nvmefc_ls_req_op *lsop; 1179 struct nvmefc_ls_req *lsreq; 1180 struct fcnvme_ls_cr_assoc_rqst *assoc_rqst; 1181 struct fcnvme_ls_cr_assoc_acc *assoc_acc; 1182 int ret, fcret = 0; 1183 1184 lsop = kzalloc((sizeof(*lsop) + 1185 ctrl->lport->ops->lsrqst_priv_sz + 1186 sizeof(*assoc_rqst) + sizeof(*assoc_acc)), GFP_KERNEL); 1187 if (!lsop) { 1188 ret = -ENOMEM; 1189 goto out_no_memory; 1190 } 1191 lsreq = &lsop->ls_req; 1192 1193 lsreq->private = (void *)&lsop[1]; 1194 assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *) 1195 (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); 1196 assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1]; 1197 1198 assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION; 1199 assoc_rqst->desc_list_len = 1200 cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)); 1201 1202 assoc_rqst->assoc_cmd.desc_tag = 1203 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD); 1204 assoc_rqst->assoc_cmd.desc_len = 1205 fcnvme_lsdesc_len( 1206 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)); 1207 1208 assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio); 1209 assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1); 1210 /* Linux supports only Dynamic controllers */ 1211 assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff); 1212 uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id); 1213 strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn, 1214 min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE)); 1215 strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn, 1216 min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE)); 1217 1218 lsop->queue = queue; 1219 lsreq->rqstaddr = assoc_rqst; 1220 lsreq->rqstlen = sizeof(*assoc_rqst); 1221 lsreq->rspaddr = assoc_acc; 1222 lsreq->rsplen = sizeof(*assoc_acc); 1223 lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; 1224 1225 ret = nvme_fc_send_ls_req(ctrl->rport, lsop); 1226 if (ret) 1227 goto out_free_buffer; 1228 1229 /* process connect LS completion */ 1230 1231 /* validate the ACC response */ 1232 if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC) 1233 fcret = VERR_LSACC; 1234 else if (assoc_acc->hdr.desc_list_len != 1235 fcnvme_lsdesc_len( 1236 sizeof(struct fcnvme_ls_cr_assoc_acc))) 1237 fcret = VERR_CR_ASSOC_ACC_LEN; 1238 else if (assoc_acc->hdr.rqst.desc_tag != 1239 cpu_to_be32(FCNVME_LSDESC_RQST)) 1240 fcret = VERR_LSDESC_RQST; 1241 else if (assoc_acc->hdr.rqst.desc_len != 1242 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst))) 1243 fcret = VERR_LSDESC_RQST_LEN; 1244 else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION) 1245 fcret = VERR_CR_ASSOC; 1246 else if (assoc_acc->associd.desc_tag != 1247 cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1248 fcret = VERR_ASSOC_ID; 1249 else if (assoc_acc->associd.desc_len != 1250 fcnvme_lsdesc_len( 1251 sizeof(struct fcnvme_lsdesc_assoc_id))) 1252 fcret = VERR_ASSOC_ID_LEN; 1253 else if (assoc_acc->connectid.desc_tag != 1254 cpu_to_be32(FCNVME_LSDESC_CONN_ID)) 1255 fcret = VERR_CONN_ID; 1256 else if (assoc_acc->connectid.desc_len != 1257 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id))) 1258 fcret = VERR_CONN_ID_LEN; 1259 1260 if (fcret) { 1261 ret = -EBADF; 1262 dev_err(ctrl->dev, 1263 "q %d connect failed: %s\n", 1264 queue->qnum, validation_errors[fcret]); 1265 } else { 1266 ctrl->association_id = 1267 be64_to_cpu(assoc_acc->associd.association_id); 1268 queue->connection_id = 1269 be64_to_cpu(assoc_acc->connectid.connection_id); 1270 set_bit(NVME_FC_Q_CONNECTED, &queue->flags); 1271 } 1272 1273 out_free_buffer: 1274 kfree(lsop); 1275 out_no_memory: 1276 if (ret) 1277 dev_err(ctrl->dev, 1278 "queue %d connect admin queue failed (%d).\n", 1279 queue->qnum, ret); 1280 return ret; 1281 } 1282 1283 static int 1284 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, 1285 u16 qsize, u16 ersp_ratio) 1286 { 1287 struct nvmefc_ls_req_op *lsop; 1288 struct nvmefc_ls_req *lsreq; 1289 struct fcnvme_ls_cr_conn_rqst *conn_rqst; 1290 struct fcnvme_ls_cr_conn_acc *conn_acc; 1291 int ret, fcret = 0; 1292 1293 lsop = kzalloc((sizeof(*lsop) + 1294 ctrl->lport->ops->lsrqst_priv_sz + 1295 sizeof(*conn_rqst) + sizeof(*conn_acc)), GFP_KERNEL); 1296 if (!lsop) { 1297 ret = -ENOMEM; 1298 goto out_no_memory; 1299 } 1300 lsreq = &lsop->ls_req; 1301 1302 lsreq->private = (void *)&lsop[1]; 1303 conn_rqst = (struct fcnvme_ls_cr_conn_rqst *) 1304 (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); 1305 conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1]; 1306 1307 conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION; 1308 conn_rqst->desc_list_len = cpu_to_be32( 1309 sizeof(struct fcnvme_lsdesc_assoc_id) + 1310 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)); 1311 1312 conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1313 conn_rqst->associd.desc_len = 1314 fcnvme_lsdesc_len( 1315 sizeof(struct fcnvme_lsdesc_assoc_id)); 1316 conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id); 1317 conn_rqst->connect_cmd.desc_tag = 1318 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD); 1319 conn_rqst->connect_cmd.desc_len = 1320 fcnvme_lsdesc_len( 1321 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)); 1322 conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio); 1323 conn_rqst->connect_cmd.qid = cpu_to_be16(queue->qnum); 1324 conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1); 1325 1326 lsop->queue = queue; 1327 lsreq->rqstaddr = conn_rqst; 1328 lsreq->rqstlen = sizeof(*conn_rqst); 1329 lsreq->rspaddr = conn_acc; 1330 lsreq->rsplen = sizeof(*conn_acc); 1331 lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; 1332 1333 ret = nvme_fc_send_ls_req(ctrl->rport, lsop); 1334 if (ret) 1335 goto out_free_buffer; 1336 1337 /* process connect LS completion */ 1338 1339 /* validate the ACC response */ 1340 if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC) 1341 fcret = VERR_LSACC; 1342 else if (conn_acc->hdr.desc_list_len != 1343 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc))) 1344 fcret = VERR_CR_CONN_ACC_LEN; 1345 else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST)) 1346 fcret = VERR_LSDESC_RQST; 1347 else if (conn_acc->hdr.rqst.desc_len != 1348 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst))) 1349 fcret = VERR_LSDESC_RQST_LEN; 1350 else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION) 1351 fcret = VERR_CR_CONN; 1352 else if (conn_acc->connectid.desc_tag != 1353 cpu_to_be32(FCNVME_LSDESC_CONN_ID)) 1354 fcret = VERR_CONN_ID; 1355 else if (conn_acc->connectid.desc_len != 1356 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id))) 1357 fcret = VERR_CONN_ID_LEN; 1358 1359 if (fcret) { 1360 ret = -EBADF; 1361 dev_err(ctrl->dev, 1362 "q %d connect failed: %s\n", 1363 queue->qnum, validation_errors[fcret]); 1364 } else { 1365 queue->connection_id = 1366 be64_to_cpu(conn_acc->connectid.connection_id); 1367 set_bit(NVME_FC_Q_CONNECTED, &queue->flags); 1368 } 1369 1370 out_free_buffer: 1371 kfree(lsop); 1372 out_no_memory: 1373 if (ret) 1374 dev_err(ctrl->dev, 1375 "queue %d connect command failed (%d).\n", 1376 queue->qnum, ret); 1377 return ret; 1378 } 1379 1380 static void 1381 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) 1382 { 1383 struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq); 1384 1385 __nvme_fc_finish_ls_req(lsop); 1386 1387 /* fc-nvme iniator doesn't care about success or failure of cmd */ 1388 1389 kfree(lsop); 1390 } 1391 1392 /* 1393 * This routine sends a FC-NVME LS to disconnect (aka terminate) 1394 * the FC-NVME Association. Terminating the association also 1395 * terminates the FC-NVME connections (per queue, both admin and io 1396 * queues) that are part of the association. E.g. things are torn 1397 * down, and the related FC-NVME Association ID and Connection IDs 1398 * become invalid. 1399 * 1400 * The behavior of the fc-nvme initiator is such that it's 1401 * understanding of the association and connections will implicitly 1402 * be torn down. The action is implicit as it may be due to a loss of 1403 * connectivity with the fc-nvme target, so you may never get a 1404 * response even if you tried. As such, the action of this routine 1405 * is to asynchronously send the LS, ignore any results of the LS, and 1406 * continue on with terminating the association. If the fc-nvme target 1407 * is present and receives the LS, it too can tear down. 1408 */ 1409 static void 1410 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) 1411 { 1412 struct fcnvme_ls_disconnect_rqst *discon_rqst; 1413 struct fcnvme_ls_disconnect_acc *discon_acc; 1414 struct nvmefc_ls_req_op *lsop; 1415 struct nvmefc_ls_req *lsreq; 1416 int ret; 1417 1418 lsop = kzalloc((sizeof(*lsop) + 1419 ctrl->lport->ops->lsrqst_priv_sz + 1420 sizeof(*discon_rqst) + sizeof(*discon_acc)), 1421 GFP_KERNEL); 1422 if (!lsop) 1423 /* couldn't sent it... too bad */ 1424 return; 1425 1426 lsreq = &lsop->ls_req; 1427 1428 lsreq->private = (void *)&lsop[1]; 1429 discon_rqst = (struct fcnvme_ls_disconnect_rqst *) 1430 (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); 1431 discon_acc = (struct fcnvme_ls_disconnect_acc *)&discon_rqst[1]; 1432 1433 discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT; 1434 discon_rqst->desc_list_len = cpu_to_be32( 1435 sizeof(struct fcnvme_lsdesc_assoc_id) + 1436 sizeof(struct fcnvme_lsdesc_disconn_cmd)); 1437 1438 discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1439 discon_rqst->associd.desc_len = 1440 fcnvme_lsdesc_len( 1441 sizeof(struct fcnvme_lsdesc_assoc_id)); 1442 1443 discon_rqst->associd.association_id = cpu_to_be64(ctrl->association_id); 1444 1445 discon_rqst->discon_cmd.desc_tag = cpu_to_be32( 1446 FCNVME_LSDESC_DISCONN_CMD); 1447 discon_rqst->discon_cmd.desc_len = 1448 fcnvme_lsdesc_len( 1449 sizeof(struct fcnvme_lsdesc_disconn_cmd)); 1450 discon_rqst->discon_cmd.scope = FCNVME_DISCONN_ASSOCIATION; 1451 discon_rqst->discon_cmd.id = cpu_to_be64(ctrl->association_id); 1452 1453 lsreq->rqstaddr = discon_rqst; 1454 lsreq->rqstlen = sizeof(*discon_rqst); 1455 lsreq->rspaddr = discon_acc; 1456 lsreq->rsplen = sizeof(*discon_acc); 1457 lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC; 1458 1459 ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop, 1460 nvme_fc_disconnect_assoc_done); 1461 if (ret) 1462 kfree(lsop); 1463 1464 /* only meaningful part to terminating the association */ 1465 ctrl->association_id = 0; 1466 } 1467 1468 1469 /* *********************** NVME Ctrl Routines **************************** */ 1470 1471 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); 1472 1473 static int 1474 nvme_fc_reinit_request(void *data, struct request *rq) 1475 { 1476 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 1477 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 1478 1479 memset(cmdiu, 0, sizeof(*cmdiu)); 1480 cmdiu->scsi_id = NVME_CMD_SCSI_ID; 1481 cmdiu->fc_id = NVME_CMD_FC_ID; 1482 cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32)); 1483 memset(&op->rsp_iu, 0, sizeof(op->rsp_iu)); 1484 1485 return 0; 1486 } 1487 1488 static void 1489 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl, 1490 struct nvme_fc_fcp_op *op) 1491 { 1492 fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma, 1493 sizeof(op->rsp_iu), DMA_FROM_DEVICE); 1494 fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, 1495 sizeof(op->cmd_iu), DMA_TO_DEVICE); 1496 1497 atomic_set(&op->state, FCPOP_STATE_UNINIT); 1498 } 1499 1500 static void 1501 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq, 1502 unsigned int hctx_idx) 1503 { 1504 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 1505 1506 return __nvme_fc_exit_request(set->driver_data, op); 1507 } 1508 1509 static int 1510 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op) 1511 { 1512 unsigned long flags; 1513 int opstate; 1514 1515 spin_lock_irqsave(&ctrl->lock, flags); 1516 opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED); 1517 if (opstate != FCPOP_STATE_ACTIVE) 1518 atomic_set(&op->state, opstate); 1519 else if (ctrl->flags & FCCTRL_TERMIO) 1520 ctrl->iocnt++; 1521 spin_unlock_irqrestore(&ctrl->lock, flags); 1522 1523 if (opstate != FCPOP_STATE_ACTIVE) 1524 return -ECANCELED; 1525 1526 ctrl->lport->ops->fcp_abort(&ctrl->lport->localport, 1527 &ctrl->rport->remoteport, 1528 op->queue->lldd_handle, 1529 &op->fcp_req); 1530 1531 return 0; 1532 } 1533 1534 static void 1535 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl) 1536 { 1537 struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops; 1538 int i; 1539 1540 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) 1541 __nvme_fc_abort_op(ctrl, aen_op); 1542 } 1543 1544 static inline void 1545 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl, 1546 struct nvme_fc_fcp_op *op, int opstate) 1547 { 1548 unsigned long flags; 1549 1550 if (opstate == FCPOP_STATE_ABORTED) { 1551 spin_lock_irqsave(&ctrl->lock, flags); 1552 if (ctrl->flags & FCCTRL_TERMIO) { 1553 if (!--ctrl->iocnt) 1554 wake_up(&ctrl->ioabort_wait); 1555 } 1556 spin_unlock_irqrestore(&ctrl->lock, flags); 1557 } 1558 } 1559 1560 static void 1561 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) 1562 { 1563 struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req); 1564 struct request *rq = op->rq; 1565 struct nvmefc_fcp_req *freq = &op->fcp_req; 1566 struct nvme_fc_ctrl *ctrl = op->ctrl; 1567 struct nvme_fc_queue *queue = op->queue; 1568 struct nvme_completion *cqe = &op->rsp_iu.cqe; 1569 struct nvme_command *sqe = &op->cmd_iu.sqe; 1570 __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1); 1571 union nvme_result result; 1572 bool terminate_assoc = true; 1573 int opstate; 1574 1575 /* 1576 * WARNING: 1577 * The current linux implementation of a nvme controller 1578 * allocates a single tag set for all io queues and sizes 1579 * the io queues to fully hold all possible tags. Thus, the 1580 * implementation does not reference or care about the sqhd 1581 * value as it never needs to use the sqhd/sqtail pointers 1582 * for submission pacing. 1583 * 1584 * This affects the FC-NVME implementation in two ways: 1585 * 1) As the value doesn't matter, we don't need to waste 1586 * cycles extracting it from ERSPs and stamping it in the 1587 * cases where the transport fabricates CQEs on successful 1588 * completions. 1589 * 2) The FC-NVME implementation requires that delivery of 1590 * ERSP completions are to go back to the nvme layer in order 1591 * relative to the rsn, such that the sqhd value will always 1592 * be "in order" for the nvme layer. As the nvme layer in 1593 * linux doesn't care about sqhd, there's no need to return 1594 * them in order. 1595 * 1596 * Additionally: 1597 * As the core nvme layer in linux currently does not look at 1598 * every field in the cqe - in cases where the FC transport must 1599 * fabricate a CQE, the following fields will not be set as they 1600 * are not referenced: 1601 * cqe.sqid, cqe.sqhd, cqe.command_id 1602 * 1603 * Failure or error of an individual i/o, in a transport 1604 * detected fashion unrelated to the nvme completion status, 1605 * potentially cause the initiator and target sides to get out 1606 * of sync on SQ head/tail (aka outstanding io count allowed). 1607 * Per FC-NVME spec, failure of an individual command requires 1608 * the connection to be terminated, which in turn requires the 1609 * association to be terminated. 1610 */ 1611 1612 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); 1613 1614 fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma, 1615 sizeof(op->rsp_iu), DMA_FROM_DEVICE); 1616 1617 if (opstate == FCPOP_STATE_ABORTED) 1618 status = cpu_to_le16(NVME_SC_ABORT_REQ << 1); 1619 else if (freq->status) 1620 status = cpu_to_le16(NVME_SC_INTERNAL << 1); 1621 1622 /* 1623 * For the linux implementation, if we have an unsuccesful 1624 * status, they blk-mq layer can typically be called with the 1625 * non-zero status and the content of the cqe isn't important. 1626 */ 1627 if (status) 1628 goto done; 1629 1630 /* 1631 * command completed successfully relative to the wire 1632 * protocol. However, validate anything received and 1633 * extract the status and result from the cqe (create it 1634 * where necessary). 1635 */ 1636 1637 switch (freq->rcv_rsplen) { 1638 1639 case 0: 1640 case NVME_FC_SIZEOF_ZEROS_RSP: 1641 /* 1642 * No response payload or 12 bytes of payload (which 1643 * should all be zeros) are considered successful and 1644 * no payload in the CQE by the transport. 1645 */ 1646 if (freq->transferred_length != 1647 be32_to_cpu(op->cmd_iu.data_len)) { 1648 status = cpu_to_le16(NVME_SC_INTERNAL << 1); 1649 goto done; 1650 } 1651 result.u64 = 0; 1652 break; 1653 1654 case sizeof(struct nvme_fc_ersp_iu): 1655 /* 1656 * The ERSP IU contains a full completion with CQE. 1657 * Validate ERSP IU and look at cqe. 1658 */ 1659 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) != 1660 (freq->rcv_rsplen / 4) || 1661 be32_to_cpu(op->rsp_iu.xfrd_len) != 1662 freq->transferred_length || 1663 op->rsp_iu.status_code || 1664 sqe->common.command_id != cqe->command_id)) { 1665 status = cpu_to_le16(NVME_SC_INTERNAL << 1); 1666 goto done; 1667 } 1668 result = cqe->result; 1669 status = cqe->status; 1670 break; 1671 1672 default: 1673 status = cpu_to_le16(NVME_SC_INTERNAL << 1); 1674 goto done; 1675 } 1676 1677 terminate_assoc = false; 1678 1679 done: 1680 if (op->flags & FCOP_FLAGS_AEN) { 1681 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result); 1682 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 1683 atomic_set(&op->state, FCPOP_STATE_IDLE); 1684 op->flags = FCOP_FLAGS_AEN; /* clear other flags */ 1685 nvme_fc_ctrl_put(ctrl); 1686 goto check_error; 1687 } 1688 1689 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 1690 nvme_end_request(rq, status, result); 1691 1692 check_error: 1693 if (terminate_assoc) 1694 nvme_fc_error_recovery(ctrl, "transport detected io error"); 1695 } 1696 1697 static int 1698 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, 1699 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op, 1700 struct request *rq, u32 rqno) 1701 { 1702 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 1703 int ret = 0; 1704 1705 memset(op, 0, sizeof(*op)); 1706 op->fcp_req.cmdaddr = &op->cmd_iu; 1707 op->fcp_req.cmdlen = sizeof(op->cmd_iu); 1708 op->fcp_req.rspaddr = &op->rsp_iu; 1709 op->fcp_req.rsplen = sizeof(op->rsp_iu); 1710 op->fcp_req.done = nvme_fc_fcpio_done; 1711 op->fcp_req.first_sgl = (struct scatterlist *)&op[1]; 1712 op->fcp_req.private = &op->fcp_req.first_sgl[SG_CHUNK_SIZE]; 1713 op->ctrl = ctrl; 1714 op->queue = queue; 1715 op->rq = rq; 1716 op->rqno = rqno; 1717 1718 cmdiu->scsi_id = NVME_CMD_SCSI_ID; 1719 cmdiu->fc_id = NVME_CMD_FC_ID; 1720 cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32)); 1721 1722 op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev, 1723 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE); 1724 if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) { 1725 dev_err(ctrl->dev, 1726 "FCP Op failed - cmdiu dma mapping failed.\n"); 1727 ret = EFAULT; 1728 goto out_on_error; 1729 } 1730 1731 op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev, 1732 &op->rsp_iu, sizeof(op->rsp_iu), 1733 DMA_FROM_DEVICE); 1734 if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) { 1735 dev_err(ctrl->dev, 1736 "FCP Op failed - rspiu dma mapping failed.\n"); 1737 ret = EFAULT; 1738 } 1739 1740 atomic_set(&op->state, FCPOP_STATE_IDLE); 1741 out_on_error: 1742 return ret; 1743 } 1744 1745 static int 1746 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq, 1747 unsigned int hctx_idx, unsigned int numa_node) 1748 { 1749 struct nvme_fc_ctrl *ctrl = set->driver_data; 1750 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 1751 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 1752 struct nvme_fc_queue *queue = &ctrl->queues[queue_idx]; 1753 1754 return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++); 1755 } 1756 1757 static int 1758 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl) 1759 { 1760 struct nvme_fc_fcp_op *aen_op; 1761 struct nvme_fc_cmd_iu *cmdiu; 1762 struct nvme_command *sqe; 1763 void *private; 1764 int i, ret; 1765 1766 aen_op = ctrl->aen_ops; 1767 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { 1768 private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz, 1769 GFP_KERNEL); 1770 if (!private) 1771 return -ENOMEM; 1772 1773 cmdiu = &aen_op->cmd_iu; 1774 sqe = &cmdiu->sqe; 1775 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0], 1776 aen_op, (struct request *)NULL, 1777 (NVME_AQ_BLK_MQ_DEPTH + i)); 1778 if (ret) { 1779 kfree(private); 1780 return ret; 1781 } 1782 1783 aen_op->flags = FCOP_FLAGS_AEN; 1784 aen_op->fcp_req.first_sgl = NULL; /* no sg list */ 1785 aen_op->fcp_req.private = private; 1786 1787 memset(sqe, 0, sizeof(*sqe)); 1788 sqe->common.opcode = nvme_admin_async_event; 1789 /* Note: core layer may overwrite the sqe.command_id value */ 1790 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i; 1791 } 1792 return 0; 1793 } 1794 1795 static void 1796 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl) 1797 { 1798 struct nvme_fc_fcp_op *aen_op; 1799 int i; 1800 1801 aen_op = ctrl->aen_ops; 1802 for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { 1803 if (!aen_op->fcp_req.private) 1804 continue; 1805 1806 __nvme_fc_exit_request(ctrl, aen_op); 1807 1808 kfree(aen_op->fcp_req.private); 1809 aen_op->fcp_req.private = NULL; 1810 } 1811 } 1812 1813 static inline void 1814 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl, 1815 unsigned int qidx) 1816 { 1817 struct nvme_fc_queue *queue = &ctrl->queues[qidx]; 1818 1819 hctx->driver_data = queue; 1820 queue->hctx = hctx; 1821 } 1822 1823 static int 1824 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 1825 unsigned int hctx_idx) 1826 { 1827 struct nvme_fc_ctrl *ctrl = data; 1828 1829 __nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1); 1830 1831 return 0; 1832 } 1833 1834 static int 1835 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 1836 unsigned int hctx_idx) 1837 { 1838 struct nvme_fc_ctrl *ctrl = data; 1839 1840 __nvme_fc_init_hctx(hctx, ctrl, hctx_idx); 1841 1842 return 0; 1843 } 1844 1845 static void 1846 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx) 1847 { 1848 struct nvme_fc_queue *queue; 1849 1850 queue = &ctrl->queues[idx]; 1851 memset(queue, 0, sizeof(*queue)); 1852 queue->ctrl = ctrl; 1853 queue->qnum = idx; 1854 atomic_set(&queue->csn, 1); 1855 queue->dev = ctrl->dev; 1856 1857 if (idx > 0) 1858 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16; 1859 else 1860 queue->cmnd_capsule_len = sizeof(struct nvme_command); 1861 1862 /* 1863 * Considered whether we should allocate buffers for all SQEs 1864 * and CQEs and dma map them - mapping their respective entries 1865 * into the request structures (kernel vm addr and dma address) 1866 * thus the driver could use the buffers/mappings directly. 1867 * It only makes sense if the LLDD would use them for its 1868 * messaging api. It's very unlikely most adapter api's would use 1869 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload 1870 * structures were used instead. 1871 */ 1872 } 1873 1874 /* 1875 * This routine terminates a queue at the transport level. 1876 * The transport has already ensured that all outstanding ios on 1877 * the queue have been terminated. 1878 * The transport will send a Disconnect LS request to terminate 1879 * the queue's connection. Termination of the admin queue will also 1880 * terminate the association at the target. 1881 */ 1882 static void 1883 nvme_fc_free_queue(struct nvme_fc_queue *queue) 1884 { 1885 if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags)) 1886 return; 1887 1888 clear_bit(NVME_FC_Q_LIVE, &queue->flags); 1889 /* 1890 * Current implementation never disconnects a single queue. 1891 * It always terminates a whole association. So there is never 1892 * a disconnect(queue) LS sent to the target. 1893 */ 1894 1895 queue->connection_id = 0; 1896 } 1897 1898 static void 1899 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl, 1900 struct nvme_fc_queue *queue, unsigned int qidx) 1901 { 1902 if (ctrl->lport->ops->delete_queue) 1903 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx, 1904 queue->lldd_handle); 1905 queue->lldd_handle = NULL; 1906 } 1907 1908 static void 1909 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl) 1910 { 1911 int i; 1912 1913 for (i = 1; i < ctrl->ctrl.queue_count; i++) 1914 nvme_fc_free_queue(&ctrl->queues[i]); 1915 } 1916 1917 static int 1918 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl, 1919 struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize) 1920 { 1921 int ret = 0; 1922 1923 queue->lldd_handle = NULL; 1924 if (ctrl->lport->ops->create_queue) 1925 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport, 1926 qidx, qsize, &queue->lldd_handle); 1927 1928 return ret; 1929 } 1930 1931 static void 1932 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl) 1933 { 1934 struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1]; 1935 int i; 1936 1937 for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--) 1938 __nvme_fc_delete_hw_queue(ctrl, queue, i); 1939 } 1940 1941 static int 1942 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 1943 { 1944 struct nvme_fc_queue *queue = &ctrl->queues[1]; 1945 int i, ret; 1946 1947 for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) { 1948 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize); 1949 if (ret) 1950 goto delete_queues; 1951 } 1952 1953 return 0; 1954 1955 delete_queues: 1956 for (; i >= 0; i--) 1957 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); 1958 return ret; 1959 } 1960 1961 static int 1962 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 1963 { 1964 int i, ret = 0; 1965 1966 for (i = 1; i < ctrl->ctrl.queue_count; i++) { 1967 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize, 1968 (qsize / 5)); 1969 if (ret) 1970 break; 1971 ret = nvmf_connect_io_queue(&ctrl->ctrl, i); 1972 if (ret) 1973 break; 1974 1975 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags); 1976 } 1977 1978 return ret; 1979 } 1980 1981 static void 1982 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl) 1983 { 1984 int i; 1985 1986 for (i = 1; i < ctrl->ctrl.queue_count; i++) 1987 nvme_fc_init_queue(ctrl, i); 1988 } 1989 1990 static void 1991 nvme_fc_ctrl_free(struct kref *ref) 1992 { 1993 struct nvme_fc_ctrl *ctrl = 1994 container_of(ref, struct nvme_fc_ctrl, ref); 1995 unsigned long flags; 1996 1997 if (ctrl->ctrl.tagset) { 1998 blk_cleanup_queue(ctrl->ctrl.connect_q); 1999 blk_mq_free_tag_set(&ctrl->tag_set); 2000 } 2001 2002 /* remove from rport list */ 2003 spin_lock_irqsave(&ctrl->rport->lock, flags); 2004 list_del(&ctrl->ctrl_list); 2005 spin_unlock_irqrestore(&ctrl->rport->lock, flags); 2006 2007 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); 2008 blk_cleanup_queue(ctrl->ctrl.admin_q); 2009 blk_mq_free_tag_set(&ctrl->admin_tag_set); 2010 2011 kfree(ctrl->queues); 2012 2013 put_device(ctrl->dev); 2014 nvme_fc_rport_put(ctrl->rport); 2015 2016 ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum); 2017 if (ctrl->ctrl.opts) 2018 nvmf_free_options(ctrl->ctrl.opts); 2019 kfree(ctrl); 2020 } 2021 2022 static void 2023 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl) 2024 { 2025 kref_put(&ctrl->ref, nvme_fc_ctrl_free); 2026 } 2027 2028 static int 2029 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl) 2030 { 2031 return kref_get_unless_zero(&ctrl->ref); 2032 } 2033 2034 /* 2035 * All accesses from nvme core layer done - can now free the 2036 * controller. Called after last nvme_put_ctrl() call 2037 */ 2038 static void 2039 nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl) 2040 { 2041 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 2042 2043 WARN_ON(nctrl != &ctrl->ctrl); 2044 2045 nvme_fc_ctrl_put(ctrl); 2046 } 2047 2048 static void 2049 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg) 2050 { 2051 /* only proceed if in LIVE state - e.g. on first error */ 2052 if (ctrl->ctrl.state != NVME_CTRL_LIVE) 2053 return; 2054 2055 dev_warn(ctrl->ctrl.device, 2056 "NVME-FC{%d}: transport association error detected: %s\n", 2057 ctrl->cnum, errmsg); 2058 dev_warn(ctrl->ctrl.device, 2059 "NVME-FC{%d}: resetting controller\n", ctrl->cnum); 2060 2061 nvme_reset_ctrl(&ctrl->ctrl); 2062 } 2063 2064 static enum blk_eh_timer_return 2065 nvme_fc_timeout(struct request *rq, bool reserved) 2066 { 2067 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2068 struct nvme_fc_ctrl *ctrl = op->ctrl; 2069 2070 /* 2071 * we can't individually ABTS an io without affecting the queue, 2072 * thus killing the queue, and thus the association. 2073 * So resolve by performing a controller reset, which will stop 2074 * the host/io stack, terminate the association on the link, 2075 * and recreate an association on the link. 2076 */ 2077 nvme_fc_error_recovery(ctrl, "io timeout error"); 2078 2079 /* 2080 * the io abort has been initiated. Have the reset timer 2081 * restarted and the abort completion will complete the io 2082 * shortly. Avoids a synchronous wait while the abort finishes. 2083 */ 2084 return BLK_EH_RESET_TIMER; 2085 } 2086 2087 static int 2088 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq, 2089 struct nvme_fc_fcp_op *op) 2090 { 2091 struct nvmefc_fcp_req *freq = &op->fcp_req; 2092 enum dma_data_direction dir; 2093 int ret; 2094 2095 freq->sg_cnt = 0; 2096 2097 if (!blk_rq_payload_bytes(rq)) 2098 return 0; 2099 2100 freq->sg_table.sgl = freq->first_sgl; 2101 ret = sg_alloc_table_chained(&freq->sg_table, 2102 blk_rq_nr_phys_segments(rq), freq->sg_table.sgl); 2103 if (ret) 2104 return -ENOMEM; 2105 2106 op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl); 2107 WARN_ON(op->nents > blk_rq_nr_phys_segments(rq)); 2108 dir = (rq_data_dir(rq) == WRITE) ? DMA_TO_DEVICE : DMA_FROM_DEVICE; 2109 freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl, 2110 op->nents, dir); 2111 if (unlikely(freq->sg_cnt <= 0)) { 2112 sg_free_table_chained(&freq->sg_table, true); 2113 freq->sg_cnt = 0; 2114 return -EFAULT; 2115 } 2116 2117 /* 2118 * TODO: blk_integrity_rq(rq) for DIF 2119 */ 2120 return 0; 2121 } 2122 2123 static void 2124 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq, 2125 struct nvme_fc_fcp_op *op) 2126 { 2127 struct nvmefc_fcp_req *freq = &op->fcp_req; 2128 2129 if (!freq->sg_cnt) 2130 return; 2131 2132 fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents, 2133 ((rq_data_dir(rq) == WRITE) ? 2134 DMA_TO_DEVICE : DMA_FROM_DEVICE)); 2135 2136 nvme_cleanup_cmd(rq); 2137 2138 sg_free_table_chained(&freq->sg_table, true); 2139 2140 freq->sg_cnt = 0; 2141 } 2142 2143 /* 2144 * In FC, the queue is a logical thing. At transport connect, the target 2145 * creates its "queue" and returns a handle that is to be given to the 2146 * target whenever it posts something to the corresponding SQ. When an 2147 * SQE is sent on a SQ, FC effectively considers the SQE, or rather the 2148 * command contained within the SQE, an io, and assigns a FC exchange 2149 * to it. The SQE and the associated SQ handle are sent in the initial 2150 * CMD IU sents on the exchange. All transfers relative to the io occur 2151 * as part of the exchange. The CQE is the last thing for the io, 2152 * which is transferred (explicitly or implicitly) with the RSP IU 2153 * sent on the exchange. After the CQE is received, the FC exchange is 2154 * terminaed and the Exchange may be used on a different io. 2155 * 2156 * The transport to LLDD api has the transport making a request for a 2157 * new fcp io request to the LLDD. The LLDD then allocates a FC exchange 2158 * resource and transfers the command. The LLDD will then process all 2159 * steps to complete the io. Upon completion, the transport done routine 2160 * is called. 2161 * 2162 * So - while the operation is outstanding to the LLDD, there is a link 2163 * level FC exchange resource that is also outstanding. This must be 2164 * considered in all cleanup operations. 2165 */ 2166 static blk_status_t 2167 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, 2168 struct nvme_fc_fcp_op *op, u32 data_len, 2169 enum nvmefc_fcp_datadir io_dir) 2170 { 2171 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 2172 struct nvme_command *sqe = &cmdiu->sqe; 2173 u32 csn; 2174 int ret, opstate; 2175 2176 /* 2177 * before attempting to send the io, check to see if we believe 2178 * the target device is present 2179 */ 2180 if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 2181 return BLK_STS_RESOURCE; 2182 2183 if (!nvme_fc_ctrl_get(ctrl)) 2184 return BLK_STS_IOERR; 2185 2186 /* format the FC-NVME CMD IU and fcp_req */ 2187 cmdiu->connection_id = cpu_to_be64(queue->connection_id); 2188 csn = atomic_inc_return(&queue->csn); 2189 cmdiu->csn = cpu_to_be32(csn); 2190 cmdiu->data_len = cpu_to_be32(data_len); 2191 switch (io_dir) { 2192 case NVMEFC_FCP_WRITE: 2193 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE; 2194 break; 2195 case NVMEFC_FCP_READ: 2196 cmdiu->flags = FCNVME_CMD_FLAGS_READ; 2197 break; 2198 case NVMEFC_FCP_NODATA: 2199 cmdiu->flags = 0; 2200 break; 2201 } 2202 op->fcp_req.payload_length = data_len; 2203 op->fcp_req.io_dir = io_dir; 2204 op->fcp_req.transferred_length = 0; 2205 op->fcp_req.rcv_rsplen = 0; 2206 op->fcp_req.status = NVME_SC_SUCCESS; 2207 op->fcp_req.sqid = cpu_to_le16(queue->qnum); 2208 2209 /* 2210 * validate per fabric rules, set fields mandated by fabric spec 2211 * as well as those by FC-NVME spec. 2212 */ 2213 WARN_ON_ONCE(sqe->common.metadata); 2214 sqe->common.flags |= NVME_CMD_SGL_METABUF; 2215 2216 /* 2217 * format SQE DPTR field per FC-NVME rules: 2218 * type=0x5 Transport SGL Data Block Descriptor 2219 * subtype=0xA Transport-specific value 2220 * address=0 2221 * length=length of the data series 2222 */ 2223 sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2224 NVME_SGL_FMT_TRANSPORT_A; 2225 sqe->rw.dptr.sgl.length = cpu_to_le32(data_len); 2226 sqe->rw.dptr.sgl.addr = 0; 2227 2228 if (!(op->flags & FCOP_FLAGS_AEN)) { 2229 ret = nvme_fc_map_data(ctrl, op->rq, op); 2230 if (ret < 0) { 2231 nvme_cleanup_cmd(op->rq); 2232 nvme_fc_ctrl_put(ctrl); 2233 if (ret == -ENOMEM || ret == -EAGAIN) 2234 return BLK_STS_RESOURCE; 2235 return BLK_STS_IOERR; 2236 } 2237 } 2238 2239 fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma, 2240 sizeof(op->cmd_iu), DMA_TO_DEVICE); 2241 2242 atomic_set(&op->state, FCPOP_STATE_ACTIVE); 2243 2244 if (!(op->flags & FCOP_FLAGS_AEN)) 2245 blk_mq_start_request(op->rq); 2246 2247 ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport, 2248 &ctrl->rport->remoteport, 2249 queue->lldd_handle, &op->fcp_req); 2250 2251 if (ret) { 2252 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); 2253 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); 2254 2255 if (!(op->flags & FCOP_FLAGS_AEN)) 2256 nvme_fc_unmap_data(ctrl, op->rq, op); 2257 2258 nvme_fc_ctrl_put(ctrl); 2259 2260 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE && 2261 ret != -EBUSY) 2262 return BLK_STS_IOERR; 2263 2264 return BLK_STS_RESOURCE; 2265 } 2266 2267 return BLK_STS_OK; 2268 } 2269 2270 static blk_status_t 2271 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx, 2272 const struct blk_mq_queue_data *bd) 2273 { 2274 struct nvme_ns *ns = hctx->queue->queuedata; 2275 struct nvme_fc_queue *queue = hctx->driver_data; 2276 struct nvme_fc_ctrl *ctrl = queue->ctrl; 2277 struct request *rq = bd->rq; 2278 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2279 struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; 2280 struct nvme_command *sqe = &cmdiu->sqe; 2281 enum nvmefc_fcp_datadir io_dir; 2282 u32 data_len; 2283 blk_status_t ret; 2284 2285 ret = nvmf_check_if_ready(&queue->ctrl->ctrl, rq, 2286 test_bit(NVME_FC_Q_LIVE, &queue->flags), 2287 ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE); 2288 if (unlikely(ret)) 2289 return ret; 2290 2291 ret = nvme_setup_cmd(ns, rq, sqe); 2292 if (ret) 2293 return ret; 2294 2295 data_len = blk_rq_payload_bytes(rq); 2296 if (data_len) 2297 io_dir = ((rq_data_dir(rq) == WRITE) ? 2298 NVMEFC_FCP_WRITE : NVMEFC_FCP_READ); 2299 else 2300 io_dir = NVMEFC_FCP_NODATA; 2301 2302 return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir); 2303 } 2304 2305 static struct blk_mq_tags * 2306 nvme_fc_tagset(struct nvme_fc_queue *queue) 2307 { 2308 if (queue->qnum == 0) 2309 return queue->ctrl->admin_tag_set.tags[queue->qnum]; 2310 2311 return queue->ctrl->tag_set.tags[queue->qnum - 1]; 2312 } 2313 2314 static int 2315 nvme_fc_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag) 2316 2317 { 2318 struct nvme_fc_queue *queue = hctx->driver_data; 2319 struct nvme_fc_ctrl *ctrl = queue->ctrl; 2320 struct request *req; 2321 struct nvme_fc_fcp_op *op; 2322 2323 req = blk_mq_tag_to_rq(nvme_fc_tagset(queue), tag); 2324 if (!req) 2325 return 0; 2326 2327 op = blk_mq_rq_to_pdu(req); 2328 2329 if ((atomic_read(&op->state) == FCPOP_STATE_ACTIVE) && 2330 (ctrl->lport->ops->poll_queue)) 2331 ctrl->lport->ops->poll_queue(&ctrl->lport->localport, 2332 queue->lldd_handle); 2333 2334 return ((atomic_read(&op->state) != FCPOP_STATE_ACTIVE)); 2335 } 2336 2337 static void 2338 nvme_fc_submit_async_event(struct nvme_ctrl *arg) 2339 { 2340 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg); 2341 struct nvme_fc_fcp_op *aen_op; 2342 unsigned long flags; 2343 bool terminating = false; 2344 blk_status_t ret; 2345 2346 spin_lock_irqsave(&ctrl->lock, flags); 2347 if (ctrl->flags & FCCTRL_TERMIO) 2348 terminating = true; 2349 spin_unlock_irqrestore(&ctrl->lock, flags); 2350 2351 if (terminating) 2352 return; 2353 2354 aen_op = &ctrl->aen_ops[0]; 2355 2356 ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0, 2357 NVMEFC_FCP_NODATA); 2358 if (ret) 2359 dev_err(ctrl->ctrl.device, 2360 "failed async event work\n"); 2361 } 2362 2363 static void 2364 nvme_fc_complete_rq(struct request *rq) 2365 { 2366 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); 2367 struct nvme_fc_ctrl *ctrl = op->ctrl; 2368 2369 atomic_set(&op->state, FCPOP_STATE_IDLE); 2370 2371 nvme_fc_unmap_data(ctrl, rq, op); 2372 nvme_complete_rq(rq); 2373 nvme_fc_ctrl_put(ctrl); 2374 } 2375 2376 /* 2377 * This routine is used by the transport when it needs to find active 2378 * io on a queue that is to be terminated. The transport uses 2379 * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke 2380 * this routine to kill them on a 1 by 1 basis. 2381 * 2382 * As FC allocates FC exchange for each io, the transport must contact 2383 * the LLDD to terminate the exchange, thus releasing the FC exchange. 2384 * After terminating the exchange the LLDD will call the transport's 2385 * normal io done path for the request, but it will have an aborted 2386 * status. The done path will return the io request back to the block 2387 * layer with an error status. 2388 */ 2389 static void 2390 nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved) 2391 { 2392 struct nvme_ctrl *nctrl = data; 2393 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 2394 struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req); 2395 2396 __nvme_fc_abort_op(ctrl, op); 2397 } 2398 2399 2400 static const struct blk_mq_ops nvme_fc_mq_ops = { 2401 .queue_rq = nvme_fc_queue_rq, 2402 .complete = nvme_fc_complete_rq, 2403 .init_request = nvme_fc_init_request, 2404 .exit_request = nvme_fc_exit_request, 2405 .init_hctx = nvme_fc_init_hctx, 2406 .poll = nvme_fc_poll, 2407 .timeout = nvme_fc_timeout, 2408 }; 2409 2410 static int 2411 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) 2412 { 2413 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2414 unsigned int nr_io_queues; 2415 int ret; 2416 2417 nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()), 2418 ctrl->lport->ops->max_hw_queues); 2419 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 2420 if (ret) { 2421 dev_info(ctrl->ctrl.device, 2422 "set_queue_count failed: %d\n", ret); 2423 return ret; 2424 } 2425 2426 ctrl->ctrl.queue_count = nr_io_queues + 1; 2427 if (!nr_io_queues) 2428 return 0; 2429 2430 nvme_fc_init_io_queues(ctrl); 2431 2432 memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set)); 2433 ctrl->tag_set.ops = &nvme_fc_mq_ops; 2434 ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size; 2435 ctrl->tag_set.reserved_tags = 1; /* fabric connect */ 2436 ctrl->tag_set.numa_node = NUMA_NO_NODE; 2437 ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 2438 ctrl->tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) + 2439 (SG_CHUNK_SIZE * 2440 sizeof(struct scatterlist)) + 2441 ctrl->lport->ops->fcprqst_priv_sz; 2442 ctrl->tag_set.driver_data = ctrl; 2443 ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1; 2444 ctrl->tag_set.timeout = NVME_IO_TIMEOUT; 2445 2446 ret = blk_mq_alloc_tag_set(&ctrl->tag_set); 2447 if (ret) 2448 return ret; 2449 2450 ctrl->ctrl.tagset = &ctrl->tag_set; 2451 2452 ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set); 2453 if (IS_ERR(ctrl->ctrl.connect_q)) { 2454 ret = PTR_ERR(ctrl->ctrl.connect_q); 2455 goto out_free_tag_set; 2456 } 2457 2458 ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2459 if (ret) 2460 goto out_cleanup_blk_queue; 2461 2462 ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2463 if (ret) 2464 goto out_delete_hw_queues; 2465 2466 return 0; 2467 2468 out_delete_hw_queues: 2469 nvme_fc_delete_hw_io_queues(ctrl); 2470 out_cleanup_blk_queue: 2471 blk_cleanup_queue(ctrl->ctrl.connect_q); 2472 out_free_tag_set: 2473 blk_mq_free_tag_set(&ctrl->tag_set); 2474 nvme_fc_free_io_queues(ctrl); 2475 2476 /* force put free routine to ignore io queues */ 2477 ctrl->ctrl.tagset = NULL; 2478 2479 return ret; 2480 } 2481 2482 static int 2483 nvme_fc_reinit_io_queues(struct nvme_fc_ctrl *ctrl) 2484 { 2485 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2486 unsigned int nr_io_queues; 2487 int ret; 2488 2489 nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()), 2490 ctrl->lport->ops->max_hw_queues); 2491 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 2492 if (ret) { 2493 dev_info(ctrl->ctrl.device, 2494 "set_queue_count failed: %d\n", ret); 2495 return ret; 2496 } 2497 2498 ctrl->ctrl.queue_count = nr_io_queues + 1; 2499 /* check for io queues existing */ 2500 if (ctrl->ctrl.queue_count == 1) 2501 return 0; 2502 2503 nvme_fc_init_io_queues(ctrl); 2504 2505 ret = nvme_reinit_tagset(&ctrl->ctrl, ctrl->ctrl.tagset); 2506 if (ret) 2507 goto out_free_io_queues; 2508 2509 ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2510 if (ret) 2511 goto out_free_io_queues; 2512 2513 ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1); 2514 if (ret) 2515 goto out_delete_hw_queues; 2516 2517 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues); 2518 2519 return 0; 2520 2521 out_delete_hw_queues: 2522 nvme_fc_delete_hw_io_queues(ctrl); 2523 out_free_io_queues: 2524 nvme_fc_free_io_queues(ctrl); 2525 return ret; 2526 } 2527 2528 static void 2529 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport) 2530 { 2531 struct nvme_fc_lport *lport = rport->lport; 2532 2533 atomic_inc(&lport->act_rport_cnt); 2534 } 2535 2536 static void 2537 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport) 2538 { 2539 struct nvme_fc_lport *lport = rport->lport; 2540 u32 cnt; 2541 2542 cnt = atomic_dec_return(&lport->act_rport_cnt); 2543 if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED) 2544 lport->ops->localport_delete(&lport->localport); 2545 } 2546 2547 static int 2548 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl) 2549 { 2550 struct nvme_fc_rport *rport = ctrl->rport; 2551 u32 cnt; 2552 2553 if (ctrl->assoc_active) 2554 return 1; 2555 2556 ctrl->assoc_active = true; 2557 cnt = atomic_inc_return(&rport->act_ctrl_cnt); 2558 if (cnt == 1) 2559 nvme_fc_rport_active_on_lport(rport); 2560 2561 return 0; 2562 } 2563 2564 static int 2565 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl) 2566 { 2567 struct nvme_fc_rport *rport = ctrl->rport; 2568 struct nvme_fc_lport *lport = rport->lport; 2569 u32 cnt; 2570 2571 /* ctrl->assoc_active=false will be set independently */ 2572 2573 cnt = atomic_dec_return(&rport->act_ctrl_cnt); 2574 if (cnt == 0) { 2575 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED) 2576 lport->ops->remoteport_delete(&rport->remoteport); 2577 nvme_fc_rport_inactive_on_lport(rport); 2578 } 2579 2580 return 0; 2581 } 2582 2583 /* 2584 * This routine restarts the controller on the host side, and 2585 * on the link side, recreates the controller association. 2586 */ 2587 static int 2588 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 2589 { 2590 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2591 int ret; 2592 bool changed; 2593 2594 ++ctrl->ctrl.nr_reconnects; 2595 2596 if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) 2597 return -ENODEV; 2598 2599 if (nvme_fc_ctlr_active_on_rport(ctrl)) 2600 return -ENOTUNIQ; 2601 2602 /* 2603 * Create the admin queue 2604 */ 2605 2606 nvme_fc_init_queue(ctrl, 0); 2607 2608 ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0, 2609 NVME_AQ_DEPTH); 2610 if (ret) 2611 goto out_free_queue; 2612 2613 ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0], 2614 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4)); 2615 if (ret) 2616 goto out_delete_hw_queue; 2617 2618 if (ctrl->ctrl.state != NVME_CTRL_NEW) 2619 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); 2620 2621 ret = nvmf_connect_admin_queue(&ctrl->ctrl); 2622 if (ret) 2623 goto out_disconnect_admin_queue; 2624 2625 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags); 2626 2627 /* 2628 * Check controller capabilities 2629 * 2630 * todo:- add code to check if ctrl attributes changed from 2631 * prior connection values 2632 */ 2633 2634 ret = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap); 2635 if (ret) { 2636 dev_err(ctrl->ctrl.device, 2637 "prop_get NVME_REG_CAP failed\n"); 2638 goto out_disconnect_admin_queue; 2639 } 2640 2641 ctrl->ctrl.sqsize = 2642 min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize); 2643 2644 ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap); 2645 if (ret) 2646 goto out_disconnect_admin_queue; 2647 2648 ctrl->ctrl.max_hw_sectors = 2649 (ctrl->lport->ops->max_sgl_segments - 1) << (PAGE_SHIFT - 9); 2650 2651 ret = nvme_init_identify(&ctrl->ctrl); 2652 if (ret) 2653 goto out_disconnect_admin_queue; 2654 2655 /* sanity checks */ 2656 2657 /* FC-NVME does not have other data in the capsule */ 2658 if (ctrl->ctrl.icdoff) { 2659 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n", 2660 ctrl->ctrl.icdoff); 2661 goto out_disconnect_admin_queue; 2662 } 2663 2664 /* FC-NVME supports normal SGL Data Block Descriptors */ 2665 2666 if (opts->queue_size > ctrl->ctrl.maxcmd) { 2667 /* warn if maxcmd is lower than queue_size */ 2668 dev_warn(ctrl->ctrl.device, 2669 "queue_size %zu > ctrl maxcmd %u, reducing " 2670 "to queue_size\n", 2671 opts->queue_size, ctrl->ctrl.maxcmd); 2672 opts->queue_size = ctrl->ctrl.maxcmd; 2673 } 2674 2675 if (opts->queue_size > ctrl->ctrl.sqsize + 1) { 2676 /* warn if sqsize is lower than queue_size */ 2677 dev_warn(ctrl->ctrl.device, 2678 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2679 opts->queue_size, ctrl->ctrl.sqsize + 1); 2680 opts->queue_size = ctrl->ctrl.sqsize + 1; 2681 } 2682 2683 ret = nvme_fc_init_aen_ops(ctrl); 2684 if (ret) 2685 goto out_term_aen_ops; 2686 2687 /* 2688 * Create the io queues 2689 */ 2690 2691 if (ctrl->ctrl.queue_count > 1) { 2692 if (ctrl->ctrl.state == NVME_CTRL_NEW) 2693 ret = nvme_fc_create_io_queues(ctrl); 2694 else 2695 ret = nvme_fc_reinit_io_queues(ctrl); 2696 if (ret) 2697 goto out_term_aen_ops; 2698 } 2699 2700 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); 2701 2702 ctrl->ctrl.nr_reconnects = 0; 2703 2704 if (changed) 2705 nvme_start_ctrl(&ctrl->ctrl); 2706 2707 return 0; /* Success */ 2708 2709 out_term_aen_ops: 2710 nvme_fc_term_aen_ops(ctrl); 2711 out_disconnect_admin_queue: 2712 /* send a Disconnect(association) LS to fc-nvme target */ 2713 nvme_fc_xmt_disconnect_assoc(ctrl); 2714 out_delete_hw_queue: 2715 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); 2716 out_free_queue: 2717 nvme_fc_free_queue(&ctrl->queues[0]); 2718 ctrl->assoc_active = false; 2719 nvme_fc_ctlr_inactive_on_rport(ctrl); 2720 2721 return ret; 2722 } 2723 2724 /* 2725 * This routine stops operation of the controller on the host side. 2726 * On the host os stack side: Admin and IO queues are stopped, 2727 * outstanding ios on them terminated via FC ABTS. 2728 * On the link side: the association is terminated. 2729 */ 2730 static void 2731 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) 2732 { 2733 unsigned long flags; 2734 2735 if (!ctrl->assoc_active) 2736 return; 2737 ctrl->assoc_active = false; 2738 2739 spin_lock_irqsave(&ctrl->lock, flags); 2740 ctrl->flags |= FCCTRL_TERMIO; 2741 ctrl->iocnt = 0; 2742 spin_unlock_irqrestore(&ctrl->lock, flags); 2743 2744 /* 2745 * If io queues are present, stop them and terminate all outstanding 2746 * ios on them. As FC allocates FC exchange for each io, the 2747 * transport must contact the LLDD to terminate the exchange, 2748 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr() 2749 * to tell us what io's are busy and invoke a transport routine 2750 * to kill them with the LLDD. After terminating the exchange 2751 * the LLDD will call the transport's normal io done path, but it 2752 * will have an aborted status. The done path will return the 2753 * io requests back to the block layer as part of normal completions 2754 * (but with error status). 2755 */ 2756 if (ctrl->ctrl.queue_count > 1) { 2757 nvme_stop_queues(&ctrl->ctrl); 2758 blk_mq_tagset_busy_iter(&ctrl->tag_set, 2759 nvme_fc_terminate_exchange, &ctrl->ctrl); 2760 } 2761 2762 /* 2763 * Other transports, which don't have link-level contexts bound 2764 * to sqe's, would try to gracefully shutdown the controller by 2765 * writing the registers for shutdown and polling (call 2766 * nvme_shutdown_ctrl()). Given a bunch of i/o was potentially 2767 * just aborted and we will wait on those contexts, and given 2768 * there was no indication of how live the controlelr is on the 2769 * link, don't send more io to create more contexts for the 2770 * shutdown. Let the controller fail via keepalive failure if 2771 * its still present. 2772 */ 2773 2774 /* 2775 * clean up the admin queue. Same thing as above. 2776 * use blk_mq_tagset_busy_itr() and the transport routine to 2777 * terminate the exchanges. 2778 */ 2779 if (ctrl->ctrl.state != NVME_CTRL_NEW) 2780 blk_mq_quiesce_queue(ctrl->ctrl.admin_q); 2781 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, 2782 nvme_fc_terminate_exchange, &ctrl->ctrl); 2783 2784 /* kill the aens as they are a separate path */ 2785 nvme_fc_abort_aen_ops(ctrl); 2786 2787 /* wait for all io that had to be aborted */ 2788 spin_lock_irq(&ctrl->lock); 2789 wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock); 2790 ctrl->flags &= ~FCCTRL_TERMIO; 2791 spin_unlock_irq(&ctrl->lock); 2792 2793 nvme_fc_term_aen_ops(ctrl); 2794 2795 /* 2796 * send a Disconnect(association) LS to fc-nvme target 2797 * Note: could have been sent at top of process, but 2798 * cleaner on link traffic if after the aborts complete. 2799 * Note: if association doesn't exist, association_id will be 0 2800 */ 2801 if (ctrl->association_id) 2802 nvme_fc_xmt_disconnect_assoc(ctrl); 2803 2804 if (ctrl->ctrl.tagset) { 2805 nvme_fc_delete_hw_io_queues(ctrl); 2806 nvme_fc_free_io_queues(ctrl); 2807 } 2808 2809 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); 2810 nvme_fc_free_queue(&ctrl->queues[0]); 2811 2812 /* re-enable the admin_q so anything new can fast fail */ 2813 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); 2814 2815 nvme_fc_ctlr_inactive_on_rport(ctrl); 2816 } 2817 2818 static void 2819 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl) 2820 { 2821 struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl); 2822 2823 cancel_delayed_work_sync(&ctrl->connect_work); 2824 /* 2825 * kill the association on the link side. this will block 2826 * waiting for io to terminate 2827 */ 2828 nvme_fc_delete_association(ctrl); 2829 2830 /* resume the io queues so that things will fast fail */ 2831 nvme_start_queues(nctrl); 2832 } 2833 2834 static void 2835 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status) 2836 { 2837 struct nvme_fc_rport *rport = ctrl->rport; 2838 struct nvme_fc_remote_port *portptr = &rport->remoteport; 2839 unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ; 2840 bool recon = true; 2841 2842 if (ctrl->ctrl.state != NVME_CTRL_CONNECTING) 2843 return; 2844 2845 if (portptr->port_state == FC_OBJSTATE_ONLINE) 2846 dev_info(ctrl->ctrl.device, 2847 "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n", 2848 ctrl->cnum, status); 2849 else if (time_after_eq(jiffies, rport->dev_loss_end)) 2850 recon = false; 2851 2852 if (recon && nvmf_should_reconnect(&ctrl->ctrl)) { 2853 if (portptr->port_state == FC_OBJSTATE_ONLINE) 2854 dev_info(ctrl->ctrl.device, 2855 "NVME-FC{%d}: Reconnect attempt in %ld " 2856 "seconds\n", 2857 ctrl->cnum, recon_delay / HZ); 2858 else if (time_after(jiffies + recon_delay, rport->dev_loss_end)) 2859 recon_delay = rport->dev_loss_end - jiffies; 2860 2861 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay); 2862 } else { 2863 if (portptr->port_state == FC_OBJSTATE_ONLINE) 2864 dev_warn(ctrl->ctrl.device, 2865 "NVME-FC{%d}: Max reconnect attempts (%d) " 2866 "reached.\n", 2867 ctrl->cnum, ctrl->ctrl.nr_reconnects); 2868 else 2869 dev_warn(ctrl->ctrl.device, 2870 "NVME-FC{%d}: dev_loss_tmo (%d) expired " 2871 "while waiting for remoteport connectivity.\n", 2872 ctrl->cnum, portptr->dev_loss_tmo); 2873 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl)); 2874 } 2875 } 2876 2877 static void 2878 nvme_fc_reset_ctrl_work(struct work_struct *work) 2879 { 2880 struct nvme_fc_ctrl *ctrl = 2881 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work); 2882 int ret; 2883 2884 nvme_stop_ctrl(&ctrl->ctrl); 2885 2886 /* will block will waiting for io to terminate */ 2887 nvme_fc_delete_association(ctrl); 2888 2889 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2890 dev_err(ctrl->ctrl.device, 2891 "NVME-FC{%d}: error_recovery: Couldn't change state " 2892 "to CONNECTING\n", ctrl->cnum); 2893 return; 2894 } 2895 2896 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) 2897 ret = nvme_fc_create_association(ctrl); 2898 else 2899 ret = -ENOTCONN; 2900 2901 if (ret) 2902 nvme_fc_reconnect_or_delete(ctrl, ret); 2903 else 2904 dev_info(ctrl->ctrl.device, 2905 "NVME-FC{%d}: controller reset complete\n", 2906 ctrl->cnum); 2907 } 2908 2909 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = { 2910 .name = "fc", 2911 .module = THIS_MODULE, 2912 .flags = NVME_F_FABRICS, 2913 .reg_read32 = nvmf_reg_read32, 2914 .reg_read64 = nvmf_reg_read64, 2915 .reg_write32 = nvmf_reg_write32, 2916 .free_ctrl = nvme_fc_nvme_ctrl_freed, 2917 .submit_async_event = nvme_fc_submit_async_event, 2918 .delete_ctrl = nvme_fc_delete_ctrl, 2919 .get_address = nvmf_get_address, 2920 .reinit_request = nvme_fc_reinit_request, 2921 }; 2922 2923 static void 2924 nvme_fc_connect_ctrl_work(struct work_struct *work) 2925 { 2926 int ret; 2927 2928 struct nvme_fc_ctrl *ctrl = 2929 container_of(to_delayed_work(work), 2930 struct nvme_fc_ctrl, connect_work); 2931 2932 ret = nvme_fc_create_association(ctrl); 2933 if (ret) 2934 nvme_fc_reconnect_or_delete(ctrl, ret); 2935 else 2936 dev_info(ctrl->ctrl.device, 2937 "NVME-FC{%d}: controller reconnect complete\n", 2938 ctrl->cnum); 2939 } 2940 2941 2942 static const struct blk_mq_ops nvme_fc_admin_mq_ops = { 2943 .queue_rq = nvme_fc_queue_rq, 2944 .complete = nvme_fc_complete_rq, 2945 .init_request = nvme_fc_init_request, 2946 .exit_request = nvme_fc_exit_request, 2947 .init_hctx = nvme_fc_init_admin_hctx, 2948 .timeout = nvme_fc_timeout, 2949 }; 2950 2951 2952 /* 2953 * Fails a controller request if it matches an existing controller 2954 * (association) with the same tuple: 2955 * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN> 2956 * 2957 * The ports don't need to be compared as they are intrinsically 2958 * already matched by the port pointers supplied. 2959 */ 2960 static bool 2961 nvme_fc_existing_controller(struct nvme_fc_rport *rport, 2962 struct nvmf_ctrl_options *opts) 2963 { 2964 struct nvme_fc_ctrl *ctrl; 2965 unsigned long flags; 2966 bool found = false; 2967 2968 spin_lock_irqsave(&rport->lock, flags); 2969 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { 2970 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts); 2971 if (found) 2972 break; 2973 } 2974 spin_unlock_irqrestore(&rport->lock, flags); 2975 2976 return found; 2977 } 2978 2979 static struct nvme_ctrl * 2980 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, 2981 struct nvme_fc_lport *lport, struct nvme_fc_rport *rport) 2982 { 2983 struct nvme_fc_ctrl *ctrl; 2984 unsigned long flags; 2985 int ret, idx, retry; 2986 2987 if (!(rport->remoteport.port_role & 2988 (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) { 2989 ret = -EBADR; 2990 goto out_fail; 2991 } 2992 2993 if (!opts->duplicate_connect && 2994 nvme_fc_existing_controller(rport, opts)) { 2995 ret = -EALREADY; 2996 goto out_fail; 2997 } 2998 2999 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 3000 if (!ctrl) { 3001 ret = -ENOMEM; 3002 goto out_fail; 3003 } 3004 3005 idx = ida_simple_get(&nvme_fc_ctrl_cnt, 0, 0, GFP_KERNEL); 3006 if (idx < 0) { 3007 ret = -ENOSPC; 3008 goto out_free_ctrl; 3009 } 3010 3011 ctrl->ctrl.opts = opts; 3012 INIT_LIST_HEAD(&ctrl->ctrl_list); 3013 ctrl->lport = lport; 3014 ctrl->rport = rport; 3015 ctrl->dev = lport->dev; 3016 ctrl->cnum = idx; 3017 ctrl->assoc_active = false; 3018 init_waitqueue_head(&ctrl->ioabort_wait); 3019 3020 get_device(ctrl->dev); 3021 kref_init(&ctrl->ref); 3022 3023 INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work); 3024 INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work); 3025 spin_lock_init(&ctrl->lock); 3026 3027 /* io queue count */ 3028 ctrl->ctrl.queue_count = min_t(unsigned int, 3029 opts->nr_io_queues, 3030 lport->ops->max_hw_queues); 3031 ctrl->ctrl.queue_count++; /* +1 for admin queue */ 3032 3033 ctrl->ctrl.sqsize = opts->queue_size - 1; 3034 ctrl->ctrl.kato = opts->kato; 3035 3036 ret = -ENOMEM; 3037 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, 3038 sizeof(struct nvme_fc_queue), GFP_KERNEL); 3039 if (!ctrl->queues) 3040 goto out_free_ida; 3041 3042 memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); 3043 ctrl->admin_tag_set.ops = &nvme_fc_admin_mq_ops; 3044 ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; 3045 ctrl->admin_tag_set.reserved_tags = 2; /* fabric connect + Keep-Alive */ 3046 ctrl->admin_tag_set.numa_node = NUMA_NO_NODE; 3047 ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) + 3048 (SG_CHUNK_SIZE * 3049 sizeof(struct scatterlist)) + 3050 ctrl->lport->ops->fcprqst_priv_sz; 3051 ctrl->admin_tag_set.driver_data = ctrl; 3052 ctrl->admin_tag_set.nr_hw_queues = 1; 3053 ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT; 3054 ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED; 3055 3056 ret = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); 3057 if (ret) 3058 goto out_free_queues; 3059 ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set; 3060 3061 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); 3062 if (IS_ERR(ctrl->ctrl.admin_q)) { 3063 ret = PTR_ERR(ctrl->ctrl.admin_q); 3064 goto out_free_admin_tag_set; 3065 } 3066 3067 /* 3068 * Would have been nice to init io queues tag set as well. 3069 * However, we require interaction from the controller 3070 * for max io queue count before we can do so. 3071 * Defer this to the connect path. 3072 */ 3073 3074 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0); 3075 if (ret) 3076 goto out_cleanup_admin_q; 3077 3078 /* at this point, teardown path changes to ref counting on nvme ctrl */ 3079 3080 spin_lock_irqsave(&rport->lock, flags); 3081 list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list); 3082 spin_unlock_irqrestore(&rport->lock, flags); 3083 3084 /* 3085 * It's possible that transactions used to create the association 3086 * may fail. Examples: CreateAssociation LS or CreateIOConnection 3087 * LS gets dropped/corrupted/fails; or a frame gets dropped or a 3088 * command times out for one of the actions to init the controller 3089 * (Connect, Get/Set_Property, Set_Features, etc). Many of these 3090 * transport errors (frame drop, LS failure) inherently must kill 3091 * the association. The transport is coded so that any command used 3092 * to create the association (prior to a LIVE state transition 3093 * while NEW or CONNECTING) will fail if it completes in error or 3094 * times out. 3095 * 3096 * As such: as the connect request was mostly likely due to a 3097 * udev event that discovered the remote port, meaning there is 3098 * not an admin or script there to restart if the connect 3099 * request fails, retry the initial connection creation up to 3100 * three times before giving up and declaring failure. 3101 */ 3102 for (retry = 0; retry < 3; retry++) { 3103 ret = nvme_fc_create_association(ctrl); 3104 if (!ret) 3105 break; 3106 } 3107 3108 if (ret) { 3109 nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING); 3110 cancel_work_sync(&ctrl->ctrl.reset_work); 3111 cancel_delayed_work_sync(&ctrl->connect_work); 3112 3113 /* couldn't schedule retry - fail out */ 3114 dev_err(ctrl->ctrl.device, 3115 "NVME-FC{%d}: Connect retry failed\n", ctrl->cnum); 3116 3117 ctrl->ctrl.opts = NULL; 3118 3119 /* initiate nvme ctrl ref counting teardown */ 3120 nvme_uninit_ctrl(&ctrl->ctrl); 3121 3122 /* Remove core ctrl ref. */ 3123 nvme_put_ctrl(&ctrl->ctrl); 3124 3125 /* as we're past the point where we transition to the ref 3126 * counting teardown path, if we return a bad pointer here, 3127 * the calling routine, thinking it's prior to the 3128 * transition, will do an rport put. Since the teardown 3129 * path also does a rport put, we do an extra get here to 3130 * so proper order/teardown happens. 3131 */ 3132 nvme_fc_rport_get(rport); 3133 3134 if (ret > 0) 3135 ret = -EIO; 3136 return ERR_PTR(ret); 3137 } 3138 3139 nvme_get_ctrl(&ctrl->ctrl); 3140 3141 dev_info(ctrl->ctrl.device, 3142 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n", 3143 ctrl->cnum, ctrl->ctrl.opts->subsysnqn); 3144 3145 return &ctrl->ctrl; 3146 3147 out_cleanup_admin_q: 3148 blk_cleanup_queue(ctrl->ctrl.admin_q); 3149 out_free_admin_tag_set: 3150 blk_mq_free_tag_set(&ctrl->admin_tag_set); 3151 out_free_queues: 3152 kfree(ctrl->queues); 3153 out_free_ida: 3154 put_device(ctrl->dev); 3155 ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum); 3156 out_free_ctrl: 3157 kfree(ctrl); 3158 out_fail: 3159 /* exit via here doesn't follow ctlr ref points */ 3160 return ERR_PTR(ret); 3161 } 3162 3163 3164 struct nvmet_fc_traddr { 3165 u64 nn; 3166 u64 pn; 3167 }; 3168 3169 static int 3170 __nvme_fc_parse_u64(substring_t *sstr, u64 *val) 3171 { 3172 u64 token64; 3173 3174 if (match_u64(sstr, &token64)) 3175 return -EINVAL; 3176 *val = token64; 3177 3178 return 0; 3179 } 3180 3181 /* 3182 * This routine validates and extracts the WWN's from the TRADDR string. 3183 * As kernel parsers need the 0x to determine number base, universally 3184 * build string to parse with 0x prefix before parsing name strings. 3185 */ 3186 static int 3187 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) 3188 { 3189 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; 3190 substring_t wwn = { name, &name[sizeof(name)-1] }; 3191 int nnoffset, pnoffset; 3192 3193 /* validate it string one of the 2 allowed formats */ 3194 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && 3195 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && 3196 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], 3197 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { 3198 nnoffset = NVME_FC_TRADDR_OXNNLEN; 3199 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + 3200 NVME_FC_TRADDR_OXNNLEN; 3201 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && 3202 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && 3203 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], 3204 "pn-", NVME_FC_TRADDR_NNLEN))) { 3205 nnoffset = NVME_FC_TRADDR_NNLEN; 3206 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; 3207 } else 3208 goto out_einval; 3209 3210 name[0] = '0'; 3211 name[1] = 'x'; 3212 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; 3213 3214 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); 3215 if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) 3216 goto out_einval; 3217 3218 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); 3219 if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) 3220 goto out_einval; 3221 3222 return 0; 3223 3224 out_einval: 3225 pr_warn("%s: bad traddr string\n", __func__); 3226 return -EINVAL; 3227 } 3228 3229 static struct nvme_ctrl * 3230 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts) 3231 { 3232 struct nvme_fc_lport *lport; 3233 struct nvme_fc_rport *rport; 3234 struct nvme_ctrl *ctrl; 3235 struct nvmet_fc_traddr laddr = { 0L, 0L }; 3236 struct nvmet_fc_traddr raddr = { 0L, 0L }; 3237 unsigned long flags; 3238 int ret; 3239 3240 ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE); 3241 if (ret || !raddr.nn || !raddr.pn) 3242 return ERR_PTR(-EINVAL); 3243 3244 ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE); 3245 if (ret || !laddr.nn || !laddr.pn) 3246 return ERR_PTR(-EINVAL); 3247 3248 /* find the host and remote ports to connect together */ 3249 spin_lock_irqsave(&nvme_fc_lock, flags); 3250 list_for_each_entry(lport, &nvme_fc_lport_list, port_list) { 3251 if (lport->localport.node_name != laddr.nn || 3252 lport->localport.port_name != laddr.pn) 3253 continue; 3254 3255 list_for_each_entry(rport, &lport->endp_list, endp_list) { 3256 if (rport->remoteport.node_name != raddr.nn || 3257 rport->remoteport.port_name != raddr.pn) 3258 continue; 3259 3260 /* if fail to get reference fall through. Will error */ 3261 if (!nvme_fc_rport_get(rport)) 3262 break; 3263 3264 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3265 3266 ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport); 3267 if (IS_ERR(ctrl)) 3268 nvme_fc_rport_put(rport); 3269 return ctrl; 3270 } 3271 } 3272 spin_unlock_irqrestore(&nvme_fc_lock, flags); 3273 3274 pr_warn("%s: %s - %s combination not found\n", 3275 __func__, opts->traddr, opts->host_traddr); 3276 return ERR_PTR(-ENOENT); 3277 } 3278 3279 3280 static struct nvmf_transport_ops nvme_fc_transport = { 3281 .name = "fc", 3282 .module = THIS_MODULE, 3283 .required_opts = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR, 3284 .allowed_opts = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO, 3285 .create_ctrl = nvme_fc_create_ctrl, 3286 }; 3287 3288 static int __init nvme_fc_init_module(void) 3289 { 3290 int ret; 3291 3292 /* 3293 * NOTE: 3294 * It is expected that in the future the kernel will combine 3295 * the FC-isms that are currently under scsi and now being 3296 * added to by NVME into a new standalone FC class. The SCSI 3297 * and NVME protocols and their devices would be under this 3298 * new FC class. 3299 * 3300 * As we need something to post FC-specific udev events to, 3301 * specifically for nvme probe events, start by creating the 3302 * new device class. When the new standalone FC class is 3303 * put in place, this code will move to a more generic 3304 * location for the class. 3305 */ 3306 fc_class = class_create(THIS_MODULE, "fc"); 3307 if (IS_ERR(fc_class)) { 3308 pr_err("couldn't register class fc\n"); 3309 return PTR_ERR(fc_class); 3310 } 3311 3312 /* 3313 * Create a device for the FC-centric udev events 3314 */ 3315 fc_udev_device = device_create(fc_class, NULL, MKDEV(0, 0), NULL, 3316 "fc_udev_device"); 3317 if (IS_ERR(fc_udev_device)) { 3318 pr_err("couldn't create fc_udev device!\n"); 3319 ret = PTR_ERR(fc_udev_device); 3320 goto out_destroy_class; 3321 } 3322 3323 ret = nvmf_register_transport(&nvme_fc_transport); 3324 if (ret) 3325 goto out_destroy_device; 3326 3327 return 0; 3328 3329 out_destroy_device: 3330 device_destroy(fc_class, MKDEV(0, 0)); 3331 out_destroy_class: 3332 class_destroy(fc_class); 3333 return ret; 3334 } 3335 3336 static void __exit nvme_fc_exit_module(void) 3337 { 3338 /* sanity check - all lports should be removed */ 3339 if (!list_empty(&nvme_fc_lport_list)) 3340 pr_warn("%s: localport list not empty\n", __func__); 3341 3342 nvmf_unregister_transport(&nvme_fc_transport); 3343 3344 ida_destroy(&nvme_fc_local_port_cnt); 3345 ida_destroy(&nvme_fc_ctrl_cnt); 3346 3347 device_destroy(fc_class, MKDEV(0, 0)); 3348 class_destroy(fc_class); 3349 } 3350 3351 module_init(nvme_fc_init_module); 3352 module_exit(nvme_fc_exit_module); 3353 3354 MODULE_LICENSE("GPL v2"); 3355