1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016 Avago Technologies. All rights reserved. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 #include <linux/module.h> 7 #include <linux/slab.h> 8 #include <linux/blk-mq.h> 9 #include <linux/parser.h> 10 #include <linux/random.h> 11 #include <uapi/scsi/fc/fc_fs.h> 12 #include <uapi/scsi/fc/fc_els.h> 13 14 #include "nvmet.h" 15 #include <linux/nvme-fc-driver.h> 16 #include <linux/nvme-fc.h> 17 #include "../host/fc.h" 18 19 20 /* *************************** Data Structures/Defines ****************** */ 21 22 23 #define NVMET_LS_CTX_COUNT 256 24 25 struct nvmet_fc_tgtport; 26 struct nvmet_fc_tgt_assoc; 27 28 struct nvmet_fc_ls_iod { /* for an LS RQST RCV */ 29 struct nvmefc_ls_rsp *lsrsp; 30 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ 31 32 struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */ 33 34 struct nvmet_fc_tgtport *tgtport; 35 struct nvmet_fc_tgt_assoc *assoc; 36 void *hosthandle; 37 38 union nvmefc_ls_requests *rqstbuf; 39 union nvmefc_ls_responses *rspbuf; 40 u16 rqstdatalen; 41 dma_addr_t rspdma; 42 43 struct scatterlist sg[2]; 44 45 struct work_struct work; 46 } __aligned(sizeof(unsigned long long)); 47 48 struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */ 49 struct nvmefc_ls_req ls_req; 50 51 struct nvmet_fc_tgtport *tgtport; 52 void *hosthandle; 53 54 int ls_error; 55 struct list_head lsreq_list; /* tgtport->ls_req_list */ 56 bool req_queued; 57 }; 58 59 60 /* desired maximum for a single sequence - if sg list allows it */ 61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024) 62 63 enum nvmet_fcp_datadir { 64 NVMET_FCP_NODATA, 65 NVMET_FCP_WRITE, 66 NVMET_FCP_READ, 67 NVMET_FCP_ABORTED, 68 }; 69 70 struct nvmet_fc_fcp_iod { 71 struct nvmefc_tgt_fcp_req *fcpreq; 72 73 struct nvme_fc_cmd_iu cmdiubuf; 74 struct nvme_fc_ersp_iu rspiubuf; 75 dma_addr_t rspdma; 76 struct scatterlist *next_sg; 77 struct scatterlist *data_sg; 78 int data_sg_cnt; 79 u32 offset; 80 enum nvmet_fcp_datadir io_dir; 81 bool active; 82 bool abort; 83 bool aborted; 84 bool writedataactive; 85 spinlock_t flock; 86 87 struct nvmet_req req; 88 struct work_struct defer_work; 89 90 struct nvmet_fc_tgtport *tgtport; 91 struct nvmet_fc_tgt_queue *queue; 92 93 struct list_head fcp_list; /* tgtport->fcp_list */ 94 }; 95 96 struct nvmet_fc_tgtport { 97 struct nvmet_fc_target_port fc_target_port; 98 99 struct list_head tgt_list; /* nvmet_fc_target_list */ 100 struct device *dev; /* dev for dma mapping */ 101 struct nvmet_fc_target_template *ops; 102 103 struct nvmet_fc_ls_iod *iod; 104 spinlock_t lock; 105 struct list_head ls_rcv_list; 106 struct list_head ls_req_list; 107 struct list_head ls_busylist; 108 struct list_head assoc_list; 109 struct list_head host_list; 110 struct ida assoc_cnt; 111 struct nvmet_fc_port_entry *pe; 112 struct kref ref; 113 u32 max_sg_cnt; 114 115 struct work_struct put_work; 116 }; 117 118 struct nvmet_fc_port_entry { 119 struct nvmet_fc_tgtport *tgtport; 120 struct nvmet_port *port; 121 u64 node_name; 122 u64 port_name; 123 struct list_head pe_list; 124 }; 125 126 struct nvmet_fc_defer_fcp_req { 127 struct list_head req_list; 128 struct nvmefc_tgt_fcp_req *fcp_req; 129 }; 130 131 struct nvmet_fc_tgt_queue { 132 bool ninetypercent; 133 u16 qid; 134 u16 sqsize; 135 u16 ersp_ratio; 136 __le16 sqhd; 137 atomic_t connected; 138 atomic_t sqtail; 139 atomic_t zrspcnt; 140 atomic_t rsn; 141 spinlock_t qlock; 142 struct nvmet_cq nvme_cq; 143 struct nvmet_sq nvme_sq; 144 struct nvmet_fc_tgt_assoc *assoc; 145 struct list_head fod_list; 146 struct list_head pending_cmd_list; 147 struct list_head avail_defer_list; 148 struct workqueue_struct *work_q; 149 struct kref ref; 150 /* array of fcp_iods */ 151 struct nvmet_fc_fcp_iod fod[] /* __counted_by(sqsize) */; 152 } __aligned(sizeof(unsigned long long)); 153 154 struct nvmet_fc_hostport { 155 struct nvmet_fc_tgtport *tgtport; 156 void *hosthandle; 157 struct list_head host_list; 158 struct kref ref; 159 u8 invalid; 160 }; 161 162 struct nvmet_fc_tgt_assoc { 163 u64 association_id; 164 u32 a_id; 165 atomic_t terminating; 166 struct nvmet_fc_tgtport *tgtport; 167 struct nvmet_fc_hostport *hostport; 168 struct nvmet_fc_ls_iod *rcv_disconn; 169 struct list_head a_list; 170 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1]; 171 struct kref ref; 172 struct work_struct del_work; 173 }; 174 175 /* 176 * Association and Connection IDs: 177 * 178 * Association ID will have random number in upper 6 bytes and zero 179 * in lower 2 bytes 180 * 181 * Connection IDs will be Association ID with QID or'd in lower 2 bytes 182 * 183 * note: Association ID = Connection ID for queue 0 184 */ 185 #define BYTES_FOR_QID sizeof(u16) 186 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8) 187 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1)) 188 189 static inline u64 190 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid) 191 { 192 return (assoc->association_id | qid); 193 } 194 195 static inline u64 196 nvmet_fc_getassociationid(u64 connectionid) 197 { 198 return connectionid & ~NVMET_FC_QUEUEID_MASK; 199 } 200 201 static inline u16 202 nvmet_fc_getqueueid(u64 connectionid) 203 { 204 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK); 205 } 206 207 static inline struct nvmet_fc_tgtport * 208 targetport_to_tgtport(struct nvmet_fc_target_port *targetport) 209 { 210 return container_of(targetport, struct nvmet_fc_tgtport, 211 fc_target_port); 212 } 213 214 static inline struct nvmet_fc_fcp_iod * 215 nvmet_req_to_fod(struct nvmet_req *nvme_req) 216 { 217 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req); 218 } 219 220 221 /* *************************** Globals **************************** */ 222 223 224 static DEFINE_SPINLOCK(nvmet_fc_tgtlock); 225 226 static LIST_HEAD(nvmet_fc_target_list); 227 static DEFINE_IDA(nvmet_fc_tgtport_cnt); 228 static LIST_HEAD(nvmet_fc_portentry_list); 229 230 231 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work); 232 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work); 233 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc); 234 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc); 235 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue); 236 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue); 237 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport); 238 static void nvmet_fc_put_tgtport_work(struct work_struct *work) 239 { 240 struct nvmet_fc_tgtport *tgtport = 241 container_of(work, struct nvmet_fc_tgtport, put_work); 242 243 nvmet_fc_tgtport_put(tgtport); 244 } 245 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); 246 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 247 struct nvmet_fc_fcp_iod *fod); 248 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc); 249 static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, 250 struct nvmet_fc_ls_iod *iod); 251 252 253 /* *********************** FC-NVME DMA Handling **************************** */ 254 255 /* 256 * The fcloop device passes in a NULL device pointer. Real LLD's will 257 * pass in a valid device pointer. If NULL is passed to the dma mapping 258 * routines, depending on the platform, it may or may not succeed, and 259 * may crash. 260 * 261 * As such: 262 * Wrapper all the dma routines and check the dev pointer. 263 * 264 * If simple mappings (return just a dma address, we'll noop them, 265 * returning a dma address of 0. 266 * 267 * On more complex mappings (dma_map_sg), a pseudo routine fills 268 * in the scatter list, setting all dma addresses to 0. 269 */ 270 271 static inline dma_addr_t 272 fc_dma_map_single(struct device *dev, void *ptr, size_t size, 273 enum dma_data_direction dir) 274 { 275 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; 276 } 277 278 static inline int 279 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 280 { 281 return dev ? dma_mapping_error(dev, dma_addr) : 0; 282 } 283 284 static inline void 285 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, 286 enum dma_data_direction dir) 287 { 288 if (dev) 289 dma_unmap_single(dev, addr, size, dir); 290 } 291 292 static inline void 293 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 294 enum dma_data_direction dir) 295 { 296 if (dev) 297 dma_sync_single_for_cpu(dev, addr, size, dir); 298 } 299 300 static inline void 301 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, 302 enum dma_data_direction dir) 303 { 304 if (dev) 305 dma_sync_single_for_device(dev, addr, size, dir); 306 } 307 308 /* pseudo dma_map_sg call */ 309 static int 310 fc_map_sg(struct scatterlist *sg, int nents) 311 { 312 struct scatterlist *s; 313 int i; 314 315 WARN_ON(nents == 0 || sg[0].length == 0); 316 317 for_each_sg(sg, s, nents, i) { 318 s->dma_address = 0L; 319 #ifdef CONFIG_NEED_SG_DMA_LENGTH 320 s->dma_length = s->length; 321 #endif 322 } 323 return nents; 324 } 325 326 static inline int 327 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 328 enum dma_data_direction dir) 329 { 330 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); 331 } 332 333 static inline void 334 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 335 enum dma_data_direction dir) 336 { 337 if (dev) 338 dma_unmap_sg(dev, sg, nents, dir); 339 } 340 341 342 /* ********************** FC-NVME LS XMT Handling ************************* */ 343 344 345 static void 346 __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) 347 { 348 struct nvmet_fc_tgtport *tgtport = lsop->tgtport; 349 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 350 unsigned long flags; 351 352 spin_lock_irqsave(&tgtport->lock, flags); 353 354 if (!lsop->req_queued) { 355 spin_unlock_irqrestore(&tgtport->lock, flags); 356 goto out_putwork; 357 } 358 359 list_del(&lsop->lsreq_list); 360 361 lsop->req_queued = false; 362 363 spin_unlock_irqrestore(&tgtport->lock, flags); 364 365 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, 366 (lsreq->rqstlen + lsreq->rsplen), 367 DMA_BIDIRECTIONAL); 368 369 out_putwork: 370 queue_work(nvmet_wq, &tgtport->put_work); 371 } 372 373 static int 374 __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, 375 struct nvmet_fc_ls_req_op *lsop, 376 void (*done)(struct nvmefc_ls_req *req, int status)) 377 { 378 struct nvmefc_ls_req *lsreq = &lsop->ls_req; 379 unsigned long flags; 380 int ret = 0; 381 382 if (!tgtport->ops->ls_req) 383 return -EOPNOTSUPP; 384 385 if (!nvmet_fc_tgtport_get(tgtport)) 386 return -ESHUTDOWN; 387 388 lsreq->done = done; 389 lsop->req_queued = false; 390 INIT_LIST_HEAD(&lsop->lsreq_list); 391 392 lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr, 393 lsreq->rqstlen + lsreq->rsplen, 394 DMA_BIDIRECTIONAL); 395 if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) { 396 ret = -EFAULT; 397 goto out_puttgtport; 398 } 399 lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; 400 401 spin_lock_irqsave(&tgtport->lock, flags); 402 403 list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list); 404 405 lsop->req_queued = true; 406 407 spin_unlock_irqrestore(&tgtport->lock, flags); 408 409 ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle, 410 lsreq); 411 if (ret) 412 goto out_unlink; 413 414 return 0; 415 416 out_unlink: 417 lsop->ls_error = ret; 418 spin_lock_irqsave(&tgtport->lock, flags); 419 lsop->req_queued = false; 420 list_del(&lsop->lsreq_list); 421 spin_unlock_irqrestore(&tgtport->lock, flags); 422 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, 423 (lsreq->rqstlen + lsreq->rsplen), 424 DMA_BIDIRECTIONAL); 425 out_puttgtport: 426 nvmet_fc_tgtport_put(tgtport); 427 428 return ret; 429 } 430 431 static int 432 nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport, 433 struct nvmet_fc_ls_req_op *lsop, 434 void (*done)(struct nvmefc_ls_req *req, int status)) 435 { 436 /* don't wait for completion */ 437 438 return __nvmet_fc_send_ls_req(tgtport, lsop, done); 439 } 440 441 static void 442 nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) 443 { 444 struct nvmet_fc_ls_req_op *lsop = 445 container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req); 446 447 __nvmet_fc_finish_ls_req(lsop); 448 449 /* fc-nvme target doesn't care about success or failure of cmd */ 450 451 kfree(lsop); 452 } 453 454 /* 455 * This routine sends a FC-NVME LS to disconnect (aka terminate) 456 * the FC-NVME Association. Terminating the association also 457 * terminates the FC-NVME connections (per queue, both admin and io 458 * queues) that are part of the association. E.g. things are torn 459 * down, and the related FC-NVME Association ID and Connection IDs 460 * become invalid. 461 * 462 * The behavior of the fc-nvme target is such that it's 463 * understanding of the association and connections will implicitly 464 * be torn down. The action is implicit as it may be due to a loss of 465 * connectivity with the fc-nvme host, so the target may never get a 466 * response even if it tried. As such, the action of this routine 467 * is to asynchronously send the LS, ignore any results of the LS, and 468 * continue on with terminating the association. If the fc-nvme host 469 * is present and receives the LS, it too can tear down. 470 */ 471 static void 472 nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc) 473 { 474 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 475 struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst; 476 struct fcnvme_ls_disconnect_assoc_acc *discon_acc; 477 struct nvmet_fc_ls_req_op *lsop; 478 struct nvmefc_ls_req *lsreq; 479 int ret; 480 481 /* 482 * If ls_req is NULL or no hosthandle, it's an older lldd and no 483 * message is normal. Otherwise, send unless the hostport has 484 * already been invalidated by the lldd. 485 */ 486 if (!tgtport->ops->ls_req || assoc->hostport->invalid) 487 return; 488 489 lsop = kzalloc((sizeof(*lsop) + 490 sizeof(*discon_rqst) + sizeof(*discon_acc) + 491 tgtport->ops->lsrqst_priv_sz), GFP_KERNEL); 492 if (!lsop) { 493 dev_info(tgtport->dev, 494 "{%d:%d} send Disconnect Association failed: ENOMEM\n", 495 tgtport->fc_target_port.port_num, assoc->a_id); 496 return; 497 } 498 499 discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1]; 500 discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1]; 501 lsreq = &lsop->ls_req; 502 if (tgtport->ops->lsrqst_priv_sz) 503 lsreq->private = (void *)&discon_acc[1]; 504 else 505 lsreq->private = NULL; 506 507 lsop->tgtport = tgtport; 508 lsop->hosthandle = assoc->hostport->hosthandle; 509 510 nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc, 511 assoc->association_id); 512 513 ret = nvmet_fc_send_ls_req_async(tgtport, lsop, 514 nvmet_fc_disconnect_assoc_done); 515 if (ret) { 516 dev_info(tgtport->dev, 517 "{%d:%d} XMT Disconnect Association failed: %d\n", 518 tgtport->fc_target_port.port_num, assoc->a_id, ret); 519 kfree(lsop); 520 } 521 } 522 523 524 /* *********************** FC-NVME Port Management ************************ */ 525 526 527 static int 528 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 529 { 530 struct nvmet_fc_ls_iod *iod; 531 int i; 532 533 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod), 534 GFP_KERNEL); 535 if (!iod) 536 return -ENOMEM; 537 538 tgtport->iod = iod; 539 540 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 541 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work); 542 iod->tgtport = tgtport; 543 list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list); 544 545 iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) + 546 sizeof(union nvmefc_ls_responses), 547 GFP_KERNEL); 548 if (!iod->rqstbuf) 549 goto out_fail; 550 551 iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1]; 552 553 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf, 554 sizeof(*iod->rspbuf), 555 DMA_TO_DEVICE); 556 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma)) 557 goto out_fail; 558 } 559 560 return 0; 561 562 out_fail: 563 kfree(iod->rqstbuf); 564 list_del(&iod->ls_rcv_list); 565 for (iod--, i--; i >= 0; iod--, i--) { 566 fc_dma_unmap_single(tgtport->dev, iod->rspdma, 567 sizeof(*iod->rspbuf), DMA_TO_DEVICE); 568 kfree(iod->rqstbuf); 569 list_del(&iod->ls_rcv_list); 570 } 571 572 kfree(iod); 573 574 return -EFAULT; 575 } 576 577 static void 578 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 579 { 580 struct nvmet_fc_ls_iod *iod = tgtport->iod; 581 int i; 582 583 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 584 fc_dma_unmap_single(tgtport->dev, 585 iod->rspdma, sizeof(*iod->rspbuf), 586 DMA_TO_DEVICE); 587 kfree(iod->rqstbuf); 588 list_del(&iod->ls_rcv_list); 589 } 590 kfree(tgtport->iod); 591 } 592 593 static struct nvmet_fc_ls_iod * 594 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport) 595 { 596 struct nvmet_fc_ls_iod *iod; 597 unsigned long flags; 598 599 spin_lock_irqsave(&tgtport->lock, flags); 600 iod = list_first_entry_or_null(&tgtport->ls_rcv_list, 601 struct nvmet_fc_ls_iod, ls_rcv_list); 602 if (iod) 603 list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist); 604 spin_unlock_irqrestore(&tgtport->lock, flags); 605 return iod; 606 } 607 608 609 static void 610 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport, 611 struct nvmet_fc_ls_iod *iod) 612 { 613 unsigned long flags; 614 615 spin_lock_irqsave(&tgtport->lock, flags); 616 list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list); 617 spin_unlock_irqrestore(&tgtport->lock, flags); 618 } 619 620 static void 621 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 622 struct nvmet_fc_tgt_queue *queue) 623 { 624 struct nvmet_fc_fcp_iod *fod = queue->fod; 625 int i; 626 627 for (i = 0; i < queue->sqsize; fod++, i++) { 628 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work); 629 fod->tgtport = tgtport; 630 fod->queue = queue; 631 fod->active = false; 632 fod->abort = false; 633 fod->aborted = false; 634 fod->fcpreq = NULL; 635 list_add_tail(&fod->fcp_list, &queue->fod_list); 636 spin_lock_init(&fod->flock); 637 638 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf, 639 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 640 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) { 641 list_del(&fod->fcp_list); 642 for (fod--, i--; i >= 0; fod--, i--) { 643 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 644 sizeof(fod->rspiubuf), 645 DMA_TO_DEVICE); 646 fod->rspdma = 0L; 647 list_del(&fod->fcp_list); 648 } 649 650 return; 651 } 652 } 653 } 654 655 static void 656 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 657 struct nvmet_fc_tgt_queue *queue) 658 { 659 struct nvmet_fc_fcp_iod *fod = queue->fod; 660 int i; 661 662 for (i = 0; i < queue->sqsize; fod++, i++) { 663 if (fod->rspdma) 664 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 665 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 666 } 667 } 668 669 static struct nvmet_fc_fcp_iod * 670 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue) 671 { 672 struct nvmet_fc_fcp_iod *fod; 673 674 lockdep_assert_held(&queue->qlock); 675 676 fod = list_first_entry_or_null(&queue->fod_list, 677 struct nvmet_fc_fcp_iod, fcp_list); 678 if (fod) { 679 list_del(&fod->fcp_list); 680 fod->active = true; 681 /* 682 * no queue reference is taken, as it was taken by the 683 * queue lookup just prior to the allocation. The iod 684 * will "inherit" that reference. 685 */ 686 } 687 return fod; 688 } 689 690 691 static void 692 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport, 693 struct nvmet_fc_tgt_queue *queue, 694 struct nvmefc_tgt_fcp_req *fcpreq) 695 { 696 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 697 698 /* 699 * put all admin cmds on hw queue id 0. All io commands go to 700 * the respective hw queue based on a modulo basis 701 */ 702 fcpreq->hwqid = queue->qid ? 703 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0; 704 705 nvmet_fc_handle_fcp_rqst(tgtport, fod); 706 } 707 708 static void 709 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work) 710 { 711 struct nvmet_fc_fcp_iod *fod = 712 container_of(work, struct nvmet_fc_fcp_iod, defer_work); 713 714 /* Submit deferred IO for processing */ 715 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq); 716 717 } 718 719 static void 720 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue, 721 struct nvmet_fc_fcp_iod *fod) 722 { 723 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 724 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 725 struct nvmet_fc_defer_fcp_req *deferfcp; 726 unsigned long flags; 727 728 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma, 729 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 730 731 fcpreq->nvmet_fc_private = NULL; 732 733 fod->active = false; 734 fod->abort = false; 735 fod->aborted = false; 736 fod->writedataactive = false; 737 fod->fcpreq = NULL; 738 739 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq); 740 741 /* release the queue lookup reference on the completed IO */ 742 nvmet_fc_tgt_q_put(queue); 743 744 spin_lock_irqsave(&queue->qlock, flags); 745 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 746 struct nvmet_fc_defer_fcp_req, req_list); 747 if (!deferfcp) { 748 list_add_tail(&fod->fcp_list, &fod->queue->fod_list); 749 spin_unlock_irqrestore(&queue->qlock, flags); 750 return; 751 } 752 753 /* Re-use the fod for the next pending cmd that was deferred */ 754 list_del(&deferfcp->req_list); 755 756 fcpreq = deferfcp->fcp_req; 757 758 /* deferfcp can be reused for another IO at a later date */ 759 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list); 760 761 spin_unlock_irqrestore(&queue->qlock, flags); 762 763 /* Save NVME CMD IO in fod */ 764 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen); 765 766 /* Setup new fcpreq to be processed */ 767 fcpreq->rspaddr = NULL; 768 fcpreq->rsplen = 0; 769 fcpreq->nvmet_fc_private = fod; 770 fod->fcpreq = fcpreq; 771 fod->active = true; 772 773 /* inform LLDD IO is now being processed */ 774 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq); 775 776 /* 777 * Leave the queue lookup get reference taken when 778 * fod was originally allocated. 779 */ 780 781 queue_work(queue->work_q, &fod->defer_work); 782 } 783 784 static struct nvmet_fc_tgt_queue * 785 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, 786 u16 qid, u16 sqsize) 787 { 788 struct nvmet_fc_tgt_queue *queue; 789 int ret; 790 791 if (qid > NVMET_NR_QUEUES) 792 return NULL; 793 794 queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL); 795 if (!queue) 796 return NULL; 797 798 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0, 799 assoc->tgtport->fc_target_port.port_num, 800 assoc->a_id, qid); 801 if (!queue->work_q) 802 goto out_free_queue; 803 804 queue->qid = qid; 805 queue->sqsize = sqsize; 806 queue->assoc = assoc; 807 INIT_LIST_HEAD(&queue->fod_list); 808 INIT_LIST_HEAD(&queue->avail_defer_list); 809 INIT_LIST_HEAD(&queue->pending_cmd_list); 810 atomic_set(&queue->connected, 0); 811 atomic_set(&queue->sqtail, 0); 812 atomic_set(&queue->rsn, 1); 813 atomic_set(&queue->zrspcnt, 0); 814 spin_lock_init(&queue->qlock); 815 kref_init(&queue->ref); 816 817 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue); 818 819 nvmet_cq_init(&queue->nvme_cq); 820 ret = nvmet_sq_init(&queue->nvme_sq, &queue->nvme_cq); 821 if (ret) 822 goto out_fail_iodlist; 823 824 WARN_ON(assoc->queues[qid]); 825 assoc->queues[qid] = queue; 826 827 return queue; 828 829 out_fail_iodlist: 830 nvmet_cq_put(&queue->nvme_cq); 831 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue); 832 destroy_workqueue(queue->work_q); 833 out_free_queue: 834 kfree(queue); 835 return NULL; 836 } 837 838 839 static void 840 nvmet_fc_tgt_queue_free(struct kref *ref) 841 { 842 struct nvmet_fc_tgt_queue *queue = 843 container_of(ref, struct nvmet_fc_tgt_queue, ref); 844 845 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue); 846 847 destroy_workqueue(queue->work_q); 848 849 kfree(queue); 850 } 851 852 static void 853 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue) 854 { 855 kref_put(&queue->ref, nvmet_fc_tgt_queue_free); 856 } 857 858 static int 859 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue) 860 { 861 return kref_get_unless_zero(&queue->ref); 862 } 863 864 865 static void 866 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) 867 { 868 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport; 869 struct nvmet_fc_fcp_iod *fod = queue->fod; 870 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr; 871 unsigned long flags; 872 int i; 873 bool disconnect; 874 875 disconnect = atomic_xchg(&queue->connected, 0); 876 877 /* if not connected, nothing to do */ 878 if (!disconnect) 879 return; 880 881 spin_lock_irqsave(&queue->qlock, flags); 882 /* abort outstanding io's */ 883 for (i = 0; i < queue->sqsize; fod++, i++) { 884 if (fod->active) { 885 spin_lock(&fod->flock); 886 fod->abort = true; 887 /* 888 * only call lldd abort routine if waiting for 889 * writedata. other outstanding ops should finish 890 * on their own. 891 */ 892 if (fod->writedataactive) { 893 fod->aborted = true; 894 spin_unlock(&fod->flock); 895 tgtport->ops->fcp_abort( 896 &tgtport->fc_target_port, fod->fcpreq); 897 } else 898 spin_unlock(&fod->flock); 899 } 900 } 901 902 /* Cleanup defer'ed IOs in queue */ 903 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list, 904 req_list) { 905 list_del(&deferfcp->req_list); 906 kfree(deferfcp); 907 } 908 909 for (;;) { 910 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 911 struct nvmet_fc_defer_fcp_req, req_list); 912 if (!deferfcp) 913 break; 914 915 list_del(&deferfcp->req_list); 916 spin_unlock_irqrestore(&queue->qlock, flags); 917 918 tgtport->ops->defer_rcv(&tgtport->fc_target_port, 919 deferfcp->fcp_req); 920 921 tgtport->ops->fcp_abort(&tgtport->fc_target_port, 922 deferfcp->fcp_req); 923 924 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, 925 deferfcp->fcp_req); 926 927 /* release the queue lookup reference */ 928 nvmet_fc_tgt_q_put(queue); 929 930 kfree(deferfcp); 931 932 spin_lock_irqsave(&queue->qlock, flags); 933 } 934 spin_unlock_irqrestore(&queue->qlock, flags); 935 936 flush_workqueue(queue->work_q); 937 938 nvmet_sq_destroy(&queue->nvme_sq); 939 nvmet_cq_put(&queue->nvme_cq); 940 941 nvmet_fc_tgt_q_put(queue); 942 } 943 944 static struct nvmet_fc_tgt_queue * 945 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport, 946 u64 connection_id) 947 { 948 struct nvmet_fc_tgt_assoc *assoc; 949 struct nvmet_fc_tgt_queue *queue; 950 u64 association_id = nvmet_fc_getassociationid(connection_id); 951 u16 qid = nvmet_fc_getqueueid(connection_id); 952 953 if (qid > NVMET_NR_QUEUES) 954 return NULL; 955 956 rcu_read_lock(); 957 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { 958 if (association_id == assoc->association_id) { 959 queue = assoc->queues[qid]; 960 if (queue && 961 (!atomic_read(&queue->connected) || 962 !nvmet_fc_tgt_q_get(queue))) 963 queue = NULL; 964 rcu_read_unlock(); 965 return queue; 966 } 967 } 968 rcu_read_unlock(); 969 return NULL; 970 } 971 972 static void 973 nvmet_fc_hostport_free(struct kref *ref) 974 { 975 struct nvmet_fc_hostport *hostport = 976 container_of(ref, struct nvmet_fc_hostport, ref); 977 struct nvmet_fc_tgtport *tgtport = hostport->tgtport; 978 unsigned long flags; 979 980 spin_lock_irqsave(&tgtport->lock, flags); 981 list_del(&hostport->host_list); 982 spin_unlock_irqrestore(&tgtport->lock, flags); 983 if (tgtport->ops->host_release && hostport->invalid) 984 tgtport->ops->host_release(hostport->hosthandle); 985 kfree(hostport); 986 nvmet_fc_tgtport_put(tgtport); 987 } 988 989 static void 990 nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport) 991 { 992 kref_put(&hostport->ref, nvmet_fc_hostport_free); 993 } 994 995 static int 996 nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport) 997 { 998 return kref_get_unless_zero(&hostport->ref); 999 } 1000 1001 static struct nvmet_fc_hostport * 1002 nvmet_fc_match_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) 1003 { 1004 struct nvmet_fc_hostport *host; 1005 1006 lockdep_assert_held(&tgtport->lock); 1007 1008 list_for_each_entry(host, &tgtport->host_list, host_list) { 1009 if (host->hosthandle == hosthandle && !host->invalid) { 1010 if (nvmet_fc_hostport_get(host)) 1011 return host; 1012 } 1013 } 1014 1015 return NULL; 1016 } 1017 1018 static struct nvmet_fc_hostport * 1019 nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) 1020 { 1021 struct nvmet_fc_hostport *newhost, *match = NULL; 1022 unsigned long flags; 1023 1024 /* 1025 * Caller holds a reference on tgtport. 1026 */ 1027 1028 /* if LLDD not implemented, leave as NULL */ 1029 if (!hosthandle) 1030 return NULL; 1031 1032 spin_lock_irqsave(&tgtport->lock, flags); 1033 match = nvmet_fc_match_hostport(tgtport, hosthandle); 1034 spin_unlock_irqrestore(&tgtport->lock, flags); 1035 1036 if (match) 1037 return match; 1038 1039 newhost = kzalloc(sizeof(*newhost), GFP_KERNEL); 1040 if (!newhost) 1041 return ERR_PTR(-ENOMEM); 1042 1043 spin_lock_irqsave(&tgtport->lock, flags); 1044 match = nvmet_fc_match_hostport(tgtport, hosthandle); 1045 if (match) { 1046 /* new allocation not needed */ 1047 kfree(newhost); 1048 newhost = match; 1049 } else { 1050 nvmet_fc_tgtport_get(tgtport); 1051 newhost->tgtport = tgtport; 1052 newhost->hosthandle = hosthandle; 1053 INIT_LIST_HEAD(&newhost->host_list); 1054 kref_init(&newhost->ref); 1055 1056 list_add_tail(&newhost->host_list, &tgtport->host_list); 1057 } 1058 spin_unlock_irqrestore(&tgtport->lock, flags); 1059 1060 return newhost; 1061 } 1062 1063 static void 1064 nvmet_fc_delete_assoc_work(struct work_struct *work) 1065 { 1066 struct nvmet_fc_tgt_assoc *assoc = 1067 container_of(work, struct nvmet_fc_tgt_assoc, del_work); 1068 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 1069 1070 nvmet_fc_delete_target_assoc(assoc); 1071 nvmet_fc_tgt_a_put(assoc); 1072 nvmet_fc_tgtport_put(tgtport); 1073 } 1074 1075 static void 1076 nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc *assoc) 1077 { 1078 nvmet_fc_tgtport_get(assoc->tgtport); 1079 if (!queue_work(nvmet_wq, &assoc->del_work)) 1080 nvmet_fc_tgtport_put(assoc->tgtport); 1081 } 1082 1083 static bool 1084 nvmet_fc_assoc_exists(struct nvmet_fc_tgtport *tgtport, u64 association_id) 1085 { 1086 struct nvmet_fc_tgt_assoc *a; 1087 bool found = false; 1088 1089 rcu_read_lock(); 1090 list_for_each_entry_rcu(a, &tgtport->assoc_list, a_list) { 1091 if (association_id == a->association_id) { 1092 found = true; 1093 break; 1094 } 1095 } 1096 rcu_read_unlock(); 1097 1098 return found; 1099 } 1100 1101 static struct nvmet_fc_tgt_assoc * 1102 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) 1103 { 1104 struct nvmet_fc_tgt_assoc *assoc; 1105 unsigned long flags; 1106 bool done; 1107 u64 ran; 1108 int idx; 1109 1110 if (!tgtport->pe) 1111 return NULL; 1112 1113 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL); 1114 if (!assoc) 1115 return NULL; 1116 1117 idx = ida_alloc(&tgtport->assoc_cnt, GFP_KERNEL); 1118 if (idx < 0) 1119 goto out_free_assoc; 1120 1121 assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle); 1122 if (IS_ERR(assoc->hostport)) 1123 goto out_ida; 1124 1125 assoc->tgtport = tgtport; 1126 nvmet_fc_tgtport_get(tgtport); 1127 assoc->a_id = idx; 1128 INIT_LIST_HEAD(&assoc->a_list); 1129 kref_init(&assoc->ref); 1130 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc_work); 1131 atomic_set(&assoc->terminating, 0); 1132 1133 done = false; 1134 do { 1135 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); 1136 ran = ran << BYTES_FOR_QID_SHIFT; 1137 1138 spin_lock_irqsave(&tgtport->lock, flags); 1139 if (!nvmet_fc_assoc_exists(tgtport, ran)) { 1140 assoc->association_id = ran; 1141 list_add_tail_rcu(&assoc->a_list, &tgtport->assoc_list); 1142 done = true; 1143 } 1144 spin_unlock_irqrestore(&tgtport->lock, flags); 1145 } while (!done); 1146 1147 return assoc; 1148 1149 out_ida: 1150 ida_free(&tgtport->assoc_cnt, idx); 1151 out_free_assoc: 1152 kfree(assoc); 1153 return NULL; 1154 } 1155 1156 static void 1157 nvmet_fc_target_assoc_free(struct kref *ref) 1158 { 1159 struct nvmet_fc_tgt_assoc *assoc = 1160 container_of(ref, struct nvmet_fc_tgt_assoc, ref); 1161 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 1162 struct nvmet_fc_ls_iod *oldls; 1163 unsigned long flags; 1164 int i; 1165 1166 for (i = NVMET_NR_QUEUES; i >= 0; i--) { 1167 if (assoc->queues[i]) 1168 nvmet_fc_delete_target_queue(assoc->queues[i]); 1169 } 1170 1171 /* Send Disconnect now that all i/o has completed */ 1172 nvmet_fc_xmt_disconnect_assoc(assoc); 1173 1174 nvmet_fc_hostport_put(assoc->hostport); 1175 spin_lock_irqsave(&tgtport->lock, flags); 1176 oldls = assoc->rcv_disconn; 1177 spin_unlock_irqrestore(&tgtport->lock, flags); 1178 /* if pending Rcv Disconnect Association LS, send rsp now */ 1179 if (oldls) 1180 nvmet_fc_xmt_ls_rsp(tgtport, oldls); 1181 ida_free(&tgtport->assoc_cnt, assoc->a_id); 1182 dev_info(tgtport->dev, 1183 "{%d:%d} Association freed\n", 1184 tgtport->fc_target_port.port_num, assoc->a_id); 1185 kfree(assoc); 1186 } 1187 1188 static void 1189 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc) 1190 { 1191 kref_put(&assoc->ref, nvmet_fc_target_assoc_free); 1192 } 1193 1194 static int 1195 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc) 1196 { 1197 return kref_get_unless_zero(&assoc->ref); 1198 } 1199 1200 static void 1201 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) 1202 { 1203 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 1204 unsigned long flags; 1205 int i, terminating; 1206 1207 terminating = atomic_xchg(&assoc->terminating, 1); 1208 1209 /* if already terminating, do nothing */ 1210 if (terminating) 1211 return; 1212 1213 spin_lock_irqsave(&tgtport->lock, flags); 1214 list_del_rcu(&assoc->a_list); 1215 spin_unlock_irqrestore(&tgtport->lock, flags); 1216 1217 synchronize_rcu(); 1218 1219 /* ensure all in-flight I/Os have been processed */ 1220 for (i = NVMET_NR_QUEUES; i >= 0; i--) { 1221 if (assoc->queues[i]) 1222 flush_workqueue(assoc->queues[i]->work_q); 1223 } 1224 1225 dev_info(tgtport->dev, 1226 "{%d:%d} Association deleted\n", 1227 tgtport->fc_target_port.port_num, assoc->a_id); 1228 1229 nvmet_fc_tgtport_put(tgtport); 1230 } 1231 1232 static struct nvmet_fc_tgt_assoc * 1233 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport, 1234 u64 association_id) 1235 { 1236 struct nvmet_fc_tgt_assoc *assoc; 1237 struct nvmet_fc_tgt_assoc *ret = NULL; 1238 1239 rcu_read_lock(); 1240 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { 1241 if (association_id == assoc->association_id) { 1242 ret = assoc; 1243 if (!nvmet_fc_tgt_a_get(assoc)) 1244 ret = NULL; 1245 break; 1246 } 1247 } 1248 rcu_read_unlock(); 1249 1250 return ret; 1251 } 1252 1253 static void 1254 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport, 1255 struct nvmet_fc_port_entry *pe, 1256 struct nvmet_port *port) 1257 { 1258 lockdep_assert_held(&nvmet_fc_tgtlock); 1259 1260 nvmet_fc_tgtport_get(tgtport); 1261 pe->tgtport = tgtport; 1262 tgtport->pe = pe; 1263 1264 pe->port = port; 1265 port->priv = pe; 1266 1267 pe->node_name = tgtport->fc_target_port.node_name; 1268 pe->port_name = tgtport->fc_target_port.port_name; 1269 INIT_LIST_HEAD(&pe->pe_list); 1270 1271 list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list); 1272 } 1273 1274 static void 1275 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe) 1276 { 1277 unsigned long flags; 1278 1279 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1280 if (pe->tgtport) { 1281 nvmet_fc_tgtport_put(pe->tgtport); 1282 pe->tgtport->pe = NULL; 1283 } 1284 list_del(&pe->pe_list); 1285 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1286 } 1287 1288 /* 1289 * called when a targetport deregisters. Breaks the relationship 1290 * with the nvmet port, but leaves the port_entry in place so that 1291 * re-registration can resume operation. 1292 */ 1293 static void 1294 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport) 1295 { 1296 struct nvmet_fc_port_entry *pe; 1297 unsigned long flags; 1298 1299 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1300 pe = tgtport->pe; 1301 if (pe) { 1302 nvmet_fc_tgtport_put(pe->tgtport); 1303 pe->tgtport = NULL; 1304 } 1305 tgtport->pe = NULL; 1306 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1307 } 1308 1309 /* 1310 * called when a new targetport is registered. Looks in the 1311 * existing nvmet port_entries to see if the nvmet layer is 1312 * configured for the targetport's wwn's. (the targetport existed, 1313 * nvmet configured, the lldd unregistered the tgtport, and is now 1314 * reregistering the same targetport). If so, set the nvmet port 1315 * port entry on the targetport. 1316 */ 1317 static void 1318 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport) 1319 { 1320 struct nvmet_fc_port_entry *pe; 1321 unsigned long flags; 1322 1323 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1324 list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) { 1325 if (tgtport->fc_target_port.node_name == pe->node_name && 1326 tgtport->fc_target_port.port_name == pe->port_name) { 1327 if (!nvmet_fc_tgtport_get(tgtport)) 1328 continue; 1329 1330 WARN_ON(pe->tgtport); 1331 tgtport->pe = pe; 1332 pe->tgtport = tgtport; 1333 break; 1334 } 1335 } 1336 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1337 } 1338 1339 /** 1340 * nvmet_fc_register_targetport - transport entry point called by an 1341 * LLDD to register the existence of a local 1342 * NVME subsystem FC port. 1343 * @pinfo: pointer to information about the port to be registered 1344 * @template: LLDD entrypoints and operational parameters for the port 1345 * @dev: physical hardware device node port corresponds to. Will be 1346 * used for DMA mappings 1347 * @portptr: pointer to a local port pointer. Upon success, the routine 1348 * will allocate a nvme_fc_local_port structure and place its 1349 * address in the local port pointer. Upon failure, local port 1350 * pointer will be set to NULL. 1351 * 1352 * Returns: 1353 * a completion status. Must be 0 upon success; a negative errno 1354 * (ex: -ENXIO) upon failure. 1355 */ 1356 int 1357 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, 1358 struct nvmet_fc_target_template *template, 1359 struct device *dev, 1360 struct nvmet_fc_target_port **portptr) 1361 { 1362 struct nvmet_fc_tgtport *newrec; 1363 unsigned long flags; 1364 int ret, idx; 1365 1366 if (!template->xmt_ls_rsp || !template->fcp_op || 1367 !template->fcp_abort || 1368 !template->fcp_req_release || !template->targetport_delete || 1369 !template->max_hw_queues || !template->max_sgl_segments || 1370 !template->max_dif_sgl_segments || !template->dma_boundary) { 1371 ret = -EINVAL; 1372 goto out_regtgt_failed; 1373 } 1374 1375 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz), 1376 GFP_KERNEL); 1377 if (!newrec) { 1378 ret = -ENOMEM; 1379 goto out_regtgt_failed; 1380 } 1381 1382 idx = ida_alloc(&nvmet_fc_tgtport_cnt, GFP_KERNEL); 1383 if (idx < 0) { 1384 ret = -ENOSPC; 1385 goto out_fail_kfree; 1386 } 1387 1388 if (!get_device(dev) && dev) { 1389 ret = -ENODEV; 1390 goto out_ida_put; 1391 } 1392 1393 newrec->fc_target_port.node_name = pinfo->node_name; 1394 newrec->fc_target_port.port_name = pinfo->port_name; 1395 if (template->target_priv_sz) 1396 newrec->fc_target_port.private = &newrec[1]; 1397 else 1398 newrec->fc_target_port.private = NULL; 1399 newrec->fc_target_port.port_id = pinfo->port_id; 1400 newrec->fc_target_port.port_num = idx; 1401 INIT_LIST_HEAD(&newrec->tgt_list); 1402 newrec->dev = dev; 1403 newrec->ops = template; 1404 spin_lock_init(&newrec->lock); 1405 INIT_LIST_HEAD(&newrec->ls_rcv_list); 1406 INIT_LIST_HEAD(&newrec->ls_req_list); 1407 INIT_LIST_HEAD(&newrec->ls_busylist); 1408 INIT_LIST_HEAD(&newrec->assoc_list); 1409 INIT_LIST_HEAD(&newrec->host_list); 1410 kref_init(&newrec->ref); 1411 ida_init(&newrec->assoc_cnt); 1412 newrec->max_sg_cnt = template->max_sgl_segments; 1413 INIT_WORK(&newrec->put_work, nvmet_fc_put_tgtport_work); 1414 1415 ret = nvmet_fc_alloc_ls_iodlist(newrec); 1416 if (ret) { 1417 ret = -ENOMEM; 1418 goto out_free_newrec; 1419 } 1420 1421 nvmet_fc_portentry_rebind_tgt(newrec); 1422 1423 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1424 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list); 1425 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1426 1427 *portptr = &newrec->fc_target_port; 1428 return 0; 1429 1430 out_free_newrec: 1431 put_device(dev); 1432 out_ida_put: 1433 ida_free(&nvmet_fc_tgtport_cnt, idx); 1434 out_fail_kfree: 1435 kfree(newrec); 1436 out_regtgt_failed: 1437 *portptr = NULL; 1438 return ret; 1439 } 1440 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport); 1441 1442 1443 static void 1444 nvmet_fc_free_tgtport(struct kref *ref) 1445 { 1446 struct nvmet_fc_tgtport *tgtport = 1447 container_of(ref, struct nvmet_fc_tgtport, ref); 1448 struct device *dev = tgtport->dev; 1449 1450 nvmet_fc_free_ls_iodlist(tgtport); 1451 1452 /* let the LLDD know we've finished tearing it down */ 1453 tgtport->ops->targetport_delete(&tgtport->fc_target_port); 1454 1455 ida_free(&nvmet_fc_tgtport_cnt, 1456 tgtport->fc_target_port.port_num); 1457 1458 ida_destroy(&tgtport->assoc_cnt); 1459 1460 kfree(tgtport); 1461 1462 put_device(dev); 1463 } 1464 1465 static void 1466 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport) 1467 { 1468 kref_put(&tgtport->ref, nvmet_fc_free_tgtport); 1469 } 1470 1471 static int 1472 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport) 1473 { 1474 return kref_get_unless_zero(&tgtport->ref); 1475 } 1476 1477 static void 1478 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) 1479 { 1480 struct nvmet_fc_tgt_assoc *assoc; 1481 1482 rcu_read_lock(); 1483 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { 1484 if (!nvmet_fc_tgt_a_get(assoc)) 1485 continue; 1486 nvmet_fc_schedule_delete_assoc(assoc); 1487 nvmet_fc_tgt_a_put(assoc); 1488 } 1489 rcu_read_unlock(); 1490 } 1491 1492 /** 1493 * nvmet_fc_invalidate_host - transport entry point called by an LLDD 1494 * to remove references to a hosthandle for LS's. 1495 * 1496 * The nvmet-fc layer ensures that any references to the hosthandle 1497 * on the targetport are forgotten (set to NULL). The LLDD will 1498 * typically call this when a login with a remote host port has been 1499 * lost, thus LS's for the remote host port are no longer possible. 1500 * 1501 * If an LS request is outstanding to the targetport/hosthandle (or 1502 * issued concurrently with the call to invalidate the host), the 1503 * LLDD is responsible for terminating/aborting the LS and completing 1504 * the LS request. It is recommended that these terminations/aborts 1505 * occur after calling to invalidate the host handle to avoid additional 1506 * retries by the nvmet-fc transport. The nvmet-fc transport may 1507 * continue to reference host handle while it cleans up outstanding 1508 * NVME associations. The nvmet-fc transport will call the 1509 * ops->host_release() callback to notify the LLDD that all references 1510 * are complete and the related host handle can be recovered. 1511 * Note: if there are no references, the callback may be called before 1512 * the invalidate host call returns. 1513 * 1514 * @target_port: pointer to the (registered) target port that a prior 1515 * LS was received on and which supplied the transport the 1516 * hosthandle. 1517 * @hosthandle: the handle (pointer) that represents the host port 1518 * that no longer has connectivity and that LS's should 1519 * no longer be directed to. 1520 */ 1521 void 1522 nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port, 1523 void *hosthandle) 1524 { 1525 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1526 struct nvmet_fc_tgt_assoc *assoc, *next; 1527 unsigned long flags; 1528 bool noassoc = true; 1529 1530 spin_lock_irqsave(&tgtport->lock, flags); 1531 list_for_each_entry_safe(assoc, next, 1532 &tgtport->assoc_list, a_list) { 1533 if (assoc->hostport->hosthandle != hosthandle) 1534 continue; 1535 if (!nvmet_fc_tgt_a_get(assoc)) 1536 continue; 1537 assoc->hostport->invalid = 1; 1538 noassoc = false; 1539 nvmet_fc_schedule_delete_assoc(assoc); 1540 nvmet_fc_tgt_a_put(assoc); 1541 } 1542 spin_unlock_irqrestore(&tgtport->lock, flags); 1543 1544 /* if there's nothing to wait for - call the callback */ 1545 if (noassoc && tgtport->ops->host_release) 1546 tgtport->ops->host_release(hosthandle); 1547 } 1548 EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host); 1549 1550 /* 1551 * nvmet layer has called to terminate an association 1552 */ 1553 static void 1554 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) 1555 { 1556 struct nvmet_fc_tgtport *tgtport, *next; 1557 struct nvmet_fc_tgt_assoc *assoc; 1558 struct nvmet_fc_tgt_queue *queue; 1559 unsigned long flags; 1560 bool found_ctrl = false; 1561 1562 /* this is a bit ugly, but don't want to make locks layered */ 1563 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1564 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list, 1565 tgt_list) { 1566 if (!nvmet_fc_tgtport_get(tgtport)) 1567 continue; 1568 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1569 1570 rcu_read_lock(); 1571 list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { 1572 queue = assoc->queues[0]; 1573 if (queue && queue->nvme_sq.ctrl == ctrl) { 1574 if (nvmet_fc_tgt_a_get(assoc)) 1575 found_ctrl = true; 1576 break; 1577 } 1578 } 1579 rcu_read_unlock(); 1580 1581 nvmet_fc_tgtport_put(tgtport); 1582 1583 if (found_ctrl) { 1584 nvmet_fc_schedule_delete_assoc(assoc); 1585 nvmet_fc_tgt_a_put(assoc); 1586 return; 1587 } 1588 1589 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1590 } 1591 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1592 } 1593 1594 static void 1595 nvmet_fc_free_pending_reqs(struct nvmet_fc_tgtport *tgtport) 1596 { 1597 struct nvmet_fc_ls_req_op *lsop; 1598 struct nvmefc_ls_req *lsreq; 1599 struct nvmet_fc_ls_iod *iod; 1600 int i; 1601 1602 iod = tgtport->iod; 1603 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) 1604 cancel_work(&iod->work); 1605 1606 /* 1607 * After this point the connection is lost and thus any pending 1608 * request can't be processed by the normal completion path. This 1609 * is likely a request from nvmet_fc_send_ls_req_async. 1610 */ 1611 while ((lsop = list_first_entry_or_null(&tgtport->ls_req_list, 1612 struct nvmet_fc_ls_req_op, lsreq_list))) { 1613 list_del(&lsop->lsreq_list); 1614 1615 if (!lsop->req_queued) 1616 continue; 1617 1618 lsreq = &lsop->ls_req; 1619 fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, 1620 (lsreq->rqstlen + lsreq->rsplen), 1621 DMA_BIDIRECTIONAL); 1622 nvmet_fc_tgtport_put(tgtport); 1623 kfree(lsop); 1624 } 1625 } 1626 1627 /** 1628 * nvmet_fc_unregister_targetport - transport entry point called by an 1629 * LLDD to deregister/remove a previously 1630 * registered a local NVME subsystem FC port. 1631 * @target_port: pointer to the (registered) target port that is to be 1632 * deregistered. 1633 * 1634 * Returns: 1635 * a completion status. Must be 0 upon success; a negative errno 1636 * (ex: -ENXIO) upon failure. 1637 */ 1638 int 1639 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) 1640 { 1641 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1642 unsigned long flags; 1643 1644 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1645 list_del(&tgtport->tgt_list); 1646 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1647 1648 nvmet_fc_portentry_unbind_tgt(tgtport); 1649 1650 /* terminate any outstanding associations */ 1651 __nvmet_fc_free_assocs(tgtport); 1652 1653 flush_workqueue(nvmet_wq); 1654 1655 nvmet_fc_free_pending_reqs(tgtport); 1656 nvmet_fc_tgtport_put(tgtport); 1657 1658 return 0; 1659 } 1660 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); 1661 1662 1663 /* ********************** FC-NVME LS RCV Handling ************************* */ 1664 1665 1666 static void 1667 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, 1668 struct nvmet_fc_ls_iod *iod) 1669 { 1670 struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc; 1671 struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc; 1672 struct nvmet_fc_tgt_queue *queue; 1673 int ret = 0; 1674 1675 memset(acc, 0, sizeof(*acc)); 1676 1677 /* 1678 * FC-NVME spec changes. There are initiators sending different 1679 * lengths as padding sizes for Create Association Cmd descriptor 1680 * was incorrect. 1681 * Accept anything of "minimum" length. Assume format per 1.15 1682 * spec (with HOSTID reduced to 16 bytes), ignore how long the 1683 * trailing pad length is. 1684 */ 1685 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN) 1686 ret = VERR_CR_ASSOC_LEN; 1687 else if (be32_to_cpu(rqst->desc_list_len) < 1688 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN) 1689 ret = VERR_CR_ASSOC_RQST_LEN; 1690 else if (rqst->assoc_cmd.desc_tag != 1691 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD)) 1692 ret = VERR_CR_ASSOC_CMD; 1693 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) < 1694 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN) 1695 ret = VERR_CR_ASSOC_CMD_LEN; 1696 else if (!rqst->assoc_cmd.ersp_ratio || 1697 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >= 1698 be16_to_cpu(rqst->assoc_cmd.sqsize))) 1699 ret = VERR_ERSP_RATIO; 1700 1701 else { 1702 /* new association w/ admin queue */ 1703 iod->assoc = nvmet_fc_alloc_target_assoc( 1704 tgtport, iod->hosthandle); 1705 if (!iod->assoc) 1706 ret = VERR_ASSOC_ALLOC_FAIL; 1707 else { 1708 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0, 1709 be16_to_cpu(rqst->assoc_cmd.sqsize)); 1710 if (!queue) { 1711 ret = VERR_QUEUE_ALLOC_FAIL; 1712 nvmet_fc_tgt_a_put(iod->assoc); 1713 } 1714 } 1715 } 1716 1717 if (ret) { 1718 dev_err(tgtport->dev, 1719 "Create Association LS failed: %s\n", 1720 validation_errors[ret]); 1721 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, 1722 sizeof(*acc), rqst->w0.ls_cmd, 1723 FCNVME_RJT_RC_LOGIC, 1724 FCNVME_RJT_EXP_NONE, 0); 1725 return; 1726 } 1727 1728 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio); 1729 atomic_set(&queue->connected, 1); 1730 queue->sqhd = 0; /* best place to init value */ 1731 1732 dev_info(tgtport->dev, 1733 "{%d:%d} Association created\n", 1734 tgtport->fc_target_port.port_num, iod->assoc->a_id); 1735 1736 /* format a response */ 1737 1738 iod->lsrsp->rsplen = sizeof(*acc); 1739 1740 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1741 fcnvme_lsdesc_len( 1742 sizeof(struct fcnvme_ls_cr_assoc_acc)), 1743 FCNVME_LS_CREATE_ASSOCIATION); 1744 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1745 acc->associd.desc_len = 1746 fcnvme_lsdesc_len( 1747 sizeof(struct fcnvme_lsdesc_assoc_id)); 1748 acc->associd.association_id = 1749 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0)); 1750 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1751 acc->connectid.desc_len = 1752 fcnvme_lsdesc_len( 1753 sizeof(struct fcnvme_lsdesc_conn_id)); 1754 acc->connectid.connection_id = acc->associd.association_id; 1755 } 1756 1757 static void 1758 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, 1759 struct nvmet_fc_ls_iod *iod) 1760 { 1761 struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn; 1762 struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn; 1763 struct nvmet_fc_tgt_queue *queue; 1764 int ret = 0; 1765 1766 memset(acc, 0, sizeof(*acc)); 1767 1768 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst)) 1769 ret = VERR_CR_CONN_LEN; 1770 else if (rqst->desc_list_len != 1771 fcnvme_lsdesc_len( 1772 sizeof(struct fcnvme_ls_cr_conn_rqst))) 1773 ret = VERR_CR_CONN_RQST_LEN; 1774 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1775 ret = VERR_ASSOC_ID; 1776 else if (rqst->associd.desc_len != 1777 fcnvme_lsdesc_len( 1778 sizeof(struct fcnvme_lsdesc_assoc_id))) 1779 ret = VERR_ASSOC_ID_LEN; 1780 else if (rqst->connect_cmd.desc_tag != 1781 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD)) 1782 ret = VERR_CR_CONN_CMD; 1783 else if (rqst->connect_cmd.desc_len != 1784 fcnvme_lsdesc_len( 1785 sizeof(struct fcnvme_lsdesc_cr_conn_cmd))) 1786 ret = VERR_CR_CONN_CMD_LEN; 1787 else if (!rqst->connect_cmd.ersp_ratio || 1788 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >= 1789 be16_to_cpu(rqst->connect_cmd.sqsize))) 1790 ret = VERR_ERSP_RATIO; 1791 1792 else { 1793 /* new io queue */ 1794 iod->assoc = nvmet_fc_find_target_assoc(tgtport, 1795 be64_to_cpu(rqst->associd.association_id)); 1796 if (!iod->assoc) 1797 ret = VERR_NO_ASSOC; 1798 else { 1799 queue = nvmet_fc_alloc_target_queue(iod->assoc, 1800 be16_to_cpu(rqst->connect_cmd.qid), 1801 be16_to_cpu(rqst->connect_cmd.sqsize)); 1802 if (!queue) 1803 ret = VERR_QUEUE_ALLOC_FAIL; 1804 1805 /* release get taken in nvmet_fc_find_target_assoc */ 1806 nvmet_fc_tgt_a_put(iod->assoc); 1807 } 1808 } 1809 1810 if (ret) { 1811 dev_err(tgtport->dev, 1812 "Create Connection LS failed: %s\n", 1813 validation_errors[ret]); 1814 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, 1815 sizeof(*acc), rqst->w0.ls_cmd, 1816 (ret == VERR_NO_ASSOC) ? 1817 FCNVME_RJT_RC_INV_ASSOC : 1818 FCNVME_RJT_RC_LOGIC, 1819 FCNVME_RJT_EXP_NONE, 0); 1820 return; 1821 } 1822 1823 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio); 1824 atomic_set(&queue->connected, 1); 1825 queue->sqhd = 0; /* best place to init value */ 1826 1827 /* format a response */ 1828 1829 iod->lsrsp->rsplen = sizeof(*acc); 1830 1831 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1832 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), 1833 FCNVME_LS_CREATE_CONNECTION); 1834 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1835 acc->connectid.desc_len = 1836 fcnvme_lsdesc_len( 1837 sizeof(struct fcnvme_lsdesc_conn_id)); 1838 acc->connectid.connection_id = 1839 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 1840 be16_to_cpu(rqst->connect_cmd.qid))); 1841 } 1842 1843 /* 1844 * Returns true if the LS response is to be transmit 1845 * Returns false if the LS response is to be delayed 1846 */ 1847 static int 1848 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, 1849 struct nvmet_fc_ls_iod *iod) 1850 { 1851 struct fcnvme_ls_disconnect_assoc_rqst *rqst = 1852 &iod->rqstbuf->rq_dis_assoc; 1853 struct fcnvme_ls_disconnect_assoc_acc *acc = 1854 &iod->rspbuf->rsp_dis_assoc; 1855 struct nvmet_fc_tgt_assoc *assoc = NULL; 1856 struct nvmet_fc_ls_iod *oldls = NULL; 1857 unsigned long flags; 1858 int ret = 0; 1859 1860 memset(acc, 0, sizeof(*acc)); 1861 1862 ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst); 1863 if (!ret) { 1864 /* match an active association - takes an assoc ref if !NULL */ 1865 assoc = nvmet_fc_find_target_assoc(tgtport, 1866 be64_to_cpu(rqst->associd.association_id)); 1867 iod->assoc = assoc; 1868 if (!assoc) 1869 ret = VERR_NO_ASSOC; 1870 } 1871 1872 if (ret || !assoc) { 1873 dev_err(tgtport->dev, 1874 "Disconnect LS failed: %s\n", 1875 validation_errors[ret]); 1876 iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, 1877 sizeof(*acc), rqst->w0.ls_cmd, 1878 (ret == VERR_NO_ASSOC) ? 1879 FCNVME_RJT_RC_INV_ASSOC : 1880 FCNVME_RJT_RC_LOGIC, 1881 FCNVME_RJT_EXP_NONE, 0); 1882 return true; 1883 } 1884 1885 /* format a response */ 1886 1887 iod->lsrsp->rsplen = sizeof(*acc); 1888 1889 nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1890 fcnvme_lsdesc_len( 1891 sizeof(struct fcnvme_ls_disconnect_assoc_acc)), 1892 FCNVME_LS_DISCONNECT_ASSOC); 1893 1894 /* 1895 * The rules for LS response says the response cannot 1896 * go back until ABTS's have been sent for all outstanding 1897 * I/O and a Disconnect Association LS has been sent. 1898 * So... save off the Disconnect LS to send the response 1899 * later. If there was a prior LS already saved, replace 1900 * it with the newer one and send a can't perform reject 1901 * on the older one. 1902 */ 1903 spin_lock_irqsave(&tgtport->lock, flags); 1904 oldls = assoc->rcv_disconn; 1905 assoc->rcv_disconn = iod; 1906 spin_unlock_irqrestore(&tgtport->lock, flags); 1907 1908 if (oldls) { 1909 dev_info(tgtport->dev, 1910 "{%d:%d} Multiple Disconnect Association LS's " 1911 "received\n", 1912 tgtport->fc_target_port.port_num, assoc->a_id); 1913 /* overwrite good response with bogus failure */ 1914 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf, 1915 sizeof(*iod->rspbuf), 1916 /* ok to use rqst, LS is same */ 1917 rqst->w0.ls_cmd, 1918 FCNVME_RJT_RC_UNAB, 1919 FCNVME_RJT_EXP_NONE, 0); 1920 nvmet_fc_xmt_ls_rsp(tgtport, oldls); 1921 } 1922 1923 nvmet_fc_schedule_delete_assoc(assoc); 1924 nvmet_fc_tgt_a_put(assoc); 1925 1926 return false; 1927 } 1928 1929 1930 /* *********************** NVME Ctrl Routines **************************** */ 1931 1932 1933 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req); 1934 1935 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; 1936 1937 static void 1938 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) 1939 { 1940 struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private; 1941 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 1942 1943 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, 1944 sizeof(*iod->rspbuf), DMA_TO_DEVICE); 1945 nvmet_fc_free_ls_iod(tgtport, iod); 1946 nvmet_fc_tgtport_put(tgtport); 1947 } 1948 1949 static void 1950 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, 1951 struct nvmet_fc_ls_iod *iod) 1952 { 1953 int ret; 1954 1955 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, 1956 sizeof(*iod->rspbuf), DMA_TO_DEVICE); 1957 1958 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp); 1959 if (ret) 1960 nvmet_fc_xmt_ls_rsp_done(iod->lsrsp); 1961 } 1962 1963 /* 1964 * Actual processing routine for received FC-NVME LS Requests from the LLD 1965 */ 1966 static void 1967 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, 1968 struct nvmet_fc_ls_iod *iod) 1969 { 1970 struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0; 1971 bool sendrsp = true; 1972 1973 iod->lsrsp->nvme_fc_private = iod; 1974 iod->lsrsp->rspbuf = iod->rspbuf; 1975 iod->lsrsp->rspdma = iod->rspdma; 1976 iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done; 1977 /* Be preventative. handlers will later set to valid length */ 1978 iod->lsrsp->rsplen = 0; 1979 1980 iod->assoc = NULL; 1981 1982 /* 1983 * handlers: 1984 * parse request input, execute the request, and format the 1985 * LS response 1986 */ 1987 switch (w0->ls_cmd) { 1988 case FCNVME_LS_CREATE_ASSOCIATION: 1989 /* Creates Association and initial Admin Queue/Connection */ 1990 nvmet_fc_ls_create_association(tgtport, iod); 1991 break; 1992 case FCNVME_LS_CREATE_CONNECTION: 1993 /* Creates an IO Queue/Connection */ 1994 nvmet_fc_ls_create_connection(tgtport, iod); 1995 break; 1996 case FCNVME_LS_DISCONNECT_ASSOC: 1997 /* Terminate a Queue/Connection or the Association */ 1998 sendrsp = nvmet_fc_ls_disconnect(tgtport, iod); 1999 break; 2000 default: 2001 iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf, 2002 sizeof(*iod->rspbuf), w0->ls_cmd, 2003 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); 2004 } 2005 2006 if (sendrsp) 2007 nvmet_fc_xmt_ls_rsp(tgtport, iod); 2008 } 2009 2010 /* 2011 * Actual processing routine for received FC-NVME LS Requests from the LLD 2012 */ 2013 static void 2014 nvmet_fc_handle_ls_rqst_work(struct work_struct *work) 2015 { 2016 struct nvmet_fc_ls_iod *iod = 2017 container_of(work, struct nvmet_fc_ls_iod, work); 2018 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 2019 2020 nvmet_fc_handle_ls_rqst(tgtport, iod); 2021 } 2022 2023 2024 /** 2025 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD 2026 * upon the reception of a NVME LS request. 2027 * 2028 * The nvmet-fc layer will copy payload to an internal structure for 2029 * processing. As such, upon completion of the routine, the LLDD may 2030 * immediately free/reuse the LS request buffer passed in the call. 2031 * 2032 * If this routine returns error, the LLDD should abort the exchange. 2033 * 2034 * @target_port: pointer to the (registered) target port the LS was 2035 * received on. 2036 * @hosthandle: pointer to the host specific data, gets stored in iod. 2037 * @lsrsp: pointer to a lsrsp structure to be used to reference 2038 * the exchange corresponding to the LS. 2039 * @lsreqbuf: pointer to the buffer containing the LS Request 2040 * @lsreqbuf_len: length, in bytes, of the received LS request 2041 */ 2042 int 2043 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, 2044 void *hosthandle, 2045 struct nvmefc_ls_rsp *lsrsp, 2046 void *lsreqbuf, u32 lsreqbuf_len) 2047 { 2048 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 2049 struct nvmet_fc_ls_iod *iod; 2050 struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf; 2051 2052 if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) { 2053 dev_info(tgtport->dev, 2054 "RCV %s LS failed: payload too large (%d)\n", 2055 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 2056 nvmefc_ls_names[w0->ls_cmd] : "", 2057 lsreqbuf_len); 2058 return -E2BIG; 2059 } 2060 2061 if (!nvmet_fc_tgtport_get(tgtport)) { 2062 dev_info(tgtport->dev, 2063 "RCV %s LS failed: target deleting\n", 2064 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 2065 nvmefc_ls_names[w0->ls_cmd] : ""); 2066 return -ESHUTDOWN; 2067 } 2068 2069 iod = nvmet_fc_alloc_ls_iod(tgtport); 2070 if (!iod) { 2071 dev_info(tgtport->dev, 2072 "RCV %s LS failed: context allocation failed\n", 2073 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? 2074 nvmefc_ls_names[w0->ls_cmd] : ""); 2075 nvmet_fc_tgtport_put(tgtport); 2076 return -ENOENT; 2077 } 2078 2079 iod->lsrsp = lsrsp; 2080 iod->fcpreq = NULL; 2081 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); 2082 iod->rqstdatalen = lsreqbuf_len; 2083 iod->hosthandle = hosthandle; 2084 2085 queue_work(nvmet_wq, &iod->work); 2086 2087 return 0; 2088 } 2089 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req); 2090 2091 2092 /* 2093 * ********************** 2094 * Start of FCP handling 2095 * ********************** 2096 */ 2097 2098 static int 2099 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 2100 { 2101 struct scatterlist *sg; 2102 unsigned int nent; 2103 2104 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent); 2105 if (!sg) 2106 goto out; 2107 2108 fod->data_sg = sg; 2109 fod->data_sg_cnt = nent; 2110 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent, 2111 ((fod->io_dir == NVMET_FCP_WRITE) ? 2112 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 2113 /* note: write from initiator perspective */ 2114 fod->next_sg = fod->data_sg; 2115 2116 return 0; 2117 2118 out: 2119 return NVME_SC_INTERNAL; 2120 } 2121 2122 static void 2123 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 2124 { 2125 if (!fod->data_sg || !fod->data_sg_cnt) 2126 return; 2127 2128 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt, 2129 ((fod->io_dir == NVMET_FCP_WRITE) ? 2130 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 2131 sgl_free(fod->data_sg); 2132 fod->data_sg = NULL; 2133 fod->data_sg_cnt = 0; 2134 } 2135 2136 2137 static bool 2138 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd) 2139 { 2140 u32 sqtail, used; 2141 2142 /* egad, this is ugly. And sqtail is just a best guess */ 2143 sqtail = atomic_read(&q->sqtail) % q->sqsize; 2144 2145 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd); 2146 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9))); 2147 } 2148 2149 /* 2150 * Prep RSP payload. 2151 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op 2152 */ 2153 static void 2154 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 2155 struct nvmet_fc_fcp_iod *fod) 2156 { 2157 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf; 2158 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 2159 struct nvme_completion *cqe = &ersp->cqe; 2160 u32 *cqewd = (u32 *)cqe; 2161 bool send_ersp = false; 2162 u32 rsn, rspcnt, xfr_length; 2163 2164 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP) 2165 xfr_length = fod->req.transfer_len; 2166 else 2167 xfr_length = fod->offset; 2168 2169 /* 2170 * check to see if we can send a 0's rsp. 2171 * Note: to send a 0's response, the NVME-FC host transport will 2172 * recreate the CQE. The host transport knows: sq id, SQHD (last 2173 * seen in an ersp), and command_id. Thus it will create a 2174 * zero-filled CQE with those known fields filled in. Transport 2175 * must send an ersp for any condition where the cqe won't match 2176 * this. 2177 * 2178 * Here are the FC-NVME mandated cases where we must send an ersp: 2179 * every N responses, where N=ersp_ratio 2180 * force fabric commands to send ersp's (not in FC-NVME but good 2181 * practice) 2182 * normal cmds: any time status is non-zero, or status is zero 2183 * but words 0 or 1 are non-zero. 2184 * the SQ is 90% or more full 2185 * the cmd is a fused command 2186 * transferred data length not equal to cmd iu length 2187 */ 2188 rspcnt = atomic_inc_return(&fod->queue->zrspcnt); 2189 if (!(rspcnt % fod->queue->ersp_ratio) || 2190 nvme_is_fabrics((struct nvme_command *) sqe) || 2191 xfr_length != fod->req.transfer_len || 2192 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] || 2193 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) || 2194 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head))) 2195 send_ersp = true; 2196 2197 /* re-set the fields */ 2198 fod->fcpreq->rspaddr = ersp; 2199 fod->fcpreq->rspdma = fod->rspdma; 2200 2201 if (!send_ersp) { 2202 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP); 2203 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP; 2204 } else { 2205 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32)); 2206 rsn = atomic_inc_return(&fod->queue->rsn); 2207 ersp->rsn = cpu_to_be32(rsn); 2208 ersp->xfrd_len = cpu_to_be32(xfr_length); 2209 fod->fcpreq->rsplen = sizeof(*ersp); 2210 } 2211 2212 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma, 2213 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 2214 } 2215 2216 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq); 2217 2218 static void 2219 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport, 2220 struct nvmet_fc_fcp_iod *fod) 2221 { 2222 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 2223 2224 /* data no longer needed */ 2225 nvmet_fc_free_tgt_pgs(fod); 2226 2227 /* 2228 * if an ABTS was received or we issued the fcp_abort early 2229 * don't call abort routine again. 2230 */ 2231 /* no need to take lock - lock was taken earlier to get here */ 2232 if (!fod->aborted) 2233 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq); 2234 2235 nvmet_fc_free_fcp_iod(fod->queue, fod); 2236 } 2237 2238 static void 2239 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 2240 struct nvmet_fc_fcp_iod *fod) 2241 { 2242 int ret; 2243 2244 fod->fcpreq->op = NVMET_FCOP_RSP; 2245 fod->fcpreq->timeout = 0; 2246 2247 nvmet_fc_prep_fcp_rsp(tgtport, fod); 2248 2249 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 2250 if (ret) 2251 nvmet_fc_abort_op(tgtport, fod); 2252 } 2253 2254 static void 2255 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport, 2256 struct nvmet_fc_fcp_iod *fod, u8 op) 2257 { 2258 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 2259 struct scatterlist *sg = fod->next_sg; 2260 unsigned long flags; 2261 u32 remaininglen = fod->req.transfer_len - fod->offset; 2262 u32 tlen = 0; 2263 int ret; 2264 2265 fcpreq->op = op; 2266 fcpreq->offset = fod->offset; 2267 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC; 2268 2269 /* 2270 * for next sequence: 2271 * break at a sg element boundary 2272 * attempt to keep sequence length capped at 2273 * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to 2274 * be longer if a single sg element is larger 2275 * than that amount. This is done to avoid creating 2276 * a new sg list to use for the tgtport api. 2277 */ 2278 fcpreq->sg = sg; 2279 fcpreq->sg_cnt = 0; 2280 while (tlen < remaininglen && 2281 fcpreq->sg_cnt < tgtport->max_sg_cnt && 2282 tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) { 2283 fcpreq->sg_cnt++; 2284 tlen += sg_dma_len(sg); 2285 sg = sg_next(sg); 2286 } 2287 if (tlen < remaininglen && fcpreq->sg_cnt == 0) { 2288 fcpreq->sg_cnt++; 2289 tlen += min_t(u32, sg_dma_len(sg), remaininglen); 2290 sg = sg_next(sg); 2291 } 2292 if (tlen < remaininglen) 2293 fod->next_sg = sg; 2294 else 2295 fod->next_sg = NULL; 2296 2297 fcpreq->transfer_length = tlen; 2298 fcpreq->transferred_length = 0; 2299 fcpreq->fcp_error = 0; 2300 fcpreq->rsplen = 0; 2301 2302 /* 2303 * If the last READDATA request: check if LLDD supports 2304 * combined xfr with response. 2305 */ 2306 if ((op == NVMET_FCOP_READDATA) && 2307 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) && 2308 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) { 2309 fcpreq->op = NVMET_FCOP_READDATA_RSP; 2310 nvmet_fc_prep_fcp_rsp(tgtport, fod); 2311 } 2312 2313 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 2314 if (ret) { 2315 /* 2316 * should be ok to set w/o lock as its in the thread of 2317 * execution (not an async timer routine) and doesn't 2318 * contend with any clearing action 2319 */ 2320 fod->abort = true; 2321 2322 if (op == NVMET_FCOP_WRITEDATA) { 2323 spin_lock_irqsave(&fod->flock, flags); 2324 fod->writedataactive = false; 2325 spin_unlock_irqrestore(&fod->flock, flags); 2326 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 2327 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ { 2328 fcpreq->fcp_error = ret; 2329 fcpreq->transferred_length = 0; 2330 nvmet_fc_xmt_fcp_op_done(fod->fcpreq); 2331 } 2332 } 2333 } 2334 2335 static inline bool 2336 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort) 2337 { 2338 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 2339 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2340 2341 /* if in the middle of an io and we need to tear down */ 2342 if (abort) { 2343 if (fcpreq->op == NVMET_FCOP_WRITEDATA) { 2344 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 2345 return true; 2346 } 2347 2348 nvmet_fc_abort_op(tgtport, fod); 2349 return true; 2350 } 2351 2352 return false; 2353 } 2354 2355 /* 2356 * actual done handler for FCP operations when completed by the lldd 2357 */ 2358 static void 2359 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) 2360 { 2361 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 2362 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2363 unsigned long flags; 2364 bool abort; 2365 2366 spin_lock_irqsave(&fod->flock, flags); 2367 abort = fod->abort; 2368 fod->writedataactive = false; 2369 spin_unlock_irqrestore(&fod->flock, flags); 2370 2371 switch (fcpreq->op) { 2372 2373 case NVMET_FCOP_WRITEDATA: 2374 if (__nvmet_fc_fod_op_abort(fod, abort)) 2375 return; 2376 if (fcpreq->fcp_error || 2377 fcpreq->transferred_length != fcpreq->transfer_length) { 2378 spin_lock_irqsave(&fod->flock, flags); 2379 fod->abort = true; 2380 spin_unlock_irqrestore(&fod->flock, flags); 2381 2382 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 2383 return; 2384 } 2385 2386 fod->offset += fcpreq->transferred_length; 2387 if (fod->offset != fod->req.transfer_len) { 2388 spin_lock_irqsave(&fod->flock, flags); 2389 fod->writedataactive = true; 2390 spin_unlock_irqrestore(&fod->flock, flags); 2391 2392 /* transfer the next chunk */ 2393 nvmet_fc_transfer_fcp_data(tgtport, fod, 2394 NVMET_FCOP_WRITEDATA); 2395 return; 2396 } 2397 2398 /* data transfer complete, resume with nvmet layer */ 2399 fod->req.execute(&fod->req); 2400 break; 2401 2402 case NVMET_FCOP_READDATA: 2403 case NVMET_FCOP_READDATA_RSP: 2404 if (__nvmet_fc_fod_op_abort(fod, abort)) 2405 return; 2406 if (fcpreq->fcp_error || 2407 fcpreq->transferred_length != fcpreq->transfer_length) { 2408 nvmet_fc_abort_op(tgtport, fod); 2409 return; 2410 } 2411 2412 /* success */ 2413 2414 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) { 2415 /* data no longer needed */ 2416 nvmet_fc_free_tgt_pgs(fod); 2417 nvmet_fc_free_fcp_iod(fod->queue, fod); 2418 return; 2419 } 2420 2421 fod->offset += fcpreq->transferred_length; 2422 if (fod->offset != fod->req.transfer_len) { 2423 /* transfer the next chunk */ 2424 nvmet_fc_transfer_fcp_data(tgtport, fod, 2425 NVMET_FCOP_READDATA); 2426 return; 2427 } 2428 2429 /* data transfer complete, send response */ 2430 2431 /* data no longer needed */ 2432 nvmet_fc_free_tgt_pgs(fod); 2433 2434 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2435 2436 break; 2437 2438 case NVMET_FCOP_RSP: 2439 if (__nvmet_fc_fod_op_abort(fod, abort)) 2440 return; 2441 nvmet_fc_free_fcp_iod(fod->queue, fod); 2442 break; 2443 2444 default: 2445 break; 2446 } 2447 } 2448 2449 static void 2450 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq) 2451 { 2452 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2453 2454 nvmet_fc_fod_op_done(fod); 2455 } 2456 2457 /* 2458 * actual completion handler after execution by the nvmet layer 2459 */ 2460 static void 2461 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport, 2462 struct nvmet_fc_fcp_iod *fod, int status) 2463 { 2464 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 2465 struct nvme_completion *cqe = &fod->rspiubuf.cqe; 2466 unsigned long flags; 2467 bool abort; 2468 2469 spin_lock_irqsave(&fod->flock, flags); 2470 abort = fod->abort; 2471 spin_unlock_irqrestore(&fod->flock, flags); 2472 2473 /* if we have a CQE, snoop the last sq_head value */ 2474 if (!status) 2475 fod->queue->sqhd = cqe->sq_head; 2476 2477 if (abort) { 2478 nvmet_fc_abort_op(tgtport, fod); 2479 return; 2480 } 2481 2482 /* if an error handling the cmd post initial parsing */ 2483 if (status) { 2484 /* fudge up a failed CQE status for our transport error */ 2485 memset(cqe, 0, sizeof(*cqe)); 2486 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */ 2487 cqe->sq_id = cpu_to_le16(fod->queue->qid); 2488 cqe->command_id = sqe->command_id; 2489 cqe->status = cpu_to_le16(status); 2490 } else { 2491 2492 /* 2493 * try to push the data even if the SQE status is non-zero. 2494 * There may be a status where data still was intended to 2495 * be moved 2496 */ 2497 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) { 2498 /* push the data over before sending rsp */ 2499 nvmet_fc_transfer_fcp_data(tgtport, fod, 2500 NVMET_FCOP_READDATA); 2501 return; 2502 } 2503 2504 /* writes & no data - fall thru */ 2505 } 2506 2507 /* data no longer needed */ 2508 nvmet_fc_free_tgt_pgs(fod); 2509 2510 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2511 } 2512 2513 2514 static void 2515 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req) 2516 { 2517 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req); 2518 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2519 2520 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0); 2521 } 2522 2523 2524 /* 2525 * Actual processing routine for received FC-NVME I/O Requests from the LLD 2526 */ 2527 static void 2528 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 2529 struct nvmet_fc_fcp_iod *fod) 2530 { 2531 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf; 2532 u32 xfrlen = be32_to_cpu(cmdiu->data_len); 2533 int ret; 2534 2535 /* 2536 * Fused commands are currently not supported in the linux 2537 * implementation. 2538 * 2539 * As such, the implementation of the FC transport does not 2540 * look at the fused commands and order delivery to the upper 2541 * layer until we have both based on csn. 2542 */ 2543 2544 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done; 2545 2546 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) { 2547 fod->io_dir = NVMET_FCP_WRITE; 2548 if (!nvme_is_write(&cmdiu->sqe)) 2549 goto transport_error; 2550 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) { 2551 fod->io_dir = NVMET_FCP_READ; 2552 if (nvme_is_write(&cmdiu->sqe)) 2553 goto transport_error; 2554 } else { 2555 fod->io_dir = NVMET_FCP_NODATA; 2556 if (xfrlen) 2557 goto transport_error; 2558 } 2559 2560 fod->req.cmd = &fod->cmdiubuf.sqe; 2561 fod->req.cqe = &fod->rspiubuf.cqe; 2562 if (!tgtport->pe) 2563 goto transport_error; 2564 fod->req.port = tgtport->pe->port; 2565 2566 /* clear any response payload */ 2567 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf)); 2568 2569 fod->data_sg = NULL; 2570 fod->data_sg_cnt = 0; 2571 2572 ret = nvmet_req_init(&fod->req, &fod->queue->nvme_sq, 2573 &nvmet_fc_tgt_fcp_ops); 2574 if (!ret) { 2575 /* bad SQE content or invalid ctrl state */ 2576 /* nvmet layer has already called op done to send rsp. */ 2577 return; 2578 } 2579 2580 fod->req.transfer_len = xfrlen; 2581 2582 /* keep a running counter of tail position */ 2583 atomic_inc(&fod->queue->sqtail); 2584 2585 if (fod->req.transfer_len) { 2586 ret = nvmet_fc_alloc_tgt_pgs(fod); 2587 if (ret) { 2588 nvmet_req_complete(&fod->req, ret); 2589 return; 2590 } 2591 } 2592 fod->req.sg = fod->data_sg; 2593 fod->req.sg_cnt = fod->data_sg_cnt; 2594 fod->offset = 0; 2595 2596 if (fod->io_dir == NVMET_FCP_WRITE) { 2597 /* pull the data over before invoking nvmet layer */ 2598 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA); 2599 return; 2600 } 2601 2602 /* 2603 * Reads or no data: 2604 * 2605 * can invoke the nvmet_layer now. If read data, cmd completion will 2606 * push the data 2607 */ 2608 fod->req.execute(&fod->req); 2609 return; 2610 2611 transport_error: 2612 nvmet_fc_abort_op(tgtport, fod); 2613 } 2614 2615 /** 2616 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD 2617 * upon the reception of a NVME FCP CMD IU. 2618 * 2619 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc 2620 * layer for processing. 2621 * 2622 * The nvmet_fc layer allocates a local job structure (struct 2623 * nvmet_fc_fcp_iod) from the queue for the io and copies the 2624 * CMD IU buffer to the job structure. As such, on a successful 2625 * completion (returns 0), the LLDD may immediately free/reuse 2626 * the CMD IU buffer passed in the call. 2627 * 2628 * However, in some circumstances, due to the packetized nature of FC 2629 * and the api of the FC LLDD which may issue a hw command to send the 2630 * response, but the LLDD may not get the hw completion for that command 2631 * and upcall the nvmet_fc layer before a new command may be 2632 * asynchronously received - its possible for a command to be received 2633 * before the LLDD and nvmet_fc have recycled the job structure. It gives 2634 * the appearance of more commands received than fits in the sq. 2635 * To alleviate this scenario, a temporary queue is maintained in the 2636 * transport for pending LLDD requests waiting for a queue job structure. 2637 * In these "overrun" cases, a temporary queue element is allocated 2638 * the LLDD request and CMD iu buffer information remembered, and the 2639 * routine returns a -EOVERFLOW status. Subsequently, when a queue job 2640 * structure is freed, it is immediately reallocated for anything on the 2641 * pending request list. The LLDDs defer_rcv() callback is called, 2642 * informing the LLDD that it may reuse the CMD IU buffer, and the io 2643 * is then started normally with the transport. 2644 * 2645 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat 2646 * the completion as successful but must not reuse the CMD IU buffer 2647 * until the LLDD's defer_rcv() callback has been called for the 2648 * corresponding struct nvmefc_tgt_fcp_req pointer. 2649 * 2650 * If there is any other condition in which an error occurs, the 2651 * transport will return a non-zero status indicating the error. 2652 * In all cases other than -EOVERFLOW, the transport has not accepted the 2653 * request and the LLDD should abort the exchange. 2654 * 2655 * @target_port: pointer to the (registered) target port the FCP CMD IU 2656 * was received on. 2657 * @fcpreq: pointer to a fcpreq request structure to be used to reference 2658 * the exchange corresponding to the FCP Exchange. 2659 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU 2660 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU 2661 */ 2662 int 2663 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port, 2664 struct nvmefc_tgt_fcp_req *fcpreq, 2665 void *cmdiubuf, u32 cmdiubuf_len) 2666 { 2667 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 2668 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf; 2669 struct nvmet_fc_tgt_queue *queue; 2670 struct nvmet_fc_fcp_iod *fod; 2671 struct nvmet_fc_defer_fcp_req *deferfcp; 2672 unsigned long flags; 2673 2674 /* validate iu, so the connection id can be used to find the queue */ 2675 if ((cmdiubuf_len != sizeof(*cmdiu)) || 2676 (cmdiu->format_id != NVME_CMD_FORMAT_ID) || 2677 (cmdiu->fc_id != NVME_CMD_FC_ID) || 2678 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4))) 2679 return -EIO; 2680 2681 queue = nvmet_fc_find_target_queue(tgtport, 2682 be64_to_cpu(cmdiu->connection_id)); 2683 if (!queue) 2684 return -ENOTCONN; 2685 2686 /* 2687 * note: reference taken by find_target_queue 2688 * After successful fod allocation, the fod will inherit the 2689 * ownership of that reference and will remove the reference 2690 * when the fod is freed. 2691 */ 2692 2693 spin_lock_irqsave(&queue->qlock, flags); 2694 2695 fod = nvmet_fc_alloc_fcp_iod(queue); 2696 if (fod) { 2697 spin_unlock_irqrestore(&queue->qlock, flags); 2698 2699 fcpreq->nvmet_fc_private = fod; 2700 fod->fcpreq = fcpreq; 2701 2702 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len); 2703 2704 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); 2705 2706 return 0; 2707 } 2708 2709 if (!tgtport->ops->defer_rcv) { 2710 spin_unlock_irqrestore(&queue->qlock, flags); 2711 /* release the queue lookup reference */ 2712 nvmet_fc_tgt_q_put(queue); 2713 return -ENOENT; 2714 } 2715 2716 deferfcp = list_first_entry_or_null(&queue->avail_defer_list, 2717 struct nvmet_fc_defer_fcp_req, req_list); 2718 if (deferfcp) { 2719 /* Just re-use one that was previously allocated */ 2720 list_del(&deferfcp->req_list); 2721 } else { 2722 spin_unlock_irqrestore(&queue->qlock, flags); 2723 2724 /* Now we need to dynamically allocate one */ 2725 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL); 2726 if (!deferfcp) { 2727 /* release the queue lookup reference */ 2728 nvmet_fc_tgt_q_put(queue); 2729 return -ENOMEM; 2730 } 2731 spin_lock_irqsave(&queue->qlock, flags); 2732 } 2733 2734 /* For now, use rspaddr / rsplen to save payload information */ 2735 fcpreq->rspaddr = cmdiubuf; 2736 fcpreq->rsplen = cmdiubuf_len; 2737 deferfcp->fcp_req = fcpreq; 2738 2739 /* defer processing till a fod becomes available */ 2740 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list); 2741 2742 /* NOTE: the queue lookup reference is still valid */ 2743 2744 spin_unlock_irqrestore(&queue->qlock, flags); 2745 2746 return -EOVERFLOW; 2747 } 2748 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req); 2749 2750 /** 2751 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD 2752 * upon the reception of an ABTS for a FCP command 2753 * 2754 * Notify the transport that an ABTS has been received for a FCP command 2755 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The 2756 * LLDD believes the command is still being worked on 2757 * (template_ops->fcp_req_release() has not been called). 2758 * 2759 * The transport will wait for any outstanding work (an op to the LLDD, 2760 * which the lldd should complete with error due to the ABTS; or the 2761 * completion from the nvmet layer of the nvme command), then will 2762 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to 2763 * return the i/o context to the LLDD. The LLDD may send the BA_ACC 2764 * to the ABTS either after return from this function (assuming any 2765 * outstanding op work has been terminated) or upon the callback being 2766 * called. 2767 * 2768 * @target_port: pointer to the (registered) target port the FCP CMD IU 2769 * was received on. 2770 * @fcpreq: pointer to the fcpreq request structure that corresponds 2771 * to the exchange that received the ABTS. 2772 */ 2773 void 2774 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port, 2775 struct nvmefc_tgt_fcp_req *fcpreq) 2776 { 2777 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2778 struct nvmet_fc_tgt_queue *queue; 2779 unsigned long flags; 2780 2781 if (!fod || fod->fcpreq != fcpreq) 2782 /* job appears to have already completed, ignore abort */ 2783 return; 2784 2785 queue = fod->queue; 2786 2787 spin_lock_irqsave(&queue->qlock, flags); 2788 if (fod->active) { 2789 /* 2790 * mark as abort. The abort handler, invoked upon completion 2791 * of any work, will detect the aborted status and do the 2792 * callback. 2793 */ 2794 spin_lock(&fod->flock); 2795 fod->abort = true; 2796 fod->aborted = true; 2797 spin_unlock(&fod->flock); 2798 } 2799 spin_unlock_irqrestore(&queue->qlock, flags); 2800 } 2801 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort); 2802 2803 2804 struct nvmet_fc_traddr { 2805 u64 nn; 2806 u64 pn; 2807 }; 2808 2809 static int 2810 __nvme_fc_parse_u64(substring_t *sstr, u64 *val) 2811 { 2812 u64 token64; 2813 2814 if (match_u64(sstr, &token64)) 2815 return -EINVAL; 2816 *val = token64; 2817 2818 return 0; 2819 } 2820 2821 /* 2822 * This routine validates and extracts the WWN's from the TRADDR string. 2823 * As kernel parsers need the 0x to determine number base, universally 2824 * build string to parse with 0x prefix before parsing name strings. 2825 */ 2826 static int 2827 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) 2828 { 2829 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; 2830 substring_t wwn = { name, &name[sizeof(name)-1] }; 2831 int nnoffset, pnoffset; 2832 2833 /* validate if string is one of the 2 allowed formats */ 2834 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && 2835 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && 2836 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], 2837 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { 2838 nnoffset = NVME_FC_TRADDR_OXNNLEN; 2839 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + 2840 NVME_FC_TRADDR_OXNNLEN; 2841 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && 2842 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && 2843 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], 2844 "pn-", NVME_FC_TRADDR_NNLEN))) { 2845 nnoffset = NVME_FC_TRADDR_NNLEN; 2846 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; 2847 } else 2848 goto out_einval; 2849 2850 name[0] = '0'; 2851 name[1] = 'x'; 2852 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; 2853 2854 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2855 if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) 2856 goto out_einval; 2857 2858 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2859 if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) 2860 goto out_einval; 2861 2862 return 0; 2863 2864 out_einval: 2865 pr_warn("%s: bad traddr string\n", __func__); 2866 return -EINVAL; 2867 } 2868 2869 static int 2870 nvmet_fc_add_port(struct nvmet_port *port) 2871 { 2872 struct nvmet_fc_tgtport *tgtport; 2873 struct nvmet_fc_port_entry *pe; 2874 struct nvmet_fc_traddr traddr = { 0L, 0L }; 2875 unsigned long flags; 2876 int ret; 2877 2878 /* validate the address info */ 2879 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) || 2880 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC)) 2881 return -EINVAL; 2882 2883 /* map the traddr address info to a target port */ 2884 2885 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr, 2886 sizeof(port->disc_addr.traddr)); 2887 if (ret) 2888 return ret; 2889 2890 pe = kzalloc(sizeof(*pe), GFP_KERNEL); 2891 if (!pe) 2892 return -ENOMEM; 2893 2894 ret = -ENXIO; 2895 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2896 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) { 2897 if ((tgtport->fc_target_port.node_name == traddr.nn) && 2898 (tgtport->fc_target_port.port_name == traddr.pn)) { 2899 if (!nvmet_fc_tgtport_get(tgtport)) 2900 continue; 2901 2902 /* a FC port can only be 1 nvmet port id */ 2903 if (!tgtport->pe) { 2904 nvmet_fc_portentry_bind(tgtport, pe, port); 2905 ret = 0; 2906 } else 2907 ret = -EALREADY; 2908 2909 nvmet_fc_tgtport_put(tgtport); 2910 break; 2911 } 2912 } 2913 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2914 2915 if (ret) 2916 kfree(pe); 2917 2918 return ret; 2919 } 2920 2921 static void 2922 nvmet_fc_remove_port(struct nvmet_port *port) 2923 { 2924 struct nvmet_fc_port_entry *pe = port->priv; 2925 struct nvmet_fc_tgtport *tgtport = NULL; 2926 unsigned long flags; 2927 2928 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2929 if (pe->tgtport && nvmet_fc_tgtport_get(pe->tgtport)) 2930 tgtport = pe->tgtport; 2931 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2932 2933 nvmet_fc_portentry_unbind(pe); 2934 2935 if (tgtport) { 2936 /* terminate any outstanding associations */ 2937 __nvmet_fc_free_assocs(tgtport); 2938 nvmet_fc_tgtport_put(tgtport); 2939 } 2940 2941 kfree(pe); 2942 } 2943 2944 static void 2945 nvmet_fc_discovery_chg(struct nvmet_port *port) 2946 { 2947 struct nvmet_fc_port_entry *pe = port->priv; 2948 struct nvmet_fc_tgtport *tgtport = NULL; 2949 unsigned long flags; 2950 2951 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2952 if (pe->tgtport && nvmet_fc_tgtport_get(pe->tgtport)) 2953 tgtport = pe->tgtport; 2954 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2955 2956 if (!tgtport) 2957 return; 2958 2959 if (tgtport && tgtport->ops->discovery_event) 2960 tgtport->ops->discovery_event(&tgtport->fc_target_port); 2961 2962 nvmet_fc_tgtport_put(tgtport); 2963 } 2964 2965 static ssize_t 2966 nvmet_fc_host_traddr(struct nvmet_ctrl *ctrl, 2967 char *traddr, size_t traddr_size) 2968 { 2969 struct nvmet_sq *sq = ctrl->sqs[0]; 2970 struct nvmet_fc_tgt_queue *queue = 2971 container_of(sq, struct nvmet_fc_tgt_queue, nvme_sq); 2972 struct nvmet_fc_tgtport *tgtport = queue->assoc ? queue->assoc->tgtport : NULL; 2973 struct nvmet_fc_hostport *hostport = queue->assoc ? queue->assoc->hostport : NULL; 2974 u64 wwnn, wwpn; 2975 ssize_t ret = 0; 2976 2977 if (!tgtport || !nvmet_fc_tgtport_get(tgtport)) 2978 return -ENODEV; 2979 if (!hostport || !nvmet_fc_hostport_get(hostport)) { 2980 ret = -ENODEV; 2981 goto out_put; 2982 } 2983 2984 if (tgtport->ops->host_traddr) { 2985 ret = tgtport->ops->host_traddr(hostport->hosthandle, &wwnn, &wwpn); 2986 if (ret) 2987 goto out_put_host; 2988 ret = snprintf(traddr, traddr_size, "nn-0x%llx:pn-0x%llx", wwnn, wwpn); 2989 } 2990 out_put_host: 2991 nvmet_fc_hostport_put(hostport); 2992 out_put: 2993 nvmet_fc_tgtport_put(tgtport); 2994 return ret; 2995 } 2996 2997 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { 2998 .owner = THIS_MODULE, 2999 .type = NVMF_TRTYPE_FC, 3000 .msdbd = 1, 3001 .add_port = nvmet_fc_add_port, 3002 .remove_port = nvmet_fc_remove_port, 3003 .queue_response = nvmet_fc_fcp_nvme_cmd_done, 3004 .delete_ctrl = nvmet_fc_delete_ctrl, 3005 .discovery_chg = nvmet_fc_discovery_chg, 3006 .host_traddr = nvmet_fc_host_traddr, 3007 }; 3008 3009 static int __init nvmet_fc_init_module(void) 3010 { 3011 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops); 3012 } 3013 3014 static void __exit nvmet_fc_exit_module(void) 3015 { 3016 /* ensure any shutdown operation, e.g. delete ctrls have finished */ 3017 flush_workqueue(nvmet_wq); 3018 3019 /* sanity check - all lports should be removed */ 3020 if (!list_empty(&nvmet_fc_target_list)) 3021 pr_warn("%s: targetport list not empty\n", __func__); 3022 3023 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops); 3024 3025 ida_destroy(&nvmet_fc_tgtport_cnt); 3026 } 3027 3028 module_init(nvmet_fc_init_module); 3029 module_exit(nvmet_fc_exit_module); 3030 3031 MODULE_DESCRIPTION("NVMe target FC transport driver"); 3032 MODULE_LICENSE("GPL v2"); 3033