1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2008 Cisco Systems, Inc. All rights reserved. 4 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 5 */ 6 #include <linux/mempool.h> 7 #include <linux/errno.h> 8 #include <linux/init.h> 9 #include <linux/workqueue.h> 10 #include <linux/pci.h> 11 #include <linux/scatterlist.h> 12 #include <linux/skbuff.h> 13 #include <linux/spinlock.h> 14 #include <linux/kthread.h> 15 #include <linux/if_ether.h> 16 #include <linux/if_vlan.h> 17 #include <linux/delay.h> 18 #include <linux/gfp.h> 19 #include <linux/nvme.h> 20 #include <linux/nvme-fc.h> 21 #include "fnic.h" 22 #include "fnic_trace.h" 23 #include "fdls_fc.h" 24 25 #if IS_REACHABLE(CONFIG_NVME_FC) 26 27 static bool nvfnic_ls_req_cleanup(struct fnic_iport_s *iport, 28 struct nvmefc_ls_req *lsreq, 29 uint16_t oxid); 30 31 int nvfnic_get_sg_count(struct fnic_io_req *io_req) 32 { 33 return io_req->fcp_req->sg_cnt; 34 } 35 36 int nvfnic_dma_map_sgl(struct fnic *fnic, struct fnic_io_req *io_req, 37 int sg_count) 38 { 39 io_req->sgl_list_pa = dma_map_single(&fnic->pdev->dev, 40 io_req->sgl_list, 41 sizeof(io_req->sgl_list[0]) * 42 sg_count, DMA_TO_DEVICE); 43 if (dma_mapping_error(&fnic->pdev->dev, io_req->sgl_list_pa)) { 44 dev_err(&fnic->pdev->dev, "DMA mapping failed\n"); 45 io_req->sgl_list_pa = 0; 46 io_req->sgl_mapped = 0; 47 return -EBUSY; 48 } 49 50 io_req->sgl_mapped = 1; 51 return 0; 52 } 53 54 void nvfnic_dma_unmap_sgl(struct fnic *fnic, struct fnic_io_req *io_req) 55 { 56 if (io_req->sgl_mapped) { 57 dma_unmap_single(&fnic->pdev->dev, io_req->sgl_list_pa, 58 sizeof(io_req->sgl_list[0]) * io_req->sgl_cnt, 59 DMA_TO_DEVICE); 60 io_req->sgl_mapped = 0; 61 io_req->sgl_list_pa = 0; 62 } 63 } 64 65 static void nvfnic_update_io_bytes(struct fnic *fnic, 66 struct fnic_io_req *io_req, u8 opcode) 67 { 68 if (opcode == nvme_cmd_read) 69 fnic->fcp_input_bytes += io_req->fcp_req->transferred_length; 70 else if (opcode == nvme_cmd_write) 71 fnic->fcp_output_bytes += io_req->fcp_req->transferred_length; 72 } 73 74 static void nvfnic_update_io_stats(struct fnic *fnic, u8 opcode) 75 { 76 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 77 78 switch (opcode) { 79 case nvme_cmd_read: 80 atomic64_inc(&fnic_stats->nvme_stats.nvme_input_requests); 81 break; 82 case nvme_cmd_write: 83 atomic64_inc(&fnic_stats->nvme_stats.nvme_output_requests); 84 break; 85 default: 86 atomic64_inc(&fnic_stats->nvme_stats.nvme_control_requests); 87 break; 88 } 89 } 90 91 static void nvfnic_update_cmpl_stats(struct fnic *fnic, 92 struct fnic_io_req *io_req) 93 { 94 struct io_path_stats *io_stats = &fnic->fnic_stats.io_stats; 95 atomic64_t *duration_stat; 96 unsigned long io_duration_time; 97 98 atomic64_dec(&io_stats->active_ios); 99 if (atomic64_read(&fnic->io_cmpl_skip)) 100 atomic64_dec(&fnic->io_cmpl_skip); 101 else 102 atomic64_inc(&io_stats->io_completions); 103 104 io_duration_time = jiffies_to_msecs(jiffies - io_req->start_time); 105 106 if (io_duration_time <= 10) 107 duration_stat = &io_stats->io_btw_0_to_10_msec; 108 else if (io_duration_time <= 100) 109 duration_stat = &io_stats->io_btw_10_to_100_msec; 110 else if (io_duration_time <= 500) 111 duration_stat = &io_stats->io_btw_100_to_500_msec; 112 else if (io_duration_time <= 5000) 113 duration_stat = &io_stats->io_btw_500_to_5000_msec; 114 else if (io_duration_time <= 10000) 115 duration_stat = &io_stats->io_btw_5000_to_10000_msec; 116 else if (io_duration_time <= 30000) 117 duration_stat = &io_stats->io_btw_10000_to_30000_msec; 118 else { 119 duration_stat = &io_stats->io_greater_than_30000_msec; 120 if (io_duration_time > 121 atomic64_read(&io_stats->current_max_io_time)) 122 atomic64_set(&io_stats->current_max_io_time, 123 io_duration_time); 124 } 125 126 atomic64_inc(duration_stat); 127 } 128 129 int 130 nvfnic_alloc_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req) 131 { 132 struct fnic *fnic = iport->fnic; 133 int tag; 134 135 tag = sbitmap_get(&fnic->nvfnic_tag_map); 136 if (tag >= 0) { 137 WRITE_ONCE(io_req->tag, tag); 138 fnic->sw_copy_wq[0].io_req_table[tag] = io_req; 139 return tag; 140 } 141 return FNIC_NVME_NO_FREE_TAG; 142 } 143 144 void 145 nvfnic_free_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req) 146 { 147 struct fnic *fnic = iport->fnic; 148 uint16_t tag = io_req->tag; 149 150 if (tag == FNIC_NVME_NO_FREE_TAG) { 151 FNIC_NVME_DBG(KERN_ERR, fnic, 152 "Attempting to free invalid tag: 0x%x\n", tag); 153 return; 154 } 155 fnic->sw_copy_wq[0].io_req_table[tag] = NULL; 156 WRITE_ONCE(io_req->tag, FNIC_NVME_NO_FREE_TAG); 157 sbitmap_clear_bit(&fnic->nvfnic_tag_map, tag); 158 } 159 160 void 161 nvfnic_reset_fcpio_tag_pool(struct fnic_iport_s *iport) 162 { 163 WARN_ON(sbitmap_weight(&iport->fnic->nvfnic_tag_map)); 164 } 165 166 struct fnic_io_req * 167 nvfnic_find_io_req_by_tag(struct fnic *fnic, uint16_t tag) 168 { 169 if (tag == FNIC_NVME_NO_FREE_TAG || 170 !sbitmap_test_bit(&fnic->nvfnic_tag_map, tag)) 171 return NULL; 172 return fnic->sw_copy_wq[0].io_req_table[tag]; 173 } 174 175 /* 176 * Unmap the data buffer and sense buffer for an io_req, 177 * also unmap and free the device-private scatter/gather list. 178 */ 179 void nvfnic_release_nvme_ioreq_buf(struct fnic_iport_s *iport, 180 struct fnic_io_req *io_req) 181 { 182 struct fnic *fnic = iport->fnic; 183 184 nvfnic_dma_unmap_sgl(fnic, io_req); 185 186 if (io_req->sgl_cnt) 187 mempool_free(io_req->sgl_list_alloc, 188 fnic->io_sgl_pool[io_req->sgl_type]); 189 } 190 191 int nvfnic_get_nvmef_info(struct fnic *fnic, struct fnic_nvmef_info *info) 192 { 193 int len = 0; 194 struct fnic_iport_s *iport = &fnic->iport; 195 int buf_size = info->buf_size; 196 struct fnic_tport_s *tport; 197 struct fnic_tport_s *next; 198 unsigned long flags; 199 200 if (buf_size <= 0) 201 return 0; 202 203 len += scnprintf(info->info_buffer + len, buf_size - len, 204 "lport wwpn 0x%llx wwnn 0x%llx fcid 0x%06x\n", 205 iport->wwpn, iport->wwnn, iport->fcid); 206 207 spin_lock_irqsave(&fnic->fnic_lock, flags); 208 list_for_each_entry_safe(tport, next, &iport->tport_list, links) { 209 if (len >= buf_size - 1) 210 break; 211 212 len += scnprintf(info->info_buffer + len, buf_size - len, 213 "tport wwpn 0x%llx wwnn 0x%llx fcid 0x%06x\n", 214 tport->wwpn, tport->wwnn, tport->fcid); 215 } 216 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 217 218 return len; 219 } 220 221 inline int nvfnic_queue_wq_nvme_copy_desc(struct fnic *fnic, 222 struct vnic_wq_copy *wq, 223 struct fnic_io_req *io_req, 224 int sg_count) 225 { 226 struct scatterlist *sg; 227 struct fnic_tport_s *tport = io_req->tport; 228 struct host_sg_desc *desc; 229 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; 230 unsigned int i; 231 unsigned long intr_flags; 232 int flags; 233 u8 exch_flags; 234 struct scatterlist *sgl; 235 int idx; 236 int ret = 0; 237 238 if (sg_count) { 239 /* For each SGE, create a device desc entry */ 240 desc = io_req->sgl_list; 241 sgl = io_req->fcp_req->first_sgl; 242 for_each_sg(sgl, sg, sg_count, i) { 243 desc->addr = cpu_to_le64(sg_dma_address(sg)); 244 desc->len = cpu_to_le32(sg_dma_len(sg)); 245 desc->_resvd = 0; 246 desc++; 247 } 248 249 ret = nvfnic_dma_map_sgl(fnic, io_req, sg_count); 250 if (ret) 251 return ret; 252 } 253 254 idx = (struct vnic_wq_copy *)wq - &fnic->hw_copy_wq[0]; 255 256 /* Enqueue the descriptor in the Copy WQ */ 257 spin_lock_irqsave(&fnic->wq_copy_lock[idx], intr_flags); 258 259 if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[idx]) 260 free_wq_copy_descs(fnic, wq, idx); 261 262 if (unlikely(!vnic_wq_copy_desc_avail(wq))) { 263 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags); 264 FNIC_NVME_DBG(KERN_ERR, fnic, 265 "Enqueue failure: No descriptors\n"); 266 atomic64_inc(&misc_stats->io_cpwq_alloc_failures); 267 return -EBUSY; 268 } 269 270 flags = 0; 271 if (io_req->fcp_req->io_dir == NVMEFC_FCP_READ) 272 flags = FCPIO_ICMND_RDDATA; 273 else if (io_req->fcp_req->io_dir == NVMEFC_FCP_WRITE) 274 flags = FCPIO_ICMND_WRDATA; 275 276 exch_flags = 0; 277 278 fnic_queue_wq_copy_desc_nvme_io(wq, io_req->tag, 279 exch_flags, io_req->sgl_cnt, 280 io_req->sgl_list_pa, flags, 281 io_req->fcp_req->cmdaddr, 282 io_req->fcp_req->cmdlen, 283 io_req->fcp_req->payload_length, 284 io_req->port_id, 285 tport->max_payload_size, tport->r_a_tov, 286 tport->e_d_tov); 287 288 atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); 289 if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) > 290 atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs)) 291 atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, 292 atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); 293 294 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags); 295 return 0; 296 } 297 298 bool 299 nvfnic_transport_ready(struct fnic_iport_s *iport, struct fnic_tport_s *tport) 300 { 301 struct fnic *fnic = iport->fnic; 302 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 303 304 if (tport == NULL) 305 return false; 306 307 if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN || 308 iport->state != FNIC_IPORT_STATE_READY) { 309 atomic64_inc(&fnic_stats->misc_stats.iport_not_ready); 310 return false; 311 } 312 313 if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) 314 return false; 315 316 if (fdls_tport_is_offline(tport)) { 317 atomic64_inc(&fnic_stats->misc_stats.tport_not_ready); 318 return false; 319 } 320 321 return true; 322 } 323 324 int nvfnic_queuecommand(struct fnic_io_req *io_req) 325 { 326 struct fnic_iport_s *iport = io_req->iport; 327 struct fnic *fnic = iport->fnic; 328 struct fnic_tport_s *tport = io_req->tport; 329 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 330 struct vnic_wq_copy *wq = io_req->wq; 331 int ret = 0; 332 int sg_count = 0; 333 unsigned long ptr; 334 unsigned char *lba; 335 u64 cmd_trace; 336 int idx; 337 struct nvme_fc_cmd_iu *cmdiu = io_req->fcp_req->cmdaddr; 338 339 io_req->cmd_state = FNIC_IOREQ_NOT_INITED; 340 io_req->cmd_flags = FNIC_NO_FLAGS; 341 /* Map the data buffer */ 342 sg_count = nvfnic_get_sg_count(io_req); 343 if (sg_count < 0) { 344 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num, 345 io_req->tag, io_req, 0, io_req->fcp_req->io_dir, 346 sg_count, io_req->cmd_state); 347 FNIC_NVME_DBG(KERN_INFO, fnic, "sg count is less-than-zero\n"); 348 ret = -1; 349 goto out; 350 } 351 352 /* Determine the type of scatter/gather list we need */ 353 io_req->sgl_cnt = sg_count; 354 io_req->sgl_type = FNIC_SGL_CACHE_DFLT; 355 if (sg_count > FNIC_DFLT_SG_DESC_CNT) 356 io_req->sgl_type = FNIC_SGL_CACHE_MAX; 357 358 if (sg_count) { 359 io_req->sgl_list = 360 mempool_alloc(fnic->io_sgl_pool[io_req->sgl_type], 361 GFP_ATOMIC); 362 if (!io_req->sgl_list) { 363 atomic64_inc(&fnic_stats->io_stats.alloc_failures); 364 FNIC_NVME_DBG(KERN_INFO, fnic, 365 "Unable to alloc SGLs\n"); 366 ret = -ENOMEM; 367 goto out; 368 } 369 370 /* Cache sgl list allocated address before alignment */ 371 io_req->sgl_list_alloc = io_req->sgl_list; 372 ptr = (unsigned long)io_req->sgl_list; 373 if (ptr % FNIC_SG_DESC_ALIGN) { 374 io_req->sgl_list = (struct host_sg_desc *) 375 (((unsigned long)ptr + FNIC_SG_DESC_ALIGN - 1) 376 & ~(FNIC_SG_DESC_ALIGN - 1)); 377 } 378 } 379 380 io_req->port_id = tport->fcid; 381 io_req->start_time = jiffies; 382 io_req->cmd_state = FNIC_IOREQ_CMD_PENDING; 383 io_req->cmd_flags = FNIC_IO_INITIALIZED; 384 385 /* create copy wq desc and enqueue it */ 386 idx = wq - &fnic->hw_copy_wq[0]; 387 atomic64_inc(&fnic_stats->io_stats.ios[idx]); 388 ret = nvfnic_queue_wq_nvme_copy_desc(fnic, io_req->wq, io_req, sg_count); 389 if (ret) { 390 FNIC_NVME_DBG(KERN_ERR, fnic, "Unable to queue frame\n"); 391 /* 392 * In case another thread cancelled the request, 393 * refetch the pointer under the lock. 394 */ 395 nvfnic_release_nvme_ioreq_buf(iport, io_req); 396 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num, 397 io_req->tag, io_req->fcp_req, 0, 0, 0, 398 (((u64)io_req->cmd_flags << 32) | io_req->cmd_state)); 399 return ret; 400 } 401 402 atomic64_inc(&fnic_stats->io_stats.active_ios); 403 atomic64_inc(&fnic_stats->io_stats.num_ios); 404 if (atomic64_read(&fnic_stats->io_stats.active_ios) > 405 atomic64_read(&fnic_stats->io_stats.max_active_ios)) 406 atomic64_set(&fnic_stats->io_stats.max_active_ios, 407 atomic64_read(&fnic_stats->io_stats.active_ios)); 408 io_req->cmd_flags |= FNIC_IO_ISSUED; 409 out: 410 lba = (char *)&cmdiu->sqe.rw.slba; 411 cmd_trace = ((u64) cmdiu->sqe.rw.opcode << 56 | (u64) lba[4] << 40 | 412 (u64) lba[5] << 32 | (u64) lba[0] << 24 | 413 (u64) lba[1] << 16 | (u64) lba[2] << 8 | lba[3]); 414 415 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num, 416 io_req->tag, 0, io_req, 417 sg_count, cmd_trace, 418 (((u64)io_req->cmd_flags >> 32) | 419 io_req->cmd_state)); 420 421 return ret; 422 } 423 424 int nvfnic_fcpio_send(struct nvme_fc_local_port *lport, 425 struct nvme_fc_remote_port *rport, void *hw_queue_handle, 426 struct nvmefc_fcp_req *fcp_req) 427 { 428 struct fnic_iport_s *iport = lport->private; 429 struct fnic_io_req *io_req; 430 int ret; 431 struct fnic *fnic = iport->fnic; 432 unsigned long flags = 0; 433 struct fnic_tport_s *tport; 434 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 435 436 spin_lock_irqsave(&fnic->fnic_lock, flags); 437 438 tport = (struct fnic_tport_s *)rport->private; 439 atomic64_inc(&fnic_stats->io_stats.nvme_io_reqs_rcvd); 440 if (!nvfnic_transport_ready(iport, tport)) { 441 atomic64_inc(&fnic_stats->io_stats.nvme_io_rsps_sent); 442 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 443 if (tport != NULL) 444 FNIC_NVME_DBG(KERN_INFO, fnic, 445 "iport: 0x%x tport: 0x%x not ready\n", 446 iport->fcid, tport->fcid); 447 else 448 FNIC_NVME_DBG(KERN_INFO, fnic, 449 "iport: 0x%x tport not ready\n", 450 iport->fcid); 451 return -ENODEV; 452 } 453 atomic_inc(&fnic->in_flight); 454 455 io_req = (struct fnic_io_req *) fcp_req->private; 456 io_req->iport = iport; 457 io_req->tport = (struct fnic_tport_s *)rport->private; 458 io_req->fcp_req = fcp_req; 459 io_req->done = nvfnic_fcpio_cmpl; 460 io_req->sgl_list = NULL; 461 io_req->sgl_list_alloc = NULL; 462 io_req->sgl_list_pa = 0; 463 io_req->sgl_cnt = 0; 464 io_req->sgl_mapped = 0; 465 io_req->wq = hw_queue_handle; 466 init_llist_node(&io_req->nvfnic_io_cmpl); 467 468 io_req->tag = nvfnic_alloc_fcpio_tag(iport, io_req); 469 if (io_req->tag == FNIC_NVME_NO_FREE_TAG) { 470 FNIC_NVME_DBG(KERN_ERR, fnic, 471 "No free tag available. Failing IO\n"); 472 atomic64_inc(&fnic_stats->io_stats.alloc_failures); 473 atomic_dec(&fnic->in_flight); 474 atomic64_inc(&fnic_stats->io_stats.nvme_io_rsps_sent); 475 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 476 return -EBUSY; 477 } 478 479 ret = nvfnic_queuecommand(io_req); 480 if (ret) { 481 FNIC_NVME_DBG(KERN_ERR, fnic, 482 "Queuecommand failed tag: 0x%x\n", 483 io_req->tag); 484 nvfnic_free_fcpio_tag(iport, io_req); 485 } 486 487 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 488 atomic_dec(&fnic->in_flight); 489 return ret; 490 } 491 492 void nvfnic_fcpio_nvme_fast_cmpl_handler(struct fnic *fnic, 493 struct fcpio_fw_req *desc) 494 { 495 u8 type; 496 u8 hdr_status; 497 struct fcpio_tag ftag; 498 u32 id; 499 struct fnic_io_req *io_req; 500 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 501 unsigned long start_time; 502 u64 cmd_trace; 503 char *lba; 504 struct nvme_fc_cmd_iu *cmdiu; 505 struct fnic_iport_s *iport; 506 unsigned int tag; 507 508 /* Decode the cmpl description to get the io_req id */ 509 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); 510 fcpio_tag_id_dec(&ftag, &id); 511 tag = id & FNIC_TAG_MASK; 512 513 if (tag >= fnic->fnic_max_tag_id) { 514 FNIC_NVME_DBG(KERN_ERR, fnic, 515 "Tag out of range tag: 0x%x hdr status: %s\n", tag, 516 fnic_fcpio_status_to_str(hdr_status)); 517 return; 518 } 519 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 520 521 io_req = nvfnic_find_io_req_by_tag(fnic, tag); 522 523 WARN_ON_ONCE(!io_req); 524 if (!io_req) { 525 atomic64_inc(&fnic_stats->io_stats.ioreq_null); 526 FNIC_NVME_DBG(KERN_ERR, fnic, 527 "IO req null hdr: %s tag: 0x%x desc: 0x%p\n", 528 fnic_fcpio_status_to_str(hdr_status), id, desc); 529 FNIC_NVME_DBG(KERN_ERR, fnic, 530 "type: 0x%x status: 0x%x rsvd: 0x%x tag: 0x%x\n", 531 desc->hdr.type, desc->hdr.status, desc->hdr._resvd, 532 id); 533 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 534 return; 535 } 536 537 cmdiu = io_req->fcp_req->cmdaddr; 538 if (io_req->tag != tag) { 539 FNIC_NVME_DBG(KERN_INFO, fnic, 540 "Tag mismatch tag:%d io:0x%x id:0x%x csn:%08x\n", 541 tag, io_req->tag, id, be32_to_cpu(cmdiu->csn)); 542 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 543 return; 544 } 545 iport = io_req->iport; 546 547 start_time = io_req->start_time; 548 549 /* firmware completed the io */ 550 io_req->io_completed = 1; 551 if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) { 552 /* 553 * set the FNIC_IO_DONE so that this doesn't get 554 * flagged as 'out of order' if it was not aborted 555 */ 556 io_req->cmd_flags |= FNIC_IO_DONE; 557 io_req->cmd_flags |= FNIC_IO_ABTS_PENDING; 558 if (hdr_status == FCPIO_ABORTED) 559 io_req->cmd_flags |= FNIC_IO_ABORTED; 560 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 561 562 FNIC_NVME_DBG(KERN_INFO, fnic, 563 "icmnd abts hdr:%d %s tag:0x%x io:%p", 564 hdr_status, fnic_fcpio_status_to_str(hdr_status), 565 id, io_req); 566 return; 567 } 568 569 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) { 570 FNIC_NVME_DBG(KERN_INFO, fnic, 571 "IO freed id:%d tag:0x%x st:0x%x csn:%08x\n", 572 id, io_req->tag, io_req->cmd_state, 573 be32_to_cpu(cmdiu->csn)); 574 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 575 return; 576 } 577 578 /* Mark the IO as complete */ 579 io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE; 580 switch (hdr_status) { 581 case FCPIO_SUCCESS: 582 io_req->fcp_req->status = 0; 583 io_req->fcp_req->transferred_length = 584 io_req->fcp_req->payload_length; 585 io_req->fcp_req->rcv_rsplen = 12; 586 break; 587 default: 588 FNIC_NVME_DBG(KERN_ERR, fnic, "HDR status is non-zero\n"); 589 io_req->fcp_req->status = NVME_SC_INTERNAL; 590 break; 591 } 592 593 if (hdr_status != FCPIO_SUCCESS) { 594 atomic64_inc(&fnic_stats->io_stats.io_failures); 595 FNIC_NVME_DBG(KERN_INFO, fnic, "hdr status: %s\n", 596 fnic_fcpio_status_to_str(hdr_status)); 597 } 598 599 io_req->cmd_flags |= FNIC_IO_DONE; 600 601 cmdiu = io_req->fcp_req->cmdaddr; 602 lba = (char *)&cmdiu->sqe.rw.slba; 603 cmd_trace = ((u64) hdr_status << 56) | 604 (u64) cmdiu->sqe.rw.opcode << 32 | 605 (u64) lba[0] << 24 | (u64) lba[1] << 16 | 606 (u64) lba[2] << 8 | lba[3]; 607 608 FNIC_TRACE(nvfnic_fcpio_nvme_fast_cmpl_handler, fnic->fnic_num, 609 tag, io_req, 610 jiffies_to_msecs(jiffies - start_time), 611 desc, cmd_trace, 612 (((u64) io_req->cmd_flags << 32) | 613 io_req->cmd_state)); 614 615 nvfnic_update_io_stats(fnic, cmdiu->sqe.rw.opcode); 616 nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode); 617 nvfnic_update_cmpl_stats(fnic, io_req); 618 619 nvfnic_release_nvme_ioreq_buf(iport, io_req); 620 if (io_req->done) 621 io_req->done(io_req); 622 623 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 624 } 625 626 void nvfnic_fcpio_ersp_cmpl_handler(struct fnic *fnic, 627 struct fcpio_fw_req *desc, int sw_flag) 628 { 629 u8 type; 630 u8 hdr_status; 631 struct fcpio_tag ftag; 632 u32 id; 633 struct fcpio_nvme_cmpl *nvme_cmpl; 634 struct fnic_io_req *io_req; 635 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 636 unsigned long start_time; 637 uint32_t rsplen; 638 struct nvme_fc_ersp_iu *ersp; 639 struct nvme_fc_ersp_iu *nrsp; 640 struct nvme_fc_cmd_iu *cmdiu; 641 struct nvme_command *sqe; 642 struct nvme_completion *cqe; 643 u64 cmd_trace; 644 struct fnic_iport_s *iport; 645 unsigned int tag; 646 char *lba; 647 uint64_t tport_wwpn = 0; 648 649 /* Decode the cmpl description to get the io_req id */ 650 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); 651 fcpio_tag_id_dec(&ftag, &id); 652 nvme_cmpl = &desc->u.nvme_cmpl; 653 ersp = (struct nvme_fc_ersp_iu *) nvme_cmpl->resp_bytes; 654 tag = id & FNIC_TAG_MASK; 655 656 if (tag >= fnic->fnic_max_tag_id) { 657 FNIC_NVME_DBG(KERN_ERR, fnic, 658 "Tag out of range tag:0x%x hdr:%s\n", tag, 659 fnic_fcpio_status_to_str(hdr_status)); 660 return; 661 } 662 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 663 664 io_req = nvfnic_find_io_req_by_tag(fnic, tag); 665 if (!io_req) { 666 atomic64_inc(&fnic_stats->io_stats.ioreq_null); 667 FNIC_NVME_DBG(KERN_ERR, fnic, 668 "IOREQ is null hdr status: %s tag: 0x%x desc: %p\n", 669 fnic_fcpio_status_to_str(hdr_status), tag, desc); 670 FNIC_NVME_DBG(KERN_ERR, fnic, 671 "type: 0x%x status: 0x%x rsvd: 0x%x\n", 672 desc->hdr.type, desc->hdr.status, desc->hdr._resvd); 673 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 674 return; 675 } 676 iport = io_req->iport; 677 if (io_req->tport != NULL) 678 tport_wwpn = io_req->tport->wwpn; 679 cmdiu = io_req->fcp_req->cmdaddr; 680 681 if (io_req->tag != tag) { 682 FNIC_NVME_DBG(KERN_ERR, fnic, 683 "Tag mismatch io:0x%x tag:0x%x id:0x%x\n", 684 io_req->tag, tag, id); 685 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 686 return; 687 } 688 689 nrsp = (struct nvme_fc_ersp_iu *)io_req->fcp_req->rspaddr; 690 cmdiu = (struct nvme_fc_cmd_iu *)io_req->fcp_req->cmdaddr; 691 sqe = &cmdiu->sqe; 692 cqe = &nrsp->cqe; 693 start_time = io_req->start_time; 694 695 /* firmware completed the io */ 696 io_req->io_completed = 1; 697 698 if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) { 699 /* 700 * set the FNIC_IO_DONE so that this doesn't get 701 * flagged as 'out of order' if it was not aborted 702 */ 703 io_req->cmd_flags |= FNIC_IO_DONE; 704 io_req->cmd_flags |= FNIC_IO_ABTS_PENDING; 705 if (hdr_status == FCPIO_ABORTED) 706 io_req->cmd_flags |= FNIC_IO_ABORTED; 707 708 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 709 FNIC_NVME_DBG(KERN_INFO, fnic, 710 "ABTS pending hdr status: %s tag: 0x%x", 711 fnic_fcpio_status_to_str(hdr_status), tag); 712 return; 713 } 714 715 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) { 716 FNIC_NVME_DBG(KERN_ERR, fnic, 717 "IO already freed by abort. tag: 0x%x id: 0x%x\n", 718 io_req->tag, id); 719 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 720 return; 721 } 722 723 /* Mark the IO as complete */ 724 io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE; 725 726 switch (hdr_status) { 727 case FCPIO_SUCCESS: 728 io_req->fcp_req->status = 0; 729 if (!sw_flag) { 730 io_req->fcp_req->transferred_length = 731 io_req->fcp_req->payload_length; 732 rsplen = 32; 733 nrsp->iu_len = 734 cpu_to_be16(sizeof(struct nvme_fc_ersp_iu) / 4); 735 nrsp->xfrd_len = 736 cpu_to_be32(io_req->fcp_req->payload_length); 737 738 nrsp->ersp_result = 0; 739 cqe->command_id = sqe->common.command_id; 740 cqe->status = 0; 741 cqe->result.u64 = 0; 742 } else { 743 io_req->fcp_req->transferred_length = 744 be32_to_cpu(ersp->xfrd_len); 745 rsplen = be16_to_cpu(ersp->iu_len) * 4; 746 if (rsplen > sizeof(nvme_cmpl->resp_bytes) || 747 rsplen > io_req->fcp_req->rsplen) { 748 FNIC_NVME_DBG(KERN_ERR, fnic, 749 "tport wwpn 0x%llx ERSP len %u desc %u req %u\n", 750 tport_wwpn, rsplen, 751 (u32)sizeof(nvme_cmpl->resp_bytes), 752 io_req->fcp_req->rsplen); 753 io_req->fcp_req->status = NVME_SC_INTERNAL; 754 io_req->fcp_req->rcv_rsplen = 0; 755 break; 756 } 757 memcpy(io_req->fcp_req->rspaddr, ersp, rsplen); 758 } 759 atomic64_inc(&fnic_stats->nvme_stats.nvme_ersps); 760 io_req->fcp_req->rcv_rsplen = rsplen; 761 break; 762 763 default: 764 FNIC_NVME_DBG(KERN_ERR, fnic, 765 "Unexpected header status: %d\n", hdr_status); 766 io_req->fcp_req->status = NVME_SC_INTERNAL; 767 break; 768 } 769 770 if (hdr_status != FCPIO_SUCCESS) { 771 atomic64_inc(&fnic_stats->io_stats.io_failures); 772 FNIC_NVME_DBG(KERN_ERR, fnic, "hdr status: %s tag: 0x%x\n", 773 fnic_fcpio_status_to_str(hdr_status), tag); 774 } 775 776 io_req->cmd_flags |= FNIC_IO_DONE; 777 778 lba = (char *) &cmdiu->sqe.rw.slba; 779 cmd_trace = ((u64) hdr_status << 56) | 780 (u64) ersp->ersp_result << 48 | 781 (u64) cmdiu->sqe.rw.opcode << 32 | 782 (u64) lba[0] << 24 | (u64) lba[1] << 16 | 783 (u64) lba[2] << 8 | lba[3]; 784 785 FNIC_TRACE(nvfnic_fcpio_ersp_cmpl_handler, fnic->fnic_num, 786 tag, io_req, 787 ((u64) nvme_cmpl->resvd[1] << 56 | 788 (u64) nvme_cmpl->resvd[0] << 48 | 789 jiffies_to_msecs(jiffies - start_time)), 790 desc, cmd_trace, 791 (((u64) io_req->cmd_flags << 32) | 792 io_req->cmd_state)); 793 794 nvfnic_update_io_stats(fnic, cmdiu->sqe.rw.opcode); 795 nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode); 796 nvfnic_update_cmpl_stats(fnic, io_req); 797 798 nvfnic_release_nvme_ioreq_buf(iport, io_req); 799 800 /* Call NVME completion function to complete the IO */ 801 if (io_req->done) 802 io_req->done(io_req); 803 804 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 805 } 806 807 void nvfnic_fcpio_nvme_itmf_cmpl_handler(struct fnic *fnic, 808 struct fcpio_fw_req *desc) 809 { 810 u8 type; 811 u8 hdr_status; 812 struct fcpio_tag ftag; 813 u32 id; 814 unsigned int tag; 815 struct fnic_io_req *io_req; 816 struct nvme_fc_cmd_iu *cmd_iu; 817 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 818 struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats; 819 struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; 820 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; 821 struct fnic_iport_s *iport; 822 823 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); 824 fcpio_tag_id_dec(&ftag, &id); 825 tag = id & FNIC_TAG_MASK; 826 827 if (tag >= fnic->fnic_max_tag_id) { 828 FNIC_NVME_DBG(KERN_ERR, fnic, 829 "Tag out of range id:0x%x tag:0x%x hdr:%s\n", 830 id, tag, fnic_fcpio_status_to_str(hdr_status)); 831 return; 832 } 833 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 834 835 io_req = nvfnic_find_io_req_by_tag(fnic, tag); 836 WARN_ON_ONCE(!io_req); 837 if (!io_req) { 838 atomic64_inc(&fnic_stats->io_stats.ioreq_null); 839 FNIC_NVME_DBG(KERN_ERR, fnic, 840 "IOREQ null hdr:%s tag:0x%x desc:%p\n", 841 fnic_fcpio_status_to_str(hdr_status), tag, desc); 842 FNIC_NVME_DBG(KERN_ERR, fnic, 843 "type: 0x%x status: 0x%x rsvd: 0x%x\n", 844 desc->hdr.type, desc->hdr.status, desc->hdr._resvd); 845 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 846 return; 847 } 848 849 cmd_iu = io_req->fcp_req->cmdaddr; 850 FNIC_NVME_DBG(KERN_INFO, fnic, 851 "Received ITMF completion tag: 0x%x hdr_status: %d csn: 0x%08x\n", 852 tag, hdr_status, be32_to_cpu(cmd_iu->csn)); 853 854 iport = io_req->iport; 855 856 /* Completion of abort cmd */ 857 switch (hdr_status) { 858 case FCPIO_SUCCESS: 859 FNIC_NVME_DBG(KERN_DEBUG, fnic, 860 "Abort success received tag: 0x%x id: 0x%x\n", 861 tag, id); 862 break; 863 case FCPIO_TIMEOUT: 864 FNIC_NVME_DBG(KERN_ERR, fnic, 865 "Abort timeout received tag: 0x%x id: 0x%x\n", 866 tag, id); 867 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED) 868 atomic64_inc(&abts_stats->abort_fw_timeouts); 869 else 870 atomic64_inc(&term_stats->terminate_fw_timeouts); 871 break; 872 case FCPIO_ITMF_REJECTED: 873 FNIC_NVME_DBG(KERN_ERR, fnic, 874 "Abort reject received tag: 0x%x id: 0x%x\n", 875 tag, id); 876 break; 877 878 case FCPIO_IO_NOT_FOUND: 879 FNIC_NVME_DBG(KERN_ERR, fnic, 880 "Abort IO not found tag:0x%x id:0x%x\n", 881 tag, id); 882 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED) 883 atomic64_inc(&abts_stats->abort_io_not_found); 884 else 885 atomic64_inc(&term_stats->terminate_io_not_found); 886 break; 887 default: 888 FNIC_NVME_DBG(KERN_ERR, fnic, 889 "Abort unknown received tag: 0x%x id: 0x%x\n", 890 tag, id); 891 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED) 892 atomic64_inc(&abts_stats->abort_failures); 893 else 894 atomic64_inc(&term_stats->terminate_failures); 895 break; 896 } 897 898 if (io_req->cmd_state != FNIC_IOREQ_ABTS_PENDING) { 899 FNIC_NVME_DBG(KERN_ERR, fnic, 900 "Abort late completion tag:0x%x id:0x%x\n", 901 tag, id); 902 /* This is a late completion. Ignore it */ 903 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 904 return; 905 } 906 907 io_req->abts_state = hdr_status; 908 909 /* If the status is IO not found consider it as success. 910 * NVME sends abort even if rport is down in which case 911 * we will get FCPIO_TIMEOUT. Consider this as success. 912 */ 913 if ((hdr_status == FCPIO_IO_NOT_FOUND) || 914 (hdr_status == FCPIO_TIMEOUT) || 915 (hdr_status == FCPIO_ITMF_REJECTED)) 916 io_req->abts_state = FCPIO_SUCCESS; 917 918 io_req->cmd_flags |= FNIC_IO_ABT_TERM_DONE; 919 920 if (!(io_req->cmd_flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) 921 atomic64_inc(&misc_stats->no_icmnd_itmf_cmpls); 922 923 io_req->fcp_req->transferred_length = 0; 924 io_req->fcp_req->rcv_rsplen = 0; 925 if (io_req->abts_state == FCPIO_SUCCESS) 926 io_req->fcp_req->status = NVME_SC_ABORT_REQ; 927 else 928 io_req->fcp_req->status = NVME_SC_INTERNAL; 929 930 atomic64_dec(&fnic_stats->io_stats.active_ios); 931 if (atomic64_read(&fnic->io_cmpl_skip)) 932 atomic64_dec(&fnic->io_cmpl_skip); 933 else 934 atomic64_inc(&fnic_stats->io_stats.io_completions); 935 936 nvfnic_release_nvme_ioreq_buf(iport, io_req); 937 if (io_req->done) 938 io_req->done(io_req); 939 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 940 } 941 942 bool _cleanup_tport_io(struct sbitmap *map, unsigned int tag, 943 void *data) 944 { 945 struct fnic_tport_s *tport = data; 946 struct fnic_iport_s *iport = tport->iport; 947 struct fnic *fnic = iport->fnic; 948 struct fnic_io_req *io_req; 949 enum fnic_ioreq_state old_ioreq_state; 950 unsigned long flags; 951 952 spin_lock_irqsave(&fnic->fnic_lock, flags); 953 io_req = nvfnic_find_io_req_by_tag(fnic, tag); 954 if (!io_req || io_req->tport != tport) { 955 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 956 return true; 957 } 958 959 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) || 960 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) { 961 FNIC_NVME_DBG(KERN_INFO, fnic, 962 "Abort already pending 0x%x\n", io_req->tag); 963 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 964 return true; 965 } 966 967 FNIC_NVME_DBG(KERN_ERR, fnic, 968 "io_req tag: 0x%x abort after unregister timeout\n", 969 io_req->tag); 970 971 old_ioreq_state = io_req->cmd_state; 972 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING; 973 io_req->abts_state = FCPIO_INVALID_CODE; 974 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 975 976 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag, 977 FCPIO_ITMF_ABT_TASK_TERM, 978 io_req)) { 979 FNIC_NVME_DBG(KERN_ERR, fnic, 980 "Failed to enqueue abort for ioreq tag: 0x%x\n", 981 io_req->tag); 982 spin_lock_irqsave(&fnic->fnic_lock, flags); 983 io_req->cmd_state = old_ioreq_state; 984 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 985 } 986 return true; 987 } 988 989 void 990 nvfnic_cleanup_tport_io(struct fnic *fnic, struct fnic_tport_s *tport) 991 { 992 unsigned long flags; 993 struct nvfnic_ls_req *nvfnic_ls_req, *next; 994 struct nvmefc_ls_req *lsreq; 995 uint16_t oxid; 996 LIST_HEAD(done_reqs); 997 998 spin_lock_irqsave(&fnic->fnic_lock, flags); 999 list_for_each_entry_safe(nvfnic_ls_req, next, 1000 &(tport->ls_req_list), list) { 1001 lsreq = nvfnic_ls_req->ls_req; 1002 if (!lsreq || (lsreq->private == NULL)) { 1003 FNIC_NVME_DBG(KERN_INFO, fnic, 1004 "fnic_cleanup_tport_io lsreq NULL\n"); 1005 continue; 1006 } 1007 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED) { 1008 FNIC_NVME_DBG(KERN_INFO, fnic, 1009 "fnic_cleanup_tport_io lsreq abort started\n"); 1010 continue; 1011 } 1012 list_del_init(&nvfnic_ls_req->list); 1013 lsreq->private = NULL; 1014 oxid = nvfnic_ls_req->oxid; 1015 fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid); 1016 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE; 1017 list_add_tail(&nvfnic_ls_req->list, &done_reqs); 1018 } 1019 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1020 1021 list_for_each_entry_safe(nvfnic_ls_req, next, &done_reqs, list) { 1022 list_del_init(&nvfnic_ls_req->list); 1023 lsreq = nvfnic_ls_req->ls_req; 1024 timer_delete_sync(&nvfnic_ls_req->ls_req_timer); 1025 lsreq->done(lsreq, -ENXIO); 1026 } 1027 1028 /* For link-down, IOs are freed by firmware reset completion */ 1029 if (fdls_get_state(&fnic->iport.fabric) == FDLS_STATE_LINKDOWN) 1030 return; 1031 1032 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_tport_io, tport); 1033 } 1034 1035 void 1036 nvfnic_terminate_tport_ls_reqs(struct fnic *fnic, struct fnic_tport_s *tport) 1037 { 1038 struct nvmefc_ls_req *lsreq; 1039 struct nvfnic_ls_req *nvfnic_ls_req, *next; 1040 int count = 0; 1041 uint16_t oxid; 1042 LIST_HEAD(done_reqs); 1043 1044 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1045 list_for_each_entry_safe(nvfnic_ls_req, next, 1046 &(tport->ls_req_list), list) { 1047 1048 lsreq = nvfnic_ls_req->ls_req; 1049 if (!lsreq || (lsreq->private == NULL)) { 1050 FNIC_NVME_DBG(KERN_ERR, fnic, 1051 "lsreq is NULL\n"); 1052 continue; 1053 } 1054 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED) { 1055 FNIC_NVME_DBG(KERN_INFO, fnic, 1056 "lsreq abort started\n"); 1057 continue; 1058 } 1059 oxid = nvfnic_ls_req->oxid; 1060 list_del_init(&nvfnic_ls_req->list); 1061 lsreq->private = NULL; 1062 fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid); 1063 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE; 1064 list_add_tail(&nvfnic_ls_req->list, &done_reqs); 1065 } 1066 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1067 1068 list_for_each_entry_safe(nvfnic_ls_req, next, &done_reqs, list) { 1069 list_del_init(&nvfnic_ls_req->list); 1070 lsreq = nvfnic_ls_req->ls_req; 1071 timer_delete_sync(&nvfnic_ls_req->ls_req_timer); 1072 lsreq->done(lsreq, -ENXIO); 1073 count++; 1074 } 1075 1076 FNIC_NVME_DBG(KERN_INFO, fnic, 1077 "fnic_terminate_tport_lsreqs tport: 0x%x: freed lsreq: %d\n", 1078 tport->fcid, count); 1079 } 1080 1081 bool _terminate_tport_ios(struct sbitmap *map, unsigned int tag, 1082 void *data) 1083 { 1084 struct fnic_tport_s *tport = data; 1085 struct fnic_iport_s *iport = tport->iport; 1086 struct fnic *fnic = iport->fnic; 1087 struct fnic_io_req *io_req; 1088 enum fnic_ioreq_state old_ioreq_state; 1089 unsigned long flags; 1090 1091 spin_lock_irqsave(&fnic->fnic_lock, flags); 1092 io_req = fnic->sw_copy_wq[0].io_req_table[tag]; 1093 if (!io_req) { 1094 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1095 return true; 1096 } 1097 1098 if (io_req->tport != tport) { 1099 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1100 return true; 1101 } 1102 1103 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) || 1104 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) { 1105 FNIC_NVME_DBG(KERN_INFO, fnic, 1106 "Abort already pending 0x%x\n", io_req->tag); 1107 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1108 return true; 1109 } 1110 1111 FNIC_NVME_DBG(KERN_INFO, fnic, 1112 "Terminate tag: 0x%x (tport fcid 0x%x)\n", 1113 io_req->tag, io_req->tport->fcid); 1114 1115 old_ioreq_state = io_req->cmd_state; 1116 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING; 1117 io_req->abts_state = FCPIO_INVALID_CODE; 1118 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1119 1120 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag, 1121 FCPIO_ITMF_ABT_TASK_TERM, io_req)) { 1122 FNIC_NVME_DBG(KERN_ERR, fnic, 1123 "Failed to enqueue abort for ioreq tag: 0x%x\n", 1124 io_req->tag); 1125 spin_lock_irqsave(&fnic->fnic_lock, flags); 1126 io_req->cmd_state = old_ioreq_state; 1127 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1128 } 1129 return true; 1130 } 1131 1132 void nvfnic_terminate_tport_ios(struct fnic *fnic, 1133 struct fnic_tport_s *tport) 1134 { 1135 struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats; 1136 1137 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _terminate_tport_ios, tport); 1138 1139 FNIC_NVME_DBG(KERN_INFO, fnic, 1140 "tport: 0x%x aborted %lld in_flight %d\n", 1141 tport->fcid, atomic64_read(&abts_stats->aborts), 1142 atomic_read(&fnic->in_flight)); 1143 } 1144 1145 bool _cleanup_all_nvme_io(struct sbitmap *map, unsigned int tag, 1146 void *data) 1147 { 1148 struct fnic_iport_s *iport = data; 1149 struct fnic_io_req *io_req; 1150 1151 io_req = iport->fnic->sw_copy_wq[0].io_req_table[tag]; 1152 if (!io_req) 1153 return true; 1154 1155 io_req->cmd_state = FNIC_DEV_RST_TERM_ISSUED; 1156 io_req->fcp_req->status = NVME_SC_INTERNAL; 1157 io_req->fcp_req->transferred_length = 0; 1158 io_req->fcp_req->rcv_rsplen = 0; 1159 nvfnic_release_nvme_ioreq_buf(iport, io_req); 1160 io_req->done(io_req); 1161 return true; 1162 } 1163 1164 void nvfnic_cleanup_all_nvme_ios(struct fnic *fnic) 1165 { 1166 struct fnic_iport_s *iport = &fnic->iport; 1167 1168 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1169 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_all_nvme_io, 1170 iport); 1171 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1172 } 1173 1174 void nvfnic_nvme_zero_devloss_tports(struct fnic *fnic) 1175 { 1176 struct fnic_tport_s *tport, *next; 1177 1178 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1179 list_for_each_entry_safe(tport, next, &fnic->iport.tport_list, links) { 1180 if (tport->flags & FNIC_FDLS_NVME_REGISTERED) { 1181 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1182 nvme_fc_set_remoteport_devloss(tport->nv_rport, 0); 1183 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1184 } 1185 } 1186 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1187 } 1188 1189 void nvfnic_nvme_unload(struct fnic *fnic) 1190 { 1191 int ret = 0; 1192 struct fnic_iport_s *iport = &fnic->iport; 1193 unsigned long flags; 1194 unsigned int time_wait = FNIC_NVME_LPORT_REMOVE_WAIT; 1195 unsigned int time_remain; 1196 DECLARE_COMPLETION_ONSTACK(nvme_lport_unreg_done); 1197 1198 /* Mark iport state as INIT so that no IOs can be issued from this point */ 1199 spin_lock_irqsave(&fnic->fnic_lock, flags); 1200 fnic->in_remove = 1; 1201 fnic->iport.state = FNIC_IPORT_STATE_LINK_WAIT; 1202 fnic->nvme_lport_unreg_done = &nvme_lport_unreg_done; 1203 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1204 1205 /* 1206 * If fnic is already processing link-down or fnic is held 1207 * in disabled state following a reboot we dont need to issue 1208 * firmware reset and unregister remote ports as it is already 1209 * done as part of link down handling. 1210 */ 1211 if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN) { 1212 while (fnic->reset_in_progress == IN_PROGRESS) { 1213 wait_for_completion_timeout(&fnic->reset_completion_wait, 1214 msecs_to_jiffies(5000)); 1215 FNIC_NVME_DBG(KERN_INFO, fnic, 1216 "rmmod waiting for reset %p\n", fnic); 1217 } 1218 } else if (fdls_get_state(&iport->fabric) != FDLS_STATE_INIT) 1219 fnic_fcpio_reset(fnic); 1220 1221 /* 1222 * Mark state so that the workqueue thread stops forwarding 1223 * received frames and link events to the local port. ISR and 1224 * other threads that can queue work items will also stop 1225 * creating work items on the fnic workqueue 1226 */ 1227 nvfnic_nvme_zero_devloss_tports(fnic); 1228 fnic_flush_tport_event_list(fnic); 1229 fnic_delete_fcp_tports(fnic); 1230 1231 if (iport->flags & FNIC_LPORT_NVME_REGISTERED) { 1232 ret = nvme_fc_unregister_localport(fnic->iport.nv_lport); 1233 if (ret) { 1234 FNIC_NVME_DBG(KERN_ERR, fnic, 1235 "Unregister nvme localport failed: %d\n", ret); 1236 } else { 1237 time_remain = wait_for_completion_timeout( 1238 fnic->nvme_lport_unreg_done, 1239 msecs_to_jiffies(time_wait)); 1240 if (!time_remain) { 1241 FNIC_NVME_DBG(KERN_ERR, fnic, 1242 "Local port removal timed out\n"); 1243 WARN_ON(1); 1244 } 1245 iport->flags &= ~FNIC_LPORT_NVME_REGISTERED; 1246 kfree(iport->nv_tmpl); 1247 } 1248 } 1249 1250 spin_lock_irqsave(&fnic->fnic_lock, flags); 1251 fnic->nvme_lport_unreg_done = NULL; 1252 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1253 1254 nvfnic_flush_nvme_io_list(fnic); 1255 } 1256 1257 struct nvfnic_ls_req* 1258 nvfnic_find_ls_req(struct fnic_tport_s *tport, uint16_t oxid) 1259 { 1260 struct nvfnic_ls_req *nvfnic_ls_req, *next; 1261 1262 list_for_each_entry_safe(nvfnic_ls_req, next, &(tport->ls_req_list), list) { 1263 if (nvfnic_ls_req->oxid == oxid) 1264 return nvfnic_ls_req; 1265 } 1266 return NULL; 1267 } 1268 1269 void nvfnic_fcpio_cmpl(struct fnic_io_req *io_req) 1270 { 1271 struct fnic *fnic = io_req->iport->fnic; 1272 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1273 1274 nvfnic_free_fcpio_tag(io_req->iport, io_req); 1275 atomic64_inc(&fnic_stats->io_stats.nvme_ios_queued_for_rsp); 1276 io_req->waitq_start_time = jiffies; 1277 1278 llist_add(&io_req->nvfnic_io_cmpl, &fnic->nvme_io_event_llist); 1279 atomic_inc(&fnic->nvme_io_event_queued); 1280 atomic64_inc(&fnic_stats->io_stats.nvme_num_ios_in_waitq); 1281 1282 queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work); 1283 } 1284 1285 void nvfnic_process_ls_abts_rsp(struct fnic_iport_s *iport, 1286 struct fc_frame_header *fchdr) 1287 { 1288 uint32_t tport_fcid; 1289 struct fnic_tport_s *tport; 1290 struct nvfnic_ls_req *nvfnic_ls_req; 1291 struct nvmefc_ls_req *lsreq; 1292 uint8_t *fcid; 1293 uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr); 1294 struct fnic *fnic = iport->fnic; 1295 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1296 1297 fcid = FNIC_STD_GET_S_ID(fchdr); 1298 tport_fcid = ntoh24(fcid); 1299 1300 tport = fnic_find_tport_by_fcid(iport, tport_fcid); 1301 if (tport == NULL) { 1302 FNIC_NVME_DBG(KERN_ERR, fnic, 1303 "tport: 0x%x not found\n", tport_fcid); 1304 return; 1305 } 1306 1307 nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid); 1308 if (nvfnic_ls_req == NULL) { 1309 FNIC_NVME_DBG(KERN_ERR, fnic, 1310 "tport: 0x%x lsreq oxid: 0x%x not found\n", 1311 tport_fcid, oxid); 1312 return; 1313 } 1314 1315 lsreq = nvfnic_ls_req->ls_req; 1316 if ((lsreq == NULL) || (lsreq->private == NULL)) { 1317 FNIC_NVME_DBG(KERN_INFO, fnic, 1318 "tport: 0x%x lsreq oxid: 0x%x already aborted\n", 1319 tport_fcid, oxid); 1320 return; 1321 } 1322 1323 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_abort_responses); 1324 nvfnic_ls_req->state = FNIC_LS_REQ_ABTS_COMPLETE; 1325 1326 FNIC_NVME_DBG(KERN_DEBUG, fnic, "nvme_ls_requests: %lld\n", 1327 (u64) atomic64_read(&fnic_stats->nvme_stats.nvme_ls_requests)); 1328 1329 list_del(&nvfnic_ls_req->list); 1330 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid); 1331 lsreq->private = NULL; 1332 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1333 timer_delete_sync(&nvfnic_ls_req->ls_req_timer); 1334 lsreq->done(lsreq, NVME_SC_HOST_ABORTED_CMD); 1335 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1336 } 1337 1338 /** 1339 * nvfnic_ls_rsp_recv - Handle received NVMe FC link service (LS) response 1340 * @iport: Pointer to the local FNIC port structure 1341 * @fchdr: Pointer to the Fibre Channel frame header for the 1342 * received response 1343 * @len: Length of the received frame 1344 * 1345 * This function processes link service (LS) responses received from 1346 * NVMe Discovery Controllers or regular NVMe subsystems during 1347 * association. 1348 */ 1349 void nvfnic_ls_rsp_recv(struct fnic_iport_s *iport, 1350 struct fc_frame_header *fchdr, int len) 1351 { 1352 uint8_t *fcid; 1353 uint32_t tport_fcid; 1354 struct fnic_tport_s *tport; 1355 struct nvfnic_ls_req *nvfnic_ls_req; 1356 struct nvmefc_ls_req *lsreq; 1357 uint16_t oxid; 1358 uint32_t rsp_len; 1359 int sid_len = offsetof(struct fc_frame_header, fh_s_id) + 1360 sizeof(fchdr->fh_s_id); 1361 int status = 0; 1362 struct fnic *fnic = iport->fnic; 1363 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1364 1365 if (len < (int)sizeof(*fchdr)) { 1366 if (len >= sid_len) { 1367 fcid = FNIC_STD_GET_S_ID(fchdr); 1368 tport_fcid = ntoh24(fcid); 1369 FNIC_NVME_DBG(KERN_ERR, fnic, 1370 "tport: 0x%x LS rsp len %d too short\n", 1371 tport_fcid, len); 1372 } else { 1373 FNIC_NVME_DBG(KERN_ERR, fnic, 1374 "LS response len %d too short\n", len); 1375 } 1376 return; 1377 } 1378 rsp_len = len - sizeof(*fchdr); 1379 1380 fcid = FNIC_STD_GET_S_ID(fchdr); 1381 tport_fcid = ntoh24(fcid); 1382 1383 tport = fnic_find_tport_by_fcid(iport, tport_fcid); 1384 if (!tport) { 1385 FNIC_NVME_DBG(KERN_ERR, fnic, 1386 "tport: 0x%x not found\n", tport_fcid); 1387 return; 1388 } 1389 1390 oxid = FNIC_STD_GET_OX_ID(fchdr); 1391 nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid); 1392 if (!nvfnic_ls_req) { 1393 FNIC_NVME_DBG(KERN_ERR, fnic, 1394 "tport: 0x%x no nvfnic_lsreq for oxid: 0x%x\n", 1395 tport_fcid, oxid); 1396 return; 1397 } 1398 1399 lsreq = nvfnic_ls_req->ls_req; 1400 if (!lsreq || (lsreq->private == NULL)) { 1401 FNIC_NVME_DBG(KERN_ERR, fnic, 1402 "tport:0x%x lsreq:0x%x already done\n", 1403 tport_fcid, oxid); 1404 return; 1405 } 1406 if (!lsreq->rspaddr) { 1407 FNIC_NVME_DBG(KERN_ERR, fnic, 1408 "tport 0x%x lsreq 0x%x rspaddr NULL\n", 1409 tport_fcid, oxid); 1410 return; 1411 } 1412 1413 if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) || 1414 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) { 1415 FNIC_NVME_DBG(KERN_INFO, fnic, 1416 "tport 0x%x lsreq oxid: 0x%x abts pending\n", 1417 tport_fcid, oxid); 1418 return; 1419 } 1420 1421 if (rsp_len > lsreq->rsplen) { 1422 FNIC_NVME_DBG(KERN_ERR, fnic, 1423 "tport:0x%x lsreq:0x%x rsp %u > %u\n", 1424 tport_fcid, oxid, rsp_len, lsreq->rsplen); 1425 status = -EOVERFLOW; 1426 } 1427 1428 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE; 1429 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_responses); 1430 1431 list_del_init(&nvfnic_ls_req->list); 1432 lsreq->private = NULL; 1433 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid); 1434 1435 if (status == 0) { 1436 FNIC_NVME_DBG(KERN_DEBUG, fnic, 1437 "tport:0x%x lsreq:0x%x completed\n", 1438 tport_fcid, oxid); 1439 1440 /* Copy the Response */ 1441 memcpy(lsreq->rspaddr, (uint8_t *)fchdr + sizeof(*fchdr), 1442 rsp_len); 1443 } 1444 1445 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1446 timer_delete_sync(&nvfnic_ls_req->ls_req_timer); 1447 lsreq->done(lsreq, status); 1448 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1449 } 1450 1451 static bool nvfnic_ls_req_cleanup(struct fnic_iport_s *iport, 1452 struct nvmefc_ls_req *lsreq, 1453 uint16_t oxid) 1454 { 1455 struct nvfnic_ls_req *nvfnic_ls_req = lsreq->private; 1456 1457 if (!nvfnic_ls_req) 1458 return false; 1459 1460 lsreq->private = NULL; 1461 list_del(&nvfnic_ls_req->list); 1462 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid); 1463 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE; 1464 1465 return true; 1466 } 1467 1468 void nvfnic_ls_req_timeout(struct timer_list *t) 1469 { 1470 struct nvfnic_ls_req *nvfnic_ls_req = timer_container_of(nvfnic_ls_req, 1471 t, ls_req_timer); 1472 struct fnic *fnic = nvfnic_ls_req->fnic; 1473 struct nvmefc_ls_req *ls_req = nvfnic_ls_req->ls_req; 1474 struct fnic_iport_s *iport = &fnic->iport; 1475 struct fnic_tport_s *tport = (struct fnic_tport_s *) nvfnic_ls_req->tport; 1476 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1477 uint16_t oxid = nvfnic_ls_req->oxid; 1478 int timeout; 1479 1480 FNIC_NVME_DBG(KERN_INFO, fnic, 1481 "tport: 0x%x lsreq: 0x%x state: %d timeout\n", 1482 tport->fcid, nvfnic_ls_req->oxid, 1483 nvfnic_ls_req->state); 1484 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1485 1486 if ((ls_req->private == NULL) || 1487 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) { 1488 FNIC_NVME_DBG(KERN_ERR, fnic, 1489 "tport: 0x%x lsreq: 0x%x already aborted\n", 1490 tport->fcid, nvfnic_ls_req->oxid); 1491 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1492 return; 1493 } 1494 1495 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) { 1496 FNIC_NVME_DBG(KERN_ERR, fnic, 1497 "tport: 0x%x lsreq: 0x%x abort timeout\n", 1498 tport->fcid, nvfnic_ls_req->oxid); 1499 1500 ls_req = nvfnic_ls_req->ls_req; 1501 nvfnic_ls_req_cleanup(iport, ls_req, oxid); 1502 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1503 ls_req->done(ls_req, -ETIMEDOUT); 1504 return; 1505 } else if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_PENDING) && 1506 (nvfnic_transport_ready(iport, tport))) { 1507 FNIC_NVME_DBG(KERN_ERR, fnic, 1508 "tport: 0x%x lsreq: 0x%x sending abort\n", 1509 tport->fcid, nvfnic_ls_req->oxid); 1510 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_PENDING; 1511 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_aborts); 1512 1513 if (fdls_send_ls_req_abts(iport, tport, nvfnic_ls_req->oxid) == 0) { 1514 timeout = FNIC_LS_REQ_TMO_MSECS(ls_req->timeout); 1515 mod_timer(&nvfnic_ls_req->ls_req_timer, 1516 round_jiffies(jiffies + msecs_to_jiffies(timeout))); 1517 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1518 return; 1519 } 1520 FNIC_NVME_DBG(KERN_ERR, fnic, 1521 "tport: 0x%x lsreq: 0x%x cannot send abort\n", 1522 tport->fcid, oxid); 1523 } 1524 1525 if (ls_req->private == NULL) { 1526 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1527 return; 1528 } 1529 1530 ls_req = nvfnic_ls_req->ls_req; 1531 nvfnic_ls_req_cleanup(iport, ls_req, oxid); 1532 1533 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1534 ls_req->done(ls_req, -ETIMEDOUT); 1535 } 1536 1537 /** 1538 * nvfnic_ls_req_send - Send NVMe FC link service (LS) request 1539 * @lport: Pointer to local NVMe FC port structure 1540 * @rport: Pointer to remote NVMe FC port structure 1541 * @ls_req: Pointer to the link service request structure 1542 * 1543 * This function is used to send link service (LS) commands to an NVMe 1544 * Discovery Controller for discovery operations, as well as to regular 1545 * NVMe subsystems during association. It encapsulates the logic for 1546 * transmitting LS requests over the NVMe over Fabrics (NVMe-oF) FC 1547 * transport. 1548 * 1549 * Returns: 0 on success, or a negative error code on failure. 1550 */ 1551 int nvfnic_ls_req_send(struct nvme_fc_local_port *lport, 1552 struct nvme_fc_remote_port *rport, 1553 struct nvmefc_ls_req *ls_req) 1554 { 1555 int timeout; 1556 uint8_t *frame; 1557 uint8_t fcid[3]; 1558 unsigned long flags = 0; 1559 struct fnic_iport_s *iport = lport->private; 1560 uint8_t *ls_req_payload; 1561 struct fnic *fnic = iport->fnic; 1562 struct fc_frame_header *fchdr; 1563 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1564 struct nvfnic_ls_req *nvfnic_ls_req = ls_req->private; 1565 uint16_t frame_size = FNIC_ETH_FCOE_HDRS_OFFSET + 1566 sizeof(struct fc_frame_header) + ls_req->rqstlen; 1567 struct fnic_tport_s *tport; 1568 int ret; 1569 1570 spin_lock_irqsave(&fnic->fnic_lock, flags); 1571 1572 tport = (struct fnic_tport_s *)rport->private; 1573 INIT_LIST_HEAD(&nvfnic_ls_req->list); 1574 1575 if (!nvfnic_transport_ready(iport, tport)) { 1576 if (tport != NULL) 1577 FNIC_NVME_DBG(KERN_INFO, fnic, 1578 "iport: 0x%x tport: 0x%x transport not ready\n", 1579 iport->fcid, tport->fcid); 1580 else 1581 FNIC_NVME_DBG(KERN_INFO, fnic, 1582 "iport: 0x%x transport not ready\n", 1583 iport->fcid); 1584 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1585 return -ENOLINK; 1586 } 1587 1588 frame = fdls_alloc_frame(iport); 1589 if (frame == NULL) { 1590 FNIC_NVME_DBG(KERN_ERR, fnic, 1591 "Failed to allocate frame to send NVME LS REQ"); 1592 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1593 return -ENOMEM; 1594 } 1595 1596 if (fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_NVME_LS, 1597 &nvfnic_ls_req->oxid) == FNIC_UNASSIGNED_OXID) { 1598 FNIC_FCS_DBG(KERN_INFO, fnic, 1599 "0x%x: Failed to allocate OXID to send NVME LS REQ", 1600 iport->fcid); 1601 mempool_free(frame, fnic->frame_pool); 1602 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1603 return -EAGAIN; 1604 } 1605 1606 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_requests); 1607 timer_setup(&nvfnic_ls_req->ls_req_timer, nvfnic_ls_req_timeout, 1608 0UL); 1609 1610 nvfnic_ls_req->fnic = fnic; 1611 nvfnic_ls_req->tport = tport; 1612 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_INIT; 1613 nvfnic_ls_req->ls_req = ls_req; 1614 1615 fchdr = (struct fc_frame_header *)(frame + FNIC_ETH_FCOE_HDRS_OFFSET); 1616 *fchdr = (struct fc_frame_header) { 1617 .fh_r_ctl = FC_RCTL_ELS4_REQ, 1618 .fh_type = FC_TYPE_NVME, 1619 .fh_f_ctl = {FNIC_ELS_REQ_FCTL, 0, 0}, 1620 .fh_rx_id = cpu_to_be16(FNIC_UNASSIGNED_RXID) 1621 }; 1622 1623 hton24(fcid, iport->fcid); 1624 FNIC_STD_SET_S_ID(*fchdr, fcid); 1625 1626 hton24(fcid, tport->fcid); 1627 FNIC_STD_SET_D_ID(*fchdr, fcid); 1628 1629 FNIC_STD_SET_OX_ID(*fchdr, nvfnic_ls_req->oxid); 1630 1631 ls_req_payload = frame + FNIC_ETH_FCOE_HDRS_OFFSET + sizeof(*fchdr); 1632 memcpy(ls_req_payload, ls_req->rqstaddr, ls_req->rqstlen); 1633 1634 FNIC_NVME_DBG(KERN_INFO, fnic, 1635 "0x%x: NVME send ls req with oxid: 0x%x type: 0x%02x len: %d", 1636 iport->fcid, nvfnic_ls_req->oxid, *((uint8_t *) ls_req->rqstaddr), 1637 ls_req->rqstlen); 1638 1639 FNIC_NVME_DBG(KERN_INFO, fnic, 1640 "0x%x: ls_reqs count: %lld", 1641 iport->fcid, 1642 (u64) atomic64_read(&fnic_stats->nvme_stats.nvme_ls_requests)); 1643 1644 list_add_tail(&nvfnic_ls_req->list, &tport->ls_req_list); 1645 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_PENDING; 1646 1647 ret = fnic_send_fcoe_frame(iport, frame, frame_size); 1648 if (ret) { 1649 nvfnic_ls_req_cleanup(iport, ls_req, nvfnic_ls_req->oxid); 1650 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1651 mempool_free(frame, fnic->frame_pool); 1652 return ret; 1653 } 1654 1655 timeout = FNIC_LS_REQ_TMO_MSECS(ls_req->timeout); 1656 mod_timer(&nvfnic_ls_req->ls_req_timer, 1657 round_jiffies(jiffies + msecs_to_jiffies(timeout))); 1658 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1659 1660 return 0; 1661 } 1662 1663 void nvfnic_local_port_delete(struct nvme_fc_local_port *lport) 1664 { 1665 struct fnic_iport_s *iport = (struct fnic_iport_s *) lport->private; 1666 struct fnic *fnic = iport->fnic; 1667 unsigned long flags = 0; 1668 1669 FNIC_NVME_DBG(KERN_INFO, fnic, "lport delete 0x%x\n", 1670 iport->fcid); 1671 1672 spin_lock_irqsave(&fnic->fnic_lock, flags); 1673 if (fnic->nvme_lport_unreg_done) 1674 complete(fnic->nvme_lport_unreg_done); 1675 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1676 } 1677 1678 void nvfnic_remote_port_delete(struct nvme_fc_remote_port *rport) 1679 { 1680 /* 1681 * Read rport->private without the lock only to find fnic. 1682 * Re-read it under fnic_lock to claim this delete callback, since 1683 * another callback may already have cleared it. 1684 */ 1685 struct fnic_tport_s *tport = (struct fnic_tport_s *)rport->private; 1686 struct fnic_iport_s *iport; 1687 struct fnic *fnic = NULL; 1688 unsigned long flags = 0; 1689 1690 if (tport == NULL) { 1691 pr_err("Attempt to delete already deleted tport\n"); 1692 return; 1693 } 1694 1695 iport = tport->iport; 1696 fnic = iport->fnic; 1697 FNIC_NVME_DBG(KERN_INFO, fnic, "0x%x tport 0x%x\n", 1698 iport->fcid, tport->fcid); 1699 1700 spin_lock_irqsave(&fnic->fnic_lock, flags); 1701 tport = (struct fnic_tport_s *)rport->private; 1702 if (tport == NULL) { 1703 FNIC_NVME_DBG(KERN_ERR, fnic, 1704 "NVMe tport callback after NULL set %p\n", 1705 rport); 1706 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1707 return; 1708 } 1709 1710 iport = tport->iport; 1711 rport->private = NULL; 1712 1713 if (tport->timer_pending) { 1714 FNIC_NVME_DBG(KERN_INFO, fnic, 1715 "tport: 0x%x canceling discovery timer\n", 1716 tport->fcid); 1717 /* 1718 * The retry timer callback takes fnic_lock and dereferences 1719 * this tport. Set del_timer_inprogress before dropping the 1720 * lock so a callback that is already running observes teardown 1721 * and exits. 1722 * 1723 * timer_delete_sync() then waits until no callback can still 1724 * hold a reference before the remoteport-delete path completes 1725 * or frees the target port. 1726 */ 1727 tport->del_timer_inprogress = 1; 1728 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1729 timer_delete_sync(&tport->retry_timer); 1730 spin_lock_irqsave(&fnic->fnic_lock, flags); 1731 tport->del_timer_inprogress = 0; 1732 tport->timer_pending = 0; 1733 } 1734 1735 if (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) { 1736 FNIC_NVME_DBG(KERN_ERR, fnic, 1737 "tport %8x waiting on clean pending\n", 1738 tport->fcid); 1739 } 1740 1741 while (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) { 1742 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1743 msleep(2000); 1744 spin_lock_irqsave(&fnic->fnic_lock, flags); 1745 } 1746 1747 list_del(&tport->links); 1748 1749 if (tport->tport_del_done) { 1750 tport->flags |= FNIC_TPORT_CAN_BE_FREED; 1751 complete(tport->tport_del_done); 1752 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1753 return; 1754 } 1755 1756 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1757 kfree(tport); 1758 } 1759 1760 int nvfnic_create_queue(struct nvme_fc_local_port *lport, 1761 unsigned int idx, u16 size, void **handle) 1762 { 1763 struct fnic_iport_s *iport = (struct fnic_iport_s *)lport->private; 1764 struct fnic *fnic = iport->fnic; 1765 1766 FNIC_NVME_DBG(KERN_DEBUG, fnic, 1767 "iport:0x%x queue:%d size:%d\n", iport->fcid, idx, size); 1768 1769 if (idx > fnic->wq_copy_count) 1770 return -EINVAL; 1771 1772 if (idx == 0) { 1773 /* Admin queue */ 1774 *handle = &fnic->hw_copy_wq[0]; 1775 } else { 1776 /* IO queues */ 1777 *handle = &fnic->hw_copy_wq[idx-1]; 1778 } 1779 1780 return 0; 1781 } 1782 1783 void nvfnic_ls_req_abort(struct nvme_fc_local_port *lport, 1784 struct nvme_fc_remote_port *rport, 1785 struct nvmefc_ls_req *lsreq) 1786 { 1787 struct fnic_iport_s *iport = lport->private; 1788 struct fnic *fnic = iport->fnic; 1789 struct fnic_tport_s *tport; 1790 struct nvfnic_ls_req *nvfnic_ls_req; 1791 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1792 uint16_t oxid; 1793 int timeout; 1794 int ret; 1795 1796 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1797 1798 tport = (struct fnic_tport_s *) rport->private; 1799 /* find the request */ 1800 nvfnic_ls_req = lsreq->private; 1801 1802 if (nvfnic_ls_req == NULL) { 1803 FNIC_NVME_DBG(KERN_ERR, fnic, 1804 "0x%x null lsreq already scheduled for abort\n", 1805 iport->fcid); 1806 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1807 return; 1808 } 1809 1810 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) { 1811 FNIC_NVME_DBG(KERN_ERR, fnic, 1812 "0x%x lsreq 0x%x already scheduled for abort\n", 1813 iport->fcid, nvfnic_ls_req->oxid); 1814 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1815 return; 1816 } 1817 1818 FNIC_NVME_DBG(KERN_INFO, fnic, 1819 "0x%x lsreq 0x%x abts\n", 1820 iport->fcid, nvfnic_ls_req->oxid); 1821 1822 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_STARTED; 1823 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1824 timer_delete_sync(&nvfnic_ls_req->ls_req_timer); 1825 1826 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags); 1827 nvfnic_ls_req = lsreq->private; 1828 1829 if ((nvfnic_ls_req == NULL) || 1830 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING)) { 1831 FNIC_NVME_DBG(KERN_ERR, fnic, 1832 "lsreq timeout raced with midlayer abort\n"); 1833 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1834 return; 1835 } 1836 1837 /* Basic validations of the state */ 1838 if (!nvfnic_transport_ready(iport, tport)) { 1839 /* If iport or tport offline, it will be handled from that event */ 1840 oxid = nvfnic_ls_req->oxid; 1841 lsreq->private = NULL; 1842 list_del(&nvfnic_ls_req->list); 1843 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid); 1844 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1845 lsreq->done(lsreq, -ENXIO); 1846 FNIC_NVME_DBG(KERN_ERR, fnic, 1847 "nvfnic_lsreq_abort transport not ready\n"); 1848 return; 1849 } 1850 1851 /* Mark the state and flags */ 1852 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_PENDING; 1853 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_aborts); 1854 oxid = nvfnic_ls_req->oxid; 1855 1856 ret = fdls_send_ls_req_abts(iport, tport, oxid); 1857 if (!ret) { 1858 timeout = FNIC_LS_REQ_TMO_MSECS(lsreq->timeout); 1859 mod_timer(&nvfnic_ls_req->ls_req_timer, 1860 round_jiffies(jiffies + msecs_to_jiffies(timeout))); 1861 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1862 return; 1863 } 1864 1865 if (!nvfnic_ls_req_cleanup(iport, lsreq, oxid)) { 1866 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1867 return; 1868 } 1869 1870 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags); 1871 lsreq->done(lsreq, -EAGAIN); 1872 } 1873 1874 bool nvfnic_queue_abort_io_req(struct fnic *fnic, int tag, 1875 u32 task_req, struct fnic_io_req *io_req) 1876 { 1877 int idx; 1878 unsigned long flags; 1879 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; 1880 1881 idx = io_req->wq - &fnic->hw_copy_wq[0]; 1882 1883 atomic_inc(&fnic->in_flight); 1884 1885 spin_lock_irqsave(&fnic->wq_copy_lock[idx], flags); 1886 1887 if (vnic_wq_copy_desc_avail(io_req->wq) <= fnic->wq_copy_desc_low[idx]) 1888 free_wq_copy_descs(fnic, io_req->wq, idx); 1889 1890 if (!vnic_wq_copy_desc_avail(io_req->wq)) { 1891 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags); 1892 atomic_dec(&fnic->in_flight); 1893 FNIC_NVME_DBG(KERN_ERR, fnic, 1894 "tag 0x%x failure: no descriptors\n", tag); 1895 atomic64_inc(&misc_stats->abts_cpwq_alloc_failures); 1896 return false; 1897 } 1898 fnic_queue_wq_copy_desc_itmf(io_req->wq, tag | FNIC_TAG_ABORT, 1899 0, task_req, tag, NULL, io_req->port_id, 1900 fnic->config.ra_tov, fnic->config.ed_tov); 1901 1902 atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); 1903 if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) > 1904 atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs)) 1905 atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, 1906 atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); 1907 1908 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags); 1909 atomic_dec(&fnic->in_flight); 1910 1911 return true; 1912 } 1913 1914 void nvfnic_fcpio_abort(struct nvme_fc_local_port *lport, 1915 struct nvme_fc_remote_port *rport, 1916 void *hw_queue_handle, struct nvmefc_fcp_req *fcp_req) 1917 { 1918 struct fnic_iport_s *iport = lport->private; 1919 struct fnic *fnic = iport->fnic; 1920 struct nvme_fc_cmd_iu *cmd_iu = fcp_req->cmdaddr; 1921 struct fnic_io_req *io_req = (struct fnic_io_req *)fcp_req->private; 1922 unsigned int tag = io_req->tag; 1923 struct fnic_stats *fnic_stats = &fnic->fnic_stats; 1924 struct abort_stats *abts_stats; 1925 unsigned long flags = 0; 1926 unsigned long abt_issued_time; 1927 unsigned int task_req; 1928 enum fnic_ioreq_state old_ioreq_state; 1929 unsigned long num_ios_waitq, waitq_2sec, waitq_max_time; 1930 1931 spin_lock_irqsave(&fnic->fnic_lock, flags); 1932 1933 if (io_req->tag == FNIC_NVME_NO_FREE_TAG) { 1934 FNIC_NVME_DBG(KERN_ERR, fnic, 1935 "tag: (0x%x) tport_fcid: 0x%x\n", 1936 io_req->tag, io_req->tport->fcid); 1937 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1938 return; 1939 } 1940 1941 if (io_req != nvfnic_find_io_req_by_tag(fnic, io_req->tag)) { 1942 FNIC_NVME_DBG(KERN_INFO, fnic, 1943 "cmd tag freed or not issued:0x%x sn:0x%08x\n", 1944 io_req->tag, be32_to_cpu(cmd_iu->csn)); 1945 num_ios_waitq = 1946 atomic64_read(&fnic_stats->io_stats.nvme_num_ios_in_waitq); 1947 waitq_2sec = 1948 atomic64_read(&fnic_stats->io_stats.nvme_ios_in_waitq_3000_msec); 1949 waitq_max_time = 1950 atomic64_read(&fnic_stats->io_stats.nvme_ios_in_waitq_max_time); 1951 FNIC_NVME_DBG(KERN_INFO, fnic, 1952 "waitq:%ld waitq_2sec:%ld max_wait:%ld\n", 1953 num_ios_waitq, waitq_2sec, waitq_max_time); 1954 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1955 return; 1956 } 1957 1958 if (io_req->cmd_state == FNIC_IOREQ_CMD_COMPLETE) { 1959 FNIC_NVME_DBG(KERN_INFO, fnic, 1960 "IO already completed before abort: 0x%x\n", tag); 1961 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1962 return; 1963 } 1964 1965 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) || 1966 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) { 1967 FNIC_NVME_DBG(KERN_INFO, fnic, "abort already pending %d\n", 1968 tag); 1969 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1970 return; 1971 } 1972 1973 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) { 1974 FNIC_NVME_DBG(KERN_INFO, fnic, 1975 "io_req completed or aborted for tag:0x%x\n", 1976 tag); 1977 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1978 return; 1979 } 1980 1981 FNIC_NVME_DBG(KERN_INFO, fnic, "in abort cmd_sn:%08x %llx tag: 0x%x\n", 1982 be32_to_cpu(cmd_iu->csn), 1983 le64_to_cpu(cmd_iu->sqe.rw.slba), io_req->tag); 1984 1985 if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { 1986 FNIC_NVME_DBG(KERN_INFO, fnic, 1987 "abort tag:0x%x returned during fw reset\n", 1988 io_req->tag); 1989 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 1990 return; 1991 } 1992 atomic_inc(&fnic->in_flight); 1993 1994 if (fdls_tport_is_offline(io_req->tport) || 1995 (io_req->cmd_state == FNIC_IOREQ_RESET_TERM)) { 1996 task_req = FCPIO_ITMF_ABT_TASK_TERM; 1997 } else { 1998 task_req = FCPIO_ITMF_ABT_TASK; 1999 } 2000 2001 abts_stats = &fnic->fnic_stats.abts_stats; 2002 atomic64_inc(&abts_stats->aborts); 2003 2004 abt_issued_time = jiffies_to_msecs(jiffies - io_req->start_time); 2005 if (abt_issued_time <= 6000) 2006 atomic64_inc(&abts_stats->abort_issued_btw_0_to_6_sec); 2007 else if (abt_issued_time > 6000 && abt_issued_time <= 20000) 2008 atomic64_inc(&abts_stats->abort_issued_btw_6_to_20_sec); 2009 else if (abt_issued_time > 20000 && abt_issued_time <= 30000) 2010 atomic64_inc(&abts_stats->abort_issued_btw_20_to_30_sec); 2011 else if (abt_issued_time > 30000 && abt_issued_time <= 40000) 2012 atomic64_inc(&abts_stats->abort_issued_btw_30_to_40_sec); 2013 else if (abt_issued_time > 40000 && abt_issued_time <= 50000) 2014 atomic64_inc(&abts_stats->abort_issued_btw_40_to_50_sec); 2015 else if (abt_issued_time > 50000 && abt_issued_time <= 60000) 2016 atomic64_inc(&abts_stats->abort_issued_btw_50_to_60_sec); 2017 else 2018 atomic64_inc(&abts_stats->abort_issued_greater_than_60_sec); 2019 2020 old_ioreq_state = io_req->cmd_state; 2021 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING; 2022 io_req->abts_state = FCPIO_INVALID_CODE; 2023 2024 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2025 2026 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag, task_req, io_req)) { 2027 FNIC_NVME_DBG(KERN_INFO, fnic, 2028 "Abort io req queue failed\n"); 2029 spin_lock_irqsave(&fnic->fnic_lock, flags); 2030 io_req->cmd_state = old_ioreq_state; 2031 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2032 } 2033 atomic_dec(&fnic->in_flight); 2034 } 2035 2036 struct 2037 nvme_fc_port_template nvfnic_port = { 2038 .localport_delete = nvfnic_local_port_delete, 2039 .remoteport_delete = nvfnic_remote_port_delete, 2040 .create_queue = nvfnic_create_queue, 2041 .delete_queue = NULL, 2042 .ls_req = nvfnic_ls_req_send, 2043 .ls_abort = nvfnic_ls_req_abort, 2044 .fcp_io = nvfnic_fcpio_send, 2045 .fcp_abort = nvfnic_fcpio_abort, 2046 .max_hw_queues = 1, 2047 .max_sgl_segments = 256, 2048 .max_dif_sgl_segments = 64, 2049 .dma_boundary = 0xFFFFFFFF, 2050 .local_priv_sz = sizeof(struct fnic_iport_s *), 2051 .remote_priv_sz = sizeof(struct fnic_tport_s *), 2052 .lsrqst_priv_sz = sizeof(struct nvfnic_ls_req), 2053 .fcprqst_priv_sz = sizeof(struct fnic_io_req), 2054 }; 2055 2056 void nvfnic_flush_nvme_io_list(struct fnic *fnic) 2057 { 2058 queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work); 2059 flush_work(&fnic->nvme_io_cmpl_work); 2060 } 2061 2062 void nvfnic_nvme_iodone_work(struct work_struct *work) 2063 { 2064 struct fnic *fnic = container_of(work, struct fnic, nvme_io_cmpl_work); 2065 struct llist_node *llnode; 2066 struct fnic_io_req *io_req, *tmp; 2067 2068 llnode = llist_del_all(&fnic->nvme_io_event_llist); 2069 llist_for_each_entry_safe(io_req, tmp, llnode, nvfnic_io_cmpl) { 2070 atomic_dec(&fnic->nvme_io_event_queued); 2071 atomic64_dec(&fnic->fnic_stats.io_stats.nvme_num_ios_in_waitq); 2072 io_req->fcp_req->done(io_req->fcp_req); 2073 } 2074 } 2075 2076 void nvfnic_exch_reset(struct fnic_iport_s *iport, struct fnic_tport_s *tport) 2077 { 2078 FNIC_NVME_DBG(KERN_DEBUG, iport->fnic, 2079 "0x%x: Exchange reset scheduled for tport: 0x%x\n", 2080 iport->fcid, tport->fcid); 2081 2082 nvfnic_terminate_tport_ls_reqs(iport->fnic, tport); 2083 nvfnic_terminate_tport_ios(iport->fnic, tport); 2084 } 2085 2086 void nvfnic_delete_tport(struct fnic_iport_s *iport, 2087 struct fnic_tport_s *tport, 2088 unsigned long flags) 2089 { 2090 struct fnic *fnic = iport->fnic; 2091 int ret; 2092 unsigned int time_wait = FNIC_NVME_TPORT_REMOVE_WAIT; 2093 unsigned int time_remain; 2094 DECLARE_COMPLETION_ONSTACK(tm_done); 2095 unsigned int fcid; 2096 int count = 0; 2097 2098 if (!tport) 2099 return; 2100 2101 fcid = tport->fcid; 2102 fdls_set_tport_state(tport, FDLS_TGT_STATE_OFFLINE); 2103 2104 FNIC_NVME_DBG(KERN_DEBUG, fnic, 2105 "0x%x: scheduled deletion for tport: 0x%x\n", 2106 iport->fcid, tport->fcid); 2107 2108 if (!(tport->flags & FNIC_FDLS_NVME_REGISTERED)) { 2109 FNIC_NVME_DBG(KERN_ERR, fnic, 2110 "0x%x: tport: 0x%x not registered. Freeing\n", 2111 iport->fcid, tport->fcid); 2112 list_del(&tport->links); 2113 kfree(tport); 2114 return; 2115 } 2116 2117 tport->tport_del_done = &tm_done; 2118 2119 tport->flags |= FNIC_FDLS_TPORT_DELETED; 2120 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2121 ret = nvme_fc_unregister_remoteport(tport->nv_rport); 2122 if (ret) { 2123 FNIC_NVME_DBG(KERN_ERR, fnic, 2124 "tport: 0x%x unregister failed %d\n", 2125 tport->fcid, ret); 2126 2127 nvfnic_terminate_tport_ls_reqs(fnic, tport); 2128 2129 spin_lock_irqsave(&fnic->fnic_lock, flags); 2130 tport->tport_del_done = NULL; 2131 if (tport->nv_rport && tport->nv_rport->private == tport) 2132 tport->nv_rport->private = NULL; 2133 list_del(&tport->links); 2134 kfree(tport); 2135 return; 2136 } 2137 time_remain = wait_for_completion_timeout(tport->tport_del_done, 2138 msecs_to_jiffies(time_wait)); 2139 2140 FNIC_NVME_DBG(KERN_DEBUG, fnic, 2141 "tport: 0x%x wait for deletion done\n", 2142 tport->fcid); 2143 2144 spin_lock_irqsave(&fnic->fnic_lock, flags); 2145 tport->tport_del_done = NULL; 2146 2147 if (!time_remain) { 2148 FNIC_NVME_DBG(KERN_ERR, fnic, 2149 "tport: 0x%x nvme midlayer completion timed out\n", 2150 tport->fcid); 2151 2152 if (tport->flags & FNIC_TPORT_CAN_BE_FREED) { 2153 kfree(tport); 2154 FNIC_NVME_DBG(KERN_INFO, fnic, 2155 "tport: 0x%x delete complete\n", fcid); 2156 return; 2157 } 2158 2159 tport->flags |= FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING; 2160 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2161 nvfnic_cleanup_tport_io(fnic, tport); 2162 spin_lock_irqsave(&fnic->fnic_lock, flags); 2163 tport->flags &= ~FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING; 2164 return; 2165 } 2166 while (!(tport->flags & FNIC_TPORT_CAN_BE_FREED) && 2167 (count < FNIC_TPORT_CLEANUP_WAIT_COUNT)) { 2168 count++; 2169 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2170 msleep(2000); 2171 spin_lock_irqsave(&fnic->fnic_lock, flags); 2172 } 2173 if (tport->flags & FNIC_TPORT_CAN_BE_FREED) 2174 kfree(tport); 2175 2176 FNIC_NVME_DBG(KERN_INFO, fnic, 2177 "tport: 0x%x delete complete\n", fcid); 2178 } 2179 2180 int nvfnic_add_tport(struct fnic *fnic, struct fnic_tport_s *tport, 2181 unsigned long flags) 2182 { 2183 struct fnic_iport_s *iport = &fnic->iport; 2184 struct nvme_fc_port_info pinfo; 2185 int ret = 0; 2186 2187 FNIC_NVME_DBG(KERN_INFO, fnic, 2188 "Adding tport to nvme wwpn: 0x%llx\n", 2189 tport->wwpn); 2190 2191 memset(&pinfo, 0, sizeof(struct nvme_fc_port_info)); 2192 2193 pinfo.port_name = tport->wwpn; 2194 pinfo.node_name = tport->wwnn; 2195 pinfo.port_role = FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET; 2196 pinfo.port_id = tport->fcid; 2197 pinfo.dev_loss_tmo = nvme_dev_loss_tmo; 2198 2199 spin_unlock_irqrestore(&fnic->fnic_lock, flags); 2200 ret = nvme_fc_register_remoteport(iport->nv_lport, &pinfo, 2201 &tport->nv_rport); 2202 spin_lock_irqsave(&fnic->fnic_lock, flags); 2203 if (ret) { 2204 FNIC_NVME_DBG(KERN_INFO, fnic, 2205 "Failed to register tport wwpn: 0x%llx ret: %d\n", 2206 tport->wwpn, ret); 2207 return ret; 2208 } 2209 tport->flags |= FNIC_FDLS_NVME_REGISTERED; 2210 tport->nv_rport->private = tport; 2211 2212 snprintf(tport->str_wwpn, sizeof(tport->str_wwpn), "0x%llx", tport->wwpn); 2213 snprintf(tport->str_wwnn, sizeof(tport->str_wwnn), "0x%llx", tport->wwnn); 2214 return ret; 2215 } 2216 2217 int nvfnic_add_lport(struct fnic *fnic) 2218 { 2219 struct nvme_fc_port_info pinfo; 2220 struct fnic_iport_s *iport = &fnic->iport; 2221 int ret = 0; 2222 2223 FNIC_NVME_DBG(KERN_INFO, fnic, 2224 "Adding lport nvme wwpn: 0x%llx\n", 2225 iport->wwpn); 2226 2227 pinfo.node_name = iport->wwnn; 2228 pinfo.port_name = iport->wwpn; 2229 pinfo.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2230 pinfo.port_id = iport->fcid; 2231 2232 nvfnic_reset_fcpio_tag_pool(iport); 2233 2234 if (!(iport->flags & FNIC_LPORT_NVME_REGISTERED)) { 2235 iport->nv_tmpl = kzalloc_obj(struct nvme_fc_port_template, GFP_ATOMIC); 2236 if (!iport->nv_tmpl) { 2237 FNIC_FCS_DBG(KERN_INFO, fnic, 2238 "iport:0x%x NVMe tmpl alloc failed\n", 2239 iport->fcid); 2240 return -ENOMEM; 2241 } 2242 memcpy(iport->nv_tmpl, &nvfnic_port, 2243 sizeof(struct nvme_fc_port_template)); 2244 iport->nv_tmpl->max_hw_queues = fnic->wq_copy_count; 2245 2246 ret = nvme_fc_register_localport(&pinfo, iport->nv_tmpl, 2247 &fnic->pdev->dev, &iport->nv_lport); 2248 if (ret) { 2249 FNIC_NVME_DBG(KERN_ERR, fnic, 2250 "Failed to add wwpn: 0x%llx ret: %d\n", 2251 iport->wwpn, ret); 2252 kfree(iport->nv_tmpl); 2253 return ret; 2254 } 2255 iport->flags |= FNIC_LPORT_NVME_REGISTERED; 2256 iport->nv_lport->private = iport; 2257 } 2258 2259 sprintf(iport->str_wwpn, "0x%llx", iport->wwpn); 2260 sprintf(iport->str_wwnn, "0x%llx", iport->wwnn); 2261 2262 FNIC_NVME_DBG(KERN_INFO, fnic, 2263 "Successfully added lport wwpn: 0x%llx\n", 2264 iport->wwpn); 2265 return 0; 2266 } 2267 2268 #endif 2269