1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2008 Cisco Systems, Inc. All rights reserved.
4 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 */
6 #include <linux/mempool.h>
7 #include <linux/errno.h>
8 #include <linux/init.h>
9 #include <linux/workqueue.h>
10 #include <linux/pci.h>
11 #include <linux/scatterlist.h>
12 #include <linux/skbuff.h>
13 #include <linux/spinlock.h>
14 #include <linux/kthread.h>
15 #include <linux/if_ether.h>
16 #include <linux/if_vlan.h>
17 #include <linux/delay.h>
18 #include <linux/gfp.h>
19 #include <linux/nvme.h>
20 #include <linux/nvme-fc.h>
21 #include "fnic.h"
22 #include "fnic_trace.h"
23 #include "fdls_fc.h"
24
25 #if IS_REACHABLE(CONFIG_NVME_FC)
26
27 static bool nvfnic_ls_req_cleanup(struct fnic_iport_s *iport,
28 struct nvmefc_ls_req *lsreq,
29 uint16_t oxid);
30
nvfnic_get_sg_count(struct fnic_io_req * io_req)31 int nvfnic_get_sg_count(struct fnic_io_req *io_req)
32 {
33 return io_req->fcp_req->sg_cnt;
34 }
35
nvfnic_dma_map_sgl(struct fnic * fnic,struct fnic_io_req * io_req,int sg_count)36 int nvfnic_dma_map_sgl(struct fnic *fnic, struct fnic_io_req *io_req,
37 int sg_count)
38 {
39 io_req->sgl_list_pa = dma_map_single(&fnic->pdev->dev,
40 io_req->sgl_list,
41 sizeof(io_req->sgl_list[0]) *
42 sg_count, DMA_TO_DEVICE);
43 if (dma_mapping_error(&fnic->pdev->dev, io_req->sgl_list_pa)) {
44 dev_err(&fnic->pdev->dev, "DMA mapping failed\n");
45 io_req->sgl_list_pa = 0;
46 io_req->sgl_mapped = 0;
47 return -EBUSY;
48 }
49
50 io_req->sgl_mapped = 1;
51 return 0;
52 }
53
nvfnic_dma_unmap_sgl(struct fnic * fnic,struct fnic_io_req * io_req)54 void nvfnic_dma_unmap_sgl(struct fnic *fnic, struct fnic_io_req *io_req)
55 {
56 if (io_req->sgl_mapped) {
57 dma_unmap_single(&fnic->pdev->dev, io_req->sgl_list_pa,
58 sizeof(io_req->sgl_list[0]) * io_req->sgl_cnt,
59 DMA_TO_DEVICE);
60 io_req->sgl_mapped = 0;
61 io_req->sgl_list_pa = 0;
62 }
63 }
64
nvfnic_update_io_bytes(struct fnic * fnic,struct fnic_io_req * io_req,u8 opcode)65 static void nvfnic_update_io_bytes(struct fnic *fnic,
66 struct fnic_io_req *io_req, u8 opcode)
67 {
68 if (opcode == nvme_cmd_read)
69 fnic->fcp_input_bytes += io_req->fcp_req->transferred_length;
70 else if (opcode == nvme_cmd_write)
71 fnic->fcp_output_bytes += io_req->fcp_req->transferred_length;
72 }
73
nvfnic_update_io_stats(struct fnic * fnic,u8 opcode)74 static void nvfnic_update_io_stats(struct fnic *fnic, u8 opcode)
75 {
76 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
77
78 switch (opcode) {
79 case nvme_cmd_read:
80 atomic64_inc(&fnic_stats->nvme_stats.nvme_input_requests);
81 break;
82 case nvme_cmd_write:
83 atomic64_inc(&fnic_stats->nvme_stats.nvme_output_requests);
84 break;
85 default:
86 atomic64_inc(&fnic_stats->nvme_stats.nvme_control_requests);
87 break;
88 }
89 }
90
nvfnic_update_cmpl_stats(struct fnic * fnic,struct fnic_io_req * io_req)91 static void nvfnic_update_cmpl_stats(struct fnic *fnic,
92 struct fnic_io_req *io_req)
93 {
94 struct io_path_stats *io_stats = &fnic->fnic_stats.io_stats;
95 atomic64_t *duration_stat;
96 unsigned long io_duration_time;
97
98 atomic64_dec(&io_stats->active_ios);
99 if (atomic64_read(&fnic->io_cmpl_skip))
100 atomic64_dec(&fnic->io_cmpl_skip);
101 else
102 atomic64_inc(&io_stats->io_completions);
103
104 io_duration_time = jiffies_to_msecs(jiffies - io_req->start_time);
105
106 if (io_duration_time <= 10)
107 duration_stat = &io_stats->io_btw_0_to_10_msec;
108 else if (io_duration_time <= 100)
109 duration_stat = &io_stats->io_btw_10_to_100_msec;
110 else if (io_duration_time <= 500)
111 duration_stat = &io_stats->io_btw_100_to_500_msec;
112 else if (io_duration_time <= 5000)
113 duration_stat = &io_stats->io_btw_500_to_5000_msec;
114 else if (io_duration_time <= 10000)
115 duration_stat = &io_stats->io_btw_5000_to_10000_msec;
116 else if (io_duration_time <= 30000)
117 duration_stat = &io_stats->io_btw_10000_to_30000_msec;
118 else {
119 duration_stat = &io_stats->io_greater_than_30000_msec;
120 if (io_duration_time >
121 atomic64_read(&io_stats->current_max_io_time))
122 atomic64_set(&io_stats->current_max_io_time,
123 io_duration_time);
124 }
125
126 atomic64_inc(duration_stat);
127 }
128
129 int
nvfnic_alloc_fcpio_tag(struct fnic_iport_s * iport,struct fnic_io_req * io_req)130 nvfnic_alloc_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req)
131 {
132 struct fnic *fnic = iport->fnic;
133 int tag;
134
135 tag = sbitmap_get(&fnic->nvfnic_tag_map);
136 if (tag >= 0) {
137 WRITE_ONCE(io_req->tag, tag);
138 fnic->sw_copy_wq[0].io_req_table[tag] = io_req;
139 return tag;
140 }
141 return FNIC_NVME_NO_FREE_TAG;
142 }
143
144 void
nvfnic_free_fcpio_tag(struct fnic_iport_s * iport,struct fnic_io_req * io_req)145 nvfnic_free_fcpio_tag(struct fnic_iport_s *iport, struct fnic_io_req *io_req)
146 {
147 struct fnic *fnic = iport->fnic;
148 uint16_t tag = io_req->tag;
149
150 if (tag == FNIC_NVME_NO_FREE_TAG) {
151 FNIC_NVME_DBG(KERN_ERR, fnic,
152 "Attempting to free invalid tag: 0x%x\n", tag);
153 return;
154 }
155 fnic->sw_copy_wq[0].io_req_table[tag] = NULL;
156 WRITE_ONCE(io_req->tag, FNIC_NVME_NO_FREE_TAG);
157 sbitmap_clear_bit(&fnic->nvfnic_tag_map, tag);
158 }
159
160 void
nvfnic_reset_fcpio_tag_pool(struct fnic_iport_s * iport)161 nvfnic_reset_fcpio_tag_pool(struct fnic_iport_s *iport)
162 {
163 WARN_ON(sbitmap_weight(&iport->fnic->nvfnic_tag_map));
164 }
165
166 struct fnic_io_req *
nvfnic_find_io_req_by_tag(struct fnic * fnic,uint16_t tag)167 nvfnic_find_io_req_by_tag(struct fnic *fnic, uint16_t tag)
168 {
169 if (tag == FNIC_NVME_NO_FREE_TAG ||
170 !sbitmap_test_bit(&fnic->nvfnic_tag_map, tag))
171 return NULL;
172 return fnic->sw_copy_wq[0].io_req_table[tag];
173 }
174
175 /*
176 * Unmap the data buffer and sense buffer for an io_req,
177 * also unmap and free the device-private scatter/gather list.
178 */
nvfnic_release_nvme_ioreq_buf(struct fnic_iport_s * iport,struct fnic_io_req * io_req)179 void nvfnic_release_nvme_ioreq_buf(struct fnic_iport_s *iport,
180 struct fnic_io_req *io_req)
181 {
182 struct fnic *fnic = iport->fnic;
183
184 nvfnic_dma_unmap_sgl(fnic, io_req);
185
186 if (io_req->sgl_cnt)
187 mempool_free(io_req->sgl_list_alloc,
188 fnic->io_sgl_pool[io_req->sgl_type]);
189 }
190
nvfnic_get_nvmef_info(struct fnic * fnic,struct fnic_nvmef_info * info)191 int nvfnic_get_nvmef_info(struct fnic *fnic, struct fnic_nvmef_info *info)
192 {
193 int len = 0;
194 struct fnic_iport_s *iport = &fnic->iport;
195 int buf_size = info->buf_size;
196 struct fnic_tport_s *tport;
197 struct fnic_tport_s *next;
198 unsigned long flags;
199
200 if (buf_size <= 0)
201 return 0;
202
203 len += scnprintf(info->info_buffer + len, buf_size - len,
204 "lport wwpn 0x%llx wwnn 0x%llx fcid 0x%06x\n",
205 iport->wwpn, iport->wwnn, iport->fcid);
206
207 spin_lock_irqsave(&fnic->fnic_lock, flags);
208 list_for_each_entry_safe(tport, next, &iport->tport_list, links) {
209 if (len >= buf_size - 1)
210 break;
211
212 len += scnprintf(info->info_buffer + len, buf_size - len,
213 "tport wwpn 0x%llx wwnn 0x%llx fcid 0x%06x\n",
214 tport->wwpn, tport->wwnn, tport->fcid);
215 }
216 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
217
218 return len;
219 }
220
nvfnic_queue_wq_nvme_copy_desc(struct fnic * fnic,struct vnic_wq_copy * wq,struct fnic_io_req * io_req,int sg_count)221 inline int nvfnic_queue_wq_nvme_copy_desc(struct fnic *fnic,
222 struct vnic_wq_copy *wq,
223 struct fnic_io_req *io_req,
224 int sg_count)
225 {
226 struct scatterlist *sg;
227 struct fnic_tport_s *tport = io_req->tport;
228 struct host_sg_desc *desc;
229 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;
230 unsigned int i;
231 unsigned long intr_flags;
232 int flags;
233 u8 exch_flags;
234 struct scatterlist *sgl;
235 int idx;
236 int ret = 0;
237
238 if (sg_count) {
239 /* For each SGE, create a device desc entry */
240 desc = io_req->sgl_list;
241 sgl = io_req->fcp_req->first_sgl;
242 for_each_sg(sgl, sg, sg_count, i) {
243 desc->addr = cpu_to_le64(sg_dma_address(sg));
244 desc->len = cpu_to_le32(sg_dma_len(sg));
245 desc->_resvd = 0;
246 desc++;
247 }
248
249 ret = nvfnic_dma_map_sgl(fnic, io_req, sg_count);
250 if (ret)
251 return ret;
252 }
253
254 idx = (struct vnic_wq_copy *)wq - &fnic->hw_copy_wq[0];
255
256 /* Enqueue the descriptor in the Copy WQ */
257 spin_lock_irqsave(&fnic->wq_copy_lock[idx], intr_flags);
258
259 if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[idx])
260 free_wq_copy_descs(fnic, wq, idx);
261
262 if (unlikely(!vnic_wq_copy_desc_avail(wq))) {
263 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags);
264 FNIC_NVME_DBG(KERN_ERR, fnic,
265 "Enqueue failure: No descriptors\n");
266 atomic64_inc(&misc_stats->io_cpwq_alloc_failures);
267 return -EBUSY;
268 }
269
270 flags = 0;
271 if (io_req->fcp_req->io_dir == NVMEFC_FCP_READ)
272 flags = FCPIO_ICMND_RDDATA;
273 else if (io_req->fcp_req->io_dir == NVMEFC_FCP_WRITE)
274 flags = FCPIO_ICMND_WRDATA;
275
276 exch_flags = 0;
277
278 fnic_queue_wq_copy_desc_nvme_io(wq, io_req->tag,
279 exch_flags, io_req->sgl_cnt,
280 io_req->sgl_list_pa, flags,
281 io_req->fcp_req->cmdaddr,
282 io_req->fcp_req->cmdlen,
283 io_req->fcp_req->payload_length,
284 io_req->port_id,
285 tport->max_payload_size, tport->r_a_tov,
286 tport->e_d_tov);
287
288 atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);
289 if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >
290 atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))
291 atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,
292 atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));
293
294 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], intr_flags);
295 return 0;
296 }
297
298 bool
nvfnic_transport_ready(struct fnic_iport_s * iport,struct fnic_tport_s * tport)299 nvfnic_transport_ready(struct fnic_iport_s *iport, struct fnic_tport_s *tport)
300 {
301 struct fnic *fnic = iport->fnic;
302 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
303
304 if (tport == NULL)
305 return false;
306
307 if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN ||
308 iport->state != FNIC_IPORT_STATE_READY) {
309 atomic64_inc(&fnic_stats->misc_stats.iport_not_ready);
310 return false;
311 }
312
313 if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED)))
314 return false;
315
316 if (fdls_tport_is_offline(tport)) {
317 atomic64_inc(&fnic_stats->misc_stats.tport_not_ready);
318 return false;
319 }
320
321 return true;
322 }
323
nvfnic_queuecommand(struct fnic_io_req * io_req)324 int nvfnic_queuecommand(struct fnic_io_req *io_req)
325 {
326 struct fnic_iport_s *iport = io_req->iport;
327 struct fnic *fnic = iport->fnic;
328 struct fnic_tport_s *tport = io_req->tport;
329 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
330 struct vnic_wq_copy *wq = io_req->wq;
331 int ret = 0;
332 int sg_count = 0;
333 unsigned long ptr;
334 unsigned char *lba;
335 u64 cmd_trace;
336 int idx;
337 struct nvme_fc_cmd_iu *cmdiu = io_req->fcp_req->cmdaddr;
338
339 io_req->cmd_state = FNIC_IOREQ_NOT_INITED;
340 io_req->cmd_flags = FNIC_NO_FLAGS;
341 /* Map the data buffer */
342 sg_count = nvfnic_get_sg_count(io_req);
343 if (sg_count < 0) {
344 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
345 io_req->tag, io_req, 0, io_req->fcp_req->io_dir,
346 sg_count, io_req->cmd_state);
347 FNIC_NVME_DBG(KERN_INFO, fnic, "sg count is less-than-zero\n");
348 ret = -1;
349 goto out;
350 }
351
352 /* Determine the type of scatter/gather list we need */
353 io_req->sgl_cnt = sg_count;
354 io_req->sgl_type = FNIC_SGL_CACHE_DFLT;
355 if (sg_count > FNIC_DFLT_SG_DESC_CNT)
356 io_req->sgl_type = FNIC_SGL_CACHE_MAX;
357
358 if (sg_count) {
359 io_req->sgl_list =
360 mempool_alloc(fnic->io_sgl_pool[io_req->sgl_type],
361 GFP_ATOMIC);
362 if (!io_req->sgl_list) {
363 atomic64_inc(&fnic_stats->io_stats.alloc_failures);
364 FNIC_NVME_DBG(KERN_INFO, fnic,
365 "Unable to alloc SGLs\n");
366 ret = -ENOMEM;
367 goto out;
368 }
369
370 /* Cache sgl list allocated address before alignment */
371 io_req->sgl_list_alloc = io_req->sgl_list;
372 ptr = (unsigned long)io_req->sgl_list;
373 if (ptr % FNIC_SG_DESC_ALIGN) {
374 io_req->sgl_list = (struct host_sg_desc *)
375 (((unsigned long)ptr + FNIC_SG_DESC_ALIGN - 1)
376 & ~(FNIC_SG_DESC_ALIGN - 1));
377 }
378 }
379
380 io_req->port_id = tport->fcid;
381 io_req->start_time = jiffies;
382 io_req->cmd_state = FNIC_IOREQ_CMD_PENDING;
383 io_req->cmd_flags = FNIC_IO_INITIALIZED;
384
385 /* create copy wq desc and enqueue it */
386 idx = wq - &fnic->hw_copy_wq[0];
387 atomic64_inc(&fnic_stats->io_stats.ios[idx]);
388 ret = nvfnic_queue_wq_nvme_copy_desc(fnic, io_req->wq, io_req, sg_count);
389 if (ret) {
390 FNIC_NVME_DBG(KERN_ERR, fnic, "Unable to queue frame\n");
391 /*
392 * In case another thread cancelled the request,
393 * refetch the pointer under the lock.
394 */
395 nvfnic_release_nvme_ioreq_buf(iport, io_req);
396 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
397 io_req->tag, io_req->fcp_req, 0, 0, 0,
398 (((u64)io_req->cmd_flags << 32) | io_req->cmd_state));
399 return ret;
400 }
401
402 atomic64_inc(&fnic_stats->io_stats.active_ios);
403 atomic64_inc(&fnic_stats->io_stats.num_ios);
404 if (atomic64_read(&fnic_stats->io_stats.active_ios) >
405 atomic64_read(&fnic_stats->io_stats.max_active_ios))
406 atomic64_set(&fnic_stats->io_stats.max_active_ios,
407 atomic64_read(&fnic_stats->io_stats.active_ios));
408 io_req->cmd_flags |= FNIC_IO_ISSUED;
409 out:
410 lba = (char *)&cmdiu->sqe.rw.slba;
411 cmd_trace = ((u64) cmdiu->sqe.rw.opcode << 56 | (u64) lba[4] << 40 |
412 (u64) lba[5] << 32 | (u64) lba[0] << 24 |
413 (u64) lba[1] << 16 | (u64) lba[2] << 8 | lba[3]);
414
415 FNIC_TRACE(nvfnic_queuecommand, fnic->fnic_num,
416 io_req->tag, 0, io_req,
417 sg_count, cmd_trace,
418 (((u64)io_req->cmd_flags >> 32) |
419 io_req->cmd_state));
420
421 return ret;
422 }
423
nvfnic_fcpio_send(struct nvme_fc_local_port * lport,struct nvme_fc_remote_port * rport,void * hw_queue_handle,struct nvmefc_fcp_req * fcp_req)424 int nvfnic_fcpio_send(struct nvme_fc_local_port *lport,
425 struct nvme_fc_remote_port *rport, void *hw_queue_handle,
426 struct nvmefc_fcp_req *fcp_req)
427 {
428 struct fnic_iport_s *iport = lport->private;
429 struct fnic_io_req *io_req;
430 int ret;
431 struct fnic *fnic = iport->fnic;
432 unsigned long flags = 0;
433 struct fnic_tport_s *tport;
434 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
435
436 spin_lock_irqsave(&fnic->fnic_lock, flags);
437
438 tport = (struct fnic_tport_s *)rport->private;
439 atomic64_inc(&fnic_stats->io_stats.nvme_io_reqs_rcvd);
440 if (!nvfnic_transport_ready(iport, tport)) {
441 atomic64_inc(&fnic_stats->io_stats.nvme_io_rsps_sent);
442 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
443 if (tport != NULL)
444 FNIC_NVME_DBG(KERN_INFO, fnic,
445 "iport: 0x%x tport: 0x%x not ready\n",
446 iport->fcid, tport->fcid);
447 else
448 FNIC_NVME_DBG(KERN_INFO, fnic,
449 "iport: 0x%x tport not ready\n",
450 iport->fcid);
451 return -ENODEV;
452 }
453 atomic_inc(&fnic->in_flight);
454
455 io_req = (struct fnic_io_req *) fcp_req->private;
456 io_req->iport = iport;
457 io_req->tport = (struct fnic_tport_s *)rport->private;
458 io_req->fcp_req = fcp_req;
459 io_req->done = nvfnic_fcpio_cmpl;
460 io_req->sgl_list = NULL;
461 io_req->sgl_list_alloc = NULL;
462 io_req->sgl_list_pa = 0;
463 io_req->sgl_cnt = 0;
464 io_req->sgl_mapped = 0;
465 io_req->wq = hw_queue_handle;
466 init_llist_node(&io_req->nvfnic_io_cmpl);
467
468 io_req->tag = nvfnic_alloc_fcpio_tag(iport, io_req);
469 if (io_req->tag == FNIC_NVME_NO_FREE_TAG) {
470 FNIC_NVME_DBG(KERN_ERR, fnic,
471 "No free tag available. Failing IO\n");
472 atomic64_inc(&fnic_stats->io_stats.alloc_failures);
473 atomic_dec(&fnic->in_flight);
474 atomic64_inc(&fnic_stats->io_stats.nvme_io_rsps_sent);
475 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
476 return -EBUSY;
477 }
478
479 ret = nvfnic_queuecommand(io_req);
480 if (ret) {
481 FNIC_NVME_DBG(KERN_ERR, fnic,
482 "Queuecommand failed tag: 0x%x\n",
483 io_req->tag);
484 nvfnic_free_fcpio_tag(iport, io_req);
485 }
486
487 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
488 atomic_dec(&fnic->in_flight);
489 return ret;
490 }
491
nvfnic_fcpio_nvme_fast_cmpl_handler(struct fnic * fnic,struct fcpio_fw_req * desc)492 void nvfnic_fcpio_nvme_fast_cmpl_handler(struct fnic *fnic,
493 struct fcpio_fw_req *desc)
494 {
495 u8 type;
496 u8 hdr_status;
497 struct fcpio_tag ftag;
498 u32 id;
499 struct fnic_io_req *io_req;
500 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
501 unsigned long start_time;
502 u64 cmd_trace;
503 char *lba;
504 struct nvme_fc_cmd_iu *cmdiu;
505 struct fnic_iport_s *iport;
506 unsigned int tag;
507
508 /* Decode the cmpl description to get the io_req id */
509 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
510 fcpio_tag_id_dec(&ftag, &id);
511 tag = id & FNIC_TAG_MASK;
512
513 if (tag >= fnic->fnic_max_tag_id) {
514 FNIC_NVME_DBG(KERN_ERR, fnic,
515 "Tag out of range tag: 0x%x hdr status: %s\n", tag,
516 fnic_fcpio_status_to_str(hdr_status));
517 return;
518 }
519 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
520
521 io_req = nvfnic_find_io_req_by_tag(fnic, tag);
522
523 WARN_ON_ONCE(!io_req);
524 if (!io_req) {
525 atomic64_inc(&fnic_stats->io_stats.ioreq_null);
526 FNIC_NVME_DBG(KERN_ERR, fnic,
527 "IO req null hdr: %s tag: 0x%x desc: 0x%p\n",
528 fnic_fcpio_status_to_str(hdr_status), id, desc);
529 FNIC_NVME_DBG(KERN_ERR, fnic,
530 "type: 0x%x status: 0x%x rsvd: 0x%x tag: 0x%x\n",
531 desc->hdr.type, desc->hdr.status, desc->hdr._resvd,
532 id);
533 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
534 return;
535 }
536
537 cmdiu = io_req->fcp_req->cmdaddr;
538 if (io_req->tag != tag) {
539 FNIC_NVME_DBG(KERN_INFO, fnic,
540 "Tag mismatch tag:%d io:0x%x id:0x%x csn:%08x\n",
541 tag, io_req->tag, id, be32_to_cpu(cmdiu->csn));
542 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
543 return;
544 }
545 iport = io_req->iport;
546
547 start_time = io_req->start_time;
548
549 /* firmware completed the io */
550 io_req->io_completed = 1;
551 if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) {
552 /*
553 * set the FNIC_IO_DONE so that this doesn't get
554 * flagged as 'out of order' if it was not aborted
555 */
556 io_req->cmd_flags |= FNIC_IO_DONE;
557 io_req->cmd_flags |= FNIC_IO_ABTS_PENDING;
558 if (hdr_status == FCPIO_ABORTED)
559 io_req->cmd_flags |= FNIC_IO_ABORTED;
560 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
561
562 FNIC_NVME_DBG(KERN_INFO, fnic,
563 "icmnd abts hdr:%d %s tag:0x%x io:%p",
564 hdr_status, fnic_fcpio_status_to_str(hdr_status),
565 id, io_req);
566 return;
567 }
568
569 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
570 FNIC_NVME_DBG(KERN_INFO, fnic,
571 "IO freed id:%d tag:0x%x st:0x%x csn:%08x\n",
572 id, io_req->tag, io_req->cmd_state,
573 be32_to_cpu(cmdiu->csn));
574 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
575 return;
576 }
577
578 /* Mark the IO as complete */
579 io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE;
580 switch (hdr_status) {
581 case FCPIO_SUCCESS:
582 io_req->fcp_req->status = 0;
583 io_req->fcp_req->transferred_length =
584 io_req->fcp_req->payload_length;
585 io_req->fcp_req->rcv_rsplen = 12;
586 break;
587 default:
588 FNIC_NVME_DBG(KERN_ERR, fnic, "HDR status is non-zero\n");
589 io_req->fcp_req->status = NVME_SC_INTERNAL;
590 break;
591 }
592
593 if (hdr_status != FCPIO_SUCCESS) {
594 atomic64_inc(&fnic_stats->io_stats.io_failures);
595 FNIC_NVME_DBG(KERN_INFO, fnic, "hdr status: %s\n",
596 fnic_fcpio_status_to_str(hdr_status));
597 }
598
599 io_req->cmd_flags |= FNIC_IO_DONE;
600
601 cmdiu = io_req->fcp_req->cmdaddr;
602 lba = (char *)&cmdiu->sqe.rw.slba;
603 cmd_trace = ((u64) hdr_status << 56) |
604 (u64) cmdiu->sqe.rw.opcode << 32 |
605 (u64) lba[0] << 24 | (u64) lba[1] << 16 |
606 (u64) lba[2] << 8 | lba[3];
607
608 FNIC_TRACE(nvfnic_fcpio_nvme_fast_cmpl_handler, fnic->fnic_num,
609 tag, io_req,
610 jiffies_to_msecs(jiffies - start_time),
611 desc, cmd_trace,
612 (((u64) io_req->cmd_flags << 32) |
613 io_req->cmd_state));
614
615 nvfnic_update_io_stats(fnic, cmdiu->sqe.rw.opcode);
616 nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode);
617 nvfnic_update_cmpl_stats(fnic, io_req);
618
619 nvfnic_release_nvme_ioreq_buf(iport, io_req);
620 if (io_req->done)
621 io_req->done(io_req);
622
623 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
624 }
625
nvfnic_fcpio_ersp_cmpl_handler(struct fnic * fnic,struct fcpio_fw_req * desc,int sw_flag)626 void nvfnic_fcpio_ersp_cmpl_handler(struct fnic *fnic,
627 struct fcpio_fw_req *desc, int sw_flag)
628 {
629 u8 type;
630 u8 hdr_status;
631 struct fcpio_tag ftag;
632 u32 id;
633 struct fcpio_nvme_cmpl *nvme_cmpl;
634 struct fnic_io_req *io_req;
635 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
636 unsigned long start_time;
637 uint32_t rsplen;
638 struct nvme_fc_ersp_iu *ersp;
639 struct nvme_fc_ersp_iu *nrsp;
640 struct nvme_fc_cmd_iu *cmdiu;
641 struct nvme_command *sqe;
642 struct nvme_completion *cqe;
643 u64 cmd_trace;
644 struct fnic_iport_s *iport;
645 unsigned int tag;
646 char *lba;
647 uint64_t tport_wwpn = 0;
648
649 /* Decode the cmpl description to get the io_req id */
650 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
651 fcpio_tag_id_dec(&ftag, &id);
652 nvme_cmpl = &desc->u.nvme_cmpl;
653 ersp = (struct nvme_fc_ersp_iu *) nvme_cmpl->resp_bytes;
654 tag = id & FNIC_TAG_MASK;
655
656 if (tag >= fnic->fnic_max_tag_id) {
657 FNIC_NVME_DBG(KERN_ERR, fnic,
658 "Tag out of range tag:0x%x hdr:%s\n", tag,
659 fnic_fcpio_status_to_str(hdr_status));
660 return;
661 }
662 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
663
664 io_req = nvfnic_find_io_req_by_tag(fnic, tag);
665 if (!io_req) {
666 atomic64_inc(&fnic_stats->io_stats.ioreq_null);
667 FNIC_NVME_DBG(KERN_ERR, fnic,
668 "IOREQ is null hdr status: %s tag: 0x%x desc: %p\n",
669 fnic_fcpio_status_to_str(hdr_status), tag, desc);
670 FNIC_NVME_DBG(KERN_ERR, fnic,
671 "type: 0x%x status: 0x%x rsvd: 0x%x\n",
672 desc->hdr.type, desc->hdr.status, desc->hdr._resvd);
673 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
674 return;
675 }
676 iport = io_req->iport;
677 if (io_req->tport != NULL)
678 tport_wwpn = io_req->tport->wwpn;
679 cmdiu = io_req->fcp_req->cmdaddr;
680
681 if (io_req->tag != tag) {
682 FNIC_NVME_DBG(KERN_ERR, fnic,
683 "Tag mismatch io:0x%x tag:0x%x id:0x%x\n",
684 io_req->tag, tag, id);
685 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
686 return;
687 }
688
689 nrsp = (struct nvme_fc_ersp_iu *)io_req->fcp_req->rspaddr;
690 cmdiu = (struct nvme_fc_cmd_iu *)io_req->fcp_req->cmdaddr;
691 sqe = &cmdiu->sqe;
692 cqe = &nrsp->cqe;
693 start_time = io_req->start_time;
694
695 /* firmware completed the io */
696 io_req->io_completed = 1;
697
698 if (io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) {
699 /*
700 * set the FNIC_IO_DONE so that this doesn't get
701 * flagged as 'out of order' if it was not aborted
702 */
703 io_req->cmd_flags |= FNIC_IO_DONE;
704 io_req->cmd_flags |= FNIC_IO_ABTS_PENDING;
705 if (hdr_status == FCPIO_ABORTED)
706 io_req->cmd_flags |= FNIC_IO_ABORTED;
707
708 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
709 FNIC_NVME_DBG(KERN_INFO, fnic,
710 "ABTS pending hdr status: %s tag: 0x%x",
711 fnic_fcpio_status_to_str(hdr_status), tag);
712 return;
713 }
714
715 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
716 FNIC_NVME_DBG(KERN_ERR, fnic,
717 "IO already freed by abort. tag: 0x%x id: 0x%x\n",
718 io_req->tag, id);
719 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
720 return;
721 }
722
723 /* Mark the IO as complete */
724 io_req->cmd_state = FNIC_IOREQ_CMD_COMPLETE;
725
726 switch (hdr_status) {
727 case FCPIO_SUCCESS:
728 io_req->fcp_req->status = 0;
729 if (!sw_flag) {
730 io_req->fcp_req->transferred_length =
731 io_req->fcp_req->payload_length;
732 rsplen = 32;
733 nrsp->iu_len =
734 cpu_to_be16(sizeof(struct nvme_fc_ersp_iu) / 4);
735 nrsp->xfrd_len =
736 cpu_to_be32(io_req->fcp_req->payload_length);
737
738 nrsp->ersp_result = 0;
739 cqe->command_id = sqe->common.command_id;
740 cqe->status = 0;
741 cqe->result.u64 = 0;
742 } else {
743 io_req->fcp_req->transferred_length =
744 be32_to_cpu(ersp->xfrd_len);
745 rsplen = be16_to_cpu(ersp->iu_len) * 4;
746 if (rsplen > sizeof(nvme_cmpl->resp_bytes) ||
747 rsplen > io_req->fcp_req->rsplen) {
748 FNIC_NVME_DBG(KERN_ERR, fnic,
749 "tport wwpn 0x%llx ERSP len %u desc %u req %u\n",
750 tport_wwpn, rsplen,
751 (u32)sizeof(nvme_cmpl->resp_bytes),
752 io_req->fcp_req->rsplen);
753 io_req->fcp_req->status = NVME_SC_INTERNAL;
754 io_req->fcp_req->rcv_rsplen = 0;
755 break;
756 }
757 memcpy(io_req->fcp_req->rspaddr, ersp, rsplen);
758 }
759 atomic64_inc(&fnic_stats->nvme_stats.nvme_ersps);
760 io_req->fcp_req->rcv_rsplen = rsplen;
761 break;
762
763 default:
764 FNIC_NVME_DBG(KERN_ERR, fnic,
765 "Unexpected header status: %d\n", hdr_status);
766 io_req->fcp_req->status = NVME_SC_INTERNAL;
767 break;
768 }
769
770 if (hdr_status != FCPIO_SUCCESS) {
771 atomic64_inc(&fnic_stats->io_stats.io_failures);
772 FNIC_NVME_DBG(KERN_ERR, fnic, "hdr status: %s tag: 0x%x\n",
773 fnic_fcpio_status_to_str(hdr_status), tag);
774 }
775
776 io_req->cmd_flags |= FNIC_IO_DONE;
777
778 lba = (char *) &cmdiu->sqe.rw.slba;
779 cmd_trace = ((u64) hdr_status << 56) |
780 (u64) ersp->ersp_result << 48 |
781 (u64) cmdiu->sqe.rw.opcode << 32 |
782 (u64) lba[0] << 24 | (u64) lba[1] << 16 |
783 (u64) lba[2] << 8 | lba[3];
784
785 FNIC_TRACE(nvfnic_fcpio_ersp_cmpl_handler, fnic->fnic_num,
786 tag, io_req,
787 ((u64) nvme_cmpl->resvd[1] << 56 |
788 (u64) nvme_cmpl->resvd[0] << 48 |
789 jiffies_to_msecs(jiffies - start_time)),
790 desc, cmd_trace,
791 (((u64) io_req->cmd_flags << 32) |
792 io_req->cmd_state));
793
794 nvfnic_update_io_stats(fnic, cmdiu->sqe.rw.opcode);
795 nvfnic_update_io_bytes(fnic, io_req, cmdiu->sqe.rw.opcode);
796 nvfnic_update_cmpl_stats(fnic, io_req);
797
798 nvfnic_release_nvme_ioreq_buf(iport, io_req);
799
800 /* Call NVME completion function to complete the IO */
801 if (io_req->done)
802 io_req->done(io_req);
803
804 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
805 }
806
nvfnic_fcpio_nvme_itmf_cmpl_handler(struct fnic * fnic,struct fcpio_fw_req * desc)807 void nvfnic_fcpio_nvme_itmf_cmpl_handler(struct fnic *fnic,
808 struct fcpio_fw_req *desc)
809 {
810 u8 type;
811 u8 hdr_status;
812 struct fcpio_tag ftag;
813 u32 id;
814 unsigned int tag;
815 struct fnic_io_req *io_req;
816 struct nvme_fc_cmd_iu *cmd_iu;
817 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
818 struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats;
819 struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats;
820 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;
821 struct fnic_iport_s *iport;
822
823 fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag);
824 fcpio_tag_id_dec(&ftag, &id);
825 tag = id & FNIC_TAG_MASK;
826
827 if (tag >= fnic->fnic_max_tag_id) {
828 FNIC_NVME_DBG(KERN_ERR, fnic,
829 "Tag out of range id:0x%x tag:0x%x hdr:%s\n",
830 id, tag, fnic_fcpio_status_to_str(hdr_status));
831 return;
832 }
833 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
834
835 io_req = nvfnic_find_io_req_by_tag(fnic, tag);
836 WARN_ON_ONCE(!io_req);
837 if (!io_req) {
838 atomic64_inc(&fnic_stats->io_stats.ioreq_null);
839 FNIC_NVME_DBG(KERN_ERR, fnic,
840 "IOREQ null hdr:%s tag:0x%x desc:%p\n",
841 fnic_fcpio_status_to_str(hdr_status), tag, desc);
842 FNIC_NVME_DBG(KERN_ERR, fnic,
843 "type: 0x%x status: 0x%x rsvd: 0x%x\n",
844 desc->hdr.type, desc->hdr.status, desc->hdr._resvd);
845 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
846 return;
847 }
848
849 cmd_iu = io_req->fcp_req->cmdaddr;
850 FNIC_NVME_DBG(KERN_INFO, fnic,
851 "Received ITMF completion tag: 0x%x hdr_status: %d csn: 0x%08x\n",
852 tag, hdr_status, be32_to_cpu(cmd_iu->csn));
853
854 iport = io_req->iport;
855
856 /* Completion of abort cmd */
857 switch (hdr_status) {
858 case FCPIO_SUCCESS:
859 FNIC_NVME_DBG(KERN_DEBUG, fnic,
860 "Abort success received tag: 0x%x id: 0x%x\n",
861 tag, id);
862 break;
863 case FCPIO_TIMEOUT:
864 FNIC_NVME_DBG(KERN_ERR, fnic,
865 "Abort timeout received tag: 0x%x id: 0x%x\n",
866 tag, id);
867 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED)
868 atomic64_inc(&abts_stats->abort_fw_timeouts);
869 else
870 atomic64_inc(&term_stats->terminate_fw_timeouts);
871 break;
872 case FCPIO_ITMF_REJECTED:
873 FNIC_NVME_DBG(KERN_ERR, fnic,
874 "Abort reject received tag: 0x%x id: 0x%x\n",
875 tag, id);
876 break;
877
878 case FCPIO_IO_NOT_FOUND:
879 FNIC_NVME_DBG(KERN_ERR, fnic,
880 "Abort IO not found tag:0x%x id:0x%x\n",
881 tag, id);
882 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED)
883 atomic64_inc(&abts_stats->abort_io_not_found);
884 else
885 atomic64_inc(&term_stats->terminate_io_not_found);
886 break;
887 default:
888 FNIC_NVME_DBG(KERN_ERR, fnic,
889 "Abort unknown received tag: 0x%x id: 0x%x\n",
890 tag, id);
891 if (io_req->cmd_flags & FNIC_IO_ABTS_ISSUED)
892 atomic64_inc(&abts_stats->abort_failures);
893 else
894 atomic64_inc(&term_stats->terminate_failures);
895 break;
896 }
897
898 if (io_req->cmd_state != FNIC_IOREQ_ABTS_PENDING) {
899 FNIC_NVME_DBG(KERN_ERR, fnic,
900 "Abort late completion tag:0x%x id:0x%x\n",
901 tag, id);
902 /* This is a late completion. Ignore it */
903 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
904 return;
905 }
906
907 io_req->abts_state = hdr_status;
908
909 /* If the status is IO not found consider it as success.
910 * NVME sends abort even if rport is down in which case
911 * we will get FCPIO_TIMEOUT. Consider this as success.
912 */
913 if ((hdr_status == FCPIO_IO_NOT_FOUND) ||
914 (hdr_status == FCPIO_TIMEOUT) ||
915 (hdr_status == FCPIO_ITMF_REJECTED))
916 io_req->abts_state = FCPIO_SUCCESS;
917
918 io_req->cmd_flags |= FNIC_IO_ABT_TERM_DONE;
919
920 if (!(io_req->cmd_flags & (FNIC_IO_ABORTED | FNIC_IO_DONE)))
921 atomic64_inc(&misc_stats->no_icmnd_itmf_cmpls);
922
923 io_req->fcp_req->transferred_length = 0;
924 io_req->fcp_req->rcv_rsplen = 0;
925 if (io_req->abts_state == FCPIO_SUCCESS)
926 io_req->fcp_req->status = NVME_SC_ABORT_REQ;
927 else
928 io_req->fcp_req->status = NVME_SC_INTERNAL;
929
930 atomic64_dec(&fnic_stats->io_stats.active_ios);
931 if (atomic64_read(&fnic->io_cmpl_skip))
932 atomic64_dec(&fnic->io_cmpl_skip);
933 else
934 atomic64_inc(&fnic_stats->io_stats.io_completions);
935
936 nvfnic_release_nvme_ioreq_buf(iport, io_req);
937 if (io_req->done)
938 io_req->done(io_req);
939 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
940 }
941
_cleanup_tport_io(struct sbitmap * map,unsigned int tag,void * data)942 bool _cleanup_tport_io(struct sbitmap *map, unsigned int tag,
943 void *data)
944 {
945 struct fnic_tport_s *tport = data;
946 struct fnic_iport_s *iport = tport->iport;
947 struct fnic *fnic = iport->fnic;
948 struct fnic_io_req *io_req;
949 enum fnic_ioreq_state old_ioreq_state;
950 unsigned long flags;
951
952 spin_lock_irqsave(&fnic->fnic_lock, flags);
953 io_req = nvfnic_find_io_req_by_tag(fnic, tag);
954 if (!io_req || io_req->tport != tport) {
955 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
956 return true;
957 }
958
959 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) ||
960 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) {
961 FNIC_NVME_DBG(KERN_INFO, fnic,
962 "Abort already pending 0x%x\n", io_req->tag);
963 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
964 return true;
965 }
966
967 FNIC_NVME_DBG(KERN_ERR, fnic,
968 "io_req tag: 0x%x abort after unregister timeout\n",
969 io_req->tag);
970
971 old_ioreq_state = io_req->cmd_state;
972 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING;
973 io_req->abts_state = FCPIO_INVALID_CODE;
974 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
975
976 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag,
977 FCPIO_ITMF_ABT_TASK_TERM,
978 io_req)) {
979 FNIC_NVME_DBG(KERN_ERR, fnic,
980 "Failed to enqueue abort for ioreq tag: 0x%x\n",
981 io_req->tag);
982 spin_lock_irqsave(&fnic->fnic_lock, flags);
983 io_req->cmd_state = old_ioreq_state;
984 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
985 }
986 return true;
987 }
988
989 void
nvfnic_cleanup_tport_io(struct fnic * fnic,struct fnic_tport_s * tport)990 nvfnic_cleanup_tport_io(struct fnic *fnic, struct fnic_tport_s *tport)
991 {
992 unsigned long flags;
993 struct nvfnic_ls_req *nvfnic_ls_req, *next;
994 struct nvmefc_ls_req *lsreq;
995 uint16_t oxid;
996 LIST_HEAD(done_reqs);
997
998 spin_lock_irqsave(&fnic->fnic_lock, flags);
999 list_for_each_entry_safe(nvfnic_ls_req, next,
1000 &(tport->ls_req_list), list) {
1001 lsreq = nvfnic_ls_req->ls_req;
1002 if (!lsreq || (lsreq->private == NULL)) {
1003 FNIC_NVME_DBG(KERN_INFO, fnic,
1004 "fnic_cleanup_tport_io lsreq NULL\n");
1005 continue;
1006 }
1007 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED) {
1008 FNIC_NVME_DBG(KERN_INFO, fnic,
1009 "fnic_cleanup_tport_io lsreq abort started\n");
1010 continue;
1011 }
1012 list_del_init(&nvfnic_ls_req->list);
1013 lsreq->private = NULL;
1014 oxid = nvfnic_ls_req->oxid;
1015 fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid);
1016 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
1017 list_add_tail(&nvfnic_ls_req->list, &done_reqs);
1018 }
1019 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1020
1021 list_for_each_entry_safe(nvfnic_ls_req, next, &done_reqs, list) {
1022 list_del_init(&nvfnic_ls_req->list);
1023 lsreq = nvfnic_ls_req->ls_req;
1024 timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
1025 lsreq->done(lsreq, -ENXIO);
1026 }
1027
1028 /* For link-down, IOs are freed by firmware reset completion */
1029 if (fdls_get_state(&fnic->iport.fabric) == FDLS_STATE_LINKDOWN)
1030 return;
1031
1032 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_tport_io, tport);
1033 }
1034
1035 void
nvfnic_terminate_tport_ls_reqs(struct fnic * fnic,struct fnic_tport_s * tport)1036 nvfnic_terminate_tport_ls_reqs(struct fnic *fnic, struct fnic_tport_s *tport)
1037 {
1038 struct nvmefc_ls_req *lsreq;
1039 struct nvfnic_ls_req *nvfnic_ls_req, *next;
1040 int count = 0;
1041 uint16_t oxid;
1042 LIST_HEAD(done_reqs);
1043
1044 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1045 list_for_each_entry_safe(nvfnic_ls_req, next,
1046 &(tport->ls_req_list), list) {
1047
1048 lsreq = nvfnic_ls_req->ls_req;
1049 if (!lsreq || (lsreq->private == NULL)) {
1050 FNIC_NVME_DBG(KERN_ERR, fnic,
1051 "lsreq is NULL\n");
1052 continue;
1053 }
1054 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED) {
1055 FNIC_NVME_DBG(KERN_INFO, fnic,
1056 "lsreq abort started\n");
1057 continue;
1058 }
1059 oxid = nvfnic_ls_req->oxid;
1060 list_del_init(&nvfnic_ls_req->list);
1061 lsreq->private = NULL;
1062 fdls_free_oxid(&fnic->iport, oxid, &nvfnic_ls_req->oxid);
1063 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
1064 list_add_tail(&nvfnic_ls_req->list, &done_reqs);
1065 }
1066 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1067
1068 list_for_each_entry_safe(nvfnic_ls_req, next, &done_reqs, list) {
1069 list_del_init(&nvfnic_ls_req->list);
1070 lsreq = nvfnic_ls_req->ls_req;
1071 timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
1072 lsreq->done(lsreq, -ENXIO);
1073 count++;
1074 }
1075
1076 FNIC_NVME_DBG(KERN_INFO, fnic,
1077 "fnic_terminate_tport_lsreqs tport: 0x%x: freed lsreq: %d\n",
1078 tport->fcid, count);
1079 }
1080
_terminate_tport_ios(struct sbitmap * map,unsigned int tag,void * data)1081 bool _terminate_tport_ios(struct sbitmap *map, unsigned int tag,
1082 void *data)
1083 {
1084 struct fnic_tport_s *tport = data;
1085 struct fnic_iport_s *iport = tport->iport;
1086 struct fnic *fnic = iport->fnic;
1087 struct fnic_io_req *io_req;
1088 enum fnic_ioreq_state old_ioreq_state;
1089 unsigned long flags;
1090
1091 spin_lock_irqsave(&fnic->fnic_lock, flags);
1092 io_req = fnic->sw_copy_wq[0].io_req_table[tag];
1093 if (!io_req) {
1094 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1095 return true;
1096 }
1097
1098 if (io_req->tport != tport) {
1099 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1100 return true;
1101 }
1102
1103 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) ||
1104 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) {
1105 FNIC_NVME_DBG(KERN_INFO, fnic,
1106 "Abort already pending 0x%x\n", io_req->tag);
1107 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1108 return true;
1109 }
1110
1111 FNIC_NVME_DBG(KERN_INFO, fnic,
1112 "Terminate tag: 0x%x (tport fcid 0x%x)\n",
1113 io_req->tag, io_req->tport->fcid);
1114
1115 old_ioreq_state = io_req->cmd_state;
1116 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING;
1117 io_req->abts_state = FCPIO_INVALID_CODE;
1118 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1119
1120 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag,
1121 FCPIO_ITMF_ABT_TASK_TERM, io_req)) {
1122 FNIC_NVME_DBG(KERN_ERR, fnic,
1123 "Failed to enqueue abort for ioreq tag: 0x%x\n",
1124 io_req->tag);
1125 spin_lock_irqsave(&fnic->fnic_lock, flags);
1126 io_req->cmd_state = old_ioreq_state;
1127 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1128 }
1129 return true;
1130 }
1131
nvfnic_terminate_tport_ios(struct fnic * fnic,struct fnic_tport_s * tport)1132 void nvfnic_terminate_tport_ios(struct fnic *fnic,
1133 struct fnic_tport_s *tport)
1134 {
1135 struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats;
1136
1137 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _terminate_tport_ios, tport);
1138
1139 FNIC_NVME_DBG(KERN_INFO, fnic,
1140 "tport: 0x%x aborted %lld in_flight %d\n",
1141 tport->fcid, atomic64_read(&abts_stats->aborts),
1142 atomic_read(&fnic->in_flight));
1143 }
1144
_cleanup_all_nvme_io(struct sbitmap * map,unsigned int tag,void * data)1145 bool _cleanup_all_nvme_io(struct sbitmap *map, unsigned int tag,
1146 void *data)
1147 {
1148 struct fnic_iport_s *iport = data;
1149 struct fnic_io_req *io_req;
1150
1151 io_req = iport->fnic->sw_copy_wq[0].io_req_table[tag];
1152 if (!io_req)
1153 return true;
1154
1155 io_req->cmd_state = FNIC_DEV_RST_TERM_ISSUED;
1156 io_req->fcp_req->status = NVME_SC_INTERNAL;
1157 io_req->fcp_req->transferred_length = 0;
1158 io_req->fcp_req->rcv_rsplen = 0;
1159 nvfnic_release_nvme_ioreq_buf(iport, io_req);
1160 io_req->done(io_req);
1161 return true;
1162 }
1163
nvfnic_cleanup_all_nvme_ios(struct fnic * fnic)1164 void nvfnic_cleanup_all_nvme_ios(struct fnic *fnic)
1165 {
1166 struct fnic_iport_s *iport = &fnic->iport;
1167
1168 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1169 sbitmap_for_each_set(&fnic->nvfnic_tag_map, _cleanup_all_nvme_io,
1170 iport);
1171 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1172 }
1173
nvfnic_nvme_zero_devloss_tports(struct fnic * fnic)1174 void nvfnic_nvme_zero_devloss_tports(struct fnic *fnic)
1175 {
1176 struct fnic_tport_s *tport, *next;
1177
1178 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1179 list_for_each_entry_safe(tport, next, &fnic->iport.tport_list, links) {
1180 if (tport->flags & FNIC_FDLS_NVME_REGISTERED) {
1181 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1182 nvme_fc_set_remoteport_devloss(tport->nv_rport, 0);
1183 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1184 }
1185 }
1186 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1187 }
1188
nvfnic_nvme_unload(struct fnic * fnic)1189 void nvfnic_nvme_unload(struct fnic *fnic)
1190 {
1191 int ret = 0;
1192 struct fnic_iport_s *iport = &fnic->iport;
1193 unsigned long flags;
1194 unsigned int time_wait = FNIC_NVME_LPORT_REMOVE_WAIT;
1195 unsigned int time_remain;
1196 DECLARE_COMPLETION_ONSTACK(nvme_lport_unreg_done);
1197
1198 /* Mark iport state as INIT so that no IOs can be issued from this point */
1199 spin_lock_irqsave(&fnic->fnic_lock, flags);
1200 fnic->in_remove = 1;
1201 fnic->iport.state = FNIC_IPORT_STATE_LINK_WAIT;
1202 fnic->nvme_lport_unreg_done = &nvme_lport_unreg_done;
1203 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1204
1205 /*
1206 * If fnic is already processing link-down or fnic is held
1207 * in disabled state following a reboot we dont need to issue
1208 * firmware reset and unregister remote ports as it is already
1209 * done as part of link down handling.
1210 */
1211 if (fdls_get_state(&iport->fabric) == FDLS_STATE_LINKDOWN) {
1212 while (fnic->reset_in_progress == IN_PROGRESS) {
1213 wait_for_completion_timeout(&fnic->reset_completion_wait,
1214 msecs_to_jiffies(5000));
1215 FNIC_NVME_DBG(KERN_INFO, fnic,
1216 "rmmod waiting for reset %p\n", fnic);
1217 }
1218 } else if (fdls_get_state(&iport->fabric) != FDLS_STATE_INIT)
1219 fnic_fcpio_reset(fnic);
1220
1221 /*
1222 * Mark state so that the workqueue thread stops forwarding
1223 * received frames and link events to the local port. ISR and
1224 * other threads that can queue work items will also stop
1225 * creating work items on the fnic workqueue
1226 */
1227 nvfnic_nvme_zero_devloss_tports(fnic);
1228 fnic_flush_tport_event_list(fnic);
1229 fnic_delete_fcp_tports(fnic);
1230
1231 if (iport->flags & FNIC_LPORT_NVME_REGISTERED) {
1232 ret = nvme_fc_unregister_localport(fnic->iport.nv_lport);
1233 if (ret) {
1234 FNIC_NVME_DBG(KERN_ERR, fnic,
1235 "Unregister nvme localport failed: %d\n", ret);
1236 } else {
1237 time_remain = wait_for_completion_timeout(
1238 fnic->nvme_lport_unreg_done,
1239 msecs_to_jiffies(time_wait));
1240 if (!time_remain) {
1241 FNIC_NVME_DBG(KERN_ERR, fnic,
1242 "Local port removal timed out\n");
1243 WARN_ON(1);
1244 }
1245 iport->flags &= ~FNIC_LPORT_NVME_REGISTERED;
1246 kfree(iport->nv_tmpl);
1247 }
1248 }
1249
1250 spin_lock_irqsave(&fnic->fnic_lock, flags);
1251 fnic->nvme_lport_unreg_done = NULL;
1252 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1253
1254 nvfnic_flush_nvme_io_list(fnic);
1255 }
1256
1257 struct nvfnic_ls_req*
nvfnic_find_ls_req(struct fnic_tport_s * tport,uint16_t oxid)1258 nvfnic_find_ls_req(struct fnic_tport_s *tport, uint16_t oxid)
1259 {
1260 struct nvfnic_ls_req *nvfnic_ls_req, *next;
1261
1262 list_for_each_entry_safe(nvfnic_ls_req, next, &(tport->ls_req_list), list) {
1263 if (nvfnic_ls_req->oxid == oxid)
1264 return nvfnic_ls_req;
1265 }
1266 return NULL;
1267 }
1268
nvfnic_fcpio_cmpl(struct fnic_io_req * io_req)1269 void nvfnic_fcpio_cmpl(struct fnic_io_req *io_req)
1270 {
1271 struct fnic *fnic = io_req->iport->fnic;
1272 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1273
1274 nvfnic_free_fcpio_tag(io_req->iport, io_req);
1275 atomic64_inc(&fnic_stats->io_stats.nvme_ios_queued_for_rsp);
1276 io_req->waitq_start_time = jiffies;
1277
1278 llist_add(&io_req->nvfnic_io_cmpl, &fnic->nvme_io_event_llist);
1279 atomic_inc(&fnic->nvme_io_event_queued);
1280 atomic64_inc(&fnic_stats->io_stats.nvme_num_ios_in_waitq);
1281
1282 queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work);
1283 }
1284
nvfnic_process_ls_abts_rsp(struct fnic_iport_s * iport,struct fc_frame_header * fchdr)1285 void nvfnic_process_ls_abts_rsp(struct fnic_iport_s *iport,
1286 struct fc_frame_header *fchdr)
1287 {
1288 uint32_t tport_fcid;
1289 struct fnic_tport_s *tport;
1290 struct nvfnic_ls_req *nvfnic_ls_req;
1291 struct nvmefc_ls_req *lsreq;
1292 uint8_t *fcid;
1293 uint16_t oxid = FNIC_STD_GET_OX_ID(fchdr);
1294 struct fnic *fnic = iport->fnic;
1295 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1296
1297 fcid = FNIC_STD_GET_S_ID(fchdr);
1298 tport_fcid = ntoh24(fcid);
1299
1300 tport = fnic_find_tport_by_fcid(iport, tport_fcid);
1301 if (tport == NULL) {
1302 FNIC_NVME_DBG(KERN_ERR, fnic,
1303 "tport: 0x%x not found\n", tport_fcid);
1304 return;
1305 }
1306
1307 nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid);
1308 if (nvfnic_ls_req == NULL) {
1309 FNIC_NVME_DBG(KERN_ERR, fnic,
1310 "tport: 0x%x lsreq oxid: 0x%x not found\n",
1311 tport_fcid, oxid);
1312 return;
1313 }
1314
1315 lsreq = nvfnic_ls_req->ls_req;
1316 if ((lsreq == NULL) || (lsreq->private == NULL)) {
1317 FNIC_NVME_DBG(KERN_INFO, fnic,
1318 "tport: 0x%x lsreq oxid: 0x%x already aborted\n",
1319 tport_fcid, oxid);
1320 return;
1321 }
1322
1323 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_abort_responses);
1324 nvfnic_ls_req->state = FNIC_LS_REQ_ABTS_COMPLETE;
1325
1326 FNIC_NVME_DBG(KERN_DEBUG, fnic, "nvme_ls_requests: %lld\n",
1327 (u64) atomic64_read(&fnic_stats->nvme_stats.nvme_ls_requests));
1328
1329 list_del(&nvfnic_ls_req->list);
1330 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
1331 lsreq->private = NULL;
1332 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1333 timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
1334 lsreq->done(lsreq, NVME_SC_HOST_ABORTED_CMD);
1335 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1336 }
1337
1338 /**
1339 * nvfnic_ls_rsp_recv - Handle received NVMe FC link service (LS) response
1340 * @iport: Pointer to the local FNIC port structure
1341 * @fchdr: Pointer to the Fibre Channel frame header for the
1342 * received response
1343 * @len: Length of the received frame
1344 *
1345 * This function processes link service (LS) responses received from
1346 * NVMe Discovery Controllers or regular NVMe subsystems during
1347 * association.
1348 */
nvfnic_ls_rsp_recv(struct fnic_iport_s * iport,struct fc_frame_header * fchdr,int len)1349 void nvfnic_ls_rsp_recv(struct fnic_iport_s *iport,
1350 struct fc_frame_header *fchdr, int len)
1351 {
1352 uint8_t *fcid;
1353 uint32_t tport_fcid;
1354 struct fnic_tport_s *tport;
1355 struct nvfnic_ls_req *nvfnic_ls_req;
1356 struct nvmefc_ls_req *lsreq;
1357 uint16_t oxid;
1358 uint32_t rsp_len;
1359 int sid_len = offsetof(struct fc_frame_header, fh_s_id) +
1360 sizeof(fchdr->fh_s_id);
1361 int status = 0;
1362 struct fnic *fnic = iport->fnic;
1363 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1364
1365 if (len < (int)sizeof(*fchdr)) {
1366 if (len >= sid_len) {
1367 fcid = FNIC_STD_GET_S_ID(fchdr);
1368 tport_fcid = ntoh24(fcid);
1369 FNIC_NVME_DBG(KERN_ERR, fnic,
1370 "tport: 0x%x LS rsp len %d too short\n",
1371 tport_fcid, len);
1372 } else {
1373 FNIC_NVME_DBG(KERN_ERR, fnic,
1374 "LS response len %d too short\n", len);
1375 }
1376 return;
1377 }
1378 rsp_len = len - sizeof(*fchdr);
1379
1380 fcid = FNIC_STD_GET_S_ID(fchdr);
1381 tport_fcid = ntoh24(fcid);
1382
1383 tport = fnic_find_tport_by_fcid(iport, tport_fcid);
1384 if (!tport) {
1385 FNIC_NVME_DBG(KERN_ERR, fnic,
1386 "tport: 0x%x not found\n", tport_fcid);
1387 return;
1388 }
1389
1390 oxid = FNIC_STD_GET_OX_ID(fchdr);
1391 nvfnic_ls_req = nvfnic_find_ls_req(tport, oxid);
1392 if (!nvfnic_ls_req) {
1393 FNIC_NVME_DBG(KERN_ERR, fnic,
1394 "tport: 0x%x no nvfnic_lsreq for oxid: 0x%x\n",
1395 tport_fcid, oxid);
1396 return;
1397 }
1398
1399 lsreq = nvfnic_ls_req->ls_req;
1400 if (!lsreq || (lsreq->private == NULL)) {
1401 FNIC_NVME_DBG(KERN_ERR, fnic,
1402 "tport:0x%x lsreq:0x%x already done\n",
1403 tport_fcid, oxid);
1404 return;
1405 }
1406 if (!lsreq->rspaddr) {
1407 FNIC_NVME_DBG(KERN_ERR, fnic,
1408 "tport 0x%x lsreq 0x%x rspaddr NULL\n",
1409 tport_fcid, oxid);
1410 return;
1411 }
1412
1413 if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) ||
1414 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) {
1415 FNIC_NVME_DBG(KERN_INFO, fnic,
1416 "tport 0x%x lsreq oxid: 0x%x abts pending\n",
1417 tport_fcid, oxid);
1418 return;
1419 }
1420
1421 if (rsp_len > lsreq->rsplen) {
1422 FNIC_NVME_DBG(KERN_ERR, fnic,
1423 "tport:0x%x lsreq:0x%x rsp %u > %u\n",
1424 tport_fcid, oxid, rsp_len, lsreq->rsplen);
1425 status = -EOVERFLOW;
1426 }
1427
1428 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
1429 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_responses);
1430
1431 list_del_init(&nvfnic_ls_req->list);
1432 lsreq->private = NULL;
1433 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
1434
1435 if (status == 0) {
1436 FNIC_NVME_DBG(KERN_DEBUG, fnic,
1437 "tport:0x%x lsreq:0x%x completed\n",
1438 tport_fcid, oxid);
1439
1440 /* Copy the Response */
1441 memcpy(lsreq->rspaddr, (uint8_t *)fchdr + sizeof(*fchdr),
1442 rsp_len);
1443 }
1444
1445 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1446 timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
1447 lsreq->done(lsreq, status);
1448 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1449 }
1450
nvfnic_ls_req_cleanup(struct fnic_iport_s * iport,struct nvmefc_ls_req * lsreq,uint16_t oxid)1451 static bool nvfnic_ls_req_cleanup(struct fnic_iport_s *iport,
1452 struct nvmefc_ls_req *lsreq,
1453 uint16_t oxid)
1454 {
1455 struct nvfnic_ls_req *nvfnic_ls_req = lsreq->private;
1456
1457 if (!nvfnic_ls_req)
1458 return false;
1459
1460 lsreq->private = NULL;
1461 list_del(&nvfnic_ls_req->list);
1462 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
1463 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_COMPLETE;
1464
1465 return true;
1466 }
1467
nvfnic_ls_req_timeout(struct timer_list * t)1468 void nvfnic_ls_req_timeout(struct timer_list *t)
1469 {
1470 struct nvfnic_ls_req *nvfnic_ls_req = timer_container_of(nvfnic_ls_req,
1471 t, ls_req_timer);
1472 struct fnic *fnic = nvfnic_ls_req->fnic;
1473 struct nvmefc_ls_req *ls_req = nvfnic_ls_req->ls_req;
1474 struct fnic_iport_s *iport = &fnic->iport;
1475 struct fnic_tport_s *tport = (struct fnic_tport_s *) nvfnic_ls_req->tport;
1476 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1477 uint16_t oxid = nvfnic_ls_req->oxid;
1478 int timeout;
1479
1480 FNIC_NVME_DBG(KERN_INFO, fnic,
1481 "tport: 0x%x lsreq: 0x%x state: %d timeout\n",
1482 tport->fcid, nvfnic_ls_req->oxid,
1483 nvfnic_ls_req->state);
1484 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1485
1486 if ((ls_req->private == NULL) ||
1487 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_STARTED)) {
1488 FNIC_NVME_DBG(KERN_ERR, fnic,
1489 "tport: 0x%x lsreq: 0x%x already aborted\n",
1490 tport->fcid, nvfnic_ls_req->oxid);
1491 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1492 return;
1493 }
1494
1495 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) {
1496 FNIC_NVME_DBG(KERN_ERR, fnic,
1497 "tport: 0x%x lsreq: 0x%x abort timeout\n",
1498 tport->fcid, nvfnic_ls_req->oxid);
1499
1500 ls_req = nvfnic_ls_req->ls_req;
1501 nvfnic_ls_req_cleanup(iport, ls_req, oxid);
1502 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1503 ls_req->done(ls_req, -ETIMEDOUT);
1504 return;
1505 } else if ((nvfnic_ls_req->state == FNIC_LS_REQ_CMD_PENDING) &&
1506 (nvfnic_transport_ready(iport, tport))) {
1507 FNIC_NVME_DBG(KERN_ERR, fnic,
1508 "tport: 0x%x lsreq: 0x%x sending abort\n",
1509 tport->fcid, nvfnic_ls_req->oxid);
1510 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_PENDING;
1511 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_aborts);
1512
1513 if (fdls_send_ls_req_abts(iport, tport, nvfnic_ls_req->oxid) == 0) {
1514 timeout = FNIC_LS_REQ_TMO_MSECS(ls_req->timeout);
1515 mod_timer(&nvfnic_ls_req->ls_req_timer,
1516 round_jiffies(jiffies + msecs_to_jiffies(timeout)));
1517 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1518 return;
1519 }
1520 FNIC_NVME_DBG(KERN_ERR, fnic,
1521 "tport: 0x%x lsreq: 0x%x cannot send abort\n",
1522 tport->fcid, oxid);
1523 }
1524
1525 if (ls_req->private == NULL) {
1526 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1527 return;
1528 }
1529
1530 ls_req = nvfnic_ls_req->ls_req;
1531 nvfnic_ls_req_cleanup(iport, ls_req, oxid);
1532
1533 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1534 ls_req->done(ls_req, -ETIMEDOUT);
1535 }
1536
1537 /**
1538 * nvfnic_ls_req_send - Send NVMe FC link service (LS) request
1539 * @lport: Pointer to local NVMe FC port structure
1540 * @rport: Pointer to remote NVMe FC port structure
1541 * @ls_req: Pointer to the link service request structure
1542 *
1543 * This function is used to send link service (LS) commands to an NVMe
1544 * Discovery Controller for discovery operations, as well as to regular
1545 * NVMe subsystems during association. It encapsulates the logic for
1546 * transmitting LS requests over the NVMe over Fabrics (NVMe-oF) FC
1547 * transport.
1548 *
1549 * Returns: 0 on success, or a negative error code on failure.
1550 */
nvfnic_ls_req_send(struct nvme_fc_local_port * lport,struct nvme_fc_remote_port * rport,struct nvmefc_ls_req * ls_req)1551 int nvfnic_ls_req_send(struct nvme_fc_local_port *lport,
1552 struct nvme_fc_remote_port *rport,
1553 struct nvmefc_ls_req *ls_req)
1554 {
1555 int timeout;
1556 uint8_t *frame;
1557 uint8_t fcid[3];
1558 unsigned long flags = 0;
1559 struct fnic_iport_s *iport = lport->private;
1560 uint8_t *ls_req_payload;
1561 struct fnic *fnic = iport->fnic;
1562 struct fc_frame_header *fchdr;
1563 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1564 struct nvfnic_ls_req *nvfnic_ls_req = ls_req->private;
1565 uint16_t frame_size = FNIC_ETH_FCOE_HDRS_OFFSET +
1566 sizeof(struct fc_frame_header) + ls_req->rqstlen;
1567 struct fnic_tport_s *tport;
1568 int ret;
1569
1570 spin_lock_irqsave(&fnic->fnic_lock, flags);
1571
1572 tport = (struct fnic_tport_s *)rport->private;
1573 INIT_LIST_HEAD(&nvfnic_ls_req->list);
1574
1575 if (!nvfnic_transport_ready(iport, tport)) {
1576 if (tport != NULL)
1577 FNIC_NVME_DBG(KERN_INFO, fnic,
1578 "iport: 0x%x tport: 0x%x transport not ready\n",
1579 iport->fcid, tport->fcid);
1580 else
1581 FNIC_NVME_DBG(KERN_INFO, fnic,
1582 "iport: 0x%x transport not ready\n",
1583 iport->fcid);
1584 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1585 return -ENOLINK;
1586 }
1587
1588 frame = fdls_alloc_frame(iport);
1589 if (frame == NULL) {
1590 FNIC_NVME_DBG(KERN_ERR, fnic,
1591 "Failed to allocate frame to send NVME LS REQ");
1592 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1593 return -ENOMEM;
1594 }
1595
1596 if (fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_NVME_LS,
1597 &nvfnic_ls_req->oxid) == FNIC_UNASSIGNED_OXID) {
1598 FNIC_FCS_DBG(KERN_INFO, fnic,
1599 "0x%x: Failed to allocate OXID to send NVME LS REQ",
1600 iport->fcid);
1601 mempool_free(frame, fnic->frame_pool);
1602 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1603 return -EAGAIN;
1604 }
1605
1606 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_requests);
1607 timer_setup(&nvfnic_ls_req->ls_req_timer, nvfnic_ls_req_timeout,
1608 0UL);
1609
1610 nvfnic_ls_req->fnic = fnic;
1611 nvfnic_ls_req->tport = tport;
1612 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_INIT;
1613 nvfnic_ls_req->ls_req = ls_req;
1614
1615 fchdr = (struct fc_frame_header *)(frame + FNIC_ETH_FCOE_HDRS_OFFSET);
1616 *fchdr = (struct fc_frame_header) {
1617 .fh_r_ctl = FC_RCTL_ELS4_REQ,
1618 .fh_type = FC_TYPE_NVME,
1619 .fh_f_ctl = {FNIC_ELS_REQ_FCTL, 0, 0},
1620 .fh_rx_id = cpu_to_be16(FNIC_UNASSIGNED_RXID)
1621 };
1622
1623 hton24(fcid, iport->fcid);
1624 FNIC_STD_SET_S_ID(*fchdr, fcid);
1625
1626 hton24(fcid, tport->fcid);
1627 FNIC_STD_SET_D_ID(*fchdr, fcid);
1628
1629 FNIC_STD_SET_OX_ID(*fchdr, nvfnic_ls_req->oxid);
1630
1631 ls_req_payload = frame + FNIC_ETH_FCOE_HDRS_OFFSET + sizeof(*fchdr);
1632 memcpy(ls_req_payload, ls_req->rqstaddr, ls_req->rqstlen);
1633
1634 FNIC_NVME_DBG(KERN_INFO, fnic,
1635 "0x%x: NVME send ls req with oxid: 0x%x type: 0x%02x len: %d",
1636 iport->fcid, nvfnic_ls_req->oxid, *((uint8_t *) ls_req->rqstaddr),
1637 ls_req->rqstlen);
1638
1639 FNIC_NVME_DBG(KERN_INFO, fnic,
1640 "0x%x: ls_reqs count: %lld",
1641 iport->fcid,
1642 (u64) atomic64_read(&fnic_stats->nvme_stats.nvme_ls_requests));
1643
1644 list_add_tail(&nvfnic_ls_req->list, &tport->ls_req_list);
1645 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_PENDING;
1646
1647 ret = fnic_send_fcoe_frame(iport, frame, frame_size);
1648 if (ret) {
1649 nvfnic_ls_req_cleanup(iport, ls_req, nvfnic_ls_req->oxid);
1650 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1651 mempool_free(frame, fnic->frame_pool);
1652 return ret;
1653 }
1654
1655 timeout = FNIC_LS_REQ_TMO_MSECS(ls_req->timeout);
1656 mod_timer(&nvfnic_ls_req->ls_req_timer,
1657 round_jiffies(jiffies + msecs_to_jiffies(timeout)));
1658 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1659
1660 return 0;
1661 }
1662
nvfnic_local_port_delete(struct nvme_fc_local_port * lport)1663 void nvfnic_local_port_delete(struct nvme_fc_local_port *lport)
1664 {
1665 struct fnic_iport_s *iport = (struct fnic_iport_s *) lport->private;
1666 struct fnic *fnic = iport->fnic;
1667 unsigned long flags = 0;
1668
1669 FNIC_NVME_DBG(KERN_INFO, fnic, "lport delete 0x%x\n",
1670 iport->fcid);
1671
1672 spin_lock_irqsave(&fnic->fnic_lock, flags);
1673 if (fnic->nvme_lport_unreg_done)
1674 complete(fnic->nvme_lport_unreg_done);
1675 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1676 }
1677
nvfnic_remote_port_delete(struct nvme_fc_remote_port * rport)1678 void nvfnic_remote_port_delete(struct nvme_fc_remote_port *rport)
1679 {
1680 /*
1681 * Read rport->private without the lock only to find fnic.
1682 * Re-read it under fnic_lock to claim this delete callback, since
1683 * another callback may already have cleared it.
1684 */
1685 struct fnic_tport_s *tport = (struct fnic_tport_s *)rport->private;
1686 struct fnic_iport_s *iport;
1687 struct fnic *fnic = NULL;
1688 unsigned long flags = 0;
1689
1690 if (tport == NULL) {
1691 pr_err("Attempt to delete already deleted tport\n");
1692 return;
1693 }
1694
1695 iport = tport->iport;
1696 fnic = iport->fnic;
1697 FNIC_NVME_DBG(KERN_INFO, fnic, "0x%x tport 0x%x\n",
1698 iport->fcid, tport->fcid);
1699
1700 spin_lock_irqsave(&fnic->fnic_lock, flags);
1701 tport = (struct fnic_tport_s *)rport->private;
1702 if (tport == NULL) {
1703 FNIC_NVME_DBG(KERN_ERR, fnic,
1704 "NVMe tport callback after NULL set %p\n",
1705 rport);
1706 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1707 return;
1708 }
1709
1710 iport = tport->iport;
1711 rport->private = NULL;
1712
1713 if (tport->timer_pending) {
1714 FNIC_NVME_DBG(KERN_INFO, fnic,
1715 "tport: 0x%x canceling discovery timer\n",
1716 tport->fcid);
1717 /*
1718 * The retry timer callback takes fnic_lock and dereferences
1719 * this tport. Set del_timer_inprogress before dropping the
1720 * lock so a callback that is already running observes teardown
1721 * and exits.
1722 *
1723 * timer_delete_sync() then waits until no callback can still
1724 * hold a reference before the remoteport-delete path completes
1725 * or frees the target port.
1726 */
1727 tport->del_timer_inprogress = 1;
1728 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1729 timer_delete_sync(&tport->retry_timer);
1730 spin_lock_irqsave(&fnic->fnic_lock, flags);
1731 tport->del_timer_inprogress = 0;
1732 tport->timer_pending = 0;
1733 }
1734
1735 if (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) {
1736 FNIC_NVME_DBG(KERN_ERR, fnic,
1737 "tport %8x waiting on clean pending\n",
1738 tport->fcid);
1739 }
1740
1741 while (tport->flags & FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING) {
1742 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1743 msleep(2000);
1744 spin_lock_irqsave(&fnic->fnic_lock, flags);
1745 }
1746
1747 list_del(&tport->links);
1748
1749 if (tport->tport_del_done) {
1750 tport->flags |= FNIC_TPORT_CAN_BE_FREED;
1751 complete(tport->tport_del_done);
1752 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1753 return;
1754 }
1755
1756 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1757 kfree(tport);
1758 }
1759
nvfnic_create_queue(struct nvme_fc_local_port * lport,unsigned int idx,u16 size,void ** handle)1760 int nvfnic_create_queue(struct nvme_fc_local_port *lport,
1761 unsigned int idx, u16 size, void **handle)
1762 {
1763 struct fnic_iport_s *iport = (struct fnic_iport_s *)lport->private;
1764 struct fnic *fnic = iport->fnic;
1765
1766 FNIC_NVME_DBG(KERN_DEBUG, fnic,
1767 "iport:0x%x queue:%d size:%d\n", iport->fcid, idx, size);
1768
1769 if (idx > fnic->wq_copy_count)
1770 return -EINVAL;
1771
1772 if (idx == 0) {
1773 /* Admin queue */
1774 *handle = &fnic->hw_copy_wq[0];
1775 } else {
1776 /* IO queues */
1777 *handle = &fnic->hw_copy_wq[idx-1];
1778 }
1779
1780 return 0;
1781 }
1782
nvfnic_ls_req_abort(struct nvme_fc_local_port * lport,struct nvme_fc_remote_port * rport,struct nvmefc_ls_req * lsreq)1783 void nvfnic_ls_req_abort(struct nvme_fc_local_port *lport,
1784 struct nvme_fc_remote_port *rport,
1785 struct nvmefc_ls_req *lsreq)
1786 {
1787 struct fnic_iport_s *iport = lport->private;
1788 struct fnic *fnic = iport->fnic;
1789 struct fnic_tport_s *tport;
1790 struct nvfnic_ls_req *nvfnic_ls_req;
1791 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1792 uint16_t oxid;
1793 int timeout;
1794 int ret;
1795
1796 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1797
1798 tport = (struct fnic_tport_s *) rport->private;
1799 /* find the request */
1800 nvfnic_ls_req = lsreq->private;
1801
1802 if (nvfnic_ls_req == NULL) {
1803 FNIC_NVME_DBG(KERN_ERR, fnic,
1804 "0x%x null lsreq already scheduled for abort\n",
1805 iport->fcid);
1806 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1807 return;
1808 }
1809
1810 if (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING) {
1811 FNIC_NVME_DBG(KERN_ERR, fnic,
1812 "0x%x lsreq 0x%x already scheduled for abort\n",
1813 iport->fcid, nvfnic_ls_req->oxid);
1814 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1815 return;
1816 }
1817
1818 FNIC_NVME_DBG(KERN_INFO, fnic,
1819 "0x%x lsreq 0x%x abts\n",
1820 iport->fcid, nvfnic_ls_req->oxid);
1821
1822 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_STARTED;
1823 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1824 timer_delete_sync(&nvfnic_ls_req->ls_req_timer);
1825
1826 spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
1827 nvfnic_ls_req = lsreq->private;
1828
1829 if ((nvfnic_ls_req == NULL) ||
1830 (nvfnic_ls_req->state == FNIC_LS_REQ_CMD_ABTS_PENDING)) {
1831 FNIC_NVME_DBG(KERN_ERR, fnic,
1832 "lsreq timeout raced with midlayer abort\n");
1833 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1834 return;
1835 }
1836
1837 /* Basic validations of the state */
1838 if (!nvfnic_transport_ready(iport, tport)) {
1839 /* If iport or tport offline, it will be handled from that event */
1840 oxid = nvfnic_ls_req->oxid;
1841 lsreq->private = NULL;
1842 list_del(&nvfnic_ls_req->list);
1843 fdls_free_oxid(iport, oxid, &nvfnic_ls_req->oxid);
1844 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1845 lsreq->done(lsreq, -ENXIO);
1846 FNIC_NVME_DBG(KERN_ERR, fnic,
1847 "nvfnic_lsreq_abort transport not ready\n");
1848 return;
1849 }
1850
1851 /* Mark the state and flags */
1852 nvfnic_ls_req->state = FNIC_LS_REQ_CMD_ABTS_PENDING;
1853 atomic64_inc(&fnic_stats->nvme_stats.nvme_ls_aborts);
1854 oxid = nvfnic_ls_req->oxid;
1855
1856 ret = fdls_send_ls_req_abts(iport, tport, oxid);
1857 if (!ret) {
1858 timeout = FNIC_LS_REQ_TMO_MSECS(lsreq->timeout);
1859 mod_timer(&nvfnic_ls_req->ls_req_timer,
1860 round_jiffies(jiffies + msecs_to_jiffies(timeout)));
1861 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1862 return;
1863 }
1864
1865 if (!nvfnic_ls_req_cleanup(iport, lsreq, oxid)) {
1866 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1867 return;
1868 }
1869
1870 spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
1871 lsreq->done(lsreq, -EAGAIN);
1872 }
1873
nvfnic_queue_abort_io_req(struct fnic * fnic,int tag,u32 task_req,struct fnic_io_req * io_req)1874 bool nvfnic_queue_abort_io_req(struct fnic *fnic, int tag,
1875 u32 task_req, struct fnic_io_req *io_req)
1876 {
1877 int idx;
1878 unsigned long flags;
1879 struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;
1880
1881 idx = io_req->wq - &fnic->hw_copy_wq[0];
1882
1883 atomic_inc(&fnic->in_flight);
1884
1885 spin_lock_irqsave(&fnic->wq_copy_lock[idx], flags);
1886
1887 if (vnic_wq_copy_desc_avail(io_req->wq) <= fnic->wq_copy_desc_low[idx])
1888 free_wq_copy_descs(fnic, io_req->wq, idx);
1889
1890 if (!vnic_wq_copy_desc_avail(io_req->wq)) {
1891 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags);
1892 atomic_dec(&fnic->in_flight);
1893 FNIC_NVME_DBG(KERN_ERR, fnic,
1894 "tag 0x%x failure: no descriptors\n", tag);
1895 atomic64_inc(&misc_stats->abts_cpwq_alloc_failures);
1896 return false;
1897 }
1898 fnic_queue_wq_copy_desc_itmf(io_req->wq, tag | FNIC_TAG_ABORT,
1899 0, task_req, tag, NULL, io_req->port_id,
1900 fnic->config.ra_tov, fnic->config.ed_tov);
1901
1902 atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);
1903 if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >
1904 atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))
1905 atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,
1906 atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));
1907
1908 spin_unlock_irqrestore(&fnic->wq_copy_lock[idx], flags);
1909 atomic_dec(&fnic->in_flight);
1910
1911 return true;
1912 }
1913
nvfnic_fcpio_abort(struct nvme_fc_local_port * lport,struct nvme_fc_remote_port * rport,void * hw_queue_handle,struct nvmefc_fcp_req * fcp_req)1914 void nvfnic_fcpio_abort(struct nvme_fc_local_port *lport,
1915 struct nvme_fc_remote_port *rport,
1916 void *hw_queue_handle, struct nvmefc_fcp_req *fcp_req)
1917 {
1918 struct fnic_iport_s *iport = lport->private;
1919 struct fnic *fnic = iport->fnic;
1920 struct nvme_fc_cmd_iu *cmd_iu = fcp_req->cmdaddr;
1921 struct fnic_io_req *io_req = (struct fnic_io_req *)fcp_req->private;
1922 unsigned int tag = io_req->tag;
1923 struct fnic_stats *fnic_stats = &fnic->fnic_stats;
1924 struct abort_stats *abts_stats;
1925 unsigned long flags = 0;
1926 unsigned long abt_issued_time;
1927 unsigned int task_req;
1928 enum fnic_ioreq_state old_ioreq_state;
1929 unsigned long num_ios_waitq, waitq_2sec, waitq_max_time;
1930
1931 spin_lock_irqsave(&fnic->fnic_lock, flags);
1932
1933 if (io_req->tag == FNIC_NVME_NO_FREE_TAG) {
1934 FNIC_NVME_DBG(KERN_ERR, fnic,
1935 "tag: (0x%x) tport_fcid: 0x%x\n",
1936 io_req->tag, io_req->tport->fcid);
1937 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1938 return;
1939 }
1940
1941 if (io_req != nvfnic_find_io_req_by_tag(fnic, io_req->tag)) {
1942 FNIC_NVME_DBG(KERN_INFO, fnic,
1943 "cmd tag freed or not issued:0x%x sn:0x%08x\n",
1944 io_req->tag, be32_to_cpu(cmd_iu->csn));
1945 num_ios_waitq =
1946 atomic64_read(&fnic_stats->io_stats.nvme_num_ios_in_waitq);
1947 waitq_2sec =
1948 atomic64_read(&fnic_stats->io_stats.nvme_ios_in_waitq_3000_msec);
1949 waitq_max_time =
1950 atomic64_read(&fnic_stats->io_stats.nvme_ios_in_waitq_max_time);
1951 FNIC_NVME_DBG(KERN_INFO, fnic,
1952 "waitq:%ld waitq_2sec:%ld max_wait:%ld\n",
1953 num_ios_waitq, waitq_2sec, waitq_max_time);
1954 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1955 return;
1956 }
1957
1958 if (io_req->cmd_state == FNIC_IOREQ_CMD_COMPLETE) {
1959 FNIC_NVME_DBG(KERN_INFO, fnic,
1960 "IO already completed before abort: 0x%x\n", tag);
1961 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1962 return;
1963 }
1964
1965 if ((io_req->cmd_state == FNIC_IOREQ_ABTS_PENDING) ||
1966 (io_req->cmd_state == FNIC_DEV_RST_TERM_ISSUED)) {
1967 FNIC_NVME_DBG(KERN_INFO, fnic, "abort already pending %d\n",
1968 tag);
1969 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1970 return;
1971 }
1972
1973 if (io_req->cmd_state != FNIC_IOREQ_CMD_PENDING) {
1974 FNIC_NVME_DBG(KERN_INFO, fnic,
1975 "io_req completed or aborted for tag:0x%x\n",
1976 tag);
1977 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1978 return;
1979 }
1980
1981 FNIC_NVME_DBG(KERN_INFO, fnic, "in abort cmd_sn:%08x %llx tag: 0x%x\n",
1982 be32_to_cpu(cmd_iu->csn),
1983 le64_to_cpu(cmd_iu->sqe.rw.slba), io_req->tag);
1984
1985 if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) {
1986 FNIC_NVME_DBG(KERN_INFO, fnic,
1987 "abort tag:0x%x returned during fw reset\n",
1988 io_req->tag);
1989 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
1990 return;
1991 }
1992 atomic_inc(&fnic->in_flight);
1993
1994 if (fdls_tport_is_offline(io_req->tport) ||
1995 (io_req->cmd_state == FNIC_IOREQ_RESET_TERM)) {
1996 task_req = FCPIO_ITMF_ABT_TASK_TERM;
1997 } else {
1998 task_req = FCPIO_ITMF_ABT_TASK;
1999 }
2000
2001 abts_stats = &fnic->fnic_stats.abts_stats;
2002 atomic64_inc(&abts_stats->aborts);
2003
2004 abt_issued_time = jiffies_to_msecs(jiffies - io_req->start_time);
2005 if (abt_issued_time <= 6000)
2006 atomic64_inc(&abts_stats->abort_issued_btw_0_to_6_sec);
2007 else if (abt_issued_time > 6000 && abt_issued_time <= 20000)
2008 atomic64_inc(&abts_stats->abort_issued_btw_6_to_20_sec);
2009 else if (abt_issued_time > 20000 && abt_issued_time <= 30000)
2010 atomic64_inc(&abts_stats->abort_issued_btw_20_to_30_sec);
2011 else if (abt_issued_time > 30000 && abt_issued_time <= 40000)
2012 atomic64_inc(&abts_stats->abort_issued_btw_30_to_40_sec);
2013 else if (abt_issued_time > 40000 && abt_issued_time <= 50000)
2014 atomic64_inc(&abts_stats->abort_issued_btw_40_to_50_sec);
2015 else if (abt_issued_time > 50000 && abt_issued_time <= 60000)
2016 atomic64_inc(&abts_stats->abort_issued_btw_50_to_60_sec);
2017 else
2018 atomic64_inc(&abts_stats->abort_issued_greater_than_60_sec);
2019
2020 old_ioreq_state = io_req->cmd_state;
2021 io_req->cmd_state = FNIC_IOREQ_ABTS_PENDING;
2022 io_req->abts_state = FCPIO_INVALID_CODE;
2023
2024 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2025
2026 if (!nvfnic_queue_abort_io_req(fnic, io_req->tag, task_req, io_req)) {
2027 FNIC_NVME_DBG(KERN_INFO, fnic,
2028 "Abort io req queue failed\n");
2029 spin_lock_irqsave(&fnic->fnic_lock, flags);
2030 io_req->cmd_state = old_ioreq_state;
2031 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2032 }
2033 atomic_dec(&fnic->in_flight);
2034 }
2035
2036 struct
2037 nvme_fc_port_template nvfnic_port = {
2038 .localport_delete = nvfnic_local_port_delete,
2039 .remoteport_delete = nvfnic_remote_port_delete,
2040 .create_queue = nvfnic_create_queue,
2041 .delete_queue = NULL,
2042 .ls_req = nvfnic_ls_req_send,
2043 .ls_abort = nvfnic_ls_req_abort,
2044 .fcp_io = nvfnic_fcpio_send,
2045 .fcp_abort = nvfnic_fcpio_abort,
2046 .max_hw_queues = 1,
2047 .max_sgl_segments = 256,
2048 .max_dif_sgl_segments = 64,
2049 .dma_boundary = 0xFFFFFFFF,
2050 .local_priv_sz = sizeof(struct fnic_iport_s *),
2051 .remote_priv_sz = sizeof(struct fnic_tport_s *),
2052 .lsrqst_priv_sz = sizeof(struct nvfnic_ls_req),
2053 .fcprqst_priv_sz = sizeof(struct fnic_io_req),
2054 };
2055
nvfnic_flush_nvme_io_list(struct fnic * fnic)2056 void nvfnic_flush_nvme_io_list(struct fnic *fnic)
2057 {
2058 queue_work(fnic_cmpl_queue, &fnic->nvme_io_cmpl_work);
2059 flush_work(&fnic->nvme_io_cmpl_work);
2060 }
2061
nvfnic_nvme_iodone_work(struct work_struct * work)2062 void nvfnic_nvme_iodone_work(struct work_struct *work)
2063 {
2064 struct fnic *fnic = container_of(work, struct fnic, nvme_io_cmpl_work);
2065 struct llist_node *llnode;
2066 struct fnic_io_req *io_req, *tmp;
2067
2068 llnode = llist_del_all(&fnic->nvme_io_event_llist);
2069 llist_for_each_entry_safe(io_req, tmp, llnode, nvfnic_io_cmpl) {
2070 atomic_dec(&fnic->nvme_io_event_queued);
2071 atomic64_dec(&fnic->fnic_stats.io_stats.nvme_num_ios_in_waitq);
2072 io_req->fcp_req->done(io_req->fcp_req);
2073 }
2074 }
2075
nvfnic_exch_reset(struct fnic_iport_s * iport,struct fnic_tport_s * tport)2076 void nvfnic_exch_reset(struct fnic_iport_s *iport, struct fnic_tport_s *tport)
2077 {
2078 FNIC_NVME_DBG(KERN_DEBUG, iport->fnic,
2079 "0x%x: Exchange reset scheduled for tport: 0x%x\n",
2080 iport->fcid, tport->fcid);
2081
2082 nvfnic_terminate_tport_ls_reqs(iport->fnic, tport);
2083 nvfnic_terminate_tport_ios(iport->fnic, tport);
2084 }
2085
nvfnic_delete_tport(struct fnic_iport_s * iport,struct fnic_tport_s * tport,unsigned long flags)2086 void nvfnic_delete_tport(struct fnic_iport_s *iport,
2087 struct fnic_tport_s *tport,
2088 unsigned long flags)
2089 {
2090 struct fnic *fnic = iport->fnic;
2091 int ret;
2092 unsigned int time_wait = FNIC_NVME_TPORT_REMOVE_WAIT;
2093 unsigned int time_remain;
2094 DECLARE_COMPLETION_ONSTACK(tm_done);
2095 unsigned int fcid;
2096 int count = 0;
2097
2098 if (!tport)
2099 return;
2100
2101 fcid = tport->fcid;
2102 fdls_set_tport_state(tport, FDLS_TGT_STATE_OFFLINE);
2103
2104 FNIC_NVME_DBG(KERN_DEBUG, fnic,
2105 "0x%x: scheduled deletion for tport: 0x%x\n",
2106 iport->fcid, tport->fcid);
2107
2108 if (!(tport->flags & FNIC_FDLS_NVME_REGISTERED)) {
2109 FNIC_NVME_DBG(KERN_ERR, fnic,
2110 "0x%x: tport: 0x%x not registered. Freeing\n",
2111 iport->fcid, tport->fcid);
2112 list_del(&tport->links);
2113 kfree(tport);
2114 return;
2115 }
2116
2117 tport->tport_del_done = &tm_done;
2118
2119 tport->flags |= FNIC_FDLS_TPORT_DELETED;
2120 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2121 ret = nvme_fc_unregister_remoteport(tport->nv_rport);
2122 if (ret) {
2123 FNIC_NVME_DBG(KERN_ERR, fnic,
2124 "tport: 0x%x unregister failed %d\n",
2125 tport->fcid, ret);
2126
2127 nvfnic_terminate_tport_ls_reqs(fnic, tport);
2128
2129 spin_lock_irqsave(&fnic->fnic_lock, flags);
2130 tport->tport_del_done = NULL;
2131 if (tport->nv_rport && tport->nv_rport->private == tport)
2132 tport->nv_rport->private = NULL;
2133 list_del(&tport->links);
2134 kfree(tport);
2135 return;
2136 }
2137 time_remain = wait_for_completion_timeout(tport->tport_del_done,
2138 msecs_to_jiffies(time_wait));
2139
2140 FNIC_NVME_DBG(KERN_DEBUG, fnic,
2141 "tport: 0x%x wait for deletion done\n",
2142 tport->fcid);
2143
2144 spin_lock_irqsave(&fnic->fnic_lock, flags);
2145 tport->tport_del_done = NULL;
2146
2147 if (!time_remain) {
2148 FNIC_NVME_DBG(KERN_ERR, fnic,
2149 "tport: 0x%x nvme midlayer completion timed out\n",
2150 tport->fcid);
2151
2152 if (tport->flags & FNIC_TPORT_CAN_BE_FREED) {
2153 kfree(tport);
2154 FNIC_NVME_DBG(KERN_INFO, fnic,
2155 "tport: 0x%x delete complete\n", fcid);
2156 return;
2157 }
2158
2159 tport->flags |= FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING;
2160 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2161 nvfnic_cleanup_tport_io(fnic, tport);
2162 spin_lock_irqsave(&fnic->fnic_lock, flags);
2163 tport->flags &= ~FNIC_FDLS_NVME_TPORT_CLEANUP_PENDING;
2164 return;
2165 }
2166 while (!(tport->flags & FNIC_TPORT_CAN_BE_FREED) &&
2167 (count < FNIC_TPORT_CLEANUP_WAIT_COUNT)) {
2168 count++;
2169 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2170 msleep(2000);
2171 spin_lock_irqsave(&fnic->fnic_lock, flags);
2172 }
2173 if (tport->flags & FNIC_TPORT_CAN_BE_FREED)
2174 kfree(tport);
2175
2176 FNIC_NVME_DBG(KERN_INFO, fnic,
2177 "tport: 0x%x delete complete\n", fcid);
2178 }
2179
nvfnic_add_tport(struct fnic * fnic,struct fnic_tport_s * tport,unsigned long flags)2180 int nvfnic_add_tport(struct fnic *fnic, struct fnic_tport_s *tport,
2181 unsigned long flags)
2182 {
2183 struct fnic_iport_s *iport = &fnic->iport;
2184 struct nvme_fc_port_info pinfo;
2185 int ret = 0;
2186
2187 FNIC_NVME_DBG(KERN_INFO, fnic,
2188 "Adding tport to nvme wwpn: 0x%llx\n",
2189 tport->wwpn);
2190
2191 memset(&pinfo, 0, sizeof(struct nvme_fc_port_info));
2192
2193 pinfo.port_name = tport->wwpn;
2194 pinfo.node_name = tport->wwnn;
2195 pinfo.port_role = FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET;
2196 pinfo.port_id = tport->fcid;
2197 pinfo.dev_loss_tmo = nvme_dev_loss_tmo;
2198
2199 spin_unlock_irqrestore(&fnic->fnic_lock, flags);
2200 ret = nvme_fc_register_remoteport(iport->nv_lport, &pinfo,
2201 &tport->nv_rport);
2202 spin_lock_irqsave(&fnic->fnic_lock, flags);
2203 if (ret) {
2204 FNIC_NVME_DBG(KERN_INFO, fnic,
2205 "Failed to register tport wwpn: 0x%llx ret: %d\n",
2206 tport->wwpn, ret);
2207 return ret;
2208 }
2209 tport->flags |= FNIC_FDLS_NVME_REGISTERED;
2210 tport->nv_rport->private = tport;
2211
2212 snprintf(tport->str_wwpn, sizeof(tport->str_wwpn), "0x%llx", tport->wwpn);
2213 snprintf(tport->str_wwnn, sizeof(tport->str_wwnn), "0x%llx", tport->wwnn);
2214 return ret;
2215 }
2216
nvfnic_add_lport(struct fnic * fnic)2217 int nvfnic_add_lport(struct fnic *fnic)
2218 {
2219 struct nvme_fc_port_info pinfo = {};
2220 struct fnic_iport_s *iport = &fnic->iport;
2221 int ret = 0;
2222
2223 FNIC_NVME_DBG(KERN_INFO, fnic,
2224 "Adding lport nvme wwpn: 0x%llx\n",
2225 iport->wwpn);
2226
2227 pinfo.node_name = iport->wwnn;
2228 pinfo.port_name = iport->wwpn;
2229 pinfo.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2230 pinfo.port_id = iport->fcid;
2231
2232 nvfnic_reset_fcpio_tag_pool(iport);
2233
2234 if (!(iport->flags & FNIC_LPORT_NVME_REGISTERED)) {
2235 iport->nv_tmpl = kzalloc_obj(struct nvme_fc_port_template, GFP_ATOMIC);
2236 if (!iport->nv_tmpl) {
2237 FNIC_FCS_DBG(KERN_INFO, fnic,
2238 "iport:0x%x NVMe tmpl alloc failed\n",
2239 iport->fcid);
2240 return -ENOMEM;
2241 }
2242 memcpy(iport->nv_tmpl, &nvfnic_port,
2243 sizeof(struct nvme_fc_port_template));
2244 iport->nv_tmpl->max_hw_queues = fnic->wq_copy_count;
2245
2246 ret = nvme_fc_register_localport(&pinfo, iport->nv_tmpl,
2247 &fnic->pdev->dev, &iport->nv_lport);
2248 if (ret) {
2249 FNIC_NVME_DBG(KERN_ERR, fnic,
2250 "Failed to add wwpn: 0x%llx ret: %d\n",
2251 iport->wwpn, ret);
2252 kfree(iport->nv_tmpl);
2253 return ret;
2254 }
2255 iport->flags |= FNIC_LPORT_NVME_REGISTERED;
2256 iport->nv_lport->private = iport;
2257 }
2258
2259 sprintf(iport->str_wwpn, "0x%llx", iport->wwpn);
2260 sprintf(iport->str_wwnn, "0x%llx", iport->wwnn);
2261
2262 FNIC_NVME_DBG(KERN_INFO, fnic,
2263 "Successfully added lport wwpn: 0x%llx\n",
2264 iport->wwpn);
2265 return 0;
2266 }
2267
2268 #endif
2269