1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
5 */
6
7 #include "efct_driver.h"
8 #include "efct_hw.h"
9 #include "efct_unsol.h"
10
11 struct efct_hw_link_stat_cb_arg {
12 void (*cb)(int status, u32 num_counters,
13 struct efct_hw_link_stat_counts *counters, void *arg);
14 void *arg;
15 };
16
17 struct efct_hw_host_stat_cb_arg {
18 void (*cb)(int status, u32 num_counters,
19 struct efct_hw_host_stat_counts *counters, void *arg);
20 void *arg;
21 };
22
23 struct efct_hw_fw_wr_cb_arg {
24 void (*cb)(int status, u32 bytes_written, u32 change_status, void *arg);
25 void *arg;
26 };
27
28 struct efct_mbox_rqst_ctx {
29 int (*callback)(struct efc *efc, int status, u8 *mqe, void *arg);
30 void *arg;
31 };
32
33 static int
efct_hw_link_event_init(struct efct_hw * hw)34 efct_hw_link_event_init(struct efct_hw *hw)
35 {
36 hw->link.status = SLI4_LINK_STATUS_MAX;
37 hw->link.topology = SLI4_LINK_TOPO_NONE;
38 hw->link.medium = SLI4_LINK_MEDIUM_MAX;
39 hw->link.speed = 0;
40 hw->link.loop_map = NULL;
41 hw->link.fc_id = U32_MAX;
42
43 return 0;
44 }
45
46 static int
efct_hw_read_max_dump_size(struct efct_hw * hw)47 efct_hw_read_max_dump_size(struct efct_hw *hw)
48 {
49 u8 buf[SLI4_BMBX_SIZE];
50 struct efct *efct = hw->os;
51 int rc = 0;
52 struct sli4_rsp_cmn_set_dump_location *rsp;
53
54 /* attempt to detemine the dump size for function 0 only. */
55 if (PCI_FUNC(efct->pci->devfn) != 0)
56 return rc;
57
58 if (sli_cmd_common_set_dump_location(&hw->sli, buf, 1, 0, NULL, 0))
59 return -EIO;
60
61 rsp = (struct sli4_rsp_cmn_set_dump_location *)
62 (buf + offsetof(struct sli4_cmd_sli_config, payload.embed));
63
64 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
65 if (rc != 0) {
66 efc_log_debug(hw->os, "set dump location cmd failed\n");
67 return rc;
68 }
69
70 hw->dump_size =
71 le32_to_cpu(rsp->buffer_length_dword) & SLI4_CMN_SET_DUMP_BUFFER_LEN;
72
73 efc_log_debug(hw->os, "Dump size %x\n", hw->dump_size);
74
75 return rc;
76 }
77
78 static int
__efct_read_topology_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)79 __efct_read_topology_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
80 {
81 struct sli4_cmd_read_topology *read_topo =
82 (struct sli4_cmd_read_topology *)mqe;
83 u8 speed;
84 struct efc_domain_record drec = {0};
85 struct efct *efct = hw->os;
86
87 if (status || le16_to_cpu(read_topo->hdr.status)) {
88 efc_log_debug(hw->os, "bad status cqe=%#x mqe=%#x\n", status,
89 le16_to_cpu(read_topo->hdr.status));
90 return -EIO;
91 }
92
93 switch (le32_to_cpu(read_topo->dw2_attentype) &
94 SLI4_READTOPO_ATTEN_TYPE) {
95 case SLI4_READ_TOPOLOGY_LINK_UP:
96 hw->link.status = SLI4_LINK_STATUS_UP;
97 break;
98 case SLI4_READ_TOPOLOGY_LINK_DOWN:
99 hw->link.status = SLI4_LINK_STATUS_DOWN;
100 break;
101 case SLI4_READ_TOPOLOGY_LINK_NO_ALPA:
102 hw->link.status = SLI4_LINK_STATUS_NO_ALPA;
103 break;
104 default:
105 hw->link.status = SLI4_LINK_STATUS_MAX;
106 break;
107 }
108
109 switch (read_topo->topology) {
110 case SLI4_READ_TOPO_NON_FC_AL:
111 hw->link.topology = SLI4_LINK_TOPO_NON_FC_AL;
112 break;
113 case SLI4_READ_TOPO_FC_AL:
114 hw->link.topology = SLI4_LINK_TOPO_FC_AL;
115 if (hw->link.status == SLI4_LINK_STATUS_UP)
116 hw->link.loop_map = hw->loop_map.virt;
117 hw->link.fc_id = read_topo->acquired_al_pa;
118 break;
119 default:
120 hw->link.topology = SLI4_LINK_TOPO_MAX;
121 break;
122 }
123
124 hw->link.medium = SLI4_LINK_MEDIUM_FC;
125
126 speed = (le32_to_cpu(read_topo->currlink_state) &
127 SLI4_READTOPO_LINKSTATE_SPEED) >> 8;
128 switch (speed) {
129 case SLI4_READ_TOPOLOGY_SPEED_1G:
130 hw->link.speed = 1 * 1000;
131 break;
132 case SLI4_READ_TOPOLOGY_SPEED_2G:
133 hw->link.speed = 2 * 1000;
134 break;
135 case SLI4_READ_TOPOLOGY_SPEED_4G:
136 hw->link.speed = 4 * 1000;
137 break;
138 case SLI4_READ_TOPOLOGY_SPEED_8G:
139 hw->link.speed = 8 * 1000;
140 break;
141 case SLI4_READ_TOPOLOGY_SPEED_16G:
142 hw->link.speed = 16 * 1000;
143 break;
144 case SLI4_READ_TOPOLOGY_SPEED_32G:
145 hw->link.speed = 32 * 1000;
146 break;
147 case SLI4_READ_TOPOLOGY_SPEED_64G:
148 hw->link.speed = 64 * 1000;
149 break;
150 case SLI4_READ_TOPOLOGY_SPEED_128G:
151 hw->link.speed = 128 * 1000;
152 break;
153 }
154
155 drec.speed = hw->link.speed;
156 drec.fc_id = hw->link.fc_id;
157 drec.is_nport = true;
158 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND, &drec);
159
160 return 0;
161 }
162
163 static int
efct_hw_cb_link(void * ctx,void * e)164 efct_hw_cb_link(void *ctx, void *e)
165 {
166 struct efct_hw *hw = ctx;
167 struct sli4_link_event *event = e;
168 struct efc_domain *d = NULL;
169 int rc = 0;
170 struct efct *efct = hw->os;
171
172 efct_hw_link_event_init(hw);
173
174 switch (event->status) {
175 case SLI4_LINK_STATUS_UP:
176
177 hw->link = *event;
178 efct->efcport->link_status = EFC_LINK_STATUS_UP;
179
180 if (event->topology == SLI4_LINK_TOPO_NON_FC_AL) {
181 struct efc_domain_record drec = {0};
182
183 efc_log_info(hw->os, "Link Up, NPORT, speed is %d\n",
184 event->speed);
185 drec.speed = event->speed;
186 drec.fc_id = event->fc_id;
187 drec.is_nport = true;
188 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND,
189 &drec);
190 } else if (event->topology == SLI4_LINK_TOPO_FC_AL) {
191 u8 buf[SLI4_BMBX_SIZE];
192
193 efc_log_info(hw->os, "Link Up, LOOP, speed is %d\n",
194 event->speed);
195
196 if (!sli_cmd_read_topology(&hw->sli, buf,
197 &hw->loop_map)) {
198 rc = efct_hw_command(hw, buf, EFCT_CMD_NOWAIT,
199 __efct_read_topology_cb, NULL);
200 }
201
202 if (rc)
203 efc_log_debug(hw->os, "READ_TOPOLOGY failed\n");
204 } else {
205 efc_log_info(hw->os, "%s(%#x), speed is %d\n",
206 "Link Up, unsupported topology ",
207 event->topology, event->speed);
208 }
209 break;
210 case SLI4_LINK_STATUS_DOWN:
211 efc_log_info(hw->os, "Link down\n");
212
213 hw->link.status = event->status;
214 efct->efcport->link_status = EFC_LINK_STATUS_DOWN;
215
216 d = efct->efcport->domain;
217 if (d)
218 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_LOST, d);
219 break;
220 default:
221 efc_log_debug(hw->os, "unhandled link status %#x\n",
222 event->status);
223 break;
224 }
225
226 return 0;
227 }
228
229 int
efct_hw_setup(struct efct_hw * hw,void * os,struct pci_dev * pdev)230 efct_hw_setup(struct efct_hw *hw, void *os, struct pci_dev *pdev)
231 {
232 u32 i, max_sgl, cpus;
233
234 if (hw->hw_setup_called)
235 return 0;
236
237 /*
238 * efct_hw_init() relies on NULL pointers indicating that a structure
239 * needs allocation. If a structure is non-NULL, efct_hw_init() won't
240 * free/realloc that memory
241 */
242 memset(hw, 0, sizeof(struct efct_hw));
243
244 hw->hw_setup_called = true;
245
246 hw->os = os;
247
248 mutex_init(&hw->bmbx_lock);
249 spin_lock_init(&hw->cmd_lock);
250 INIT_LIST_HEAD(&hw->cmd_head);
251 INIT_LIST_HEAD(&hw->cmd_pending);
252 hw->cmd_head_count = 0;
253
254 /* Create mailbox command ctx pool */
255 hw->cmd_ctx_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,
256 sizeof(struct efct_command_ctx));
257 if (!hw->cmd_ctx_pool) {
258 efc_log_err(hw->os, "failed to allocate mailbox buffer pool\n");
259 return -EIO;
260 }
261
262 /* Create mailbox request ctx pool for library callback */
263 hw->mbox_rqst_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,
264 sizeof(struct efct_mbox_rqst_ctx));
265 if (!hw->mbox_rqst_pool) {
266 efc_log_err(hw->os, "failed to allocate mbox request pool\n");
267 return -EIO;
268 }
269
270 spin_lock_init(&hw->io_lock);
271 INIT_LIST_HEAD(&hw->io_inuse);
272 INIT_LIST_HEAD(&hw->io_free);
273 INIT_LIST_HEAD(&hw->io_wait_free);
274
275 atomic_set(&hw->io_alloc_failed_count, 0);
276
277 hw->config.speed = SLI4_LINK_SPEED_AUTO_16_8_4;
278 if (sli_setup(&hw->sli, hw->os, pdev, ((struct efct *)os)->reg)) {
279 efc_log_err(hw->os, "SLI setup failed\n");
280 return -EIO;
281 }
282
283 efct_hw_link_event_init(hw);
284
285 sli_callback(&hw->sli, SLI4_CB_LINK, efct_hw_cb_link, hw);
286
287 /*
288 * Set all the queue sizes to the maximum allowed.
289 */
290 for (i = 0; i < ARRAY_SIZE(hw->num_qentries); i++)
291 hw->num_qentries[i] = hw->sli.qinfo.max_qentries[i];
292 /*
293 * Adjust the size of the WQs so that the CQ is twice as big as
294 * the WQ to allow for 2 completions per IO. This allows us to
295 * handle multi-phase as well as aborts.
296 */
297 hw->num_qentries[SLI4_QTYPE_WQ] = hw->num_qentries[SLI4_QTYPE_CQ] / 2;
298
299 /*
300 * The RQ assignment for RQ pair mode.
301 */
302
303 hw->config.rq_default_buffer_size = EFCT_HW_RQ_SIZE_PAYLOAD;
304 hw->config.n_io = hw->sli.ext[SLI4_RSRC_XRI].size;
305
306 cpus = num_possible_cpus();
307 hw->config.n_eq = cpus > EFCT_HW_MAX_NUM_EQ ? EFCT_HW_MAX_NUM_EQ : cpus;
308
309 max_sgl = sli_get_max_sgl(&hw->sli) - SLI4_SGE_MAX_RESERVED;
310 max_sgl = (max_sgl > EFCT_FC_MAX_SGL) ? EFCT_FC_MAX_SGL : max_sgl;
311 hw->config.n_sgl = max_sgl;
312
313 (void)efct_hw_read_max_dump_size(hw);
314
315 return 0;
316 }
317
318 static void
efct_logfcfi(struct efct_hw * hw,u32 j,u32 i,u32 id)319 efct_logfcfi(struct efct_hw *hw, u32 j, u32 i, u32 id)
320 {
321 efc_log_info(hw->os,
322 "REG_FCFI: filter[%d] %08X -> RQ[%d] id=%d\n",
323 j, hw->config.filter_def[j], i, id);
324 }
325
326 static inline void
efct_hw_init_free_io(struct efct_hw_io * io)327 efct_hw_init_free_io(struct efct_hw_io *io)
328 {
329 /*
330 * Set io->done to NULL, to avoid any callbacks, should
331 * a completion be received for one of these IOs
332 */
333 io->done = NULL;
334 io->abort_done = NULL;
335 io->status_saved = false;
336 io->abort_in_progress = false;
337 io->type = 0xFFFF;
338 io->wq = NULL;
339 }
340
efct_hw_iotype_is_originator(u16 io_type)341 static bool efct_hw_iotype_is_originator(u16 io_type)
342 {
343 switch (io_type) {
344 case EFCT_HW_FC_CT:
345 case EFCT_HW_ELS_REQ:
346 return true;
347 default:
348 return false;
349 }
350 }
351
352 static void
efct_hw_io_restore_sgl(struct efct_hw * hw,struct efct_hw_io * io)353 efct_hw_io_restore_sgl(struct efct_hw *hw, struct efct_hw_io *io)
354 {
355 /* Restore the default */
356 io->sgl = &io->def_sgl;
357 io->sgl_count = io->def_sgl_count;
358 }
359
360 static void
efct_hw_wq_process_io(void * arg,u8 * cqe,int status)361 efct_hw_wq_process_io(void *arg, u8 *cqe, int status)
362 {
363 struct efct_hw_io *io = arg;
364 struct efct_hw *hw = io->hw;
365 struct sli4_fc_wcqe *wcqe = (void *)cqe;
366 u32 len = 0;
367 u32 ext = 0;
368
369 /* clear xbusy flag if WCQE[XB] is clear */
370 if (io->xbusy && (wcqe->flags & SLI4_WCQE_XB) == 0)
371 io->xbusy = false;
372
373 /* get extended CQE status */
374 switch (io->type) {
375 case EFCT_HW_BLS_ACC:
376 case EFCT_HW_BLS_RJT:
377 break;
378 case EFCT_HW_ELS_REQ:
379 sli_fc_els_did(&hw->sli, cqe, &ext);
380 len = sli_fc_response_length(&hw->sli, cqe);
381 break;
382 case EFCT_HW_ELS_RSP:
383 case EFCT_HW_FC_CT_RSP:
384 break;
385 case EFCT_HW_FC_CT:
386 len = sli_fc_response_length(&hw->sli, cqe);
387 break;
388 case EFCT_HW_IO_TARGET_WRITE:
389 len = sli_fc_io_length(&hw->sli, cqe);
390 break;
391 case EFCT_HW_IO_TARGET_READ:
392 len = sli_fc_io_length(&hw->sli, cqe);
393 break;
394 case EFCT_HW_IO_TARGET_RSP:
395 break;
396 case EFCT_HW_IO_DNRX_REQUEUE:
397 /* release the count for re-posting the buffer */
398 /* efct_hw_io_free(hw, io); */
399 break;
400 default:
401 efc_log_err(hw->os, "unhandled io type %#x for XRI 0x%x\n",
402 io->type, io->indicator);
403 break;
404 }
405 if (status) {
406 ext = sli_fc_ext_status(&hw->sli, cqe);
407 /*
408 * If we're not an originator IO, and XB is set, then issue
409 * abort for the IO from within the HW
410 */
411 if (efct_hw_iotype_is_originator(io->type) &&
412 wcqe->flags & SLI4_WCQE_XB) {
413 int rc;
414
415 efc_log_debug(hw->os, "aborting xri=%#x tag=%#x\n",
416 io->indicator, io->reqtag);
417
418 /*
419 * Because targets may send a response when the IO
420 * completes using the same XRI, we must wait for the
421 * XRI_ABORTED CQE to issue the IO callback
422 */
423 rc = efct_hw_io_abort(hw, io, false, NULL, NULL);
424 if (rc == 0) {
425 /*
426 * latch status to return after abort is
427 * complete
428 */
429 io->status_saved = true;
430 io->saved_status = status;
431 io->saved_ext = ext;
432 io->saved_len = len;
433 goto exit_efct_hw_wq_process_io;
434 } else if (rc == -EINPROGRESS) {
435 /*
436 * Already being aborted by someone else (ABTS
437 * perhaps). Just return original
438 * error.
439 */
440 efc_log_debug(hw->os, "%s%#x tag=%#x\n",
441 "abort in progress xri=",
442 io->indicator, io->reqtag);
443
444 } else {
445 /* Failed to abort for some other reason, log
446 * error
447 */
448 efc_log_debug(hw->os, "%s%#x tag=%#x rc=%d\n",
449 "Failed to abort xri=",
450 io->indicator, io->reqtag, rc);
451 }
452 }
453 }
454
455 if (io->done) {
456 efct_hw_done_t done = io->done;
457
458 io->done = NULL;
459
460 if (io->status_saved) {
461 /* use latched status if exists */
462 status = io->saved_status;
463 len = io->saved_len;
464 ext = io->saved_ext;
465 io->status_saved = false;
466 }
467
468 /* Restore default SGL */
469 efct_hw_io_restore_sgl(hw, io);
470 done(io, len, status, ext, io->arg);
471 }
472
473 exit_efct_hw_wq_process_io:
474 return;
475 }
476
477 static int
efct_hw_setup_io(struct efct_hw * hw)478 efct_hw_setup_io(struct efct_hw *hw)
479 {
480 u32 i = 0;
481 struct efct_hw_io *io = NULL;
482 uintptr_t xfer_virt = 0;
483 uintptr_t xfer_phys = 0;
484 u32 index;
485 bool new_alloc = true;
486 struct efc_dma *dma;
487 struct efct *efct = hw->os;
488
489 if (!hw->io) {
490 hw->io = kmalloc_objs(io, hw->config.n_io);
491 if (!hw->io)
492 return -ENOMEM;
493
494 memset(hw->io, 0, hw->config.n_io * sizeof(io));
495
496 for (i = 0; i < hw->config.n_io; i++) {
497 hw->io[i] = kzalloc_obj(*io);
498 if (!hw->io[i])
499 goto error;
500 }
501
502 /* Create WQE buffs for IO */
503 hw->wqe_buffs = kzalloc((hw->config.n_io * hw->sli.wqe_size),
504 GFP_KERNEL);
505 if (!hw->wqe_buffs) {
506 kfree(hw->io);
507 return -ENOMEM;
508 }
509
510 } else {
511 /* re-use existing IOs, including SGLs */
512 new_alloc = false;
513 }
514
515 if (new_alloc) {
516 dma = &hw->xfer_rdy;
517 dma->size = sizeof(struct fcp_txrdy) * hw->config.n_io;
518 dma->virt = dma_alloc_coherent(&efct->pci->dev,
519 dma->size, &dma->phys, GFP_KERNEL);
520 if (!dma->virt)
521 return -ENOMEM;
522 }
523 xfer_virt = (uintptr_t)hw->xfer_rdy.virt;
524 xfer_phys = hw->xfer_rdy.phys;
525
526 /* Initialize the pool of HW IO objects */
527 for (i = 0; i < hw->config.n_io; i++) {
528 struct hw_wq_callback *wqcb;
529
530 io = hw->io[i];
531
532 /* initialize IO fields */
533 io->hw = hw;
534
535 /* Assign a WQE buff */
536 io->wqe.wqebuf = &hw->wqe_buffs[i * hw->sli.wqe_size];
537
538 /* Allocate the request tag for this IO */
539 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_io, io);
540 if (!wqcb) {
541 efc_log_err(hw->os, "can't allocate request tag\n");
542 return -ENOSPC;
543 }
544 io->reqtag = wqcb->instance_index;
545
546 /* Now for the fields that are initialized on each free */
547 efct_hw_init_free_io(io);
548
549 /* The XB flag isn't cleared on IO free, so init to zero */
550 io->xbusy = 0;
551
552 if (sli_resource_alloc(&hw->sli, SLI4_RSRC_XRI,
553 &io->indicator, &index)) {
554 efc_log_err(hw->os,
555 "sli_resource_alloc failed @ %d\n", i);
556 return -ENOMEM;
557 }
558
559 if (new_alloc) {
560 dma = &io->def_sgl;
561 dma->size = hw->config.n_sgl *
562 sizeof(struct sli4_sge);
563 dma->virt = dma_alloc_coherent(&efct->pci->dev,
564 dma->size, &dma->phys,
565 GFP_KERNEL);
566 if (!dma->virt) {
567 efc_log_err(hw->os, "dma_alloc fail %d\n", i);
568 memset(&io->def_sgl, 0,
569 sizeof(struct efc_dma));
570 return -ENOMEM;
571 }
572 }
573 io->def_sgl_count = hw->config.n_sgl;
574 io->sgl = &io->def_sgl;
575 io->sgl_count = io->def_sgl_count;
576
577 if (hw->xfer_rdy.size) {
578 io->xfer_rdy.virt = (void *)xfer_virt;
579 io->xfer_rdy.phys = xfer_phys;
580 io->xfer_rdy.size = sizeof(struct fcp_txrdy);
581
582 xfer_virt += sizeof(struct fcp_txrdy);
583 xfer_phys += sizeof(struct fcp_txrdy);
584 }
585 }
586
587 return 0;
588 error:
589 for (i = 0; i < hw->config.n_io && hw->io[i]; i++) {
590 kfree(hw->io[i]);
591 hw->io[i] = NULL;
592 }
593
594 kfree(hw->io);
595 hw->io = NULL;
596
597 return -ENOMEM;
598 }
599
600 static int
efct_hw_init_prereg_io(struct efct_hw * hw)601 efct_hw_init_prereg_io(struct efct_hw *hw)
602 {
603 u32 i, idx = 0;
604 struct efct_hw_io *io = NULL;
605 u8 cmd[SLI4_BMBX_SIZE];
606 int rc = 0;
607 u32 n_rem;
608 u32 n = 0;
609 u32 sgls_per_request = 256;
610 struct efc_dma **sgls = NULL;
611 struct efc_dma req;
612 struct efct *efct = hw->os;
613
614 sgls = kmalloc_objs(*sgls, sgls_per_request);
615 if (!sgls)
616 return -ENOMEM;
617
618 memset(&req, 0, sizeof(struct efc_dma));
619 req.size = 32 + sgls_per_request * 16;
620 req.virt = dma_alloc_coherent(&efct->pci->dev, req.size, &req.phys,
621 GFP_KERNEL);
622 if (!req.virt) {
623 kfree(sgls);
624 return -ENOMEM;
625 }
626
627 for (n_rem = hw->config.n_io; n_rem; n_rem -= n) {
628 /* Copy address of SGL's into local sgls[] array, break
629 * out if the xri is not contiguous.
630 */
631 u32 min = (sgls_per_request < n_rem) ? sgls_per_request : n_rem;
632
633 for (n = 0; n < min; n++) {
634 /* Check that we have contiguous xri values */
635 if (n > 0) {
636 if (hw->io[idx + n]->indicator !=
637 hw->io[idx + n - 1]->indicator + 1)
638 break;
639 }
640
641 sgls[n] = hw->io[idx + n]->sgl;
642 }
643
644 if (sli_cmd_post_sgl_pages(&hw->sli, cmd,
645 hw->io[idx]->indicator, n, sgls, NULL, &req)) {
646 rc = -EIO;
647 break;
648 }
649
650 rc = efct_hw_command(hw, cmd, EFCT_CMD_POLL, NULL, NULL);
651 if (rc) {
652 efc_log_err(hw->os, "SGL post failed, rc=%d\n", rc);
653 break;
654 }
655
656 /* Add to tail if successful */
657 for (i = 0; i < n; i++, idx++) {
658 io = hw->io[idx];
659 io->state = EFCT_HW_IO_STATE_FREE;
660 INIT_LIST_HEAD(&io->list_entry);
661 list_add_tail(&io->list_entry, &hw->io_free);
662 }
663 }
664
665 dma_free_coherent(&efct->pci->dev, req.size, req.virt, req.phys);
666 memset(&req, 0, sizeof(struct efc_dma));
667 kfree(sgls);
668
669 return rc;
670 }
671
672 static int
efct_hw_init_io(struct efct_hw * hw)673 efct_hw_init_io(struct efct_hw *hw)
674 {
675 u32 i, idx = 0;
676 bool prereg = false;
677 struct efct_hw_io *io = NULL;
678 int rc = 0;
679
680 prereg = hw->sli.params.sgl_pre_registered;
681
682 if (prereg)
683 return efct_hw_init_prereg_io(hw);
684
685 for (i = 0; i < hw->config.n_io; i++, idx++) {
686 io = hw->io[idx];
687 io->state = EFCT_HW_IO_STATE_FREE;
688 INIT_LIST_HEAD(&io->list_entry);
689 list_add_tail(&io->list_entry, &hw->io_free);
690 }
691
692 return rc;
693 }
694
695 static int
efct_hw_config_set_fdt_xfer_hint(struct efct_hw * hw,u32 fdt_xfer_hint)696 efct_hw_config_set_fdt_xfer_hint(struct efct_hw *hw, u32 fdt_xfer_hint)
697 {
698 int rc = 0;
699 u8 buf[SLI4_BMBX_SIZE];
700 struct sli4_rqst_cmn_set_features_set_fdt_xfer_hint param;
701
702 memset(¶m, 0, sizeof(param));
703 param.fdt_xfer_hint = cpu_to_le32(fdt_xfer_hint);
704 /* build the set_features command */
705 sli_cmd_common_set_features(&hw->sli, buf,
706 SLI4_SET_FEATURES_SET_FTD_XFER_HINT, sizeof(param), ¶m);
707
708 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
709 if (rc)
710 efc_log_warn(hw->os, "set FDT hint %d failed: %d\n",
711 fdt_xfer_hint, rc);
712 else
713 efc_log_info(hw->os, "Set FTD transfer hint to %d\n",
714 le32_to_cpu(param.fdt_xfer_hint));
715
716 return rc;
717 }
718
719 static int
efct_hw_config_rq(struct efct_hw * hw)720 efct_hw_config_rq(struct efct_hw *hw)
721 {
722 u32 min_rq_count, i, rc;
723 struct sli4_cmd_rq_cfg rq_cfg[SLI4_CMD_REG_FCFI_NUM_RQ_CFG];
724 u8 buf[SLI4_BMBX_SIZE];
725
726 efc_log_info(hw->os, "using REG_FCFI standard\n");
727
728 /*
729 * Set the filter match/mask values from hw's
730 * filter_def values
731 */
732 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {
733 rq_cfg[i].rq_id = cpu_to_le16(0xffff);
734 rq_cfg[i].r_ctl_mask = (u8)hw->config.filter_def[i];
735 rq_cfg[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 8);
736 rq_cfg[i].type_mask = (u8)(hw->config.filter_def[i] >> 16);
737 rq_cfg[i].type_match = (u8)(hw->config.filter_def[i] >> 24);
738 }
739
740 /*
741 * Update the rq_id's of the FCF configuration
742 * (don't update more than the number of rq_cfg
743 * elements)
744 */
745 min_rq_count = (hw->hw_rq_count < SLI4_CMD_REG_FCFI_NUM_RQ_CFG) ?
746 hw->hw_rq_count : SLI4_CMD_REG_FCFI_NUM_RQ_CFG;
747 for (i = 0; i < min_rq_count; i++) {
748 struct hw_rq *rq = hw->hw_rq[i];
749 u32 j;
750
751 for (j = 0; j < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; j++) {
752 u32 mask = (rq->filter_mask != 0) ?
753 rq->filter_mask : 1;
754
755 if (!(mask & (1U << j)))
756 continue;
757
758 rq_cfg[i].rq_id = cpu_to_le16(rq->hdr->id);
759 efct_logfcfi(hw, j, i, rq->hdr->id);
760 }
761 }
762
763 rc = -EIO;
764 if (!sli_cmd_reg_fcfi(&hw->sli, buf, 0, rq_cfg))
765 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
766
767 if (rc != 0) {
768 efc_log_err(hw->os, "FCFI registration failed\n");
769 return rc;
770 }
771 hw->fcf_indicator =
772 le16_to_cpu(((struct sli4_cmd_reg_fcfi *)buf)->fcfi);
773
774 return rc;
775 }
776
777 static int
efct_hw_config_mrq(struct efct_hw * hw,u8 mode,u16 fcf_index)778 efct_hw_config_mrq(struct efct_hw *hw, u8 mode, u16 fcf_index)
779 {
780 u8 buf[SLI4_BMBX_SIZE], mrq_bitmask = 0;
781 struct hw_rq *rq;
782 struct sli4_cmd_reg_fcfi_mrq *rsp = NULL;
783 struct sli4_cmd_rq_cfg rq_filter[SLI4_CMD_REG_FCFI_MRQ_NUM_RQ_CFG];
784 u32 rc, i;
785
786 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)
787 goto issue_cmd;
788
789 /* Set the filter match/mask values from hw's filter_def values */
790 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {
791 rq_filter[i].rq_id = cpu_to_le16(0xffff);
792 rq_filter[i].type_mask = (u8)hw->config.filter_def[i];
793 rq_filter[i].type_match = (u8)(hw->config.filter_def[i] >> 8);
794 rq_filter[i].r_ctl_mask = (u8)(hw->config.filter_def[i] >> 16);
795 rq_filter[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 24);
796 }
797
798 rq = hw->hw_rq[0];
799 rq_filter[0].rq_id = cpu_to_le16(rq->hdr->id);
800 rq_filter[1].rq_id = cpu_to_le16(rq->hdr->id);
801
802 mrq_bitmask = 0x2;
803 issue_cmd:
804 efc_log_debug(hw->os, "Issue reg_fcfi_mrq count:%d policy:%d mode:%d\n",
805 hw->hw_rq_count, hw->config.rq_selection_policy, mode);
806 /* Invoke REG_FCFI_MRQ */
807 rc = sli_cmd_reg_fcfi_mrq(&hw->sli, buf, mode, fcf_index,
808 hw->config.rq_selection_policy, mrq_bitmask,
809 hw->hw_mrq_count, rq_filter);
810 if (rc) {
811 efc_log_err(hw->os, "sli_cmd_reg_fcfi_mrq() failed\n");
812 return -EIO;
813 }
814
815 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
816
817 rsp = (struct sli4_cmd_reg_fcfi_mrq *)buf;
818
819 if ((rc) || (le16_to_cpu(rsp->hdr.status))) {
820 efc_log_err(hw->os, "FCFI MRQ reg failed. cmd=%x status=%x\n",
821 rsp->hdr.command, le16_to_cpu(rsp->hdr.status));
822 return -EIO;
823 }
824
825 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)
826 hw->fcf_indicator = le16_to_cpu(rsp->fcfi);
827
828 return 0;
829 }
830
831 static void
efct_hw_queue_hash_add(struct efct_queue_hash * hash,u16 id,u16 index)832 efct_hw_queue_hash_add(struct efct_queue_hash *hash,
833 u16 id, u16 index)
834 {
835 u32 hash_index = id & (EFCT_HW_Q_HASH_SIZE - 1);
836
837 /*
838 * Since the hash is always bigger than the number of queues, then we
839 * never have to worry about an infinite loop.
840 */
841 while (hash[hash_index].in_use)
842 hash_index = (hash_index + 1) & (EFCT_HW_Q_HASH_SIZE - 1);
843
844 /* not used, claim the entry */
845 hash[hash_index].id = id;
846 hash[hash_index].in_use = true;
847 hash[hash_index].index = index;
848 }
849
850 static int
efct_hw_config_sli_port_health_check(struct efct_hw * hw,u8 query,u8 enable)851 efct_hw_config_sli_port_health_check(struct efct_hw *hw, u8 query, u8 enable)
852 {
853 int rc = 0;
854 u8 buf[SLI4_BMBX_SIZE];
855 struct sli4_rqst_cmn_set_features_health_check param;
856 u32 health_check_flag = 0;
857
858 memset(¶m, 0, sizeof(param));
859
860 if (enable)
861 health_check_flag |= SLI4_RQ_HEALTH_CHECK_ENABLE;
862
863 if (query)
864 health_check_flag |= SLI4_RQ_HEALTH_CHECK_QUERY;
865
866 param.health_check_dword = cpu_to_le32(health_check_flag);
867
868 /* build the set_features command */
869 sli_cmd_common_set_features(&hw->sli, buf,
870 SLI4_SET_FEATURES_SLI_PORT_HEALTH_CHECK, sizeof(param), ¶m);
871
872 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);
873 if (rc)
874 efc_log_err(hw->os, "efct_hw_command returns %d\n", rc);
875 else
876 efc_log_debug(hw->os, "SLI Port Health Check is enabled\n");
877
878 return rc;
879 }
880
881 int
efct_hw_init(struct efct_hw * hw)882 efct_hw_init(struct efct_hw *hw)
883 {
884 int rc;
885 u32 i = 0;
886 int rem_count;
887 unsigned long flags = 0;
888 struct efct_hw_io *temp;
889 struct efc_dma *dma;
890
891 /*
892 * Make sure the command lists are empty. If this is start-of-day,
893 * they'll be empty since they were just initialized in efct_hw_setup.
894 * If we've just gone through a reset, the command and command pending
895 * lists should have been cleaned up as part of the reset
896 * (efct_hw_reset()).
897 */
898 spin_lock_irqsave(&hw->cmd_lock, flags);
899 if (!list_empty(&hw->cmd_head)) {
900 spin_unlock_irqrestore(&hw->cmd_lock, flags);
901 efc_log_err(hw->os, "command found on cmd list\n");
902 return -EIO;
903 }
904 if (!list_empty(&hw->cmd_pending)) {
905 spin_unlock_irqrestore(&hw->cmd_lock, flags);
906 efc_log_err(hw->os, "command found on pending list\n");
907 return -EIO;
908 }
909 spin_unlock_irqrestore(&hw->cmd_lock, flags);
910
911 /* Free RQ buffers if prevously allocated */
912 efct_hw_rx_free(hw);
913
914 /*
915 * The IO queues must be initialized here for the reset case. The
916 * efct_hw_init_io() function will re-add the IOs to the free list.
917 * The cmd_head list should be OK since we free all entries in
918 * efct_hw_command_cancel() that is called in the efct_hw_reset().
919 */
920
921 /* If we are in this function due to a reset, there may be stale items
922 * on lists that need to be removed. Clean them up.
923 */
924 rem_count = 0;
925 while ((!list_empty(&hw->io_wait_free))) {
926 rem_count++;
927 temp = list_first_entry(&hw->io_wait_free, struct efct_hw_io,
928 list_entry);
929 list_del_init(&temp->list_entry);
930 }
931 if (rem_count > 0)
932 efc_log_debug(hw->os, "rmvd %d items from io_wait_free list\n",
933 rem_count);
934
935 rem_count = 0;
936 while ((!list_empty(&hw->io_inuse))) {
937 rem_count++;
938 temp = list_first_entry(&hw->io_inuse, struct efct_hw_io,
939 list_entry);
940 list_del_init(&temp->list_entry);
941 }
942 if (rem_count > 0)
943 efc_log_debug(hw->os, "rmvd %d items from io_inuse list\n",
944 rem_count);
945
946 rem_count = 0;
947 while ((!list_empty(&hw->io_free))) {
948 rem_count++;
949 temp = list_first_entry(&hw->io_free, struct efct_hw_io,
950 list_entry);
951 list_del_init(&temp->list_entry);
952 }
953 if (rem_count > 0)
954 efc_log_debug(hw->os, "rmvd %d items from io_free list\n",
955 rem_count);
956
957 /* If MRQ not required, Make sure we dont request feature. */
958 if (hw->config.n_rq == 1)
959 hw->sli.features &= (~SLI4_REQFEAT_MRQP);
960
961 if (sli_init(&hw->sli)) {
962 efc_log_err(hw->os, "SLI failed to initialize\n");
963 return -EIO;
964 }
965
966 if (hw->sliport_healthcheck) {
967 rc = efct_hw_config_sli_port_health_check(hw, 0, 1);
968 if (rc != 0) {
969 efc_log_err(hw->os, "Enable port Health check fail\n");
970 return rc;
971 }
972 }
973
974 /*
975 * Set FDT transfer hint, only works on Lancer
976 */
977 if (hw->sli.if_type == SLI4_INTF_IF_TYPE_2) {
978 /*
979 * Non-fatal error. In particular, we can disregard failure to
980 * set EFCT_HW_FDT_XFER_HINT on devices with legacy firmware
981 * that do not support EFCT_HW_FDT_XFER_HINT feature.
982 */
983 efct_hw_config_set_fdt_xfer_hint(hw, EFCT_HW_FDT_XFER_HINT);
984 }
985
986 /* zero the hashes */
987 memset(hw->cq_hash, 0, sizeof(hw->cq_hash));
988 efc_log_debug(hw->os, "Max CQs %d, hash size = %d\n",
989 EFCT_HW_MAX_NUM_CQ, EFCT_HW_Q_HASH_SIZE);
990
991 memset(hw->rq_hash, 0, sizeof(hw->rq_hash));
992 efc_log_debug(hw->os, "Max RQs %d, hash size = %d\n",
993 EFCT_HW_MAX_NUM_RQ, EFCT_HW_Q_HASH_SIZE);
994
995 memset(hw->wq_hash, 0, sizeof(hw->wq_hash));
996 efc_log_debug(hw->os, "Max WQs %d, hash size = %d\n",
997 EFCT_HW_MAX_NUM_WQ, EFCT_HW_Q_HASH_SIZE);
998
999 rc = efct_hw_init_queues(hw);
1000 if (rc)
1001 return rc;
1002
1003 rc = efct_hw_map_wq_cpu(hw);
1004 if (rc)
1005 return rc;
1006
1007 /* Allocate and p_st RQ buffers */
1008 rc = efct_hw_rx_allocate(hw);
1009 if (rc) {
1010 efc_log_err(hw->os, "rx_allocate failed\n");
1011 return rc;
1012 }
1013
1014 rc = efct_hw_rx_post(hw);
1015 if (rc) {
1016 efc_log_err(hw->os, "WARNING - error posting RQ buffers\n");
1017 return rc;
1018 }
1019
1020 if (hw->config.n_eq == 1) {
1021 rc = efct_hw_config_rq(hw);
1022 if (rc) {
1023 efc_log_err(hw->os, "config rq failed %d\n", rc);
1024 return rc;
1025 }
1026 } else {
1027 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_FCFI_MODE, 0);
1028 if (rc != 0) {
1029 efc_log_err(hw->os, "REG_FCFI_MRQ FCFI reg failed\n");
1030 return rc;
1031 }
1032
1033 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_MRQ_MODE, 0);
1034 if (rc != 0) {
1035 efc_log_err(hw->os, "REG_FCFI_MRQ MRQ reg failed\n");
1036 return rc;
1037 }
1038 }
1039
1040 /*
1041 * Allocate the WQ request tag pool, if not previously allocated
1042 * (the request tag value is 16 bits, thus the pool allocation size
1043 * of 64k)
1044 */
1045 hw->wq_reqtag_pool = efct_hw_reqtag_pool_alloc(hw);
1046 if (!hw->wq_reqtag_pool) {
1047 efc_log_err(hw->os, "efct_hw_reqtag_pool_alloc failed\n");
1048 return -ENOMEM;
1049 }
1050
1051 rc = efct_hw_setup_io(hw);
1052 if (rc) {
1053 efc_log_err(hw->os, "IO allocation failure\n");
1054 return rc;
1055 }
1056
1057 rc = efct_hw_init_io(hw);
1058 if (rc) {
1059 efc_log_err(hw->os, "IO initialization failure\n");
1060 return rc;
1061 }
1062
1063 dma = &hw->loop_map;
1064 dma->size = SLI4_MIN_LOOP_MAP_BYTES;
1065 dma->virt = dma_alloc_coherent(&hw->os->pci->dev, dma->size, &dma->phys,
1066 GFP_KERNEL);
1067 if (!dma->virt)
1068 return -EIO;
1069
1070 /*
1071 * Arming the EQ allows (e.g.) interrupts when CQ completions write EQ
1072 * entries
1073 */
1074 for (i = 0; i < hw->eq_count; i++)
1075 sli_queue_arm(&hw->sli, &hw->eq[i], true);
1076
1077 /*
1078 * Initialize RQ hash
1079 */
1080 for (i = 0; i < hw->rq_count; i++)
1081 efct_hw_queue_hash_add(hw->rq_hash, hw->rq[i].id, i);
1082
1083 /*
1084 * Initialize WQ hash
1085 */
1086 for (i = 0; i < hw->wq_count; i++)
1087 efct_hw_queue_hash_add(hw->wq_hash, hw->wq[i].id, i);
1088
1089 /*
1090 * Arming the CQ allows (e.g.) MQ completions to write CQ entries
1091 */
1092 for (i = 0; i < hw->cq_count; i++) {
1093 efct_hw_queue_hash_add(hw->cq_hash, hw->cq[i].id, i);
1094 sli_queue_arm(&hw->sli, &hw->cq[i], true);
1095 }
1096
1097 /* Set RQ process limit*/
1098 for (i = 0; i < hw->hw_rq_count; i++) {
1099 struct hw_rq *rq = hw->hw_rq[i];
1100
1101 hw->cq[rq->cq->instance].proc_limit = hw->config.n_io / 2;
1102 }
1103
1104 /* record the fact that the queues are functional */
1105 hw->state = EFCT_HW_STATE_ACTIVE;
1106 /*
1107 * Allocate a HW IOs for send frame.
1108 */
1109 hw->hw_wq[0]->send_frame_io = efct_hw_io_alloc(hw);
1110 if (!hw->hw_wq[0]->send_frame_io)
1111 efc_log_err(hw->os, "alloc for send_frame_io failed\n");
1112
1113 /* Initialize send frame sequence id */
1114 atomic_set(&hw->send_frame_seq_id, 0);
1115
1116 return 0;
1117 }
1118
1119 int
efct_hw_parse_filter(struct efct_hw * hw,void * value)1120 efct_hw_parse_filter(struct efct_hw *hw, void *value)
1121 {
1122 int rc = 0;
1123 char *p = NULL, *pp = NULL;
1124 char *token;
1125 u32 idx = 0;
1126
1127 for (idx = 0; idx < ARRAY_SIZE(hw->config.filter_def); idx++)
1128 hw->config.filter_def[idx] = 0;
1129
1130 p = kstrdup(value, GFP_KERNEL);
1131 if (!p || !*p) {
1132 efc_log_err(hw->os, "p is NULL\n");
1133 return -ENOMEM;
1134 }
1135 pp = p;
1136
1137 idx = 0;
1138 while ((token = strsep(&p, ",")) && *token) {
1139 if (kstrtou32(token, 0, &hw->config.filter_def[idx++]))
1140 efc_log_err(hw->os, "kstrtoint failed\n");
1141
1142 if (!p || !*p)
1143 break;
1144
1145 if (idx == ARRAY_SIZE(hw->config.filter_def))
1146 break;
1147 }
1148 kfree(pp);
1149
1150 return rc;
1151 }
1152
1153 u64
efct_get_wwnn(struct efct_hw * hw)1154 efct_get_wwnn(struct efct_hw *hw)
1155 {
1156 struct sli4 *sli = &hw->sli;
1157 u8 p[8];
1158
1159 memcpy(p, sli->wwnn, sizeof(p));
1160 return get_unaligned_be64(p);
1161 }
1162
1163 u64
efct_get_wwpn(struct efct_hw * hw)1164 efct_get_wwpn(struct efct_hw *hw)
1165 {
1166 struct sli4 *sli = &hw->sli;
1167 u8 p[8];
1168
1169 memcpy(p, sli->wwpn, sizeof(p));
1170 return get_unaligned_be64(p);
1171 }
1172
1173 static struct efc_hw_rq_buffer *
efct_hw_rx_buffer_alloc(struct efct_hw * hw,u32 rqindex,u32 count,u32 size)1174 efct_hw_rx_buffer_alloc(struct efct_hw *hw, u32 rqindex, u32 count,
1175 u32 size)
1176 {
1177 struct efct *efct = hw->os;
1178 struct efc_hw_rq_buffer *rq_buf = NULL;
1179 struct efc_hw_rq_buffer *prq;
1180 u32 i;
1181
1182 if (!count)
1183 return NULL;
1184
1185 rq_buf = kmalloc_objs(*rq_buf, count);
1186 if (!rq_buf)
1187 return NULL;
1188 memset(rq_buf, 0, sizeof(*rq_buf) * count);
1189
1190 for (i = 0, prq = rq_buf; i < count; i ++, prq++) {
1191 prq->rqindex = rqindex;
1192 prq->dma.size = size;
1193 prq->dma.virt = dma_alloc_coherent(&efct->pci->dev,
1194 prq->dma.size,
1195 &prq->dma.phys,
1196 GFP_KERNEL);
1197 if (!prq->dma.virt) {
1198 efc_log_err(hw->os, "DMA allocation failed\n");
1199 kfree(rq_buf);
1200 return NULL;
1201 }
1202 }
1203 return rq_buf;
1204 }
1205
1206 static void
efct_hw_rx_buffer_free(struct efct_hw * hw,struct efc_hw_rq_buffer * rq_buf,u32 count)1207 efct_hw_rx_buffer_free(struct efct_hw *hw,
1208 struct efc_hw_rq_buffer *rq_buf,
1209 u32 count)
1210 {
1211 struct efct *efct = hw->os;
1212 u32 i;
1213 struct efc_hw_rq_buffer *prq;
1214
1215 if (rq_buf) {
1216 for (i = 0, prq = rq_buf; i < count; i++, prq++) {
1217 dma_free_coherent(&efct->pci->dev,
1218 prq->dma.size, prq->dma.virt,
1219 prq->dma.phys);
1220 memset(&prq->dma, 0, sizeof(struct efc_dma));
1221 }
1222
1223 kfree(rq_buf);
1224 }
1225 }
1226
1227 int
efct_hw_rx_allocate(struct efct_hw * hw)1228 efct_hw_rx_allocate(struct efct_hw *hw)
1229 {
1230 struct efct *efct = hw->os;
1231 u32 i;
1232 int rc = 0;
1233 u32 rqindex = 0;
1234 u32 hdr_size = EFCT_HW_RQ_SIZE_HDR;
1235 u32 payload_size = hw->config.rq_default_buffer_size;
1236
1237 rqindex = 0;
1238
1239 for (i = 0; i < hw->hw_rq_count; i++) {
1240 struct hw_rq *rq = hw->hw_rq[i];
1241
1242 /* Allocate header buffers */
1243 rq->hdr_buf = efct_hw_rx_buffer_alloc(hw, rqindex,
1244 rq->entry_count,
1245 hdr_size);
1246 if (!rq->hdr_buf) {
1247 efc_log_err(efct, "rx_buffer_alloc hdr_buf failed\n");
1248 rc = -EIO;
1249 break;
1250 }
1251
1252 efc_log_debug(hw->os,
1253 "rq[%2d] rq_id %02d header %4d by %4d bytes\n",
1254 i, rq->hdr->id, rq->entry_count, hdr_size);
1255
1256 rqindex++;
1257
1258 /* Allocate payload buffers */
1259 rq->payload_buf = efct_hw_rx_buffer_alloc(hw, rqindex,
1260 rq->entry_count,
1261 payload_size);
1262 if (!rq->payload_buf) {
1263 efc_log_err(efct, "rx_buffer_alloc fb_buf failed\n");
1264 rc = -EIO;
1265 break;
1266 }
1267 efc_log_debug(hw->os,
1268 "rq[%2d] rq_id %02d default %4d by %4d bytes\n",
1269 i, rq->data->id, rq->entry_count, payload_size);
1270 rqindex++;
1271 }
1272
1273 return rc ? -EIO : 0;
1274 }
1275
1276 int
efct_hw_rx_post(struct efct_hw * hw)1277 efct_hw_rx_post(struct efct_hw *hw)
1278 {
1279 u32 i;
1280 u32 idx;
1281 u32 rq_idx;
1282 int rc = 0;
1283
1284 if (!hw->seq_pool) {
1285 u32 count = 0;
1286
1287 for (i = 0; i < hw->hw_rq_count; i++)
1288 count += hw->hw_rq[i]->entry_count;
1289
1290 hw->seq_pool = kmalloc_objs(struct efc_hw_sequence, count);
1291 if (!hw->seq_pool)
1292 return -ENOMEM;
1293 }
1294
1295 /*
1296 * In RQ pair mode, we MUST post the header and payload buffer at the
1297 * same time.
1298 */
1299 for (rq_idx = 0, idx = 0; rq_idx < hw->hw_rq_count; rq_idx++) {
1300 struct hw_rq *rq = hw->hw_rq[rq_idx];
1301
1302 for (i = 0; i < rq->entry_count - 1; i++) {
1303 struct efc_hw_sequence *seq;
1304
1305 seq = hw->seq_pool + idx;
1306 idx++;
1307 seq->header = &rq->hdr_buf[i];
1308 seq->payload = &rq->payload_buf[i];
1309 rc = efct_hw_sequence_free(hw, seq);
1310 if (rc)
1311 break;
1312 }
1313 if (rc)
1314 break;
1315 }
1316
1317 if (rc && hw->seq_pool)
1318 kfree(hw->seq_pool);
1319
1320 return rc;
1321 }
1322
1323 void
efct_hw_rx_free(struct efct_hw * hw)1324 efct_hw_rx_free(struct efct_hw *hw)
1325 {
1326 u32 i;
1327
1328 /* Free hw_rq buffers */
1329 for (i = 0; i < hw->hw_rq_count; i++) {
1330 struct hw_rq *rq = hw->hw_rq[i];
1331
1332 if (rq) {
1333 efct_hw_rx_buffer_free(hw, rq->hdr_buf,
1334 rq->entry_count);
1335 rq->hdr_buf = NULL;
1336 efct_hw_rx_buffer_free(hw, rq->payload_buf,
1337 rq->entry_count);
1338 rq->payload_buf = NULL;
1339 }
1340 }
1341 }
1342
1343 static int
efct_hw_cmd_submit_pending(struct efct_hw * hw)1344 efct_hw_cmd_submit_pending(struct efct_hw *hw)
1345 {
1346 int rc = 0;
1347
1348 /* Assumes lock held */
1349
1350 /* Only submit MQE if there's room */
1351 while (hw->cmd_head_count < (EFCT_HW_MQ_DEPTH - 1) &&
1352 !list_empty(&hw->cmd_pending)) {
1353 struct efct_command_ctx *ctx;
1354
1355 ctx = list_first_entry(&hw->cmd_pending,
1356 struct efct_command_ctx, list_entry);
1357 if (!ctx)
1358 break;
1359
1360 list_del_init(&ctx->list_entry);
1361
1362 list_add_tail(&ctx->list_entry, &hw->cmd_head);
1363 hw->cmd_head_count++;
1364 if (sli_mq_write(&hw->sli, hw->mq, ctx->buf) < 0) {
1365 efc_log_debug(hw->os,
1366 "sli_queue_write failed: %d\n", rc);
1367 rc = -EIO;
1368 break;
1369 }
1370 }
1371 return rc;
1372 }
1373
1374 int
efct_hw_command(struct efct_hw * hw,u8 * cmd,u32 opts,void * cb,void * arg)1375 efct_hw_command(struct efct_hw *hw, u8 *cmd, u32 opts, void *cb, void *arg)
1376 {
1377 int rc = -EIO;
1378 unsigned long flags = 0;
1379 void *bmbx = NULL;
1380
1381 /*
1382 * If the chip is in an error state (UE'd) then reject this mailbox
1383 * command.
1384 */
1385 if (sli_fw_error_status(&hw->sli) > 0) {
1386 efc_log_crit(hw->os, "Chip in an error state - reset needed\n");
1387 efc_log_crit(hw->os, "status=%#x error1=%#x error2=%#x\n",
1388 sli_reg_read_status(&hw->sli),
1389 sli_reg_read_err1(&hw->sli),
1390 sli_reg_read_err2(&hw->sli));
1391
1392 return -EIO;
1393 }
1394
1395 /*
1396 * Send a mailbox command to the hardware, and either wait for
1397 * a completion (EFCT_CMD_POLL) or get an optional asynchronous
1398 * completion (EFCT_CMD_NOWAIT).
1399 */
1400
1401 if (opts == EFCT_CMD_POLL) {
1402 mutex_lock(&hw->bmbx_lock);
1403 bmbx = hw->sli.bmbx.virt;
1404
1405 memcpy(bmbx, cmd, SLI4_BMBX_SIZE);
1406
1407 if (sli_bmbx_command(&hw->sli) == 0) {
1408 rc = 0;
1409 memcpy(cmd, bmbx, SLI4_BMBX_SIZE);
1410 }
1411 mutex_unlock(&hw->bmbx_lock);
1412 } else if (opts == EFCT_CMD_NOWAIT) {
1413 struct efct_command_ctx *ctx = NULL;
1414
1415 if (hw->state != EFCT_HW_STATE_ACTIVE) {
1416 efc_log_err(hw->os, "Can't send command, HW state=%d\n",
1417 hw->state);
1418 return -EIO;
1419 }
1420
1421 ctx = mempool_alloc(hw->cmd_ctx_pool, GFP_ATOMIC);
1422 if (!ctx)
1423 return -ENOSPC;
1424
1425 memset(ctx, 0, sizeof(struct efct_command_ctx));
1426
1427 if (cb) {
1428 ctx->cb = cb;
1429 ctx->arg = arg;
1430 }
1431
1432 memcpy(ctx->buf, cmd, SLI4_BMBX_SIZE);
1433 ctx->ctx = hw;
1434
1435 spin_lock_irqsave(&hw->cmd_lock, flags);
1436
1437 /* Add to pending list */
1438 INIT_LIST_HEAD(&ctx->list_entry);
1439 list_add_tail(&ctx->list_entry, &hw->cmd_pending);
1440
1441 /* Submit as much of the pending list as we can */
1442 rc = efct_hw_cmd_submit_pending(hw);
1443
1444 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1445 }
1446
1447 return rc;
1448 }
1449
1450 static int
efct_hw_command_process(struct efct_hw * hw,int status,u8 * mqe,size_t size)1451 efct_hw_command_process(struct efct_hw *hw, int status, u8 *mqe,
1452 size_t size)
1453 {
1454 struct efct_command_ctx *ctx = NULL;
1455 unsigned long flags = 0;
1456
1457 spin_lock_irqsave(&hw->cmd_lock, flags);
1458 if (!list_empty(&hw->cmd_head)) {
1459 ctx = list_first_entry(&hw->cmd_head,
1460 struct efct_command_ctx, list_entry);
1461 list_del_init(&ctx->list_entry);
1462 }
1463 if (!ctx) {
1464 efc_log_err(hw->os, "no command context\n");
1465 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1466 return -EIO;
1467 }
1468
1469 hw->cmd_head_count--;
1470
1471 /* Post any pending requests */
1472 efct_hw_cmd_submit_pending(hw);
1473
1474 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1475
1476 if (ctx->cb) {
1477 memcpy(ctx->buf, mqe, size);
1478 ctx->cb(hw, status, ctx->buf, ctx->arg);
1479 }
1480
1481 mempool_free(ctx, hw->cmd_ctx_pool);
1482
1483 return 0;
1484 }
1485
1486 static int
efct_hw_mq_process(struct efct_hw * hw,int status,struct sli4_queue * mq)1487 efct_hw_mq_process(struct efct_hw *hw,
1488 int status, struct sli4_queue *mq)
1489 {
1490 u8 mqe[SLI4_BMBX_SIZE];
1491 int rc;
1492
1493 rc = sli_mq_read(&hw->sli, mq, mqe);
1494 if (!rc)
1495 rc = efct_hw_command_process(hw, status, mqe, mq->size);
1496
1497 return rc;
1498 }
1499
1500 static int
efct_hw_command_cancel(struct efct_hw * hw)1501 efct_hw_command_cancel(struct efct_hw *hw)
1502 {
1503 unsigned long flags = 0;
1504 int rc = 0;
1505
1506 spin_lock_irqsave(&hw->cmd_lock, flags);
1507
1508 /*
1509 * Manually clean up remaining commands. Note: since this calls
1510 * efct_hw_command_process(), we'll also process the cmd_pending
1511 * list, so no need to manually clean that out.
1512 */
1513 while (!list_empty(&hw->cmd_head)) {
1514 u8 mqe[SLI4_BMBX_SIZE] = { 0 };
1515 struct efct_command_ctx *ctx;
1516
1517 ctx = list_first_entry(&hw->cmd_head,
1518 struct efct_command_ctx, list_entry);
1519
1520 efc_log_debug(hw->os, "hung command %08x\n",
1521 !ctx ? U32_MAX : *((u32 *)ctx->buf));
1522 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1523 rc = efct_hw_command_process(hw, -1, mqe, SLI4_BMBX_SIZE);
1524 spin_lock_irqsave(&hw->cmd_lock, flags);
1525 }
1526
1527 spin_unlock_irqrestore(&hw->cmd_lock, flags);
1528
1529 return rc;
1530 }
1531
1532 static void
efct_mbox_rsp_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)1533 efct_mbox_rsp_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
1534 {
1535 struct efct_mbox_rqst_ctx *ctx = arg;
1536
1537 if (ctx) {
1538 if (ctx->callback)
1539 (*ctx->callback)(hw->os->efcport, status, mqe,
1540 ctx->arg);
1541
1542 mempool_free(ctx, hw->mbox_rqst_pool);
1543 }
1544 }
1545
1546 int
efct_issue_mbox_rqst(void * base,void * cmd,void * cb,void * arg)1547 efct_issue_mbox_rqst(void *base, void *cmd, void *cb, void *arg)
1548 {
1549 struct efct_mbox_rqst_ctx *ctx;
1550 struct efct *efct = base;
1551 struct efct_hw *hw = &efct->hw;
1552 int rc;
1553
1554 /*
1555 * Allocate a callback context (which includes the mbox cmd buffer),
1556 * we need this to be persistent as the mbox cmd submission may be
1557 * queued and executed later execution.
1558 */
1559 ctx = mempool_alloc(hw->mbox_rqst_pool, GFP_ATOMIC);
1560 if (!ctx)
1561 return -EIO;
1562
1563 ctx->callback = cb;
1564 ctx->arg = arg;
1565
1566 rc = efct_hw_command(hw, cmd, EFCT_CMD_NOWAIT, efct_mbox_rsp_cb, ctx);
1567 if (rc) {
1568 efc_log_err(efct, "issue mbox rqst failure rc:%d\n", rc);
1569 mempool_free(ctx, hw->mbox_rqst_pool);
1570 return -EIO;
1571 }
1572
1573 return 0;
1574 }
1575
1576 static inline struct efct_hw_io *
_efct_hw_io_alloc(struct efct_hw * hw)1577 _efct_hw_io_alloc(struct efct_hw *hw)
1578 {
1579 struct efct_hw_io *io = NULL;
1580
1581 if (!list_empty(&hw->io_free)) {
1582 io = list_first_entry(&hw->io_free, struct efct_hw_io,
1583 list_entry);
1584 list_del(&io->list_entry);
1585 }
1586 if (io) {
1587 INIT_LIST_HEAD(&io->list_entry);
1588 list_add_tail(&io->list_entry, &hw->io_inuse);
1589 io->state = EFCT_HW_IO_STATE_INUSE;
1590 io->abort_reqtag = U32_MAX;
1591 io->wq = hw->wq_cpu_array[raw_smp_processor_id()];
1592 if (!io->wq) {
1593 efc_log_err(hw->os, "WQ not assigned for cpu:%d\n",
1594 raw_smp_processor_id());
1595 io->wq = hw->hw_wq[0];
1596 }
1597 kref_init(&io->ref);
1598 io->release = efct_hw_io_free_internal;
1599 } else {
1600 atomic_add(1, &hw->io_alloc_failed_count);
1601 }
1602
1603 return io;
1604 }
1605
1606 struct efct_hw_io *
efct_hw_io_alloc(struct efct_hw * hw)1607 efct_hw_io_alloc(struct efct_hw *hw)
1608 {
1609 struct efct_hw_io *io = NULL;
1610 unsigned long flags = 0;
1611
1612 spin_lock_irqsave(&hw->io_lock, flags);
1613 io = _efct_hw_io_alloc(hw);
1614 spin_unlock_irqrestore(&hw->io_lock, flags);
1615
1616 return io;
1617 }
1618
1619 static void
efct_hw_io_free_move_correct_list(struct efct_hw * hw,struct efct_hw_io * io)1620 efct_hw_io_free_move_correct_list(struct efct_hw *hw,
1621 struct efct_hw_io *io)
1622 {
1623 /*
1624 * When an IO is freed, depending on the exchange busy flag,
1625 * move it to the correct list.
1626 */
1627 if (io->xbusy) {
1628 /*
1629 * add to wait_free list and wait for XRI_ABORTED CQEs to clean
1630 * up
1631 */
1632 INIT_LIST_HEAD(&io->list_entry);
1633 list_add_tail(&io->list_entry, &hw->io_wait_free);
1634 io->state = EFCT_HW_IO_STATE_WAIT_FREE;
1635 } else {
1636 /* IO not busy, add to free list */
1637 INIT_LIST_HEAD(&io->list_entry);
1638 list_add_tail(&io->list_entry, &hw->io_free);
1639 io->state = EFCT_HW_IO_STATE_FREE;
1640 }
1641 }
1642
1643 static inline void
efct_hw_io_free_common(struct efct_hw * hw,struct efct_hw_io * io)1644 efct_hw_io_free_common(struct efct_hw *hw, struct efct_hw_io *io)
1645 {
1646 /* initialize IO fields */
1647 efct_hw_init_free_io(io);
1648
1649 /* Restore default SGL */
1650 efct_hw_io_restore_sgl(hw, io);
1651 }
1652
1653 void
efct_hw_io_free_internal(struct kref * arg)1654 efct_hw_io_free_internal(struct kref *arg)
1655 {
1656 unsigned long flags = 0;
1657 struct efct_hw_io *io = container_of(arg, struct efct_hw_io, ref);
1658 struct efct_hw *hw = io->hw;
1659
1660 /* perform common cleanup */
1661 efct_hw_io_free_common(hw, io);
1662
1663 spin_lock_irqsave(&hw->io_lock, flags);
1664 /* remove from in-use list */
1665 if (!list_empty(&io->list_entry) && !list_empty(&hw->io_inuse)) {
1666 list_del_init(&io->list_entry);
1667 efct_hw_io_free_move_correct_list(hw, io);
1668 }
1669 spin_unlock_irqrestore(&hw->io_lock, flags);
1670 }
1671
1672 int
efct_hw_io_free(struct efct_hw * hw,struct efct_hw_io * io)1673 efct_hw_io_free(struct efct_hw *hw, struct efct_hw_io *io)
1674 {
1675 return kref_put(&io->ref, io->release);
1676 }
1677
1678 struct efct_hw_io *
efct_hw_io_lookup(struct efct_hw * hw,u32 xri)1679 efct_hw_io_lookup(struct efct_hw *hw, u32 xri)
1680 {
1681 u32 ioindex;
1682
1683 ioindex = xri - hw->sli.ext[SLI4_RSRC_XRI].base[0];
1684 return hw->io[ioindex];
1685 }
1686
1687 int
efct_hw_io_init_sges(struct efct_hw * hw,struct efct_hw_io * io,enum efct_hw_io_type type)1688 efct_hw_io_init_sges(struct efct_hw *hw, struct efct_hw_io *io,
1689 enum efct_hw_io_type type)
1690 {
1691 struct sli4_sge *data = NULL;
1692 u32 i = 0;
1693 u32 skips = 0;
1694 u32 sge_flags = 0;
1695
1696 if (!io) {
1697 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n", hw, io);
1698 return -EIO;
1699 }
1700
1701 /* Clear / reset the scatter-gather list */
1702 io->sgl = &io->def_sgl;
1703 io->sgl_count = io->def_sgl_count;
1704 io->first_data_sge = 0;
1705
1706 memset(io->sgl->virt, 0, 2 * sizeof(struct sli4_sge));
1707 io->n_sge = 0;
1708 io->sge_offset = 0;
1709
1710 io->type = type;
1711
1712 data = io->sgl->virt;
1713
1714 /*
1715 * Some IO types have underlying hardware requirements on the order
1716 * of SGEs. Process all special entries here.
1717 */
1718 switch (type) {
1719 case EFCT_HW_IO_TARGET_WRITE:
1720
1721 /* populate host resident XFER_RDY buffer */
1722 sge_flags = le32_to_cpu(data->dw2_flags);
1723 sge_flags &= (~SLI4_SGE_TYPE_MASK);
1724 sge_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
1725 data->buffer_address_high =
1726 cpu_to_le32(upper_32_bits(io->xfer_rdy.phys));
1727 data->buffer_address_low =
1728 cpu_to_le32(lower_32_bits(io->xfer_rdy.phys));
1729 data->buffer_length = cpu_to_le32(io->xfer_rdy.size);
1730 data->dw2_flags = cpu_to_le32(sge_flags);
1731 data++;
1732
1733 skips = EFCT_TARGET_WRITE_SKIPS;
1734
1735 io->n_sge = 1;
1736 break;
1737 case EFCT_HW_IO_TARGET_READ:
1738 /*
1739 * For FCP_TSEND64, the first 2 entries are SKIP SGE's
1740 */
1741 skips = EFCT_TARGET_READ_SKIPS;
1742 break;
1743 case EFCT_HW_IO_TARGET_RSP:
1744 /*
1745 * No skips, etc. for FCP_TRSP64
1746 */
1747 break;
1748 default:
1749 efc_log_err(hw->os, "unsupported IO type %#x\n", type);
1750 return -EIO;
1751 }
1752
1753 /*
1754 * Write skip entries
1755 */
1756 for (i = 0; i < skips; i++) {
1757 sge_flags = le32_to_cpu(data->dw2_flags);
1758 sge_flags &= (~SLI4_SGE_TYPE_MASK);
1759 sge_flags |= (SLI4_SGE_TYPE_SKIP << SLI4_SGE_TYPE_SHIFT);
1760 data->dw2_flags = cpu_to_le32(sge_flags);
1761 data++;
1762 }
1763
1764 io->n_sge += skips;
1765
1766 /*
1767 * Set last
1768 */
1769 sge_flags = le32_to_cpu(data->dw2_flags);
1770 sge_flags |= SLI4_SGE_LAST;
1771 data->dw2_flags = cpu_to_le32(sge_flags);
1772
1773 return 0;
1774 }
1775
1776 int
efct_hw_io_add_sge(struct efct_hw * hw,struct efct_hw_io * io,uintptr_t addr,u32 length)1777 efct_hw_io_add_sge(struct efct_hw *hw, struct efct_hw_io *io,
1778 uintptr_t addr, u32 length)
1779 {
1780 struct sli4_sge *data = NULL;
1781 u32 sge_flags = 0;
1782
1783 if (!io || !addr || !length) {
1784 efc_log_err(hw->os,
1785 "bad parameter hw=%p io=%p addr=%lx length=%u\n",
1786 hw, io, addr, length);
1787 return -EIO;
1788 }
1789
1790 if (length > hw->sli.sge_supported_length) {
1791 efc_log_err(hw->os,
1792 "length of SGE %d bigger than allowed %d\n",
1793 length, hw->sli.sge_supported_length);
1794 return -EIO;
1795 }
1796
1797 data = io->sgl->virt;
1798 data += io->n_sge;
1799
1800 sge_flags = le32_to_cpu(data->dw2_flags);
1801 sge_flags &= ~SLI4_SGE_TYPE_MASK;
1802 sge_flags |= SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT;
1803 sge_flags &= ~SLI4_SGE_DATA_OFFSET_MASK;
1804 sge_flags |= SLI4_SGE_DATA_OFFSET_MASK & io->sge_offset;
1805
1806 data->buffer_address_high = cpu_to_le32(upper_32_bits(addr));
1807 data->buffer_address_low = cpu_to_le32(lower_32_bits(addr));
1808 data->buffer_length = cpu_to_le32(length);
1809
1810 /*
1811 * Always assume this is the last entry and mark as such.
1812 * If this is not the first entry unset the "last SGE"
1813 * indication for the previous entry
1814 */
1815 sge_flags |= SLI4_SGE_LAST;
1816 data->dw2_flags = cpu_to_le32(sge_flags);
1817
1818 if (io->n_sge) {
1819 sge_flags = le32_to_cpu(data[-1].dw2_flags);
1820 sge_flags &= ~SLI4_SGE_LAST;
1821 data[-1].dw2_flags = cpu_to_le32(sge_flags);
1822 }
1823
1824 /* Set first_data_bde if not previously set */
1825 if (io->first_data_sge == 0)
1826 io->first_data_sge = io->n_sge;
1827
1828 io->sge_offset += length;
1829 io->n_sge++;
1830
1831 return 0;
1832 }
1833
1834 void
efct_hw_io_abort_all(struct efct_hw * hw)1835 efct_hw_io_abort_all(struct efct_hw *hw)
1836 {
1837 struct efct_hw_io *io_to_abort = NULL;
1838 struct efct_hw_io *next_io = NULL;
1839
1840 list_for_each_entry_safe(io_to_abort, next_io,
1841 &hw->io_inuse, list_entry) {
1842 efct_hw_io_abort(hw, io_to_abort, true, NULL, NULL);
1843 }
1844 }
1845
1846 static void
efct_hw_wq_process_abort(void * arg,u8 * cqe,int status)1847 efct_hw_wq_process_abort(void *arg, u8 *cqe, int status)
1848 {
1849 struct efct_hw_io *io = arg;
1850 struct efct_hw *hw = io->hw;
1851 u32 ext = 0;
1852 u32 len = 0;
1853 struct hw_wq_callback *wqcb;
1854
1855 /*
1856 * For IOs that were aborted internally, we may need to issue the
1857 * callback here depending on whether a XRI_ABORTED CQE is expected ot
1858 * not. If the status is Local Reject/No XRI, then
1859 * issue the callback now.
1860 */
1861 ext = sli_fc_ext_status(&hw->sli, cqe);
1862 if (status == SLI4_FC_WCQE_STATUS_LOCAL_REJECT &&
1863 ext == SLI4_FC_LOCAL_REJECT_NO_XRI && io->done) {
1864 efct_hw_done_t done = io->done;
1865
1866 io->done = NULL;
1867
1868 /*
1869 * Use latched status as this is always saved for an internal
1870 * abort Note: We won't have both a done and abort_done
1871 * function, so don't worry about
1872 * clobbering the len, status and ext fields.
1873 */
1874 status = io->saved_status;
1875 len = io->saved_len;
1876 ext = io->saved_ext;
1877 io->status_saved = false;
1878 done(io, len, status, ext, io->arg);
1879 }
1880
1881 if (io->abort_done) {
1882 efct_hw_done_t done = io->abort_done;
1883
1884 io->abort_done = NULL;
1885 done(io, len, status, ext, io->abort_arg);
1886 }
1887
1888 /* clear abort bit to indicate abort is complete */
1889 io->abort_in_progress = false;
1890
1891 /* Free the WQ callback */
1892 if (io->abort_reqtag == U32_MAX) {
1893 efc_log_err(hw->os, "HW IO already freed\n");
1894 return;
1895 }
1896
1897 wqcb = efct_hw_reqtag_get_instance(hw, io->abort_reqtag);
1898 efct_hw_reqtag_free(hw, wqcb);
1899
1900 /*
1901 * Call efct_hw_io_free() because this releases the WQ reservation as
1902 * well as doing the refcount put. Don't duplicate the code here.
1903 */
1904 (void)efct_hw_io_free(hw, io);
1905 }
1906
1907 static void
efct_hw_fill_abort_wqe(struct efct_hw * hw,struct efct_hw_wqe * wqe)1908 efct_hw_fill_abort_wqe(struct efct_hw *hw, struct efct_hw_wqe *wqe)
1909 {
1910 struct sli4_abort_wqe *abort = (void *)wqe->wqebuf;
1911
1912 memset(abort, 0, hw->sli.wqe_size);
1913
1914 abort->criteria = SLI4_ABORT_CRITERIA_XRI_TAG;
1915 abort->ia_ir_byte |= wqe->send_abts ? 0 : 1;
1916
1917 /* Suppress ABTS retries */
1918 abort->ia_ir_byte |= SLI4_ABRT_WQE_IR;
1919
1920 abort->t_tag = cpu_to_le32(wqe->id);
1921 abort->command = SLI4_WQE_ABORT;
1922 abort->request_tag = cpu_to_le16(wqe->abort_reqtag);
1923
1924 abort->dw10w0_flags = cpu_to_le16(SLI4_ABRT_WQE_QOSD);
1925
1926 abort->cq_id = cpu_to_le16(SLI4_CQ_DEFAULT);
1927 }
1928
1929 int
efct_hw_io_abort(struct efct_hw * hw,struct efct_hw_io * io_to_abort,bool send_abts,void * cb,void * arg)1930 efct_hw_io_abort(struct efct_hw *hw, struct efct_hw_io *io_to_abort,
1931 bool send_abts, void *cb, void *arg)
1932 {
1933 struct hw_wq_callback *wqcb;
1934 unsigned long flags = 0;
1935
1936 if (!io_to_abort) {
1937 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n",
1938 hw, io_to_abort);
1939 return -EIO;
1940 }
1941
1942 if (hw->state != EFCT_HW_STATE_ACTIVE) {
1943 efc_log_err(hw->os, "cannot send IO abort, HW state=%d\n",
1944 hw->state);
1945 return -EIO;
1946 }
1947
1948 /* take a reference on IO being aborted */
1949 if (kref_get_unless_zero(&io_to_abort->ref) == 0) {
1950 /* command no longer active */
1951 efc_log_debug(hw->os,
1952 "io not active xri=0x%x tag=0x%x\n",
1953 io_to_abort->indicator, io_to_abort->reqtag);
1954 return -ENOENT;
1955 }
1956
1957 /* Must have a valid WQ reference */
1958 if (!io_to_abort->wq) {
1959 efc_log_debug(hw->os, "io_to_abort xri=0x%x not active on WQ\n",
1960 io_to_abort->indicator);
1961 /* efct_ref_get(): same function */
1962 kref_put(&io_to_abort->ref, io_to_abort->release);
1963 return -ENOENT;
1964 }
1965
1966 /*
1967 * Validation checks complete; now check to see if already being
1968 * aborted, if not set the flag.
1969 */
1970 if (cmpxchg(&io_to_abort->abort_in_progress, false, true)) {
1971 /* efct_ref_get(): same function */
1972 kref_put(&io_to_abort->ref, io_to_abort->release);
1973 efc_log_debug(hw->os,
1974 "io already being aborted xri=0x%x tag=0x%x\n",
1975 io_to_abort->indicator, io_to_abort->reqtag);
1976 return -EINPROGRESS;
1977 }
1978
1979 /*
1980 * If we got here, the possibilities are:
1981 * - host owned xri
1982 * - io_to_abort->wq_index != U32_MAX
1983 * - submit ABORT_WQE to same WQ
1984 * - port owned xri:
1985 * - rxri: io_to_abort->wq_index == U32_MAX
1986 * - submit ABORT_WQE to any WQ
1987 * - non-rxri
1988 * - io_to_abort->index != U32_MAX
1989 * - submit ABORT_WQE to same WQ
1990 * - io_to_abort->index == U32_MAX
1991 * - submit ABORT_WQE to any WQ
1992 */
1993 io_to_abort->abort_done = cb;
1994 io_to_abort->abort_arg = arg;
1995
1996 /* Allocate a request tag for the abort portion of this IO */
1997 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_abort, io_to_abort);
1998 if (!wqcb) {
1999 efc_log_err(hw->os, "can't allocate request tag\n");
2000 io_to_abort->abort_in_progress = false;
2001 kref_put(&io_to_abort->ref, io_to_abort->release);
2002 return -ENOSPC;
2003 }
2004
2005 io_to_abort->abort_reqtag = wqcb->instance_index;
2006 io_to_abort->wqe.send_abts = send_abts;
2007 io_to_abort->wqe.id = io_to_abort->indicator;
2008 io_to_abort->wqe.abort_reqtag = io_to_abort->abort_reqtag;
2009
2010 /*
2011 * If the wqe is on the pending list, then set this wqe to be
2012 * aborted when the IO's wqe is removed from the list.
2013 */
2014 if (io_to_abort->wq) {
2015 spin_lock_irqsave(&io_to_abort->wq->queue->lock, flags);
2016 if (io_to_abort->wqe.list_entry.next) {
2017 io_to_abort->wqe.abort_wqe_submit_needed = true;
2018 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock,
2019 flags);
2020 return 0;
2021 }
2022 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock, flags);
2023 }
2024
2025 efct_hw_fill_abort_wqe(hw, &io_to_abort->wqe);
2026
2027 /* ABORT_WQE does not actually utilize an XRI on the Port,
2028 * therefore, keep xbusy as-is to track the exchange's state,
2029 * not the ABORT_WQE's state
2030 */
2031 if (efct_hw_wq_write(io_to_abort->wq, &io_to_abort->wqe)) {
2032 io_to_abort->abort_in_progress = false;
2033 /* efct_ref_get(): same function */
2034 kref_put(&io_to_abort->ref, io_to_abort->release);
2035 return -EIO;
2036 }
2037
2038 return 0;
2039 }
2040
2041 void
efct_hw_reqtag_pool_free(struct efct_hw * hw)2042 efct_hw_reqtag_pool_free(struct efct_hw *hw)
2043 {
2044 u32 i;
2045 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2046 struct hw_wq_callback *wqcb = NULL;
2047
2048 if (reqtag_pool) {
2049 for (i = 0; i < U16_MAX; i++) {
2050 wqcb = reqtag_pool->tags[i];
2051 if (!wqcb)
2052 continue;
2053
2054 kfree(wqcb);
2055 }
2056 kfree(reqtag_pool);
2057 hw->wq_reqtag_pool = NULL;
2058 }
2059 }
2060
2061 struct reqtag_pool *
efct_hw_reqtag_pool_alloc(struct efct_hw * hw)2062 efct_hw_reqtag_pool_alloc(struct efct_hw *hw)
2063 {
2064 u32 i = 0;
2065 struct reqtag_pool *reqtag_pool;
2066 struct hw_wq_callback *wqcb;
2067
2068 reqtag_pool = kzalloc_obj(*reqtag_pool);
2069 if (!reqtag_pool)
2070 return NULL;
2071
2072 INIT_LIST_HEAD(&reqtag_pool->freelist);
2073 /* initialize reqtag pool lock */
2074 spin_lock_init(&reqtag_pool->lock);
2075 for (i = 0; i < U16_MAX; i++) {
2076 wqcb = kmalloc_obj(*wqcb);
2077 if (!wqcb)
2078 break;
2079
2080 reqtag_pool->tags[i] = wqcb;
2081 wqcb->instance_index = i;
2082 wqcb->callback = NULL;
2083 wqcb->arg = NULL;
2084 INIT_LIST_HEAD(&wqcb->list_entry);
2085 list_add_tail(&wqcb->list_entry, &reqtag_pool->freelist);
2086 }
2087
2088 return reqtag_pool;
2089 }
2090
2091 struct hw_wq_callback *
efct_hw_reqtag_alloc(struct efct_hw * hw,void (* callback)(void * arg,u8 * cqe,int status),void * arg)2092 efct_hw_reqtag_alloc(struct efct_hw *hw,
2093 void (*callback)(void *arg, u8 *cqe, int status),
2094 void *arg)
2095 {
2096 struct hw_wq_callback *wqcb = NULL;
2097 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2098 unsigned long flags = 0;
2099
2100 if (!callback)
2101 return wqcb;
2102
2103 spin_lock_irqsave(&reqtag_pool->lock, flags);
2104
2105 if (!list_empty(&reqtag_pool->freelist)) {
2106 wqcb = list_first_entry(&reqtag_pool->freelist,
2107 struct hw_wq_callback, list_entry);
2108 }
2109
2110 if (wqcb) {
2111 list_del_init(&wqcb->list_entry);
2112 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2113 wqcb->callback = callback;
2114 wqcb->arg = arg;
2115 } else {
2116 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2117 }
2118
2119 return wqcb;
2120 }
2121
2122 void
efct_hw_reqtag_free(struct efct_hw * hw,struct hw_wq_callback * wqcb)2123 efct_hw_reqtag_free(struct efct_hw *hw, struct hw_wq_callback *wqcb)
2124 {
2125 unsigned long flags = 0;
2126 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;
2127
2128 if (!wqcb->callback)
2129 efc_log_err(hw->os, "WQCB is already freed\n");
2130
2131 spin_lock_irqsave(&reqtag_pool->lock, flags);
2132 wqcb->callback = NULL;
2133 wqcb->arg = NULL;
2134 INIT_LIST_HEAD(&wqcb->list_entry);
2135 list_add(&wqcb->list_entry, &hw->wq_reqtag_pool->freelist);
2136 spin_unlock_irqrestore(&reqtag_pool->lock, flags);
2137 }
2138
2139 struct hw_wq_callback *
efct_hw_reqtag_get_instance(struct efct_hw * hw,u32 instance_index)2140 efct_hw_reqtag_get_instance(struct efct_hw *hw, u32 instance_index)
2141 {
2142 struct hw_wq_callback *wqcb;
2143
2144 wqcb = hw->wq_reqtag_pool->tags[instance_index];
2145 if (!wqcb)
2146 efc_log_err(hw->os, "wqcb for instance %d is null\n",
2147 instance_index);
2148
2149 return wqcb;
2150 }
2151
2152 int
efct_hw_queue_hash_find(struct efct_queue_hash * hash,u16 id)2153 efct_hw_queue_hash_find(struct efct_queue_hash *hash, u16 id)
2154 {
2155 int index = -1;
2156 int i = id & (EFCT_HW_Q_HASH_SIZE - 1);
2157
2158 /*
2159 * Since the hash is always bigger than the maximum number of Qs, then
2160 * we never have to worry about an infinite loop. We will always find
2161 * an unused entry.
2162 */
2163 do {
2164 if (hash[i].in_use && hash[i].id == id)
2165 index = hash[i].index;
2166 else
2167 i = (i + 1) & (EFCT_HW_Q_HASH_SIZE - 1);
2168 } while (index == -1 && hash[i].in_use);
2169
2170 return index;
2171 }
2172
2173 int
efct_hw_process(struct efct_hw * hw,u32 vector,u32 max_isr_time_msec)2174 efct_hw_process(struct efct_hw *hw, u32 vector,
2175 u32 max_isr_time_msec)
2176 {
2177 struct hw_eq *eq;
2178
2179 /*
2180 * The caller should disable interrupts if they wish to prevent us
2181 * from processing during a shutdown. The following states are defined:
2182 * EFCT_HW_STATE_UNINITIALIZED - No queues allocated
2183 * EFCT_HW_STATE_QUEUES_ALLOCATED - The state after a chip reset,
2184 * queues are cleared.
2185 * EFCT_HW_STATE_ACTIVE - Chip and queues are operational
2186 * EFCT_HW_STATE_RESET_IN_PROGRESS - reset, we still want completions
2187 * EFCT_HW_STATE_TEARDOWN_IN_PROGRESS - We still want mailbox
2188 * completions.
2189 */
2190 if (hw->state == EFCT_HW_STATE_UNINITIALIZED)
2191 return 0;
2192
2193 /* Get pointer to struct hw_eq */
2194 eq = hw->hw_eq[vector];
2195 if (!eq)
2196 return 0;
2197
2198 eq->use_count++;
2199
2200 return efct_hw_eq_process(hw, eq, max_isr_time_msec);
2201 }
2202
2203 int
efct_hw_eq_process(struct efct_hw * hw,struct hw_eq * eq,u32 max_isr_time_msec)2204 efct_hw_eq_process(struct efct_hw *hw, struct hw_eq *eq,
2205 u32 max_isr_time_msec)
2206 {
2207 u8 eqe[sizeof(struct sli4_eqe)] = { 0 };
2208 u32 tcheck_count;
2209 u64 tstart;
2210 u64 telapsed;
2211 bool done = false;
2212
2213 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;
2214 tstart = jiffies_to_msecs(jiffies);
2215
2216 while (!done && !sli_eq_read(&hw->sli, eq->queue, eqe)) {
2217 u16 cq_id = 0;
2218 int rc;
2219
2220 rc = sli_eq_parse(&hw->sli, eqe, &cq_id);
2221 if (unlikely(rc)) {
2222 if (rc == SLI4_EQE_STATUS_EQ_FULL) {
2223 u32 i;
2224
2225 /*
2226 * Received a sentinel EQE indicating the
2227 * EQ is full. Process all CQs
2228 */
2229 for (i = 0; i < hw->cq_count; i++)
2230 efct_hw_cq_process(hw, hw->hw_cq[i]);
2231 continue;
2232 } else {
2233 return rc;
2234 }
2235 } else {
2236 int index;
2237
2238 index = efct_hw_queue_hash_find(hw->cq_hash, cq_id);
2239
2240 if (likely(index >= 0))
2241 efct_hw_cq_process(hw, hw->hw_cq[index]);
2242 else
2243 efc_log_err(hw->os, "bad CQ_ID %#06x\n", cq_id);
2244 }
2245
2246 if (eq->queue->n_posted > eq->queue->posted_limit)
2247 sli_queue_arm(&hw->sli, eq->queue, false);
2248
2249 if (tcheck_count && (--tcheck_count == 0)) {
2250 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;
2251 telapsed = jiffies_to_msecs(jiffies) - tstart;
2252 if (telapsed >= max_isr_time_msec)
2253 done = true;
2254 }
2255 }
2256 sli_queue_eq_arm(&hw->sli, eq->queue, true);
2257
2258 return 0;
2259 }
2260
2261 static int
_efct_hw_wq_write(struct hw_wq * wq,struct efct_hw_wqe * wqe)2262 _efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)
2263 {
2264 int queue_rc;
2265
2266 /* Every so often, set the wqec bit to generate comsummed completions */
2267 if (wq->wqec_count)
2268 wq->wqec_count--;
2269
2270 if (wq->wqec_count == 0) {
2271 struct sli4_generic_wqe *genwqe = (void *)wqe->wqebuf;
2272
2273 genwqe->cmdtype_wqec_byte |= SLI4_GEN_WQE_WQEC;
2274 wq->wqec_count = wq->wqec_set_count;
2275 }
2276
2277 /* Decrement WQ free count */
2278 wq->free_count--;
2279
2280 queue_rc = sli_wq_write(&wq->hw->sli, wq->queue, wqe->wqebuf);
2281
2282 return (queue_rc < 0) ? -EIO : 0;
2283 }
2284
2285 static void
hw_wq_submit_pending(struct hw_wq * wq,u32 update_free_count)2286 hw_wq_submit_pending(struct hw_wq *wq, u32 update_free_count)
2287 {
2288 struct efct_hw_wqe *wqe;
2289 unsigned long flags = 0;
2290
2291 spin_lock_irqsave(&wq->queue->lock, flags);
2292
2293 /* Update free count with value passed in */
2294 wq->free_count += update_free_count;
2295
2296 while ((wq->free_count > 0) && (!list_empty(&wq->pending_list))) {
2297 wqe = list_first_entry(&wq->pending_list,
2298 struct efct_hw_wqe, list_entry);
2299 list_del_init(&wqe->list_entry);
2300 _efct_hw_wq_write(wq, wqe);
2301
2302 if (wqe->abort_wqe_submit_needed) {
2303 wqe->abort_wqe_submit_needed = false;
2304 efct_hw_fill_abort_wqe(wq->hw, wqe);
2305 INIT_LIST_HEAD(&wqe->list_entry);
2306 list_add_tail(&wqe->list_entry, &wq->pending_list);
2307 wq->wq_pending_count++;
2308 }
2309 }
2310
2311 spin_unlock_irqrestore(&wq->queue->lock, flags);
2312 }
2313
2314 void
efct_hw_cq_process(struct efct_hw * hw,struct hw_cq * cq)2315 efct_hw_cq_process(struct efct_hw *hw, struct hw_cq *cq)
2316 {
2317 u8 cqe[sizeof(struct sli4_mcqe)];
2318 u16 rid = U16_MAX;
2319 /* completion type */
2320 enum sli4_qentry ctype;
2321 u32 n_processed = 0;
2322 u32 tstart, telapsed;
2323
2324 tstart = jiffies_to_msecs(jiffies);
2325
2326 while (!sli_cq_read(&hw->sli, cq->queue, cqe)) {
2327 int status;
2328
2329 status = sli_cq_parse(&hw->sli, cq->queue, cqe, &ctype, &rid);
2330 /*
2331 * The sign of status is significant. If status is:
2332 * == 0 : call completed correctly and
2333 * the CQE indicated success
2334 * > 0 : call completed correctly and
2335 * the CQE indicated an error
2336 * < 0 : call failed and no information is available about the
2337 * CQE
2338 */
2339 if (status < 0) {
2340 if (status == SLI4_MCQE_STATUS_NOT_COMPLETED)
2341 /*
2342 * Notification that an entry was consumed,
2343 * but not completed
2344 */
2345 continue;
2346
2347 break;
2348 }
2349
2350 switch (ctype) {
2351 case SLI4_QENTRY_ASYNC:
2352 sli_cqe_async(&hw->sli, cqe);
2353 break;
2354 case SLI4_QENTRY_MQ:
2355 /*
2356 * Process MQ entry. Note there is no way to determine
2357 * the MQ_ID from the completion entry.
2358 */
2359 efct_hw_mq_process(hw, status, hw->mq);
2360 break;
2361 case SLI4_QENTRY_WQ:
2362 efct_hw_wq_process(hw, cq, cqe, status, rid);
2363 break;
2364 case SLI4_QENTRY_WQ_RELEASE: {
2365 u32 wq_id = rid;
2366 int index;
2367 struct hw_wq *wq = NULL;
2368
2369 index = efct_hw_queue_hash_find(hw->wq_hash, wq_id);
2370
2371 if (likely(index >= 0)) {
2372 wq = hw->hw_wq[index];
2373 } else {
2374 efc_log_err(hw->os, "bad WQ_ID %#06x\n", wq_id);
2375 break;
2376 }
2377 /* Submit any HW IOs that are on the WQ pending list */
2378 hw_wq_submit_pending(wq, wq->wqec_set_count);
2379
2380 break;
2381 }
2382
2383 case SLI4_QENTRY_RQ:
2384 efct_hw_rqpair_process_rq(hw, cq, cqe);
2385 break;
2386 case SLI4_QENTRY_XABT: {
2387 efct_hw_xabt_process(hw, cq, cqe, rid);
2388 break;
2389 }
2390 default:
2391 efc_log_debug(hw->os, "unhandled ctype=%#x rid=%#x\n",
2392 ctype, rid);
2393 break;
2394 }
2395
2396 n_processed++;
2397 if (n_processed == cq->queue->proc_limit)
2398 break;
2399
2400 if (cq->queue->n_posted >= cq->queue->posted_limit)
2401 sli_queue_arm(&hw->sli, cq->queue, false);
2402 }
2403
2404 sli_queue_arm(&hw->sli, cq->queue, true);
2405
2406 if (n_processed > cq->queue->max_num_processed)
2407 cq->queue->max_num_processed = n_processed;
2408 telapsed = jiffies_to_msecs(jiffies) - tstart;
2409 if (telapsed > cq->queue->max_process_time)
2410 cq->queue->max_process_time = telapsed;
2411 }
2412
2413 void
efct_hw_wq_process(struct efct_hw * hw,struct hw_cq * cq,u8 * cqe,int status,u16 rid)2414 efct_hw_wq_process(struct efct_hw *hw, struct hw_cq *cq,
2415 u8 *cqe, int status, u16 rid)
2416 {
2417 struct hw_wq_callback *wqcb;
2418
2419 if (rid == EFCT_HW_REQUE_XRI_REGTAG) {
2420 if (status)
2421 efc_log_err(hw->os, "reque xri failed, status = %d\n",
2422 status);
2423 return;
2424 }
2425
2426 wqcb = efct_hw_reqtag_get_instance(hw, rid);
2427 if (!wqcb) {
2428 efc_log_err(hw->os, "invalid request tag: x%x\n", rid);
2429 return;
2430 }
2431
2432 if (!wqcb->callback) {
2433 efc_log_err(hw->os, "wqcb callback is NULL\n");
2434 return;
2435 }
2436
2437 (*wqcb->callback)(wqcb->arg, cqe, status);
2438 }
2439
2440 void
efct_hw_xabt_process(struct efct_hw * hw,struct hw_cq * cq,u8 * cqe,u16 rid)2441 efct_hw_xabt_process(struct efct_hw *hw, struct hw_cq *cq,
2442 u8 *cqe, u16 rid)
2443 {
2444 /* search IOs wait free list */
2445 struct efct_hw_io *io = NULL;
2446 unsigned long flags = 0;
2447
2448 io = efct_hw_io_lookup(hw, rid);
2449 if (!io) {
2450 /* IO lookup failure should never happen */
2451 efc_log_err(hw->os, "xabt io lookup failed rid=%#x\n", rid);
2452 return;
2453 }
2454
2455 if (!io->xbusy)
2456 efc_log_debug(hw->os, "xabt io not busy rid=%#x\n", rid);
2457 else
2458 /* mark IO as no longer busy */
2459 io->xbusy = false;
2460
2461 /*
2462 * For IOs that were aborted internally, we need to issue any pending
2463 * callback here.
2464 */
2465 if (io->done) {
2466 efct_hw_done_t done = io->done;
2467 void *arg = io->arg;
2468
2469 /*
2470 * Use latched status as this is always saved for an internal
2471 * abort
2472 */
2473 int status = io->saved_status;
2474 u32 len = io->saved_len;
2475 u32 ext = io->saved_ext;
2476
2477 io->done = NULL;
2478 io->status_saved = false;
2479
2480 done(io, len, status, ext, arg);
2481 }
2482
2483 spin_lock_irqsave(&hw->io_lock, flags);
2484 if (io->state == EFCT_HW_IO_STATE_INUSE ||
2485 io->state == EFCT_HW_IO_STATE_WAIT_FREE) {
2486 /* if on wait_free list, caller has already freed IO;
2487 * remove from wait_free list and add to free list.
2488 * if on in-use list, already marked as no longer busy;
2489 * just leave there and wait for caller to free.
2490 */
2491 if (io->state == EFCT_HW_IO_STATE_WAIT_FREE) {
2492 io->state = EFCT_HW_IO_STATE_FREE;
2493 list_del_init(&io->list_entry);
2494 efct_hw_io_free_move_correct_list(hw, io);
2495 }
2496 }
2497 spin_unlock_irqrestore(&hw->io_lock, flags);
2498 }
2499
2500 static int
efct_hw_flush(struct efct_hw * hw)2501 efct_hw_flush(struct efct_hw *hw)
2502 {
2503 u32 i = 0;
2504
2505 /* Process any remaining completions */
2506 for (i = 0; i < hw->eq_count; i++)
2507 efct_hw_process(hw, i, ~0);
2508
2509 return 0;
2510 }
2511
2512 int
efct_hw_wq_write(struct hw_wq * wq,struct efct_hw_wqe * wqe)2513 efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)
2514 {
2515 int rc = 0;
2516 unsigned long flags = 0;
2517
2518 spin_lock_irqsave(&wq->queue->lock, flags);
2519 if (list_empty(&wq->pending_list)) {
2520 if (wq->free_count > 0) {
2521 rc = _efct_hw_wq_write(wq, wqe);
2522 } else {
2523 INIT_LIST_HEAD(&wqe->list_entry);
2524 list_add_tail(&wqe->list_entry, &wq->pending_list);
2525 wq->wq_pending_count++;
2526 }
2527
2528 spin_unlock_irqrestore(&wq->queue->lock, flags);
2529 return rc;
2530 }
2531
2532 INIT_LIST_HEAD(&wqe->list_entry);
2533 list_add_tail(&wqe->list_entry, &wq->pending_list);
2534 wq->wq_pending_count++;
2535 while (wq->free_count > 0) {
2536 wqe = list_first_entry(&wq->pending_list, struct efct_hw_wqe,
2537 list_entry);
2538 if (!wqe)
2539 break;
2540
2541 list_del_init(&wqe->list_entry);
2542 rc = _efct_hw_wq_write(wq, wqe);
2543 if (rc)
2544 break;
2545
2546 if (wqe->abort_wqe_submit_needed) {
2547 wqe->abort_wqe_submit_needed = false;
2548 efct_hw_fill_abort_wqe(wq->hw, wqe);
2549
2550 INIT_LIST_HEAD(&wqe->list_entry);
2551 list_add_tail(&wqe->list_entry, &wq->pending_list);
2552 wq->wq_pending_count++;
2553 }
2554 }
2555
2556 spin_unlock_irqrestore(&wq->queue->lock, flags);
2557
2558 return rc;
2559 }
2560
2561 int
efct_efc_bls_send(struct efc * efc,u32 type,struct sli_bls_params * bls)2562 efct_efc_bls_send(struct efc *efc, u32 type, struct sli_bls_params *bls)
2563 {
2564 struct efct *efct = efc->base;
2565
2566 return efct_hw_bls_send(efct, type, bls, NULL, NULL);
2567 }
2568
2569 int
efct_hw_bls_send(struct efct * efct,u32 type,struct sli_bls_params * bls_params,void * cb,void * arg)2570 efct_hw_bls_send(struct efct *efct, u32 type, struct sli_bls_params *bls_params,
2571 void *cb, void *arg)
2572 {
2573 struct efct_hw *hw = &efct->hw;
2574 struct efct_hw_io *hio;
2575 struct sli_bls_payload bls;
2576 int rc;
2577
2578 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2579 efc_log_err(hw->os,
2580 "cannot send BLS, HW state=%d\n", hw->state);
2581 return -EIO;
2582 }
2583
2584 hio = efct_hw_io_alloc(hw);
2585 if (!hio) {
2586 efc_log_err(hw->os, "HIO allocation failed\n");
2587 return -EIO;
2588 }
2589
2590 hio->done = cb;
2591 hio->arg = arg;
2592
2593 bls_params->xri = hio->indicator;
2594 bls_params->tag = hio->reqtag;
2595
2596 if (type == FC_RCTL_BA_ACC) {
2597 hio->type = EFCT_HW_BLS_ACC;
2598 bls.type = SLI4_SLI_BLS_ACC;
2599 memcpy(&bls.u.acc, bls_params->payload, sizeof(bls.u.acc));
2600 } else {
2601 hio->type = EFCT_HW_BLS_RJT;
2602 bls.type = SLI4_SLI_BLS_RJT;
2603 memcpy(&bls.u.rjt, bls_params->payload, sizeof(bls.u.rjt));
2604 }
2605
2606 bls.ox_id = cpu_to_le16(bls_params->ox_id);
2607 bls.rx_id = cpu_to_le16(bls_params->rx_id);
2608
2609 if (sli_xmit_bls_rsp64_wqe(&hw->sli, hio->wqe.wqebuf,
2610 &bls, bls_params)) {
2611 efc_log_err(hw->os, "XMIT_BLS_RSP64 WQE error\n");
2612 return -EIO;
2613 }
2614
2615 hio->xbusy = true;
2616
2617 /*
2618 * Add IO to active io wqe list before submitting, in case the
2619 * wcqe processing preempts this thread.
2620 */
2621 hio->wq->use_count++;
2622 rc = efct_hw_wq_write(hio->wq, &hio->wqe);
2623 if (rc >= 0) {
2624 /* non-negative return is success */
2625 rc = 0;
2626 } else {
2627 /* failed to write wqe, remove from active wqe list */
2628 efc_log_err(hw->os,
2629 "sli_queue_write failed: %d\n", rc);
2630 hio->xbusy = false;
2631 }
2632
2633 return rc;
2634 }
2635
2636 static int
efct_els_ssrs_send_cb(struct efct_hw_io * hio,u32 length,int status,u32 ext_status,void * arg)2637 efct_els_ssrs_send_cb(struct efct_hw_io *hio, u32 length, int status,
2638 u32 ext_status, void *arg)
2639 {
2640 struct efc_disc_io *io = arg;
2641
2642 efc_disc_io_complete(io, length, status, ext_status);
2643 return 0;
2644 }
2645
2646 static inline void
efct_fill_els_params(struct efc_disc_io * io,struct sli_els_params * params)2647 efct_fill_els_params(struct efc_disc_io *io, struct sli_els_params *params)
2648 {
2649 u8 *cmd = io->req.virt;
2650
2651 params->cmd = *cmd;
2652 params->s_id = io->s_id;
2653 params->d_id = io->d_id;
2654 params->ox_id = io->iparam.els.ox_id;
2655 params->rpi = io->rpi;
2656 params->vpi = io->vpi;
2657 params->rpi_registered = io->rpi_registered;
2658 params->xmit_len = io->xmit_len;
2659 params->rsp_len = io->rsp_len;
2660 params->timeout = io->iparam.els.timeout;
2661 }
2662
2663 static inline void
efct_fill_ct_params(struct efc_disc_io * io,struct sli_ct_params * params)2664 efct_fill_ct_params(struct efc_disc_io *io, struct sli_ct_params *params)
2665 {
2666 params->r_ctl = io->iparam.ct.r_ctl;
2667 params->type = io->iparam.ct.type;
2668 params->df_ctl = io->iparam.ct.df_ctl;
2669 params->d_id = io->d_id;
2670 params->ox_id = io->iparam.ct.ox_id;
2671 params->rpi = io->rpi;
2672 params->vpi = io->vpi;
2673 params->rpi_registered = io->rpi_registered;
2674 params->xmit_len = io->xmit_len;
2675 params->rsp_len = io->rsp_len;
2676 params->timeout = io->iparam.ct.timeout;
2677 }
2678
2679 /**
2680 * efct_els_hw_srrs_send() - Send a single request and response cmd.
2681 * @efc: efc library structure
2682 * @io: Discovery IO used to hold els and ct cmd context.
2683 *
2684 * This routine supports communication sequences consisting of a single
2685 * request and single response between two endpoints. Examples include:
2686 * - Sending an ELS request.
2687 * - Sending an ELS response - To send an ELS response, the caller must provide
2688 * the OX_ID from the received request.
2689 * - Sending a FC Common Transport (FC-CT) request - To send a FC-CT request,
2690 * the caller must provide the R_CTL, TYPE, and DF_CTL
2691 * values to place in the FC frame header.
2692 *
2693 * Return: Status of the request.
2694 */
2695 int
efct_els_hw_srrs_send(struct efc * efc,struct efc_disc_io * io)2696 efct_els_hw_srrs_send(struct efc *efc, struct efc_disc_io *io)
2697 {
2698 struct efct *efct = efc->base;
2699 struct efct_hw_io *hio;
2700 struct efct_hw *hw = &efct->hw;
2701 struct efc_dma *send = &io->req;
2702 struct efc_dma *receive = &io->rsp;
2703 struct sli4_sge *sge = NULL;
2704 int rc = 0;
2705 u32 len = io->xmit_len;
2706 u32 sge0_flags;
2707 u32 sge1_flags;
2708
2709 hio = efct_hw_io_alloc(hw);
2710 if (!hio) {
2711 pr_err("HIO alloc failed\n");
2712 return -EIO;
2713 }
2714
2715 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2716 efc_log_debug(hw->os,
2717 "cannot send SRRS, HW state=%d\n", hw->state);
2718 return -EIO;
2719 }
2720
2721 hio->done = efct_els_ssrs_send_cb;
2722 hio->arg = io;
2723
2724 sge = hio->sgl->virt;
2725
2726 /* clear both SGE */
2727 memset(hio->sgl->virt, 0, 2 * sizeof(struct sli4_sge));
2728
2729 sge0_flags = le32_to_cpu(sge[0].dw2_flags);
2730 sge1_flags = le32_to_cpu(sge[1].dw2_flags);
2731 if (send->size) {
2732 sge[0].buffer_address_high =
2733 cpu_to_le32(upper_32_bits(send->phys));
2734 sge[0].buffer_address_low =
2735 cpu_to_le32(lower_32_bits(send->phys));
2736
2737 sge0_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
2738
2739 sge[0].buffer_length = cpu_to_le32(len);
2740 }
2741
2742 if (io->io_type == EFC_DISC_IO_ELS_REQ ||
2743 io->io_type == EFC_DISC_IO_CT_REQ) {
2744 sge[1].buffer_address_high =
2745 cpu_to_le32(upper_32_bits(receive->phys));
2746 sge[1].buffer_address_low =
2747 cpu_to_le32(lower_32_bits(receive->phys));
2748
2749 sge1_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);
2750 sge1_flags |= SLI4_SGE_LAST;
2751
2752 sge[1].buffer_length = cpu_to_le32(receive->size);
2753 } else {
2754 sge0_flags |= SLI4_SGE_LAST;
2755 }
2756
2757 sge[0].dw2_flags = cpu_to_le32(sge0_flags);
2758 sge[1].dw2_flags = cpu_to_le32(sge1_flags);
2759
2760 switch (io->io_type) {
2761 case EFC_DISC_IO_ELS_REQ: {
2762 struct sli_els_params els_params;
2763
2764 hio->type = EFCT_HW_ELS_REQ;
2765 efct_fill_els_params(io, &els_params);
2766 els_params.xri = hio->indicator;
2767 els_params.tag = hio->reqtag;
2768
2769 if (sli_els_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2770 &els_params)) {
2771 efc_log_err(hw->os, "REQ WQE error\n");
2772 rc = -EIO;
2773 }
2774 break;
2775 }
2776 case EFC_DISC_IO_ELS_RESP: {
2777 struct sli_els_params els_params;
2778
2779 hio->type = EFCT_HW_ELS_RSP;
2780 efct_fill_els_params(io, &els_params);
2781 els_params.xri = hio->indicator;
2782 els_params.tag = hio->reqtag;
2783 if (sli_xmit_els_rsp64_wqe(&hw->sli, hio->wqe.wqebuf, send,
2784 &els_params)){
2785 efc_log_err(hw->os, "RSP WQE error\n");
2786 rc = -EIO;
2787 }
2788 break;
2789 }
2790 case EFC_DISC_IO_CT_REQ: {
2791 struct sli_ct_params ct_params;
2792
2793 hio->type = EFCT_HW_FC_CT;
2794 efct_fill_ct_params(io, &ct_params);
2795 ct_params.xri = hio->indicator;
2796 ct_params.tag = hio->reqtag;
2797 if (sli_gen_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2798 &ct_params)){
2799 efc_log_err(hw->os, "GEN WQE error\n");
2800 rc = -EIO;
2801 }
2802 break;
2803 }
2804 case EFC_DISC_IO_CT_RESP: {
2805 struct sli_ct_params ct_params;
2806
2807 hio->type = EFCT_HW_FC_CT_RSP;
2808 efct_fill_ct_params(io, &ct_params);
2809 ct_params.xri = hio->indicator;
2810 ct_params.tag = hio->reqtag;
2811 if (sli_xmit_sequence64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,
2812 &ct_params)){
2813 efc_log_err(hw->os, "XMIT SEQ WQE error\n");
2814 rc = -EIO;
2815 }
2816 break;
2817 }
2818 default:
2819 efc_log_err(hw->os, "bad SRRS type %#x\n", io->io_type);
2820 rc = -EIO;
2821 }
2822
2823 if (rc == 0) {
2824 hio->xbusy = true;
2825
2826 /*
2827 * Add IO to active io wqe list before submitting, in case the
2828 * wcqe processing preempts this thread.
2829 */
2830 hio->wq->use_count++;
2831 rc = efct_hw_wq_write(hio->wq, &hio->wqe);
2832 if (rc >= 0) {
2833 /* non-negative return is success */
2834 rc = 0;
2835 } else {
2836 /* failed to write wqe, remove from active wqe list */
2837 efc_log_err(hw->os,
2838 "sli_queue_write failed: %d\n", rc);
2839 hio->xbusy = false;
2840 }
2841 }
2842
2843 return rc;
2844 }
2845
2846 int
efct_hw_io_send(struct efct_hw * hw,enum efct_hw_io_type type,struct efct_hw_io * io,union efct_hw_io_param_u * iparam,void * cb,void * arg)2847 efct_hw_io_send(struct efct_hw *hw, enum efct_hw_io_type type,
2848 struct efct_hw_io *io, union efct_hw_io_param_u *iparam,
2849 void *cb, void *arg)
2850 {
2851 int rc = 0;
2852 bool send_wqe = true;
2853
2854 if (!io) {
2855 pr_err("bad parm hw=%p io=%p\n", hw, io);
2856 return -EIO;
2857 }
2858
2859 if (hw->state != EFCT_HW_STATE_ACTIVE) {
2860 efc_log_err(hw->os, "cannot send IO, HW state=%d\n", hw->state);
2861 return -EIO;
2862 }
2863
2864 /*
2865 * Save state needed during later stages
2866 */
2867 io->type = type;
2868 io->done = cb;
2869 io->arg = arg;
2870
2871 /*
2872 * Format the work queue entry used to send the IO
2873 */
2874 switch (type) {
2875 case EFCT_HW_IO_TARGET_WRITE: {
2876 u16 *flags = &iparam->fcp_tgt.flags;
2877 struct fcp_txrdy *xfer = io->xfer_rdy.virt;
2878
2879 /*
2880 * Fill in the XFER_RDY for IF_TYPE 0 devices
2881 */
2882 xfer->ft_data_ro = cpu_to_be32(iparam->fcp_tgt.offset);
2883 xfer->ft_burst_len = cpu_to_be32(iparam->fcp_tgt.xmit_len);
2884
2885 if (io->xbusy)
2886 *flags |= SLI4_IO_CONTINUATION;
2887 else
2888 *flags &= ~SLI4_IO_CONTINUATION;
2889 iparam->fcp_tgt.xri = io->indicator;
2890 iparam->fcp_tgt.tag = io->reqtag;
2891
2892 if (sli_fcp_treceive64_wqe(&hw->sli, io->wqe.wqebuf,
2893 &io->def_sgl, io->first_data_sge,
2894 SLI4_CQ_DEFAULT,
2895 0, 0, &iparam->fcp_tgt)) {
2896 efc_log_err(hw->os, "TRECEIVE WQE error\n");
2897 rc = -EIO;
2898 }
2899 break;
2900 }
2901 case EFCT_HW_IO_TARGET_READ: {
2902 u16 *flags = &iparam->fcp_tgt.flags;
2903
2904 if (io->xbusy)
2905 *flags |= SLI4_IO_CONTINUATION;
2906 else
2907 *flags &= ~SLI4_IO_CONTINUATION;
2908
2909 iparam->fcp_tgt.xri = io->indicator;
2910 iparam->fcp_tgt.tag = io->reqtag;
2911
2912 if (sli_fcp_tsend64_wqe(&hw->sli, io->wqe.wqebuf,
2913 &io->def_sgl, io->first_data_sge,
2914 SLI4_CQ_DEFAULT,
2915 0, 0, &iparam->fcp_tgt)) {
2916 efc_log_err(hw->os, "TSEND WQE error\n");
2917 rc = -EIO;
2918 }
2919 break;
2920 }
2921 case EFCT_HW_IO_TARGET_RSP: {
2922 u16 *flags = &iparam->fcp_tgt.flags;
2923
2924 if (io->xbusy)
2925 *flags |= SLI4_IO_CONTINUATION;
2926 else
2927 *flags &= ~SLI4_IO_CONTINUATION;
2928
2929 iparam->fcp_tgt.xri = io->indicator;
2930 iparam->fcp_tgt.tag = io->reqtag;
2931
2932 if (sli_fcp_trsp64_wqe(&hw->sli, io->wqe.wqebuf,
2933 &io->def_sgl, SLI4_CQ_DEFAULT,
2934 0, &iparam->fcp_tgt)) {
2935 efc_log_err(hw->os, "TRSP WQE error\n");
2936 rc = -EIO;
2937 }
2938
2939 break;
2940 }
2941 default:
2942 efc_log_err(hw->os, "unsupported IO type %#x\n", type);
2943 rc = -EIO;
2944 }
2945
2946 if (send_wqe && rc == 0) {
2947 io->xbusy = true;
2948
2949 /*
2950 * Add IO to active io wqe list before submitting, in case the
2951 * wcqe processing preempts this thread.
2952 */
2953 hw->tcmd_wq_submit[io->wq->instance]++;
2954 io->wq->use_count++;
2955 rc = efct_hw_wq_write(io->wq, &io->wqe);
2956 if (rc >= 0) {
2957 /* non-negative return is success */
2958 rc = 0;
2959 } else {
2960 /* failed to write wqe, remove from active wqe list */
2961 efc_log_err(hw->os,
2962 "sli_queue_write failed: %d\n", rc);
2963 io->xbusy = false;
2964 }
2965 }
2966
2967 return rc;
2968 }
2969
2970 int
efct_hw_send_frame(struct efct_hw * hw,struct fc_frame_header * hdr,u8 sof,u8 eof,struct efc_dma * payload,struct efct_hw_send_frame_context * ctx,void (* callback)(void * arg,u8 * cqe,int status),void * arg)2971 efct_hw_send_frame(struct efct_hw *hw, struct fc_frame_header *hdr,
2972 u8 sof, u8 eof, struct efc_dma *payload,
2973 struct efct_hw_send_frame_context *ctx,
2974 void (*callback)(void *arg, u8 *cqe, int status),
2975 void *arg)
2976 {
2977 int rc;
2978 struct efct_hw_wqe *wqe;
2979 u32 xri;
2980 struct hw_wq *wq;
2981
2982 wqe = &ctx->wqe;
2983
2984 /* populate the callback object */
2985 ctx->hw = hw;
2986
2987 /* Fetch and populate request tag */
2988 ctx->wqcb = efct_hw_reqtag_alloc(hw, callback, arg);
2989 if (!ctx->wqcb) {
2990 efc_log_err(hw->os, "can't allocate request tag\n");
2991 return -ENOSPC;
2992 }
2993
2994 wq = hw->hw_wq[0];
2995
2996 /* Set XRI and RX_ID in the header based on which WQ, and which
2997 * send_frame_io we are using
2998 */
2999 xri = wq->send_frame_io->indicator;
3000
3001 /* Build the send frame WQE */
3002 rc = sli_send_frame_wqe(&hw->sli, wqe->wqebuf,
3003 sof, eof, (u32 *)hdr, payload, payload->len,
3004 EFCT_HW_SEND_FRAME_TIMEOUT, xri,
3005 ctx->wqcb->instance_index);
3006 if (rc) {
3007 efc_log_err(hw->os, "sli_send_frame_wqe failed: %d\n", rc);
3008 return -EIO;
3009 }
3010
3011 /* Write to WQ */
3012 rc = efct_hw_wq_write(wq, wqe);
3013 if (rc) {
3014 efc_log_err(hw->os, "efct_hw_wq_write failed: %d\n", rc);
3015 return -EIO;
3016 }
3017
3018 wq->use_count++;
3019
3020 return 0;
3021 }
3022
3023 static int
efct_hw_cb_link_stat(struct efct_hw * hw,int status,u8 * mqe,void * arg)3024 efct_hw_cb_link_stat(struct efct_hw *hw, int status,
3025 u8 *mqe, void *arg)
3026 {
3027 struct sli4_cmd_read_link_stats *mbox_rsp;
3028 struct efct_hw_link_stat_cb_arg *cb_arg = arg;
3029 struct efct_hw_link_stat_counts counts[EFCT_HW_LINK_STAT_MAX];
3030 u32 num_counters, i;
3031 u32 mbox_rsp_flags = 0;
3032
3033 mbox_rsp = (struct sli4_cmd_read_link_stats *)mqe;
3034 mbox_rsp_flags = le32_to_cpu(mbox_rsp->dw1_flags);
3035 num_counters = (mbox_rsp_flags & SLI4_READ_LNKSTAT_GEC) ? 20 : 13;
3036 memset(counts, 0, sizeof(struct efct_hw_link_stat_counts) *
3037 EFCT_HW_LINK_STAT_MAX);
3038
3039 /* Fill overflow counts, mask starts from SLI4_READ_LNKSTAT_W02OF*/
3040 for (i = 0; i < EFCT_HW_LINK_STAT_MAX; i++)
3041 counts[i].overflow = (mbox_rsp_flags & (1 << (i + 2)));
3042
3043 counts[EFCT_HW_LINK_STAT_LINK_FAILURE_COUNT].counter =
3044 le32_to_cpu(mbox_rsp->linkfail_errcnt);
3045 counts[EFCT_HW_LINK_STAT_LOSS_OF_SYNC_COUNT].counter =
3046 le32_to_cpu(mbox_rsp->losssync_errcnt);
3047 counts[EFCT_HW_LINK_STAT_LOSS_OF_SIGNAL_COUNT].counter =
3048 le32_to_cpu(mbox_rsp->losssignal_errcnt);
3049 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_COUNT].counter =
3050 le32_to_cpu(mbox_rsp->primseq_errcnt);
3051 counts[EFCT_HW_LINK_STAT_INVALID_XMIT_WORD_COUNT].counter =
3052 le32_to_cpu(mbox_rsp->inval_txword_errcnt);
3053 counts[EFCT_HW_LINK_STAT_CRC_COUNT].counter =
3054 le32_to_cpu(mbox_rsp->crc_errcnt);
3055 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_TIMEOUT_COUNT].counter =
3056 le32_to_cpu(mbox_rsp->primseq_eventtimeout_cnt);
3057 counts[EFCT_HW_LINK_STAT_ELASTIC_BUFFER_OVERRUN_COUNT].counter =
3058 le32_to_cpu(mbox_rsp->elastic_bufoverrun_errcnt);
3059 counts[EFCT_HW_LINK_STAT_ARB_TIMEOUT_COUNT].counter =
3060 le32_to_cpu(mbox_rsp->arbit_fc_al_timeout_cnt);
3061 counts[EFCT_HW_LINK_STAT_ADVERTISED_RCV_B2B_CREDIT].counter =
3062 le32_to_cpu(mbox_rsp->adv_rx_buftor_to_buf_credit);
3063 counts[EFCT_HW_LINK_STAT_CURR_RCV_B2B_CREDIT].counter =
3064 le32_to_cpu(mbox_rsp->curr_rx_buf_to_buf_credit);
3065 counts[EFCT_HW_LINK_STAT_ADVERTISED_XMIT_B2B_CREDIT].counter =
3066 le32_to_cpu(mbox_rsp->adv_tx_buf_to_buf_credit);
3067 counts[EFCT_HW_LINK_STAT_CURR_XMIT_B2B_CREDIT].counter =
3068 le32_to_cpu(mbox_rsp->curr_tx_buf_to_buf_credit);
3069 counts[EFCT_HW_LINK_STAT_RCV_EOFA_COUNT].counter =
3070 le32_to_cpu(mbox_rsp->rx_eofa_cnt);
3071 counts[EFCT_HW_LINK_STAT_RCV_EOFDTI_COUNT].counter =
3072 le32_to_cpu(mbox_rsp->rx_eofdti_cnt);
3073 counts[EFCT_HW_LINK_STAT_RCV_EOFNI_COUNT].counter =
3074 le32_to_cpu(mbox_rsp->rx_eofni_cnt);
3075 counts[EFCT_HW_LINK_STAT_RCV_SOFF_COUNT].counter =
3076 le32_to_cpu(mbox_rsp->rx_soff_cnt);
3077 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_AER_COUNT].counter =
3078 le32_to_cpu(mbox_rsp->rx_dropped_no_aer_cnt);
3079 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_RPI_COUNT].counter =
3080 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_rpi_rescnt);
3081 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_XRI_COUNT].counter =
3082 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_xri_rescnt);
3083
3084 if (cb_arg) {
3085 if (cb_arg->cb) {
3086 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))
3087 status = le16_to_cpu(mbox_rsp->hdr.status);
3088 cb_arg->cb(status, num_counters, counts, cb_arg->arg);
3089 }
3090
3091 kfree(cb_arg);
3092 }
3093
3094 return 0;
3095 }
3096
3097 int
efct_hw_get_link_stats(struct efct_hw * hw,u8 req_ext_counters,u8 clear_overflow_flags,u8 clear_all_counters,void (* cb)(int status,u32 num_counters,struct efct_hw_link_stat_counts * counters,void * arg),void * arg)3098 efct_hw_get_link_stats(struct efct_hw *hw, u8 req_ext_counters,
3099 u8 clear_overflow_flags, u8 clear_all_counters,
3100 void (*cb)(int status, u32 num_counters,
3101 struct efct_hw_link_stat_counts *counters,
3102 void *arg),
3103 void *arg)
3104 {
3105 int rc = -EIO;
3106 struct efct_hw_link_stat_cb_arg *cb_arg;
3107 u8 mbxdata[SLI4_BMBX_SIZE];
3108
3109 cb_arg = kzalloc_obj(*cb_arg, GFP_ATOMIC);
3110 if (!cb_arg)
3111 return -ENOMEM;
3112
3113 cb_arg->cb = cb;
3114 cb_arg->arg = arg;
3115
3116 /* Send the HW command */
3117 if (!sli_cmd_read_link_stats(&hw->sli, mbxdata, req_ext_counters,
3118 clear_overflow_flags, clear_all_counters))
3119 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3120 efct_hw_cb_link_stat, cb_arg);
3121
3122 if (rc)
3123 kfree(cb_arg);
3124
3125 return rc;
3126 }
3127
3128 static int
efct_hw_cb_host_stat(struct efct_hw * hw,int status,u8 * mqe,void * arg)3129 efct_hw_cb_host_stat(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3130 {
3131 struct sli4_cmd_read_status *mbox_rsp =
3132 (struct sli4_cmd_read_status *)mqe;
3133 struct efct_hw_host_stat_cb_arg *cb_arg = arg;
3134 struct efct_hw_host_stat_counts counts[EFCT_HW_HOST_STAT_MAX];
3135 u32 num_counters = EFCT_HW_HOST_STAT_MAX;
3136
3137 memset(counts, 0, sizeof(struct efct_hw_host_stat_counts) *
3138 EFCT_HW_HOST_STAT_MAX);
3139
3140 counts[EFCT_HW_HOST_STAT_TX_KBYTE_COUNT].counter =
3141 le32_to_cpu(mbox_rsp->trans_kbyte_cnt);
3142 counts[EFCT_HW_HOST_STAT_RX_KBYTE_COUNT].counter =
3143 le32_to_cpu(mbox_rsp->recv_kbyte_cnt);
3144 counts[EFCT_HW_HOST_STAT_TX_FRAME_COUNT].counter =
3145 le32_to_cpu(mbox_rsp->trans_frame_cnt);
3146 counts[EFCT_HW_HOST_STAT_RX_FRAME_COUNT].counter =
3147 le32_to_cpu(mbox_rsp->recv_frame_cnt);
3148 counts[EFCT_HW_HOST_STAT_TX_SEQ_COUNT].counter =
3149 le32_to_cpu(mbox_rsp->trans_seq_cnt);
3150 counts[EFCT_HW_HOST_STAT_RX_SEQ_COUNT].counter =
3151 le32_to_cpu(mbox_rsp->recv_seq_cnt);
3152 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_ORIG].counter =
3153 le32_to_cpu(mbox_rsp->tot_exchanges_orig);
3154 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_RESP].counter =
3155 le32_to_cpu(mbox_rsp->tot_exchanges_resp);
3156 counts[EFCT_HW_HOSY_STAT_RX_P_BSY_COUNT].counter =
3157 le32_to_cpu(mbox_rsp->recv_p_bsy_cnt);
3158 counts[EFCT_HW_HOST_STAT_RX_F_BSY_COUNT].counter =
3159 le32_to_cpu(mbox_rsp->recv_f_bsy_cnt);
3160 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_RQ_BUF_COUNT].counter =
3161 le32_to_cpu(mbox_rsp->no_rq_buf_dropped_frames_cnt);
3162 counts[EFCT_HW_HOST_STAT_EMPTY_RQ_TIMEOUT_COUNT].counter =
3163 le32_to_cpu(mbox_rsp->empty_rq_timeout_cnt);
3164 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_XRI_COUNT].counter =
3165 le32_to_cpu(mbox_rsp->no_xri_dropped_frames_cnt);
3166 counts[EFCT_HW_HOST_STAT_EMPTY_XRI_POOL_COUNT].counter =
3167 le32_to_cpu(mbox_rsp->empty_xri_pool_cnt);
3168
3169 if (cb_arg) {
3170 if (cb_arg->cb) {
3171 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))
3172 status = le16_to_cpu(mbox_rsp->hdr.status);
3173 cb_arg->cb(status, num_counters, counts, cb_arg->arg);
3174 }
3175
3176 kfree(cb_arg);
3177 }
3178
3179 return 0;
3180 }
3181
3182 int
efct_hw_get_host_stats(struct efct_hw * hw,u8 cc,void (* cb)(int status,u32 num_counters,struct efct_hw_host_stat_counts * counters,void * arg),void * arg)3183 efct_hw_get_host_stats(struct efct_hw *hw, u8 cc,
3184 void (*cb)(int status, u32 num_counters,
3185 struct efct_hw_host_stat_counts *counters,
3186 void *arg),
3187 void *arg)
3188 {
3189 int rc = -EIO;
3190 struct efct_hw_host_stat_cb_arg *cb_arg;
3191 u8 mbxdata[SLI4_BMBX_SIZE];
3192
3193 cb_arg = kmalloc_obj(*cb_arg, GFP_ATOMIC);
3194 if (!cb_arg)
3195 return -ENOMEM;
3196
3197 cb_arg->cb = cb;
3198 cb_arg->arg = arg;
3199
3200 /* Send the HW command to get the host stats */
3201 if (!sli_cmd_read_status(&hw->sli, mbxdata, cc))
3202 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3203 efct_hw_cb_host_stat, cb_arg);
3204
3205 if (rc) {
3206 efc_log_debug(hw->os, "READ_HOST_STATS failed\n");
3207 kfree(cb_arg);
3208 }
3209
3210 return rc;
3211 }
3212
3213 struct efct_hw_async_call_ctx {
3214 efct_hw_async_cb_t callback;
3215 void *arg;
3216 u8 cmd[SLI4_BMBX_SIZE];
3217 };
3218
3219 static void
efct_hw_async_cb(struct efct_hw * hw,int status,u8 * mqe,void * arg)3220 efct_hw_async_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3221 {
3222 struct efct_hw_async_call_ctx *ctx = arg;
3223
3224 if (ctx) {
3225 if (ctx->callback)
3226 (*ctx->callback)(hw, status, mqe, ctx->arg);
3227
3228 kfree(ctx);
3229 }
3230 }
3231
3232 int
efct_hw_async_call(struct efct_hw * hw,efct_hw_async_cb_t callback,void * arg)3233 efct_hw_async_call(struct efct_hw *hw, efct_hw_async_cb_t callback, void *arg)
3234 {
3235 struct efct_hw_async_call_ctx *ctx;
3236 int rc;
3237
3238 /*
3239 * Allocate a callback context (which includes the mbox cmd buffer),
3240 * we need this to be persistent as the mbox cmd submission may be
3241 * queued and executed later execution.
3242 */
3243 ctx = kzalloc_obj(*ctx);
3244 if (!ctx)
3245 return -ENOMEM;
3246
3247 ctx->callback = callback;
3248 ctx->arg = arg;
3249
3250 /* Build and send a NOP mailbox command */
3251 if (sli_cmd_common_nop(&hw->sli, ctx->cmd, 0)) {
3252 efc_log_err(hw->os, "COMMON_NOP format failure\n");
3253 kfree(ctx);
3254 return -EIO;
3255 }
3256
3257 rc = efct_hw_command(hw, ctx->cmd, EFCT_CMD_NOWAIT, efct_hw_async_cb,
3258 ctx);
3259 if (rc) {
3260 efc_log_err(hw->os, "COMMON_NOP command failure, rc=%d\n", rc);
3261 kfree(ctx);
3262 return -EIO;
3263 }
3264 return 0;
3265 }
3266
3267 static int
efct_hw_cb_fw_write(struct efct_hw * hw,int status,u8 * mqe,void * arg)3268 efct_hw_cb_fw_write(struct efct_hw *hw, int status, u8 *mqe, void *arg)
3269 {
3270 struct sli4_cmd_sli_config *mbox_rsp =
3271 (struct sli4_cmd_sli_config *)mqe;
3272 struct sli4_rsp_cmn_write_object *wr_obj_rsp;
3273 struct efct_hw_fw_wr_cb_arg *cb_arg = arg;
3274 u32 bytes_written;
3275 u16 mbox_status;
3276 u32 change_status;
3277
3278 wr_obj_rsp = (struct sli4_rsp_cmn_write_object *)
3279 &mbox_rsp->payload.embed;
3280 bytes_written = le32_to_cpu(wr_obj_rsp->actual_write_length);
3281 mbox_status = le16_to_cpu(mbox_rsp->hdr.status);
3282 change_status = (le32_to_cpu(wr_obj_rsp->change_status_dword) &
3283 RSP_CHANGE_STATUS);
3284
3285 if (cb_arg) {
3286 if (cb_arg->cb) {
3287 if (!status && mbox_status)
3288 status = mbox_status;
3289 cb_arg->cb(status, bytes_written, change_status,
3290 cb_arg->arg);
3291 }
3292
3293 kfree(cb_arg);
3294 }
3295
3296 return 0;
3297 }
3298
3299 int
efct_hw_firmware_write(struct efct_hw * hw,struct efc_dma * dma,u32 size,u32 offset,int last,void (* cb)(int status,u32 bytes_written,u32 change_status,void * arg),void * arg)3300 efct_hw_firmware_write(struct efct_hw *hw, struct efc_dma *dma, u32 size,
3301 u32 offset, int last,
3302 void (*cb)(int status, u32 bytes_written,
3303 u32 change_status, void *arg),
3304 void *arg)
3305 {
3306 int rc = -EIO;
3307 u8 mbxdata[SLI4_BMBX_SIZE];
3308 struct efct_hw_fw_wr_cb_arg *cb_arg;
3309 int noc = 0;
3310
3311 cb_arg = kzalloc_obj(*cb_arg);
3312 if (!cb_arg)
3313 return -ENOMEM;
3314
3315 cb_arg->cb = cb;
3316 cb_arg->arg = arg;
3317
3318 /* Write a portion of a firmware image to the device */
3319 if (!sli_cmd_common_write_object(&hw->sli, mbxdata,
3320 noc, last, size, offset, "/prg/",
3321 dma))
3322 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,
3323 efct_hw_cb_fw_write, cb_arg);
3324
3325 if (rc != 0) {
3326 efc_log_debug(hw->os, "COMMON_WRITE_OBJECT failed\n");
3327 kfree(cb_arg);
3328 }
3329
3330 return rc;
3331 }
3332
3333 static int
efct_hw_cb_port_control(struct efct_hw * hw,int status,u8 * mqe,void * arg)3334 efct_hw_cb_port_control(struct efct_hw *hw, int status, u8 *mqe,
3335 void *arg)
3336 {
3337 return 0;
3338 }
3339
3340 int
efct_hw_port_control(struct efct_hw * hw,enum efct_hw_port ctrl,uintptr_t value,void (* cb)(int status,uintptr_t value,void * arg),void * arg)3341 efct_hw_port_control(struct efct_hw *hw, enum efct_hw_port ctrl,
3342 uintptr_t value,
3343 void (*cb)(int status, uintptr_t value, void *arg),
3344 void *arg)
3345 {
3346 int rc = -EIO;
3347 u8 link[SLI4_BMBX_SIZE];
3348 u32 speed = 0;
3349 u8 reset_alpa = 0;
3350
3351 switch (ctrl) {
3352 case EFCT_HW_PORT_INIT:
3353 if (!sli_cmd_config_link(&hw->sli, link))
3354 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3355 efct_hw_cb_port_control, NULL);
3356
3357 if (rc != 0) {
3358 efc_log_err(hw->os, "CONFIG_LINK failed\n");
3359 break;
3360 }
3361 speed = hw->config.speed;
3362 reset_alpa = (u8)(value & 0xff);
3363
3364 rc = -EIO;
3365 if (!sli_cmd_init_link(&hw->sli, link, speed, reset_alpa))
3366 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3367 efct_hw_cb_port_control, NULL);
3368 /* Free buffer on error, since no callback is coming */
3369 if (rc)
3370 efc_log_err(hw->os, "INIT_LINK failed\n");
3371 break;
3372
3373 case EFCT_HW_PORT_SHUTDOWN:
3374 if (!sli_cmd_down_link(&hw->sli, link))
3375 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,
3376 efct_hw_cb_port_control, NULL);
3377 /* Free buffer on error, since no callback is coming */
3378 if (rc)
3379 efc_log_err(hw->os, "DOWN_LINK failed\n");
3380 break;
3381
3382 default:
3383 efc_log_debug(hw->os, "unhandled control %#x\n", ctrl);
3384 break;
3385 }
3386
3387 return rc;
3388 }
3389
3390 void
efct_hw_teardown(struct efct_hw * hw)3391 efct_hw_teardown(struct efct_hw *hw)
3392 {
3393 u32 i = 0;
3394 u32 destroy_queues;
3395 u32 free_memory;
3396 struct efc_dma *dma;
3397 struct efct *efct = hw->os;
3398
3399 destroy_queues = (hw->state == EFCT_HW_STATE_ACTIVE);
3400 free_memory = (hw->state != EFCT_HW_STATE_UNINITIALIZED);
3401
3402 /* Cancel Sliport Healthcheck */
3403 if (hw->sliport_healthcheck) {
3404 hw->sliport_healthcheck = 0;
3405 efct_hw_config_sli_port_health_check(hw, 0, 0);
3406 }
3407
3408 if (hw->state != EFCT_HW_STATE_QUEUES_ALLOCATED) {
3409 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;
3410
3411 efct_hw_flush(hw);
3412
3413 if (list_empty(&hw->cmd_head))
3414 efc_log_debug(hw->os,
3415 "All commands completed on MQ queue\n");
3416 else
3417 efc_log_debug(hw->os,
3418 "Some cmds still pending on MQ queue\n");
3419
3420 /* Cancel any remaining commands */
3421 efct_hw_command_cancel(hw);
3422 } else {
3423 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;
3424 }
3425
3426 dma_free_coherent(&efct->pci->dev,
3427 hw->rnode_mem.size, hw->rnode_mem.virt,
3428 hw->rnode_mem.phys);
3429 memset(&hw->rnode_mem, 0, sizeof(struct efc_dma));
3430
3431 if (hw->io) {
3432 for (i = 0; i < hw->config.n_io; i++) {
3433 if (hw->io[i] && hw->io[i]->sgl &&
3434 hw->io[i]->sgl->virt) {
3435 dma_free_coherent(&efct->pci->dev,
3436 hw->io[i]->sgl->size,
3437 hw->io[i]->sgl->virt,
3438 hw->io[i]->sgl->phys);
3439 }
3440 kfree(hw->io[i]);
3441 hw->io[i] = NULL;
3442 }
3443 kfree(hw->io);
3444 hw->io = NULL;
3445 kfree(hw->wqe_buffs);
3446 hw->wqe_buffs = NULL;
3447 }
3448
3449 dma = &hw->xfer_rdy;
3450 dma_free_coherent(&efct->pci->dev,
3451 dma->size, dma->virt, dma->phys);
3452 memset(dma, 0, sizeof(struct efc_dma));
3453
3454 dma = &hw->loop_map;
3455 dma_free_coherent(&efct->pci->dev,
3456 dma->size, dma->virt, dma->phys);
3457 memset(dma, 0, sizeof(struct efc_dma));
3458
3459 for (i = 0; i < hw->wq_count; i++)
3460 sli_queue_free(&hw->sli, &hw->wq[i], destroy_queues,
3461 free_memory);
3462
3463 for (i = 0; i < hw->rq_count; i++)
3464 sli_queue_free(&hw->sli, &hw->rq[i], destroy_queues,
3465 free_memory);
3466
3467 for (i = 0; i < hw->mq_count; i++)
3468 sli_queue_free(&hw->sli, &hw->mq[i], destroy_queues,
3469 free_memory);
3470
3471 for (i = 0; i < hw->cq_count; i++)
3472 sli_queue_free(&hw->sli, &hw->cq[i], destroy_queues,
3473 free_memory);
3474
3475 for (i = 0; i < hw->eq_count; i++)
3476 sli_queue_free(&hw->sli, &hw->eq[i], destroy_queues,
3477 free_memory);
3478
3479 /* Free rq buffers */
3480 efct_hw_rx_free(hw);
3481
3482 efct_hw_queue_teardown(hw);
3483
3484 kfree(hw->wq_cpu_array);
3485
3486 sli_teardown(&hw->sli);
3487
3488 /* record the fact that the queues are non-functional */
3489 hw->state = EFCT_HW_STATE_UNINITIALIZED;
3490
3491 /* free sequence free pool */
3492 kfree(hw->seq_pool);
3493 hw->seq_pool = NULL;
3494
3495 /* free hw_wq_callback pool */
3496 efct_hw_reqtag_pool_free(hw);
3497
3498 mempool_destroy(hw->cmd_ctx_pool);
3499 mempool_destroy(hw->mbox_rqst_pool);
3500
3501 /* Mark HW setup as not having been called */
3502 hw->hw_setup_called = false;
3503 }
3504
3505 static int
efct_hw_sli_reset(struct efct_hw * hw,enum efct_hw_reset reset,enum efct_hw_state prev_state)3506 efct_hw_sli_reset(struct efct_hw *hw, enum efct_hw_reset reset,
3507 enum efct_hw_state prev_state)
3508 {
3509 int rc = 0;
3510
3511 switch (reset) {
3512 case EFCT_HW_RESET_FUNCTION:
3513 efc_log_debug(hw->os, "issuing function level reset\n");
3514 if (sli_reset(&hw->sli)) {
3515 efc_log_err(hw->os, "sli_reset failed\n");
3516 rc = -EIO;
3517 }
3518 break;
3519 case EFCT_HW_RESET_FIRMWARE:
3520 efc_log_debug(hw->os, "issuing firmware reset\n");
3521 if (sli_fw_reset(&hw->sli)) {
3522 efc_log_err(hw->os, "sli_soft_reset failed\n");
3523 rc = -EIO;
3524 }
3525 /*
3526 * Because the FW reset leaves the FW in a non-running state,
3527 * follow that with a regular reset.
3528 */
3529 efc_log_debug(hw->os, "issuing function level reset\n");
3530 if (sli_reset(&hw->sli)) {
3531 efc_log_err(hw->os, "sli_reset failed\n");
3532 rc = -EIO;
3533 }
3534 break;
3535 default:
3536 efc_log_err(hw->os, "unknown type - no reset performed\n");
3537 hw->state = prev_state;
3538 rc = -EINVAL;
3539 break;
3540 }
3541
3542 return rc;
3543 }
3544
3545 int
efct_hw_reset(struct efct_hw * hw,enum efct_hw_reset reset)3546 efct_hw_reset(struct efct_hw *hw, enum efct_hw_reset reset)
3547 {
3548 int rc = 0;
3549 enum efct_hw_state prev_state = hw->state;
3550
3551 if (hw->state != EFCT_HW_STATE_ACTIVE)
3552 efc_log_debug(hw->os,
3553 "HW state %d is not active\n", hw->state);
3554
3555 hw->state = EFCT_HW_STATE_RESET_IN_PROGRESS;
3556
3557 /*
3558 * If the prev_state is already reset/teardown in progress,
3559 * don't continue further
3560 */
3561 if (prev_state == EFCT_HW_STATE_RESET_IN_PROGRESS ||
3562 prev_state == EFCT_HW_STATE_TEARDOWN_IN_PROGRESS)
3563 return efct_hw_sli_reset(hw, reset, prev_state);
3564
3565 if (prev_state != EFCT_HW_STATE_UNINITIALIZED) {
3566 efct_hw_flush(hw);
3567
3568 if (list_empty(&hw->cmd_head))
3569 efc_log_debug(hw->os,
3570 "All commands completed on MQ queue\n");
3571 else
3572 efc_log_err(hw->os,
3573 "Some commands still pending on MQ queue\n");
3574 }
3575
3576 /* Reset the chip */
3577 rc = efct_hw_sli_reset(hw, reset, prev_state);
3578 if (rc == -EINVAL)
3579 return -EIO;
3580
3581 return rc;
3582 }
3583