1 /*
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: Fast Path Operators
29 */
30
31 #include <linux/interrupt.h>
32 #include <linux/spinlock.h>
33 #include <linux/sched.h>
34 #include <linux/slab.h>
35 #include <linux/pci.h>
36 #include <linux/delay.h>
37 #include <linux/if_ether.h>
38 #include <linux/hardirq.h>
39 #include <rdma/ib_mad.h>
40
41 #include "hsi_struct_def.h"
42 #include "qplib_tlv.h"
43 #include "qplib_res.h"
44 #include "qplib_rcfw.h"
45 #include "qplib_sp.h"
46 #include "qplib_fp.h"
47 #include "ib_verbs.h"
48
49 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
50
bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp * qp)51 static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
52 {
53 qp->sq.condition = false;
54 qp->sq.legacy_send_phantom = false;
55 qp->sq.single = false;
56 }
57
__bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp * qp)58 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
59 {
60 struct bnxt_qplib_cq *scq, *rcq;
61
62 scq = qp->scq;
63 rcq = qp->rcq;
64
65 if (!qp->sq.flushed) {
66 dev_dbg(&scq->hwq.pdev->dev,
67 "QPLIB: FP: Adding to SQ Flush list = %p\n",
68 qp);
69 bnxt_re_legacy_cancel_phantom_processing(qp);
70 list_add_tail(&qp->sq_flush, &scq->sqf_head);
71 qp->sq.flushed = true;
72 }
73 if (!qp->srq) {
74 if (!qp->rq.flushed) {
75 dev_dbg(&rcq->hwq.pdev->dev,
76 "QPLIB: FP: Adding to RQ Flush list = %p\n",
77 qp);
78 list_add_tail(&qp->rq_flush, &rcq->rqf_head);
79 qp->rq.flushed = true;
80 }
81 }
82 }
83
bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp * qp)84 static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp)
85 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
86 {
87 /* Interrupts are already disabled in calling functions */
88 spin_lock(&qp->scq->flush_lock);
89 if (qp->scq == qp->rcq)
90 __acquire(&qp->rcq->flush_lock);
91 else
92 spin_lock(&qp->rcq->flush_lock);
93 }
94
bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp * qp)95 static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp)
96 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
97 {
98 if (qp->scq == qp->rcq)
99 __release(&qp->rcq->flush_lock);
100 else
101 spin_unlock(&qp->rcq->flush_lock);
102 spin_unlock(&qp->scq->flush_lock);
103 }
104
bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp * qp)105 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
106 {
107
108 bnxt_qplib_acquire_cq_flush_locks(qp);
109 __bnxt_qplib_add_flush_qp(qp);
110 bnxt_qplib_release_cq_flush_locks(qp);
111 }
112
__bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp * qp)113 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
114 {
115 if (qp->sq.flushed) {
116 qp->sq.flushed = false;
117 list_del(&qp->sq_flush);
118 }
119 if (!qp->srq) {
120 if (qp->rq.flushed) {
121 qp->rq.flushed = false;
122 list_del(&qp->rq_flush);
123 }
124 }
125 }
126
bnxt_qplib_clean_qp(struct bnxt_qplib_qp * qp)127 void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
128 {
129
130 bnxt_qplib_acquire_cq_flush_locks(qp);
131 __clean_cq(qp->scq, (u64)(unsigned long)qp);
132 qp->sq.hwq.prod = 0;
133 qp->sq.hwq.cons = 0;
134 qp->sq.swq_start = 0;
135 qp->sq.swq_last = 0;
136 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
137 qp->rq.hwq.prod = 0;
138 qp->rq.hwq.cons = 0;
139 qp->rq.swq_start = 0;
140 qp->rq.swq_last = 0;
141
142 __bnxt_qplib_del_flush_qp(qp);
143 bnxt_qplib_release_cq_flush_locks(qp);
144 }
145
bnxt_qpn_cqn_sched_task(struct work_struct * work)146 static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
147 {
148 struct bnxt_qplib_nq_work *nq_work =
149 container_of(work, struct bnxt_qplib_nq_work, work);
150
151 struct bnxt_qplib_cq *cq = nq_work->cq;
152 struct bnxt_qplib_nq *nq = nq_work->nq;
153
154 if (cq && nq) {
155 spin_lock_bh(&cq->compl_lock);
156 if (nq->cqn_handler) {
157 dev_dbg(&nq->res->pdev->dev,
158 "%s:Trigger cq = %p event nq = %p\n",
159 __func__, cq, nq);
160 nq->cqn_handler(nq, cq);
161 }
162 spin_unlock_bh(&cq->compl_lock);
163 }
164 kfree(nq_work);
165 }
166
bnxt_qplib_put_hdr_buf(struct pci_dev * pdev,struct bnxt_qplib_hdrbuf * buf)167 static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev,
168 struct bnxt_qplib_hdrbuf *buf)
169 {
170 dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map);
171 kfree(buf);
172 }
173
bnxt_qplib_get_hdr_buf(struct pci_dev * pdev,u32 step,u32 cnt)174 static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt)
175 {
176 struct bnxt_qplib_hdrbuf *hdrbuf;
177 u32 len;
178
179 hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL);
180 if (!hdrbuf)
181 return NULL;
182
183 len = ALIGN((step * cnt), PAGE_SIZE);
184 hdrbuf->va = dma_alloc_coherent(&pdev->dev, len,
185 &hdrbuf->dma_map, GFP_KERNEL);
186 if (!hdrbuf->va)
187 goto out;
188
189 hdrbuf->len = len;
190 hdrbuf->step = step;
191 return hdrbuf;
192 out:
193 kfree(hdrbuf);
194 return NULL;
195 }
196
bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)197 void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res,
198 struct bnxt_qplib_qp *qp)
199 {
200 if (qp->rq_hdr_buf) {
201 bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf);
202 qp->rq_hdr_buf = NULL;
203 }
204
205 if (qp->sq_hdr_buf) {
206 bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf);
207 qp->sq_hdr_buf = NULL;
208 }
209 }
210
bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp,u32 sstep,u32 rstep)211 int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res,
212 struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep)
213 {
214 struct pci_dev *pdev;
215 int rc = 0;
216
217 pdev = res->pdev;
218 if (sstep) {
219 qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep,
220 qp->sq.max_wqe);
221 if (!qp->sq_hdr_buf) {
222 dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n");
223 return -ENOMEM;
224 }
225 }
226
227 if (rstep) {
228 qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep,
229 qp->rq.max_wqe);
230 if (!qp->rq_hdr_buf) {
231 rc = -ENOMEM;
232 dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n");
233 goto fail;
234 }
235 }
236
237 return 0;
238 fail:
239 bnxt_qplib_free_hdr_buf(res, qp);
240 return rc;
241 }
242
243 /*
244 * clean_nq - Invalidate cqe from given nq.
245 * @cq - Completion queue
246 *
247 * Traverse whole notification queue and invalidate any completion
248 * associated cq handler provided by caller.
249 * Note - This function traverse the hardware queue but do not update
250 * consumer index. Invalidated cqe(marked from this function) will be
251 * ignored from actual completion of notification queue.
252 */
clean_nq(struct bnxt_qplib_cq * cq)253 static void clean_nq(struct bnxt_qplib_cq *cq)
254 {
255 struct bnxt_qplib_hwq *nq_hwq = NULL;
256 struct bnxt_qplib_nq *nq = NULL;
257 struct nq_base *hw_nqe = NULL;
258 struct nq_cn *nqcne = NULL;
259 u32 peek_flags, peek_cons;
260 u64 q_handle;
261 u32 type;
262 int i;
263
264 nq = cq->nq;
265 nq_hwq = &nq->hwq;
266
267 spin_lock_bh(&nq_hwq->lock);
268 peek_flags = nq->nq_db.dbinfo.flags;
269 peek_cons = nq_hwq->cons;
270 for (i = 0; i < nq_hwq->max_elements; i++) {
271 hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL);
272 if (!NQE_CMP_VALID(hw_nqe, peek_flags))
273 break;
274
275 /* The valid test of the entry must be done first
276 * before reading any further.
277 */
278 dma_rmb();
279 type = le16_to_cpu(hw_nqe->info10_type) &
280 NQ_BASE_TYPE_MASK;
281
282 /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */
283 if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) {
284 nqcne = (struct nq_cn *)hw_nqe;
285
286 q_handle = le32_to_cpu(nqcne->cq_handle_low);
287 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
288 if (q_handle == (u64)cq) {
289 nqcne->cq_handle_low = 0;
290 nqcne->cq_handle_high = 0;
291 cq->cnq_events++;
292 }
293 }
294 bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons,
295 1, &peek_flags);
296 }
297 spin_unlock_bh(&nq_hwq->lock);
298 }
299
300 /*
301 * Wait for receiving all NQEs for this CQ.
302 * clean_nq is tried 100 times, each time clean_cq
303 * loops upto budget times. budget is based on the
304 * number of CQs shared by that NQ. So any NQE from
305 * CQ would be already in the NQ.
306 */
__wait_for_all_nqes(struct bnxt_qplib_cq * cq,u16 cnq_events)307 static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
308 {
309 u32 retry_cnt = 100;
310 u16 total_events;
311
312 if (!cnq_events) {
313 clean_nq(cq);
314 return;
315 }
316 while (retry_cnt--) {
317 total_events = cq->cnq_events;
318
319 /* Increment total_events by 1 if any CREQ event received with CQ notification */
320 if (cq->is_cq_err_event)
321 total_events++;
322
323 if (cnq_events == total_events) {
324 dev_dbg(&cq->nq->res->pdev->dev,
325 "QPLIB: NQ cleanup - Received all NQ events\n");
326 return;
327 }
328 msleep(1);
329 clean_nq(cq);
330 }
331 }
332
bnxt_qplib_service_nq(unsigned long data)333 static void bnxt_qplib_service_nq(unsigned long data)
334 {
335 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
336 struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
337 int budget = nq->budget;
338 struct bnxt_qplib_res *res;
339 struct bnxt_qplib_cq *cq;
340 struct pci_dev *pdev;
341 struct nq_base *nqe;
342 u32 hw_polled = 0;
343 u64 q_handle;
344 u32 type;
345
346 res = nq->res;
347 pdev = res->pdev;
348
349 spin_lock_bh(&nq_hwq->lock);
350 /* Service the NQ until empty or budget expired */
351 while (budget--) {
352 nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
353 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
354 break;
355 /* The valid test of the entry must be done first before
356 * reading any further.
357 */
358 dma_rmb();
359 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
360 switch (type) {
361 case NQ_BASE_TYPE_CQ_NOTIFICATION:
362 {
363 struct nq_cn *nqcne = (struct nq_cn *)nqe;
364
365 q_handle = le32_to_cpu(nqcne->cq_handle_low);
366 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
367 cq = (struct bnxt_qplib_cq *)q_handle;
368 if (!cq)
369 break;
370 cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT;
371 cq->dbinfo.toggle = cq->toggle;
372 bnxt_qplib_armen_db(&cq->dbinfo,
373 DBC_DBC_TYPE_CQ_ARMENA);
374 spin_lock_bh(&cq->compl_lock);
375 atomic_set(&cq->arm_state, 0) ;
376 if (!nq->cqn_handler(nq, (cq)))
377 nq->stats.num_cqne_processed++;
378 else
379 dev_warn(&pdev->dev,
380 "QPLIB: cqn - type 0x%x not handled\n",
381 type);
382 cq->cnq_events++;
383 spin_unlock_bh(&cq->compl_lock);
384 break;
385 }
386 case NQ_BASE_TYPE_SRQ_EVENT:
387 {
388 struct bnxt_qplib_srq *srq;
389 struct nq_srq_event *nqsrqe =
390 (struct nq_srq_event *)nqe;
391
392 q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
393 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32;
394 srq = (struct bnxt_qplib_srq *)q_handle;
395 bnxt_qplib_armen_db(&srq->dbinfo,
396 DBC_DBC_TYPE_SRQ_ARMENA);
397 if (!nq->srqn_handler(nq,
398 (struct bnxt_qplib_srq *)q_handle,
399 nqsrqe->event))
400 nq->stats.num_srqne_processed++;
401 else
402 dev_warn(&pdev->dev,
403 "QPLIB: SRQ event 0x%x not handled\n",
404 nqsrqe->event);
405 break;
406 }
407 default:
408 dev_warn(&pdev->dev,
409 "QPLIB: nqe with opcode = 0x%x not handled\n",
410 type);
411 break;
412 }
413 hw_polled++;
414 bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons,
415 1, &nq->nq_db.dbinfo.flags);
416 }
417 nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
418 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) {
419 nq->stats.num_nq_rearm++;
420 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
421 } else if (nq->requested) {
422 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
423 nq->stats.num_tasklet_resched++;
424 }
425 dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n");
426 if (hw_polled >= 0)
427 dev_dbg(&pdev->dev,
428 "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n",
429 nq->stats.num_cqne_processed, nq->stats.num_srqne_processed,
430 nq->stats.num_dbqne_processed, budget, hw_polled);
431 dev_dbg(&pdev->dev,
432 "QPLIB: resched_cnt = %llu arm_count = %llu\n",
433 nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm);
434 spin_unlock_bh(&nq_hwq->lock);
435 }
436
bnxt_qplib_nq_irq(int irq,void * dev_instance)437 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
438 {
439 struct bnxt_qplib_nq *nq = dev_instance;
440 struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
441 u32 sw_cons;
442
443 /* Prefetch the NQ element */
444 sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq);
445 if (sw_cons >= 0)
446 prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL));
447
448 bnxt_qplib_service_nq((unsigned long)nq);
449
450 return IRQ_HANDLED;
451 }
452
bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq * nq,bool kill)453 void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
454 {
455 struct bnxt_qplib_res *res;
456
457 if (!nq->requested)
458 return;
459
460 nq->requested = false;
461 res = nq->res;
462 /* Mask h/w interrupt */
463 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false);
464 /* Sync with last running IRQ handler */
465 synchronize_irq(nq->msix_vec);
466 free_irq(nq->msix_vec, nq);
467 kfree(nq->name);
468 nq->name = NULL;
469 }
470
bnxt_qplib_disable_nq(struct bnxt_qplib_nq * nq)471 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
472 {
473 if (nq->cqn_wq) {
474 destroy_workqueue(nq->cqn_wq);
475 nq->cqn_wq = NULL;
476 }
477 /* Make sure the HW is stopped! */
478 bnxt_qplib_nq_stop_irq(nq, true);
479
480 nq->nq_db.reg.bar_reg = NULL;
481 nq->nq_db.db = NULL;
482
483 nq->cqn_handler = NULL;
484 nq->srqn_handler = NULL;
485 nq->msix_vec = 0;
486 }
487
bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq * nq,int nq_indx,int msix_vector,bool need_init)488 int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
489 int msix_vector, bool need_init)
490 {
491 struct bnxt_qplib_res *res;
492 int rc;
493
494 res = nq->res;
495 if (nq->requested)
496 return -EFAULT;
497
498 nq->msix_vec = msix_vector;
499 nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n",
500 nq_indx, pci_name(res->pdev));
501 if (!nq->name)
502 return -ENOMEM;
503 rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
504 if (rc) {
505 kfree(nq->name);
506 nq->name = NULL;
507 return rc;
508 }
509 nq->requested = true;
510 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
511
512 return rc;
513 }
514
bnxt_qplib_map_nq_db(struct bnxt_qplib_nq * nq,u32 reg_offt)515 static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt)
516 {
517 struct bnxt_qplib_reg_desc *dbreg;
518 struct bnxt_qplib_nq_db *nq_db;
519 struct bnxt_qplib_res *res;
520
521 nq_db = &nq->nq_db;
522 res = nq->res;
523 dbreg = &res->dpi_tbl.ucreg;
524
525 nq_db->reg.bar_id = dbreg->bar_id;
526 nq_db->reg.bar_base = dbreg->bar_base;
527 nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
528 nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
529 sizeof(u32);
530
531 nq_db->dbinfo.db = nq_db->reg.bar_reg;
532 nq_db->dbinfo.hwq = &nq->hwq;
533 nq_db->dbinfo.xid = nq->ring_id;
534 nq_db->dbinfo.seed = nq->ring_id;
535 nq_db->dbinfo.flags = 0;
536 spin_lock_init(&nq_db->dbinfo.lock);
537 nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
538 nq_db->dbinfo.res = nq->res;
539
540 return;
541 }
542
bnxt_qplib_enable_nq(struct bnxt_qplib_nq * nq,int nq_idx,int msix_vector,int bar_reg_offset,cqn_handler_t cqn_handler,srqn_handler_t srqn_handler)543 int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx,
544 int msix_vector, int bar_reg_offset,
545 cqn_handler_t cqn_handler,
546 srqn_handler_t srqn_handler)
547 {
548 struct pci_dev *pdev;
549 int rc;
550
551 pdev = nq->res->pdev;
552 nq->cqn_handler = cqn_handler;
553 nq->srqn_handler = srqn_handler;
554 nq->load = 0;
555 mutex_init(&nq->lock);
556
557 /* Have a task to schedule CQ notifiers in post send case */
558 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n");
559 if (!nq->cqn_wq)
560 return -ENOMEM;
561
562 bnxt_qplib_map_nq_db(nq, bar_reg_offset);
563 rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
564 if (rc) {
565 dev_err(&pdev->dev,
566 "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx);
567 goto fail;
568 }
569 dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements);
570
571 return 0;
572 fail:
573 bnxt_qplib_disable_nq(nq);
574 return rc;
575 }
576
bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq * nq)577 void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq)
578 {
579 if (nq->hwq.max_elements) {
580 bnxt_qplib_free_hwq(nq->res, &nq->hwq);
581 nq->hwq.max_elements = 0;
582 }
583 }
584
bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res * res,struct bnxt_qplib_nq * nq)585 int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res,
586 struct bnxt_qplib_nq *nq)
587 {
588 struct bnxt_qplib_hwq_attr hwq_attr = {};
589 struct bnxt_qplib_sg_info sginfo = {};
590
591 nq->res = res;
592 if (!nq->hwq.max_elements ||
593 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
594 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
595
596 sginfo.pgsize = PAGE_SIZE;
597 sginfo.pgshft = PAGE_SHIFT;
598 hwq_attr.res = res;
599 hwq_attr.sginfo = &sginfo;
600 hwq_attr.depth = nq->hwq.max_elements;
601 hwq_attr.stride = sizeof(struct nq_base);
602 hwq_attr.type = _get_hwq_type(res);
603 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
604 dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n");
605 return -ENOMEM;
606 }
607 nq->budget = 8;
608 return 0;
609 }
610
611 /* SRQ */
__qplib_destroy_srq(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_srq * srq)612 static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw,
613 struct bnxt_qplib_srq *srq)
614 {
615 struct creq_destroy_srq_resp resp = {};
616 struct bnxt_qplib_cmdqmsg msg = {};
617 struct cmdq_destroy_srq req = {};
618 /* Configure the request */
619 req.srq_cid = cpu_to_le32(srq->id);
620 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ,
621 sizeof(req));
622 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
623 sizeof(resp), 0);
624 return bnxt_qplib_rcfw_send_message(rcfw, &msg);
625 }
626
bnxt_qplib_destroy_srq(struct bnxt_qplib_res * res,struct bnxt_qplib_srq * srq)627 int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
628 struct bnxt_qplib_srq *srq)
629 {
630 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
631 int rc;
632
633 rc = __qplib_destroy_srq(rcfw, srq);
634 if (rc)
635 return rc;
636 bnxt_qplib_free_hwq(res, &srq->hwq);
637 kfree(srq->swq);
638 return 0;
639 }
640
bnxt_qplib_create_srq(struct bnxt_qplib_res * res,struct bnxt_qplib_srq * srq)641 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
642 struct bnxt_qplib_srq *srq)
643 {
644 struct bnxt_qplib_hwq_attr hwq_attr = {};
645 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
646 struct creq_create_srq_resp resp = {};
647 struct bnxt_qplib_cmdqmsg msg = {};
648 struct cmdq_create_srq req = {};
649 u16 pg_sz_lvl = 0;
650 u16 srq_size;
651 int rc, idx;
652
653 hwq_attr.res = res;
654 hwq_attr.sginfo = &srq->sginfo;
655 hwq_attr.depth = srq->max_wqe;
656 hwq_attr.stride = srq->wqe_size;
657 hwq_attr.type = HWQ_TYPE_QUEUE;
658 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
659 if (rc)
660 goto exit;
661 /* Configure the request */
662 req.dpi = cpu_to_le32(srq->dpi->dpi);
663 req.srq_handle = cpu_to_le64((uintptr_t)srq);
664 srq_size = min_t(u32, srq->hwq.depth, U16_MAX);
665 req.srq_size = cpu_to_le16(srq_size);
666 pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) <<
667 CMDQ_CREATE_SRQ_PG_SIZE_SFT);
668 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK);
669 req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
670 req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq));
671 req.pd_id = cpu_to_le32(srq->pd->id);
672 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
673 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ,
674 sizeof(req));
675 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
676 sizeof(resp), 0);
677 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
678 if (rc)
679 goto fail;
680 if (!srq->is_user) {
681 srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq),
682 GFP_KERNEL);
683 if (!srq->swq)
684 goto srq_fail;
685 srq->start_idx = 0;
686 srq->last_idx = srq->hwq.depth - 1;
687 for (idx = 0; idx < srq->hwq.depth; idx++)
688 srq->swq[idx].next_idx = idx + 1;
689 srq->swq[srq->last_idx].next_idx = -1;
690 }
691
692 spin_lock_init(&srq->lock);
693 srq->id = le32_to_cpu(resp.xid);
694 srq->cctx = res->cctx;
695 srq->dbinfo.hwq = &srq->hwq;
696 srq->dbinfo.xid = srq->id;
697 srq->dbinfo.db = srq->dpi->dbr;
698 srq->dbinfo.max_slot = 1;
699 srq->dbinfo.priv_db = res->dpi_tbl.priv_db;
700 srq->dbinfo.flags = 0;
701 spin_lock_init(&srq->dbinfo.lock);
702 srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
703 srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
704 srq->dbinfo.res = res;
705 srq->dbinfo.seed = srq->id;
706 if (srq->threshold)
707 bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
708 srq->arm_req = false;
709 return 0;
710 srq_fail:
711 __qplib_destroy_srq(rcfw, srq);
712 fail:
713 bnxt_qplib_free_hwq(res, &srq->hwq);
714 exit:
715 return rc;
716 }
717
bnxt_qplib_modify_srq(struct bnxt_qplib_res * res,struct bnxt_qplib_srq * srq)718 int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
719 struct bnxt_qplib_srq *srq)
720 {
721 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
722 u32 avail = 0;
723
724 avail = __bnxt_qplib_get_avail(srq_hwq);
725 if (avail <= srq->threshold) {
726 srq->arm_req = false;
727 bnxt_qplib_srq_arm_db(&srq->dbinfo);
728 } else {
729 /* Deferred arming */
730 srq->arm_req = true;
731 }
732 return 0;
733 }
734
bnxt_qplib_query_srq(struct bnxt_qplib_res * res,struct bnxt_qplib_srq * srq)735 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
736 struct bnxt_qplib_srq *srq)
737 {
738 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
739 struct creq_query_srq_resp resp = {};
740 struct bnxt_qplib_cmdqmsg msg = {};
741 struct creq_query_srq_resp_sb *sb;
742 struct bnxt_qplib_rcfw_sbuf sbuf;
743 struct cmdq_query_srq req = {};
744 int rc = 0;
745
746 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ,
747 sizeof(req));
748 sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
749 sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
750 &sbuf.dma_addr, GFP_KERNEL);
751 if (!sbuf.sb)
752 return -ENOMEM;
753 req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
754 req.srq_cid = cpu_to_le32(srq->id);
755 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
756 sizeof(resp), 0);
757 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
758 /* TODO: What to do with the query? */
759 dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
760 sbuf.sb, sbuf.dma_addr);
761
762 return rc;
763 }
764
bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq * srq,struct bnxt_qplib_swqe * wqe)765 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
766 struct bnxt_qplib_swqe *wqe)
767 {
768 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
769 struct sq_sge *hw_sge;
770 struct rq_wqe *srqe;
771 int i, rc = 0, next;
772 u32 avail;
773
774 spin_lock(&srq_hwq->lock);
775 if (srq->start_idx == srq->last_idx) {
776 dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n",
777 srq->id);
778 rc = -EINVAL;
779 spin_unlock(&srq_hwq->lock);
780 goto done;
781 }
782 next = srq->start_idx;
783 srq->start_idx = srq->swq[next].next_idx;
784 spin_unlock(&srq_hwq->lock);
785
786 srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL);
787 memset(srqe, 0, srq->wqe_size);
788 /* Calculate wqe_size and data_len */
789 for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
790 i < wqe->num_sge; i++, hw_sge++) {
791 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
792 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
793 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
794 }
795 srqe->wqe_type = wqe->type;
796 srqe->flags = wqe->flags;
797 srqe->wqe_size = wqe->num_sge +
798 ((offsetof(typeof(*srqe), data) + 15) >> 4);
799 if (!wqe->num_sge)
800 srqe->wqe_size++;
801 srqe->wr_id |= cpu_to_le32((u32)next);
802 srq->swq[next].wr_id = wqe->wr_id;
803 bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot);
804 /* retaining srq_hwq->cons for this logic actually the lock is only
805 * required to read srq_hwq->cons.
806 */
807 spin_lock(&srq_hwq->lock);
808 avail = __bnxt_qplib_get_avail(srq_hwq);
809 spin_unlock(&srq_hwq->lock);
810 /* Ring DB */
811 bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
812 if (srq->arm_req && avail <= srq->threshold) {
813 srq->arm_req = false;
814 bnxt_qplib_srq_arm_db(&srq->dbinfo);
815 }
816 done:
817 return rc;
818 }
819
820 /* QP */
__qplib_destroy_qp(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_qp * qp)821 static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw,
822 struct bnxt_qplib_qp *qp)
823 {
824 struct creq_destroy_qp_resp resp = {};
825 struct bnxt_qplib_cmdqmsg msg = {};
826 struct cmdq_destroy_qp req = {};
827
828 req.qp_cid = cpu_to_le32(qp->id);
829 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP,
830 sizeof(req));
831 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
832 sizeof(resp), 0);
833 return bnxt_qplib_rcfw_send_message(rcfw, &msg);
834 }
835
bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q * que)836 static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
837 {
838 int rc = 0;
839 int indx;
840
841 que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
842 if (!que->swq) {
843 rc = -ENOMEM;
844 goto out;
845 }
846
847 que->swq_start = 0;
848 que->swq_last = que->max_wqe - 1;
849 for (indx = 0; indx < que->max_wqe; indx++)
850 que->swq[indx].next_idx = indx + 1;
851 que->swq[que->swq_last].next_idx = 0; /* Make it circular */
852 que->swq_last = 0;
853 out:
854 return rc;
855 }
856
bnxt_qplib_get_swqe(struct bnxt_qplib_q * que,u32 * swq_idx)857 static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que,
858 u32 *swq_idx)
859 {
860 u32 idx;
861
862 idx = que->swq_start;
863 if (swq_idx)
864 *swq_idx = idx;
865 return &que->swq[idx];
866 }
867
bnxt_qplib_swq_mod_start(struct bnxt_qplib_q * que,u32 idx)868 static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
869 {
870 que->swq_start = que->swq[idx].next_idx;
871 }
872
bnxt_qplib_get_stride(void)873 static u32 bnxt_qplib_get_stride(void)
874 {
875 return sizeof(struct sq_sge);
876 }
877
bnxt_qplib_get_depth(struct bnxt_qplib_q * que)878 static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
879 {
880 u8 stride;
881
882 stride = bnxt_qplib_get_stride();
883 return (que->wqe_size * que->max_wqe) / stride;
884 }
885
_set_sq_size(struct bnxt_qplib_q * que,u8 wqe_mode)886 static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
887 {
888 /* For Variable mode supply number of 16B slots */
889 return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
890 que->max_wqe : bnxt_qplib_get_depth(que);
891 }
892
_set_sq_max_slot(u8 wqe_mode)893 static u32 _set_sq_max_slot(u8 wqe_mode)
894 {
895 /* for static mode index divisor is 8 */
896 return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
897 sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;
898 }
899
_set_rq_max_slot(struct bnxt_qplib_q * que)900 static u32 _set_rq_max_slot(struct bnxt_qplib_q *que)
901 {
902 return (que->wqe_size / sizeof(struct sq_sge));
903 }
904
bnxt_qplib_create_qp1(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)905 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
906 {
907 struct bnxt_qplib_hwq_attr hwq_attr = {};
908 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
909 struct creq_create_qp1_resp resp = {};
910 struct bnxt_qplib_cmdqmsg msg = {};
911 struct bnxt_qplib_q *sq = &qp->sq;
912 struct bnxt_qplib_q *rq = &qp->rq;
913 struct cmdq_create_qp1 req = {};
914 struct bnxt_qplib_reftbl *tbl;
915 unsigned long flag;
916 u8 pg_sz_lvl = 0;
917 u32 qp_flags = 0;
918 int rc;
919
920 /* General */
921 req.type = qp->type;
922 req.dpi = cpu_to_le32(qp->dpi->dpi);
923 req.qp_handle = cpu_to_le64(qp->qp_handle);
924 /* SQ */
925 hwq_attr.res = res;
926 hwq_attr.sginfo = &sq->sginfo;
927 hwq_attr.stride = bnxt_qplib_get_stride();
928 hwq_attr.depth = bnxt_qplib_get_depth(sq);
929 hwq_attr.type = HWQ_TYPE_QUEUE;
930 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
931 if (rc)
932 goto exit;
933
934 req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode));
935 req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
936 pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
937 CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT;
938 pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) <<
939 CMDQ_CREATE_QP1_SQ_LVL_SFT);
940 req.sq_pg_size_sq_lvl = pg_sz_lvl;
941 req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) &
942 CMDQ_CREATE_QP1_SQ_FWO_MASK) |
943 (sq->max_sge &
944 CMDQ_CREATE_QP1_SQ_SGE_MASK));
945 req.scq_cid = cpu_to_le32(qp->scq->id);
946
947 /* RQ */
948 if (!qp->srq) {
949 hwq_attr.res = res;
950 hwq_attr.sginfo = &rq->sginfo;
951 hwq_attr.stride = bnxt_qplib_get_stride();
952 hwq_attr.depth = bnxt_qplib_get_depth(rq);
953 hwq_attr.type = HWQ_TYPE_QUEUE;
954 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
955 if (rc)
956 goto fail_sq;
957 req.rq_size = cpu_to_le32(rq->max_wqe);
958 req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
959 pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
960 CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT;
961 pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
962 CMDQ_CREATE_QP1_RQ_LVL_SFT);
963 req.rq_pg_size_rq_lvl = pg_sz_lvl;
964 req.rq_fwo_rq_sge =
965 cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) &
966 CMDQ_CREATE_QP1_RQ_FWO_MASK) |
967 (rq->max_sge &
968 CMDQ_CREATE_QP1_RQ_SGE_MASK));
969 } else {
970 /* SRQ */
971 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED;
972 req.srq_cid = cpu_to_le32(qp->srq->id);
973 }
974 req.rcq_cid = cpu_to_le32(qp->rcq->id);
975
976 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
977 req.qp_flags = cpu_to_le32(qp_flags);
978 req.pd_id = cpu_to_le32(qp->pd->id);
979
980 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1,
981 sizeof(req));
982 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
983 sizeof(resp), 0);
984 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
985 if (rc)
986 goto fail_rq;
987
988 rc = bnxt_qplib_alloc_init_swq(sq);
989 if (rc)
990 goto sq_swq;
991
992 if (!qp->srq) {
993 rc = bnxt_qplib_alloc_init_swq(rq);
994 if (rc)
995 goto rq_swq;
996 }
997
998 qp->id = le32_to_cpu(resp.xid);
999 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1000 qp->cctx = res->cctx;
1001 sq->dbinfo.hwq = &sq->hwq;
1002 sq->dbinfo.xid = qp->id;
1003 sq->dbinfo.db = qp->dpi->dbr;
1004 sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
1005 sq->dbinfo.flags = 0;
1006 spin_lock_init(&sq->dbinfo.lock);
1007 sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
1008 sq->dbinfo.res = res;
1009 if (rq->max_wqe) {
1010 rq->dbinfo.hwq = &rq->hwq;
1011 rq->dbinfo.xid = qp->id;
1012 rq->dbinfo.db = qp->dpi->dbr;
1013 rq->dbinfo.max_slot = _set_rq_max_slot(rq);
1014 rq->dbinfo.flags = 0;
1015 spin_lock_init(&rq->dbinfo.lock);
1016 rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
1017 rq->dbinfo.res = res;
1018 }
1019
1020 tbl = &res->reftbl.qpref;
1021 spin_lock_irqsave(&tbl->lock, flag);
1022 tbl->rec[tbl->max].xid = qp->id;
1023 tbl->rec[tbl->max].handle = qp;
1024 spin_unlock_irqrestore(&tbl->lock, flag);
1025
1026 return 0;
1027 rq_swq:
1028 kfree(sq->swq);
1029 sq_swq:
1030 __qplib_destroy_qp(rcfw, qp);
1031 fail_rq:
1032 bnxt_qplib_free_hwq(res, &rq->hwq);
1033 fail_sq:
1034 bnxt_qplib_free_hwq(res, &sq->hwq);
1035 exit:
1036 return rc;
1037 }
1038
bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp * qp,int size)1039 static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
1040 {
1041 struct bnxt_qplib_hwq *sq_hwq;
1042 struct bnxt_qplib_q *sq;
1043 u64 fpsne, psn_pg;
1044 u16 indx_pad = 0;
1045
1046 sq = &qp->sq;
1047 sq_hwq = &sq->hwq;
1048 /* First psn entry */
1049 fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg);
1050 if (!IS_ALIGNED(fpsne, PAGE_SIZE))
1051 indx_pad = (fpsne & ~PAGE_MASK) / size;
1052 sq_hwq->pad_pgofft = indx_pad;
1053 sq_hwq->pad_pg = (u64 *)psn_pg;
1054 sq_hwq->pad_stride = size;
1055 }
1056
bnxt_qplib_create_qp(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)1057 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1058 {
1059 struct bnxt_qplib_hwq_attr hwq_attr = {};
1060 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1061 struct bnxt_qplib_sg_info sginfo = {};
1062 struct creq_create_qp_resp resp = {};
1063 struct bnxt_qplib_cmdqmsg msg = {};
1064 struct bnxt_qplib_q *sq = &qp->sq;
1065 struct bnxt_qplib_q *rq = &qp->rq;
1066 struct cmdq_create_qp req = {};
1067 struct bnxt_qplib_reftbl *tbl;
1068 struct bnxt_qplib_hwq *xrrq;
1069 int rc, req_size, psn_sz;
1070 unsigned long flag;
1071 u8 pg_sz_lvl = 0;
1072 u32 qp_flags = 0;
1073 u32 qp_idx;
1074 u16 nsge;
1075 u32 sqsz;
1076
1077 qp->cctx = res->cctx;
1078 if (res->dattr)
1079 qp->dev_cap_flags = res->dattr->dev_cap_flags;
1080 /* General */
1081 req.type = qp->type;
1082 req.dpi = cpu_to_le32(qp->dpi->dpi);
1083 req.qp_handle = cpu_to_le64(qp->qp_handle);
1084
1085 /* SQ */
1086 if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1087 psn_sz = _is_chip_gen_p5_p7(qp->cctx) ?
1088 sizeof(struct sq_psn_search_ext) :
1089 sizeof(struct sq_psn_search);
1090 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
1091 psn_sz = sizeof(struct sq_msn_search);
1092 qp->msn = 0;
1093 }
1094 } else {
1095 psn_sz = 0;
1096 }
1097
1098 hwq_attr.res = res;
1099 hwq_attr.sginfo = &sq->sginfo;
1100 hwq_attr.stride = bnxt_qplib_get_stride();
1101 hwq_attr.depth = bnxt_qplib_get_depth(sq);
1102 hwq_attr.aux_stride = psn_sz;
1103 hwq_attr.aux_depth = (psn_sz) ?
1104 _set_sq_size(sq, qp->wqe_mode) : 0;
1105 /* Update msn tbl size */
1106 if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) {
1107 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1108 hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode));
1109 else
1110 hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2;
1111 qp->msn_tbl_sz = hwq_attr.aux_depth;
1112 qp->msn = 0;
1113 }
1114 hwq_attr.type = HWQ_TYPE_QUEUE;
1115 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
1116 if (rc)
1117 goto exit;
1118
1119 sqsz = _set_sq_size(sq, qp->wqe_mode);
1120 /* 0xffff is the max sq size hw limits to */
1121 if (sqsz > BNXT_QPLIB_MAX_SQSZ) {
1122 pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz);
1123 goto fail_sq;
1124 }
1125 req.sq_size = cpu_to_le32(sqsz);
1126 req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
1127 pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
1128 CMDQ_CREATE_QP_SQ_PG_SIZE_SFT;
1129 pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<
1130 CMDQ_CREATE_QP_SQ_LVL_SFT);
1131 req.sq_pg_size_sq_lvl = pg_sz_lvl;
1132 req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) &
1133 CMDQ_CREATE_QP_SQ_FWO_MASK) |
1134 ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ?
1135 BNXT_MSN_TBLE_SGE : sq->max_sge &
1136 CMDQ_CREATE_QP_SQ_SGE_MASK));
1137 req.scq_cid = cpu_to_le32(qp->scq->id);
1138
1139 /* RQ/SRQ */
1140 if (!qp->srq) {
1141 hwq_attr.res = res;
1142 hwq_attr.sginfo = &rq->sginfo;
1143 hwq_attr.stride = bnxt_qplib_get_stride();
1144 hwq_attr.depth = bnxt_qplib_get_depth(rq);
1145 hwq_attr.aux_stride = 0;
1146 hwq_attr.aux_depth = 0;
1147 hwq_attr.type = HWQ_TYPE_QUEUE;
1148 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
1149 if (rc)
1150 goto fail_sq;
1151 req.rq_size = cpu_to_le32(rq->max_wqe);
1152 req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
1153 pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
1154 CMDQ_CREATE_QP_RQ_PG_SIZE_SFT;
1155 pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
1156 CMDQ_CREATE_QP_RQ_LVL_SFT);
1157 req.rq_pg_size_rq_lvl = pg_sz_lvl;
1158 nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1159 res->dattr->max_qp_sges : rq->max_sge;
1160 req.rq_fwo_rq_sge =
1161 cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) &
1162 CMDQ_CREATE_QP_RQ_FWO_MASK) |
1163 (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK));
1164 } else {
1165 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
1166 req.srq_cid = cpu_to_le32(qp->srq->id);
1167 }
1168 req.rcq_cid = cpu_to_le32(qp->rcq->id);
1169
1170 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
1171 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
1172 if (qp->sig_type)
1173 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
1174 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1175 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
1176 if (res->cctx->modes.te_bypass)
1177 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED;
1178 if (res->dattr &&
1179 bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf))
1180 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED;
1181 req.qp_flags = cpu_to_le32(qp_flags);
1182
1183 /* ORRQ and IRRQ */
1184 if (psn_sz) {
1185 xrrq = &qp->orrq;
1186 xrrq->max_elements =
1187 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1188 req_size = xrrq->max_elements *
1189 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1190 req_size &= ~(PAGE_SIZE - 1);
1191 sginfo.pgsize = req_size;
1192 sginfo.pgshft = PAGE_SHIFT;
1193
1194 hwq_attr.res = res;
1195 hwq_attr.sginfo = &sginfo;
1196 hwq_attr.depth = xrrq->max_elements;
1197 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
1198 hwq_attr.aux_stride = 0;
1199 hwq_attr.aux_depth = 0;
1200 hwq_attr.type = HWQ_TYPE_CTX;
1201 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1202 if (rc)
1203 goto fail_rq;
1204 req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq));
1205
1206 xrrq = &qp->irrq;
1207 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
1208 qp->max_dest_rd_atomic);
1209 req_size = xrrq->max_elements *
1210 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1211 req_size &= ~(PAGE_SIZE - 1);
1212 sginfo.pgsize = req_size;
1213 hwq_attr.depth = xrrq->max_elements;
1214 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
1215 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1216 if (rc)
1217 goto fail_orrq;
1218 req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq));
1219 }
1220 req.pd_id = cpu_to_le32(qp->pd->id);
1221
1222 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP,
1223 sizeof(req));
1224 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
1225 sizeof(resp), 0);
1226 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
1227 if (rc)
1228 goto fail;
1229
1230 if (!qp->is_user) {
1231 rc = bnxt_qplib_alloc_init_swq(sq);
1232 if (rc)
1233 goto swq_sq;
1234 if (!qp->srq) {
1235 rc = bnxt_qplib_alloc_init_swq(rq);
1236 if (rc)
1237 goto swq_rq;
1238 }
1239 if (psn_sz)
1240 bnxt_qplib_init_psn_ptr(qp, psn_sz);
1241 }
1242 qp->id = le32_to_cpu(resp.xid);
1243 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1244 INIT_LIST_HEAD(&qp->sq_flush);
1245 INIT_LIST_HEAD(&qp->rq_flush);
1246
1247 sq->dbinfo.hwq = &sq->hwq;
1248 sq->dbinfo.xid = qp->id;
1249 sq->dbinfo.db = qp->dpi->dbr;
1250 sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
1251 sq->dbinfo.flags = 0;
1252 spin_lock_init(&sq->dbinfo.lock);
1253 sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
1254 sq->dbinfo.res = res;
1255 sq->dbinfo.seed = qp->id;
1256 if (rq->max_wqe) {
1257 rq->dbinfo.hwq = &rq->hwq;
1258 rq->dbinfo.xid = qp->id;
1259 rq->dbinfo.db = qp->dpi->dbr;
1260 rq->dbinfo.max_slot = _set_rq_max_slot(rq);
1261 rq->dbinfo.flags = 0;
1262 spin_lock_init(&rq->dbinfo.lock);
1263 rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
1264 rq->dbinfo.res = res;
1265 rq->dbinfo.seed = qp->id;
1266 }
1267
1268 tbl = &res->reftbl.qpref;
1269 qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
1270 spin_lock_irqsave(&tbl->lock, flag);
1271 tbl->rec[qp_idx].xid = qp->id;
1272 tbl->rec[qp_idx].handle = qp;
1273 spin_unlock_irqrestore(&tbl->lock, flag);
1274
1275 return 0;
1276 swq_rq:
1277 kfree(sq->swq);
1278 swq_sq:
1279 __qplib_destroy_qp(rcfw, qp);
1280 fail:
1281 bnxt_qplib_free_hwq(res, &qp->irrq);
1282 fail_orrq:
1283 bnxt_qplib_free_hwq(res, &qp->orrq);
1284 fail_rq:
1285 bnxt_qplib_free_hwq(res, &rq->hwq);
1286 fail_sq:
1287 bnxt_qplib_free_hwq(res, &sq->hwq);
1288 exit:
1289 return rc;
1290 }
1291
__filter_modify_flags(struct bnxt_qplib_qp * qp)1292 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
1293 {
1294 switch (qp->cur_qp_state) {
1295 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1296 switch (qp->state) {
1297 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1298 break;
1299 default:
1300 break;
1301 }
1302 break;
1303 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1304 switch (qp->state) {
1305 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1306 if (!(qp->modify_flags &
1307 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
1308 qp->modify_flags |=
1309 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1310 qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1311 }
1312 qp->modify_flags &=
1313 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1314 /* Bono FW requires the max_dest_rd_atomic to be >= 1 */
1315 if (qp->max_dest_rd_atomic < 1)
1316 qp->max_dest_rd_atomic = 1;
1317 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
1318 /* Bono FW 20.6.5 requires SGID_INDEX to be configured */
1319 if (!(qp->modify_flags &
1320 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
1321 qp->modify_flags |=
1322 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
1323 qp->ah.sgid_index = 0;
1324 }
1325 break;
1326 default:
1327 break;
1328 }
1329 break;
1330 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1331 switch (qp->state) {
1332 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1333 /* Bono FW requires the max_rd_atomic to be >= 1 */
1334 if (qp->max_rd_atomic < 1)
1335 qp->max_rd_atomic = 1;
1336 qp->modify_flags &=
1337 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
1338 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1339 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1340 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1341 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1342 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1343 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1344 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
1345 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
1346 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
1347 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
1348 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
1349 break;
1350 default:
1351 break;
1352 }
1353 break;
1354 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1355 break;
1356 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1357 break;
1358 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1359 break;
1360 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1361 break;
1362 default:
1363 break;
1364 }
1365 }
1366
bnxt_qplib_modify_qp(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)1367 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1368 {
1369 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1370 struct creq_modify_qp_resp resp = {};
1371 struct bnxt_qplib_cmdqmsg msg = {};
1372 struct cmdq_modify_qp req = {};
1373 bool ppp_requested = false;
1374 u32 temp32[4];
1375 u32 bmask;
1376 int rc;
1377
1378 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP,
1379 sizeof(req));
1380
1381 /* Filter out the qp_attr_mask based on the state->new transition */
1382 __filter_modify_flags(qp);
1383 bmask = qp->modify_flags;
1384 req.modify_mask = cpu_to_le32(qp->modify_flags);
1385 req.qp_cid = cpu_to_le32(qp->id);
1386 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
1387 req.network_type_en_sqd_async_notify_new_state =
1388 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
1389 (qp->en_sqd_async_notify == true ?
1390 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
1391 if (__can_request_ppp(qp)) {
1392 req.path_mtu_pingpong_push_enable =
1393 CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE;
1394 req.pingpong_push_dpi = qp->ppp.dpi;
1395 ppp_requested = true;
1396 }
1397 }
1398 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
1399
1400 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) {
1401 req.access = qp->access;
1402 }
1403 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)
1404 req.pkey = IB_DEFAULT_PKEY_FULL;
1405
1406 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) {
1407 req.qkey = cpu_to_le32(qp->qkey);
1408 }
1409 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
1410 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
1411 req.dgid[0] = cpu_to_le32(temp32[0]);
1412 req.dgid[1] = cpu_to_le32(temp32[1]);
1413 req.dgid[2] = cpu_to_le32(temp32[2]);
1414 req.dgid[3] = cpu_to_le32(temp32[3]);
1415 }
1416 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) {
1417 req.flow_label = cpu_to_le32(qp->ah.flow_label);
1418 }
1419 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) {
1420 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]);
1421 }
1422 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) {
1423 req.hop_limit = qp->ah.hop_limit;
1424 }
1425 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) {
1426 req.traffic_class = qp->ah.traffic_class;
1427 }
1428 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) {
1429 memcpy(req.dest_mac, qp->ah.dmac, 6);
1430 }
1431 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) {
1432 req.path_mtu_pingpong_push_enable = qp->path_mtu;
1433 }
1434 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) {
1435 req.timeout = qp->timeout;
1436 }
1437 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) {
1438 req.retry_cnt = qp->retry_cnt;
1439 }
1440 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) {
1441 req.rnr_retry = qp->rnr_retry;
1442 }
1443 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) {
1444 req.min_rnr_timer = qp->min_rnr_timer;
1445 }
1446 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) {
1447 req.rq_psn = cpu_to_le32(qp->rq.psn);
1448 }
1449 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) {
1450 req.sq_psn = cpu_to_le32(qp->sq.psn);
1451 }
1452 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) {
1453 req.max_rd_atomic =
1454 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1455 }
1456 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) {
1457 req.max_dest_rd_atomic =
1458 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
1459 }
1460 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
1461 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
1462 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
1463 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
1464 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
1465 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
1466 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
1467 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC)
1468 req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC);
1469 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN)
1470 req.tos_dscp_tos_ecn =
1471 ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) &
1472 CMDQ_MODIFY_QP_TOS_ECN_MASK);
1473 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP)
1474 req.tos_dscp_tos_ecn |=
1475 ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) &
1476 CMDQ_MODIFY_QP_TOS_DSCP_MASK);
1477 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
1478 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
1479 sizeof(resp), 0);
1480 msg.qp_state = qp->state;
1481
1482 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
1483 if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) {
1484 qp->cur_qp_state = qp->state;
1485 return 0;
1486 } else if (rc) {
1487 return rc;
1488 }
1489 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR)
1490 qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac);
1491
1492 if (ppp_requested)
1493 qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled;
1494
1495 qp->cur_qp_state = qp->state;
1496 return 0;
1497 }
1498
bnxt_qplib_query_qp(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)1499 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1500 {
1501 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1502 struct creq_query_qp_resp resp = {};
1503 struct bnxt_qplib_cmdqmsg msg = {};
1504 struct bnxt_qplib_rcfw_sbuf sbuf;
1505 struct creq_query_qp_resp_sb *sb;
1506 struct cmdq_query_qp req = {};
1507 u32 temp32[4];
1508 int i, rc;
1509
1510 sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
1511 sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
1512 &sbuf.dma_addr, GFP_KERNEL);
1513 if (!sbuf.sb)
1514 return -ENOMEM;
1515 sb = sbuf.sb;
1516
1517 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP,
1518 sizeof(req));
1519 req.qp_cid = cpu_to_le32(qp->id);
1520 req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
1521 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
1522 sizeof(resp), 0);
1523 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
1524 if (rc)
1525 goto bail;
1526
1527 /* Extract the context from the side buffer */
1528 qp->state = sb->en_sqd_async_notify_state &
1529 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
1530 qp->cur_qp_state = qp->state;
1531 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
1532 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
1533 true : false;
1534 qp->access = sb->access;
1535 qp->pkey_index = le16_to_cpu(sb->pkey);
1536 qp->qkey = le32_to_cpu(sb->qkey);
1537
1538 temp32[0] = le32_to_cpu(sb->dgid[0]);
1539 temp32[1] = le32_to_cpu(sb->dgid[1]);
1540 temp32[2] = le32_to_cpu(sb->dgid[2]);
1541 temp32[3] = le32_to_cpu(sb->dgid[3]);
1542 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
1543
1544 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
1545
1546 qp->ah.sgid_index = 0;
1547 for (i = 0; i < res->sgid_tbl.max; i++) {
1548 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
1549 qp->ah.sgid_index = i;
1550 break;
1551 }
1552 }
1553 if (i == res->sgid_tbl.max)
1554 dev_dbg(&res->pdev->dev,
1555 "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n",
1556 qp->id, le16_to_cpu(sb->sgid_index));
1557
1558 qp->ah.hop_limit = sb->hop_limit;
1559 qp->ah.traffic_class = sb->traffic_class;
1560 memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN);
1561 qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1562 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >>
1563 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
1564 qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1565 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK;
1566 qp->timeout = sb->timeout;
1567 qp->retry_cnt = sb->retry_cnt;
1568 qp->rnr_retry = sb->rnr_retry;
1569 qp->min_rnr_timer = sb->min_rnr_timer;
1570 qp->rq.psn = le32_to_cpu(sb->rq_psn);
1571 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
1572 qp->sq.psn = le32_to_cpu(sb->sq_psn);
1573 qp->max_dest_rd_atomic =
1574 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
1575 qp->sq.max_wqe = qp->sq.hwq.max_elements;
1576 qp->rq.max_wqe = qp->rq.hwq.max_elements;
1577 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
1578 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
1579 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
1580 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
1581 memcpy(qp->smac, sb->src_mac, ETH_ALEN);
1582 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
1583 qp->port_id = le16_to_cpu(sb->port_id);
1584 bail:
1585 dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
1586 sbuf.sb, sbuf.dma_addr);
1587 return rc;
1588 }
1589
1590
__clean_cq(struct bnxt_qplib_cq * cq,u64 qp)1591 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1592 {
1593 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1594 u32 peek_flags, peek_cons;
1595 struct cq_base *hw_cqe;
1596 int i;
1597
1598 peek_flags = cq->dbinfo.flags;
1599 peek_cons = cq_hwq->cons;
1600 for (i = 0; i < cq_hwq->depth; i++) {
1601 hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL);
1602 if (CQE_CMP_VALID(hw_cqe, peek_flags)) {
1603 dma_rmb();
1604 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
1605 case CQ_BASE_CQE_TYPE_REQ:
1606 case CQ_BASE_CQE_TYPE_TERMINAL:
1607 {
1608 struct cq_req *cqe = (struct cq_req *)hw_cqe;
1609
1610 if (qp == le64_to_cpu(cqe->qp_handle))
1611 cqe->qp_handle = 0;
1612 break;
1613 }
1614 case CQ_BASE_CQE_TYPE_RES_RC:
1615 case CQ_BASE_CQE_TYPE_RES_UD:
1616 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1617 {
1618 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1619
1620 if (qp == le64_to_cpu(cqe->qp_handle))
1621 cqe->qp_handle = 0;
1622 break;
1623 }
1624 default:
1625 break;
1626 }
1627 }
1628 bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons,
1629 1, &peek_flags);
1630 }
1631 }
1632
bnxt_qplib_destroy_qp(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)1633 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1634 struct bnxt_qplib_qp *qp)
1635 {
1636 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1637 struct bnxt_qplib_reftbl *tbl;
1638 unsigned long flags;
1639 u32 qp_idx;
1640 int rc;
1641
1642 tbl = &res->reftbl.qpref;
1643 qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
1644 spin_lock_irqsave(&tbl->lock, flags);
1645 tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID;
1646 tbl->rec[qp_idx].handle = NULL;
1647 spin_unlock_irqrestore(&tbl->lock, flags);
1648
1649 rc = __qplib_destroy_qp(rcfw, qp);
1650 if (rc) {
1651 spin_lock_irqsave(&tbl->lock, flags);
1652 tbl->rec[qp_idx].xid = qp->id;
1653 tbl->rec[qp_idx].handle = qp;
1654 spin_unlock_irqrestore(&tbl->lock, flags);
1655 return rc;
1656 }
1657
1658 return 0;
1659 }
1660
bnxt_qplib_free_qp_res(struct bnxt_qplib_res * res,struct bnxt_qplib_qp * qp)1661 void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
1662 struct bnxt_qplib_qp *qp)
1663 {
1664 if (qp->irrq.max_elements)
1665 bnxt_qplib_free_hwq(res, &qp->irrq);
1666 if (qp->orrq.max_elements)
1667 bnxt_qplib_free_hwq(res, &qp->orrq);
1668
1669 if (!qp->is_user)
1670 kfree(qp->rq.swq);
1671 bnxt_qplib_free_hwq(res, &qp->rq.hwq);
1672
1673 if (!qp->is_user)
1674 kfree(qp->sq.swq);
1675 bnxt_qplib_free_hwq(res, &qp->sq.hwq);
1676 }
1677
bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp * qp,struct bnxt_qplib_sge * sge)1678 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1679 struct bnxt_qplib_sge *sge)
1680 {
1681 struct bnxt_qplib_q *sq = &qp->sq;
1682 struct bnxt_qplib_hdrbuf *buf;
1683 u32 sw_prod;
1684
1685 memset(sge, 0, sizeof(*sge));
1686
1687 buf = qp->sq_hdr_buf;
1688 if (buf) {
1689 sw_prod = sq->swq_start;
1690 sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
1691 sge->lkey = 0xFFFFFFFF;
1692 sge->size = buf->step;
1693 return buf->va + sw_prod * sge->size;
1694 }
1695 return NULL;
1696 }
1697
bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp * qp)1698 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1699 {
1700 struct bnxt_qplib_q *rq = &qp->rq;
1701
1702 return rq->swq_start;
1703 }
1704
bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp * qp,struct bnxt_qplib_sge * sge)1705 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1706 struct bnxt_qplib_sge *sge)
1707 {
1708 struct bnxt_qplib_q *rq = &qp->rq;
1709 struct bnxt_qplib_hdrbuf *buf;
1710 u32 sw_prod;
1711
1712 memset(sge, 0, sizeof(*sge));
1713
1714 buf = qp->rq_hdr_buf;
1715 if (buf) {
1716 sw_prod = rq->swq_start;
1717 sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
1718 sge->lkey = 0xFFFFFFFF;
1719 sge->size = buf->step;
1720 return buf->va + sw_prod * sge->size;
1721 }
1722 return NULL;
1723 }
1724
1725 /* Fil the MSN table into the next psn row */
bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe,struct bnxt_qplib_swq * swq)1726 static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp,
1727 struct bnxt_qplib_swqe *wqe,
1728 struct bnxt_qplib_swq *swq)
1729 {
1730 struct sq_msn_search *msns;
1731 u32 start_psn, next_psn;
1732 u16 start_idx;
1733
1734 msns = (struct sq_msn_search *)swq->psn_search;
1735 msns->start_idx_next_psn_start_psn = 0;
1736
1737 start_psn = swq->start_psn;
1738 next_psn = swq->next_psn;
1739 start_idx = swq->slot_idx;
1740 msns->start_idx_next_psn_start_psn |=
1741 bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn);
1742 pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n",
1743 qp->msn,
1744 (u16)
1745 cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)),
1746 (u32)
1747 cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)),
1748 (u32)
1749 cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn)));
1750 qp->msn++;
1751 qp->msn %= qp->msn_tbl_sz;
1752 }
1753
bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe,struct bnxt_qplib_swq * swq)1754 static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
1755 struct bnxt_qplib_swqe *wqe,
1756 struct bnxt_qplib_swq *swq)
1757 {
1758 struct sq_psn_search_ext *psns_ext;
1759 struct sq_psn_search *psns;
1760 u32 flg_npsn;
1761 u32 op_spsn;
1762
1763 if (!swq->psn_search)
1764 return;
1765
1766 /* Handle MSN differently on cap flags */
1767 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
1768 bnxt_qplib_fill_msn_search(qp, wqe, swq);
1769 return;
1770 }
1771 psns = (struct sq_psn_search *)swq->psn_search;
1772 psns_ext = (struct sq_psn_search_ext *)swq->psn_search;
1773
1774 op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1775 SQ_PSN_SEARCH_START_PSN_MASK);
1776 op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1777 SQ_PSN_SEARCH_OPCODE_MASK);
1778 flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1779 SQ_PSN_SEARCH_NEXT_PSN_MASK);
1780
1781 if (_is_chip_gen_p5_p7(qp->cctx)) {
1782 psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
1783 psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
1784 psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
1785 } else {
1786 psns->opcode_start_psn = cpu_to_le32(op_spsn);
1787 psns->flags_next_psn = cpu_to_le32(flg_npsn);
1788 }
1789 }
1790
_calc_ilsize(struct bnxt_qplib_swqe * wqe)1791 static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe)
1792 {
1793 u16 size = 0;
1794 int indx;
1795
1796 for (indx = 0; indx < wqe->num_sge; indx++)
1797 size += wqe->sg_list[indx].size;
1798 return size;
1799 }
1800
bnxt_qplib_put_inline(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe,u32 * sw_prod)1801 static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
1802 struct bnxt_qplib_swqe *wqe,
1803 u32 *sw_prod)
1804 {
1805 struct bnxt_qplib_hwq *sq_hwq;
1806 int len, t_len, offt = 0;
1807 int t_cplen = 0, cplen;
1808 bool pull_dst = true;
1809 void *il_dst = NULL;
1810 void *il_src = NULL;
1811 int indx;
1812
1813 sq_hwq = &qp->sq.hwq;
1814 t_len = 0;
1815 for (indx = 0; indx < wqe->num_sge; indx++) {
1816 len = wqe->sg_list[indx].size;
1817 il_src = (void *)wqe->sg_list[indx].addr;
1818 t_len += len;
1819 if (t_len > qp->max_inline_data)
1820 goto bad;
1821 while (len) {
1822 if (pull_dst) {
1823 pull_dst = false;
1824 il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) %
1825 sq_hwq->depth), NULL);
1826 (*sw_prod)++;
1827 t_cplen = 0;
1828 offt = 0;
1829 }
1830 cplen = min_t(int, len, sizeof(struct sq_sge));
1831 cplen = min_t(int, cplen,
1832 (sizeof(struct sq_sge) - offt));
1833 memcpy(il_dst, il_src, cplen);
1834 t_cplen += cplen;
1835 il_src += cplen;
1836 il_dst += cplen;
1837 offt += cplen;
1838 len -= cplen;
1839 if (t_cplen == sizeof(struct sq_sge))
1840 pull_dst = true;
1841 }
1842 }
1843
1844 return t_len;
1845 bad:
1846 return -ENOMEM;
1847 }
1848
bnxt_qplib_put_sges(struct bnxt_qplib_hwq * sq_hwq,struct bnxt_qplib_sge * ssge,u32 nsge,u32 * sw_prod)1849 static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq,
1850 struct bnxt_qplib_sge *ssge,
1851 u32 nsge, u32 *sw_prod)
1852 {
1853 struct sq_sge *dsge;
1854 int indx, len = 0;
1855
1856 for (indx = 0; indx < nsge; indx++, (*sw_prod)++) {
1857 dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL);
1858 dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
1859 dsge->l_key = cpu_to_le32(ssge[indx].lkey);
1860 dsge->size = cpu_to_le32(ssge[indx].size);
1861 len += ssge[indx].size;
1862 }
1863 return len;
1864 }
1865
_calculate_wqe_byte(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe,u16 * wqe_byte)1866 static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp,
1867 struct bnxt_qplib_swqe *wqe, u16 *wqe_byte)
1868 {
1869 u16 wqe_size;
1870 u32 ilsize;
1871 u16 nsge;
1872
1873 nsge = wqe->num_sge;
1874 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1875 ilsize = _calc_ilsize(wqe);
1876 wqe_size = (ilsize > qp->max_inline_data) ?
1877 qp->max_inline_data : ilsize;
1878 wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge));
1879 } else {
1880 wqe_size = nsge * sizeof(struct sq_sge);
1881 }
1882 /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
1883 wqe_size += sizeof(struct sq_send_hdr);
1884 if (wqe_byte)
1885 *wqe_byte = wqe_size;
1886 return wqe_size / sizeof(struct sq_sge);
1887 }
1888
_translate_q_full_delta(struct bnxt_qplib_q * que,u16 wqe_bytes)1889 static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes)
1890 {
1891 /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128
1892 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.
1893 * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.
1894 * when 8916 is disabled.
1895 */
1896 return (que->q_full_delta * wqe_bytes) / que->hwq.element_size;
1897 }
1898
bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp * qp,struct bnxt_qplib_q * sq,struct bnxt_qplib_swq * swq,bool hw_retx)1899 static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq,
1900 struct bnxt_qplib_swq *swq, bool hw_retx)
1901 {
1902 struct bnxt_qplib_hwq *sq_hwq;
1903 u32 pg_num, pg_indx;
1904 void *buff;
1905 u32 tail;
1906
1907 sq_hwq = &sq->hwq;
1908 if (!sq_hwq->pad_pg)
1909 return;
1910
1911 tail = swq->slot_idx / sq->dbinfo.max_slot;
1912 if (hw_retx)
1913 tail %= qp->msn_tbl_sz;
1914 pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride);
1915 pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride);
1916 buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride);
1917 /* the start ptr for buff is same ie after the SQ */
1918 swq->psn_search = buff;
1919 }
1920
bnxt_qplib_post_send_db(struct bnxt_qplib_qp * qp)1921 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1922 {
1923 struct bnxt_qplib_q *sq = &qp->sq;
1924
1925 bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
1926 }
1927
bnxt_qplib_post_send(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe)1928 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1929 struct bnxt_qplib_swqe *wqe)
1930 {
1931 struct bnxt_qplib_nq_work *nq_work = NULL;
1932 int i, rc = 0, data_len = 0, pkt_num = 0;
1933 struct bnxt_qplib_q *sq = &qp->sq;
1934 struct bnxt_qplib_hwq *sq_hwq;
1935 struct bnxt_qplib_swq *swq;
1936 bool sch_handler = false;
1937 u16 slots_needed;
1938 void *base_hdr;
1939 void *ext_hdr;
1940 __le32 temp32;
1941 u16 qfd_slots;
1942 u8 wqe_slots;
1943 u16 wqe_size;
1944 u32 sw_prod;
1945 u32 wqe_idx;
1946
1947 sq_hwq = &sq->hwq;
1948 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
1949 qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1950 dev_err(&sq_hwq->pdev->dev,
1951 "QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
1952 qp->id, qp->state);
1953 rc = -EINVAL;
1954 goto done;
1955 }
1956
1957 wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size);
1958 slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1959 sq->dbinfo.max_slot : wqe_slots;
1960 qfd_slots = _translate_q_full_delta(sq, wqe_size);
1961 if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) {
1962 dev_err(&sq_hwq->pdev->dev,
1963 "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id);
1964 dev_err(&sq_hwq->pdev->dev,
1965 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n",
1966 HWQ_CMP(sq_hwq->prod, sq_hwq),
1967 HWQ_CMP(sq_hwq->cons, sq_hwq),
1968 sq_hwq->max_elements, qfd_slots, slots_needed);
1969 dev_err(&sq_hwq->pdev->dev,
1970 "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n",
1971 sq->phantom_wqe_cnt, sq->phantom_cqe_cnt);
1972 rc = -ENOMEM;
1973 goto done;
1974 }
1975
1976 sw_prod = sq_hwq->prod;
1977 swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
1978 swq->slot_idx = sw_prod;
1979 bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags));
1980
1981 swq->wr_id = wqe->wr_id;
1982 swq->type = wqe->type;
1983 swq->flags = wqe->flags;
1984 swq->slots = slots_needed;
1985 swq->start_psn = sq->psn & BTH_PSN_MASK;
1986 if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP)
1987 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1988
1989 dev_dbg(&sq_hwq->pdev->dev,
1990 "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n",
1991 qp->id, wqe_idx, swq->wr_id);
1992 if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1993 sch_handler = true;
1994 dev_dbg(&sq_hwq->pdev->dev,
1995 "%s Error QP. Scheduling for poll_cq\n", __func__);
1996 goto queue_err;
1997 }
1998
1999 base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL);
2000 sw_prod++;
2001 ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL);
2002 sw_prod++;
2003 memset(base_hdr, 0, sizeof(struct sq_sge));
2004 memset(ext_hdr, 0, sizeof(struct sq_sge));
2005
2006 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
2007 data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod);
2008 else
2009 data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list,
2010 wqe->num_sge, &sw_prod);
2011 if (data_len < 0)
2012 goto queue_err;
2013 /* Specifics */
2014 switch (wqe->type) {
2015 case BNXT_QPLIB_SWQE_TYPE_SEND:
2016 if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE ||
2017 qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
2018 /* Assemble info for Raw Ethertype QPs */
2019 struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
2020 struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
2021
2022 sqe->wqe_type = wqe->type;
2023 sqe->flags = wqe->flags;
2024 sqe->wqe_size = wqe_slots;
2025 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
2026 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
2027 sqe->length = cpu_to_le32(data_len);
2028 ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
2029 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
2030 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
2031
2032 dev_dbg(&sq_hwq->pdev->dev,
2033 "QPLIB: FP: RAW/QP1 Send WQE:\n"
2034 "\twqe_type = 0x%x\n"
2035 "\tflags = 0x%x\n"
2036 "\twqe_size = 0x%x\n"
2037 "\tlflags = 0x%x\n"
2038 "\tcfa_action = 0x%x\n"
2039 "\tlength = 0x%x\n"
2040 "\tcfa_meta = 0x%x\n",
2041 sqe->wqe_type, sqe->flags, sqe->wqe_size,
2042 sqe->lflags, sqe->cfa_action,
2043 sqe->length, ext_sqe->cfa_meta);
2044 break;
2045 }
2046 fallthrough;
2047 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2048 fallthrough;
2049 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
2050 {
2051 struct sq_send_hdr *sqe = base_hdr;
2052 struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
2053
2054 sqe->wqe_type = wqe->type;
2055 sqe->flags = wqe->flags;
2056 sqe->wqe_size = wqe_slots;
2057 sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
2058 if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||
2059 qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
2060 sqe->q_key = cpu_to_le32(wqe->send.q_key);
2061 sqe->length = cpu_to_le32(data_len);
2062 ext_sqe->dst_qp = cpu_to_le32(
2063 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
2064 ext_sqe->avid = cpu_to_le32(wqe->send.avid &
2065 SQ_SEND_AVID_MASK);
2066 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
2067 } else {
2068 sqe->length = cpu_to_le32(data_len);
2069 if (qp->mtu)
2070 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
2071 if (!pkt_num)
2072 pkt_num = 1;
2073 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
2074 }
2075 dev_dbg(&sq_hwq->pdev->dev,
2076 "QPLIB: FP: Send WQE:\n"
2077 "\twqe_type = 0x%x\n"
2078 "\tflags = 0x%x\n"
2079 "\twqe_size = 0x%x\n"
2080 "\tinv_key/immdata = 0x%x\n"
2081 "\tq_key = 0x%x\n"
2082 "\tdst_qp = 0x%x\n"
2083 "\tlength = 0x%x\n"
2084 "\tavid = 0x%x\n",
2085 sqe->wqe_type, sqe->flags, sqe->wqe_size,
2086 sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp,
2087 sqe->length, ext_sqe->avid);
2088 break;
2089 }
2090 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
2091 /* fall-thru */
2092 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
2093 /* fall-thru */
2094 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
2095 {
2096 struct sq_rdma_hdr *sqe = base_hdr;
2097 struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
2098
2099 sqe->wqe_type = wqe->type;
2100 sqe->flags = wqe->flags;
2101 sqe->wqe_size = wqe_slots;
2102 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
2103 sqe->length = cpu_to_le32((u32)data_len);
2104 ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
2105 ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
2106 if (qp->mtu)
2107 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
2108 if (!pkt_num)
2109 pkt_num = 1;
2110 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
2111
2112 dev_dbg(&sq_hwq->pdev->dev,
2113 "QPLIB: FP: RDMA WQE:\n"
2114 "\twqe_type = 0x%x\n"
2115 "\tflags = 0x%x\n"
2116 "\twqe_size = 0x%x\n"
2117 "\timmdata = 0x%x\n"
2118 "\tlength = 0x%x\n"
2119 "\tremote_va = 0x%llx\n"
2120 "\tremote_key = 0x%x\n",
2121 sqe->wqe_type, sqe->flags, sqe->wqe_size,
2122 sqe->imm_data, sqe->length, ext_sqe->remote_va,
2123 ext_sqe->remote_key);
2124 break;
2125 }
2126 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
2127 /* fall-thru */
2128 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
2129 {
2130 struct sq_atomic_hdr *sqe = base_hdr;
2131 struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
2132
2133 sqe->wqe_type = wqe->type;
2134 sqe->flags = wqe->flags;
2135 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
2136 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
2137 ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
2138 ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
2139 if (qp->mtu)
2140 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
2141 if (!pkt_num)
2142 pkt_num = 1;
2143 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
2144 break;
2145 }
2146 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
2147 {
2148 struct sq_localinvalidate_hdr *sqe = base_hdr;
2149
2150 sqe->wqe_type = wqe->type;
2151 sqe->flags = wqe->flags;
2152 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
2153
2154 dev_dbg(&sq_hwq->pdev->dev,
2155 "QPLIB: FP: LOCAL INV WQE:\n"
2156 "\twqe_type = 0x%x\n"
2157 "\tflags = 0x%x\n"
2158 "\tinv_l_key = 0x%x\n",
2159 sqe->wqe_type, sqe->flags, sqe->inv_l_key);
2160 break;
2161 }
2162 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
2163 {
2164 struct sq_fr_pmr_hdr *sqe = base_hdr;
2165 struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
2166
2167 sqe->wqe_type = wqe->type;
2168 sqe->flags = wqe->flags;
2169 sqe->access_cntl = wqe->frmr.access_cntl |
2170 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2171 sqe->zero_based_page_size_log =
2172 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
2173 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
2174 (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0);
2175 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
2176 /* TODO: OFED only provides length of MR up to 32-bits for FRMR */
2177 temp32 = cpu_to_le32(wqe->frmr.length);
2178 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
2179 sqe->numlevels_pbl_page_size_log =
2180 ((wqe->frmr.pbl_pg_sz_log <<
2181 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
2182 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
2183 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
2184 SQ_FR_PMR_NUMLEVELS_MASK);
2185 if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) {
2186 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]);
2187 } else {
2188 for (i = 0; i < wqe->frmr.page_list_len; i++)
2189 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
2190 wqe->frmr.page_list[i] |
2191 PTU_PTE_VALID);
2192 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
2193 }
2194 ext_sqe->va = cpu_to_le64(wqe->frmr.va);
2195 dev_dbg(&sq_hwq->pdev->dev,
2196 "QPLIB: FP: FRMR WQE:\n"
2197 "\twqe_type = 0x%x\n"
2198 "\tflags = 0x%x\n"
2199 "\taccess_cntl = 0x%x\n"
2200 "\tzero_based_page_size_log = 0x%x\n"
2201 "\tl_key = 0x%x\n"
2202 "\tlength = 0x%x\n"
2203 "\tnumlevels_pbl_page_size_log = 0x%x\n"
2204 "\tpblptr = 0x%llx\n"
2205 "\tva = 0x%llx\n",
2206 sqe->wqe_type, sqe->flags, sqe->access_cntl,
2207 sqe->zero_based_page_size_log, sqe->l_key,
2208 *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log,
2209 ext_sqe->pblptr, ext_sqe->va);
2210 break;
2211 }
2212 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
2213 {
2214 struct sq_bind_hdr *sqe = base_hdr;
2215 struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
2216
2217 sqe->wqe_type = wqe->type;
2218 sqe->flags = wqe->flags;
2219 sqe->access_cntl = wqe->bind.access_cntl;
2220 sqe->mw_type_zero_based = wqe->bind.mw_type |
2221 (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0);
2222 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
2223 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
2224 ext_sqe->va = cpu_to_le64(wqe->bind.va);
2225 ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
2226 dev_dbg(&sq_hwq->pdev->dev,
2227 "QPLIB: FP: BIND WQE:\n"
2228 "\twqe_type = 0x%x\n"
2229 "\tflags = 0x%x\n"
2230 "\taccess_cntl = 0x%x\n"
2231 "\tmw_type_zero_based = 0x%x\n"
2232 "\tparent_l_key = 0x%x\n"
2233 "\tl_key = 0x%x\n"
2234 "\tva = 0x%llx\n"
2235 "\tlength = 0x%x\n",
2236 sqe->wqe_type, sqe->flags, sqe->access_cntl,
2237 sqe->mw_type_zero_based, sqe->parent_l_key,
2238 sqe->l_key, sqe->va, ext_sqe->length_lo);
2239 break;
2240 }
2241 default:
2242 /* Bad wqe, return error */
2243 rc = -EINVAL;
2244 goto done;
2245 }
2246 swq->next_psn = sq->psn & BTH_PSN_MASK;
2247 bnxt_qplib_fill_psn_search(qp, wqe, swq);
2248
2249 queue_err:
2250 bnxt_qplib_swq_mod_start(sq, wqe_idx);
2251 bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots);
2252 qp->wqe_cnt++;
2253 done:
2254 if (sch_handler) {
2255 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
2256 if (nq_work) {
2257 nq_work->cq = qp->scq;
2258 nq_work->nq = qp->scq->nq;
2259 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
2260 queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
2261 } else {
2262 dev_err(&sq->hwq.pdev->dev,
2263 "QPLIB: FP: Failed to allocate SQ nq_work!\n");
2264 rc = -ENOMEM;
2265 }
2266 }
2267 return rc;
2268 }
2269
bnxt_qplib_post_recv_db(struct bnxt_qplib_qp * qp)2270 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
2271 {
2272 struct bnxt_qplib_q *rq = &qp->rq;
2273
2274 bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
2275 }
2276
bnxt_re_handle_cqn(struct bnxt_qplib_cq * cq)2277 void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq)
2278 {
2279 struct bnxt_qplib_nq *nq;
2280
2281 if (!(cq && cq->nq))
2282 return;
2283
2284 nq = cq->nq;
2285 spin_lock_bh(&cq->compl_lock);
2286 if (nq->cqn_handler) {
2287 dev_dbg(&nq->res->pdev->dev,
2288 "%s:Trigger cq = %p event nq = %p\n",
2289 __func__, cq, nq);
2290 nq->cqn_handler(nq, cq);
2291 }
2292 spin_unlock_bh(&cq->compl_lock);
2293 }
2294
bnxt_qplib_post_recv(struct bnxt_qplib_qp * qp,struct bnxt_qplib_swqe * wqe)2295 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
2296 struct bnxt_qplib_swqe *wqe)
2297 {
2298 struct bnxt_qplib_nq_work *nq_work = NULL;
2299 struct bnxt_qplib_q *rq = &qp->rq;
2300 struct bnxt_qplib_hwq *rq_hwq;
2301 struct bnxt_qplib_swq *swq;
2302 bool sch_handler = false;
2303 struct rq_wqe_hdr *base_hdr;
2304 struct rq_ext_hdr *ext_hdr;
2305 struct sq_sge *dsge;
2306 u8 wqe_slots;
2307 u32 wqe_idx;
2308 u32 sw_prod;
2309 int rc = 0;
2310
2311 rq_hwq = &rq->hwq;
2312 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2313 dev_err(&rq_hwq->pdev->dev,
2314 "QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
2315 qp->id, qp->state);
2316 rc = -EINVAL;
2317 goto done;
2318 }
2319
2320 wqe_slots = _calculate_wqe_byte(qp, wqe, NULL);
2321 if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) {
2322 dev_err(&rq_hwq->pdev->dev,
2323 "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id);
2324 rc = -EINVAL;
2325 goto done;
2326 }
2327
2328 swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
2329 swq->wr_id = wqe->wr_id;
2330 swq->slots = rq->dbinfo.max_slot;
2331 dev_dbg(&rq_hwq->pdev->dev,
2332 "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n",
2333 wqe_idx, swq->wr_id);
2334 if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2335 sch_handler = true;
2336 dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n",
2337 __func__);
2338 goto queue_err;
2339 }
2340
2341 sw_prod = rq_hwq->prod;
2342 base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL);
2343 sw_prod++;
2344 ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
2345 sw_prod++;
2346 memset(base_hdr, 0, sizeof(struct sq_sge));
2347 memset(ext_hdr, 0, sizeof(struct sq_sge));
2348
2349 if (!wqe->num_sge) {
2350 dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
2351 dsge->size = 0;
2352 wqe_slots++;
2353 } else {
2354 bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod);
2355 }
2356 base_hdr->wqe_type = wqe->type;
2357 base_hdr->flags = wqe->flags;
2358 base_hdr->wqe_size = wqe_slots;
2359 base_hdr->wr_id |= cpu_to_le32(wqe_idx);
2360 queue_err:
2361 bnxt_qplib_swq_mod_start(rq, wqe_idx);
2362 bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots);
2363 done:
2364 if (sch_handler) {
2365 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
2366 if (nq_work) {
2367 nq_work->cq = qp->rcq;
2368 nq_work->nq = qp->rcq->nq;
2369 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
2370 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
2371 } else {
2372 dev_err(&rq->hwq.pdev->dev,
2373 "QPLIB: FP: Failed to allocate RQ nq_work!\n");
2374 rc = -ENOMEM;
2375 }
2376 }
2377 return rc;
2378 }
2379
2380 /* CQ */
bnxt_qplib_create_cq(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq)2381 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2382 {
2383 struct bnxt_qplib_hwq_attr hwq_attr = {};
2384 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
2385 struct creq_create_cq_resp resp = {};
2386 struct bnxt_qplib_cmdqmsg msg = {};
2387 struct cmdq_create_cq req = {};
2388 struct bnxt_qplib_reftbl *tbl;
2389 unsigned long flag;
2390 u32 pg_sz_lvl = 0;
2391 int rc;
2392
2393 hwq_attr.res = res;
2394 hwq_attr.depth = cq->max_wqe;
2395 hwq_attr.stride = sizeof(struct cq_base);
2396 hwq_attr.type = HWQ_TYPE_QUEUE;
2397 hwq_attr.sginfo = &cq->sginfo;
2398 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
2399 if (rc)
2400 goto exit;
2401
2402 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ,
2403 sizeof(req));
2404
2405 if (!cq->dpi) {
2406 dev_err(&rcfw->pdev->dev,
2407 "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n");
2408 return -EINVAL;
2409 }
2410 req.dpi = cpu_to_le32(cq->dpi->dpi);
2411 req.cq_handle = cpu_to_le64(cq->cq_handle);
2412
2413 req.cq_size = cpu_to_le32(cq->max_wqe);
2414 req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq));
2415 pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT;
2416 pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
2417 CMDQ_CREATE_CQ_LVL_SFT);
2418 req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
2419
2420 req.cq_fco_cnq_id = cpu_to_le32(
2421 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
2422 CMDQ_CREATE_CQ_CNQ_ID_SFT);
2423 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
2424 sizeof(resp), 0);
2425 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
2426 if (rc)
2427 goto fail;
2428 cq->id = le32_to_cpu(resp.xid);
2429 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
2430 init_waitqueue_head(&cq->waitq);
2431 INIT_LIST_HEAD(&cq->sqf_head);
2432 INIT_LIST_HEAD(&cq->rqf_head);
2433 spin_lock_init(&cq->flush_lock);
2434 spin_lock_init(&cq->compl_lock);
2435
2436 /* init dbinfo */
2437 cq->cctx = res->cctx;
2438 cq->dbinfo.hwq = &cq->hwq;
2439 cq->dbinfo.xid = cq->id;
2440 cq->dbinfo.db = cq->dpi->dbr;
2441 cq->dbinfo.priv_db = res->dpi_tbl.priv_db;
2442 cq->dbinfo.flags = 0;
2443 cq->dbinfo.toggle = 0;
2444 cq->dbinfo.res = res;
2445 cq->dbinfo.seed = cq->id;
2446 spin_lock_init(&cq->dbinfo.lock);
2447 cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
2448 cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
2449
2450 tbl = &res->reftbl.cqref;
2451 spin_lock_irqsave(&tbl->lock, flag);
2452 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id;
2453 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq;
2454 spin_unlock_irqrestore(&tbl->lock, flag);
2455
2456 bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
2457 return 0;
2458
2459 fail:
2460 bnxt_qplib_free_hwq(res, &cq->hwq);
2461 exit:
2462 return rc;
2463 }
2464
bnxt_qplib_modify_cq(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq)2465 int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2466 {
2467 /* TODO: Modify CQ threshold are passed to the HW via DBR */
2468 return 0;
2469 }
2470
bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq)2471 void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
2472 struct bnxt_qplib_cq *cq)
2473 {
2474 bnxt_qplib_free_hwq(res, &cq->hwq);
2475 memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
2476 /* Reset only the cons bit in the flags */
2477 cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);
2478
2479 /* Tell HW to switch over to the new CQ */
2480 if (!cq->resize_hwq.is_user)
2481 bnxt_qplib_cq_coffack_db(&cq->dbinfo);
2482 }
2483
bnxt_qplib_resize_cq(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq,int new_cqes)2484 int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
2485 int new_cqes)
2486 {
2487 struct bnxt_qplib_hwq_attr hwq_attr = {};
2488 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
2489 struct creq_resize_cq_resp resp = {};
2490 struct bnxt_qplib_cmdqmsg msg = {};
2491 struct cmdq_resize_cq req = {};
2492 u32 pgsz = 0, lvl = 0, nsz = 0;
2493 struct bnxt_qplib_pbl *pbl;
2494 u16 count = -1;
2495 int rc;
2496
2497 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ,
2498 sizeof(req));
2499
2500 hwq_attr.sginfo = &cq->sginfo;
2501 hwq_attr.res = res;
2502 hwq_attr.depth = new_cqes;
2503 hwq_attr.stride = sizeof(struct cq_base);
2504 hwq_attr.type = HWQ_TYPE_QUEUE;
2505 rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);
2506 if (rc)
2507 return rc;
2508
2509 dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__,
2510 cq->resize_hwq.level);
2511 req.cq_cid = cpu_to_le32(cq->id);
2512 pbl = &cq->resize_hwq.pbl[PBL_LVL_0];
2513 pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K :
2514 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K :
2515 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K :
2516 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M :
2517 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M :
2518 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G :
2519 CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK);
2520 lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &
2521 CMDQ_RESIZE_CQ_LVL_MASK;
2522 nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &
2523 CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;
2524 req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl);
2525 req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
2526
2527 if (!cq->resize_hwq.is_user)
2528 set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2529
2530 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
2531 sizeof(resp), 0);
2532 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
2533 if (rc)
2534 goto fail;
2535
2536 if (!cq->resize_hwq.is_user) {
2537 wait:
2538 /* Wait here for the HW to switch the CQ over */
2539 if (wait_event_interruptible_timeout(cq->waitq,
2540 !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags),
2541 msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) ==
2542 -ERESTARTSYS && count--)
2543 goto wait;
2544
2545 if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) {
2546 dev_err(&rcfw->pdev->dev,
2547 "QPLIB: FP: RESIZE_CQ timed out\n");
2548 rc = -ETIMEDOUT;
2549 goto fail;
2550 }
2551
2552 bnxt_qplib_resize_cq_complete(res, cq);
2553 }
2554
2555 return 0;
2556 fail:
2557 if (!cq->resize_hwq.is_user) {
2558 bnxt_qplib_free_hwq(res, &cq->resize_hwq);
2559 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2560 }
2561 return rc;
2562 }
2563
bnxt_qplib_free_cq(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq)2564 void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2565 {
2566 bnxt_qplib_free_hwq(res, &cq->hwq);
2567 }
2568
bnxt_qplib_sync_cq(struct bnxt_qplib_cq * cq)2569 static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq)
2570 {
2571 struct bnxt_qplib_nq *nq = cq->nq;
2572 /* Flush any pending work and synchronize irq */
2573 flush_workqueue(cq->nq->cqn_wq);
2574 mutex_lock(&nq->lock);
2575 if (nq->requested)
2576 synchronize_irq(nq->msix_vec);
2577 mutex_unlock(&nq->lock);
2578 }
2579
bnxt_qplib_destroy_cq(struct bnxt_qplib_res * res,struct bnxt_qplib_cq * cq)2580 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
2581 {
2582 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
2583 struct creq_destroy_cq_resp resp = {};
2584 struct bnxt_qplib_cmdqmsg msg = {};
2585 struct cmdq_destroy_cq req = {};
2586 struct bnxt_qplib_reftbl *tbl;
2587 u16 total_cnq_events;
2588 unsigned long flag;
2589 int rc;
2590
2591 tbl = &res->reftbl.cqref;
2592 spin_lock_irqsave(&tbl->lock, flag);
2593 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL;
2594 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0;
2595 spin_unlock_irqrestore(&tbl->lock, flag);
2596
2597 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ,
2598 sizeof(req));
2599
2600 req.cq_cid = cpu_to_le32(cq->id);
2601 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
2602 sizeof(resp), 0);
2603 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
2604 if (rc)
2605 return rc;
2606
2607 total_cnq_events = le16_to_cpu(resp.total_cnq_events);
2608 if (total_cnq_events >= 0)
2609 dev_dbg(&rcfw->pdev->dev,
2610 "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n",
2611 __func__, cq->id, cq, total_cnq_events);
2612 __wait_for_all_nqes(cq, total_cnq_events);
2613 bnxt_qplib_sync_cq(cq);
2614 bnxt_qplib_free_hwq(res, &cq->hwq);
2615 return 0;
2616 }
2617
__flush_sq(struct bnxt_qplib_q * sq,struct bnxt_qplib_qp * qp,struct bnxt_qplib_cqe ** pcqe,int * budget)2618 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
2619 struct bnxt_qplib_cqe **pcqe, int *budget)
2620 {
2621 struct bnxt_qplib_cqe *cqe;
2622 u32 start, last;
2623 int rc = 0;
2624
2625 /* Now complete all outstanding SQEs with FLUSHED_ERR */
2626 start = sq->swq_start;
2627 cqe = *pcqe;
2628 while (*budget) {
2629 last = sq->swq_last;
2630 if (start == last) {
2631 break;
2632 }
2633 /* Skip the FENCE WQE completions */
2634 if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
2635 bnxt_re_legacy_cancel_phantom_processing(qp);
2636 goto skip_compl;
2637 }
2638
2639 memset(cqe, 0, sizeof(*cqe));
2640 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2641 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2642 cqe->qp_handle = (u64)qp;
2643 cqe->wr_id = sq->swq[last].wr_id;
2644 cqe->src_qp = qp->id;
2645 cqe->type = sq->swq[last].type;
2646 dev_dbg(&sq->hwq.pdev->dev,
2647 "QPLIB: FP: CQ Processed terminal Req \n");
2648 dev_dbg(&sq->hwq.pdev->dev,
2649 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
2650 last, cqe->wr_id, cqe->status);
2651 cqe++;
2652 (*budget)--;
2653 skip_compl:
2654 bnxt_qplib_hwq_incr_cons(sq->hwq.depth,
2655 &sq->hwq.cons,
2656 sq->swq[last].slots,
2657 &sq->dbinfo.flags);
2658 sq->swq_last = sq->swq[last].next_idx;
2659 }
2660 *pcqe = cqe;
2661 if (!*budget && sq->swq_last != start)
2662 /* Out of budget */
2663 rc = -EAGAIN;
2664 dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc);
2665
2666 return rc;
2667 }
2668
__flush_rq(struct bnxt_qplib_q * rq,struct bnxt_qplib_qp * qp,struct bnxt_qplib_cqe ** pcqe,int * budget)2669 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
2670 struct bnxt_qplib_cqe **pcqe, int *budget)
2671 {
2672 struct bnxt_qplib_cqe *cqe;
2673 u32 start, last;
2674 int opcode = 0;
2675 int rc = 0;
2676
2677 switch (qp->type) {
2678 case CMDQ_CREATE_QP1_TYPE_GSI:
2679 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2680 break;
2681 case CMDQ_CREATE_QP_TYPE_RC:
2682 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2683 break;
2684 case CMDQ_CREATE_QP_TYPE_UD:
2685 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2686 break;
2687 }
2688
2689 /* Flush the rest of the RQ */
2690 start = rq->swq_start;
2691 cqe = *pcqe;
2692 while (*budget) {
2693 last = rq->swq_last;
2694 if (last == start)
2695 break;
2696 memset(cqe, 0, sizeof(*cqe));
2697 cqe->status =
2698 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2699 cqe->opcode = opcode;
2700 cqe->qp_handle = (u64)qp;
2701 cqe->wr_id = rq->swq[last].wr_id;
2702 dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n");
2703 dev_dbg(&rq->hwq.pdev->dev,
2704 "QPLIB: rq[%d] = 0x%llx with status 0x%x\n",
2705 last, cqe->wr_id, cqe->status);
2706 cqe++;
2707 (*budget)--;
2708 bnxt_qplib_hwq_incr_cons(rq->hwq.depth,
2709 &rq->hwq.cons,
2710 rq->swq[last].slots,
2711 &rq->dbinfo.flags);
2712 rq->swq_last = rq->swq[last].next_idx;
2713 }
2714 *pcqe = cqe;
2715 if (!*budget && rq->swq_last != start)
2716 /* Out of budget */
2717 rc = -EAGAIN;
2718
2719 dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc);
2720 return rc;
2721 }
2722
bnxt_qplib_mark_qp_error(void * qp_handle)2723 void bnxt_qplib_mark_qp_error(void *qp_handle)
2724 {
2725 struct bnxt_qplib_qp *qp = qp_handle;
2726
2727 if (!qp)
2728 return;
2729
2730 /* Must block new posting of SQ and RQ */
2731 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2732 qp->state = qp->cur_qp_state;
2733
2734 /* Add qp to flush list of the CQ */
2735 if (!qp->is_user)
2736 bnxt_qplib_add_flush_qp(qp);
2737 }
2738
2739 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
2740 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2741 */
bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp * qp,struct bnxt_qplib_cq * cq,u32 cq_cons,u32 swq_last,u32 cqe_sq_cons)2742 static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp,
2743 struct bnxt_qplib_cq *cq,
2744 u32 cq_cons, u32 swq_last,
2745 u32 cqe_sq_cons)
2746 {
2747 struct bnxt_qplib_q *sq = &qp->sq;
2748 struct bnxt_qplib_swq *swq;
2749 u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags;
2750 struct cq_terminal *peek_term_hwcqe;
2751 struct cq_req *peek_req_hwcqe;
2752 struct bnxt_qplib_qp *peek_qp;
2753 struct bnxt_qplib_q *peek_sq;
2754 struct cq_base *peek_hwcqe;
2755 int i, rc = 0;
2756
2757 /* Check for the psn_search marking before completing */
2758 swq = &sq->swq[swq_last];
2759 if (swq->psn_search &&
2760 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2761 /* Unmark */
2762 swq->psn_search->flags_next_psn = cpu_to_le32
2763 (le32_to_cpu(swq->psn_search->flags_next_psn)
2764 & ~0x80000000);
2765 dev_dbg(&cq->hwq.pdev->dev,
2766 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2767 cq_cons, qp->id, swq_last, cqe_sq_cons);
2768 sq->condition = true;
2769 sq->legacy_send_phantom = true;
2770
2771 /* TODO: Only ARM if the previous SQE is ARMALL */
2772 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
2773
2774 rc = -EAGAIN;
2775 goto out;
2776 }
2777 if (sq->condition == true) {
2778 /* Peek at the completions */
2779 peek_flags = cq->dbinfo.flags;
2780 peek_sw_cq_cons = cq_cons;
2781 i = cq->hwq.depth;
2782 while (i--) {
2783 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
2784 peek_sw_cq_cons, NULL);
2785 /* If the next hwcqe is VALID */
2786 if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) {
2787 /* If the next hwcqe is a REQ */
2788 dma_rmb();
2789 switch (peek_hwcqe->cqe_type_toggle &
2790 CQ_BASE_CQE_TYPE_MASK) {
2791 case CQ_BASE_CQE_TYPE_REQ:
2792 peek_req_hwcqe = (struct cq_req *)
2793 peek_hwcqe;
2794 peek_qp = (struct bnxt_qplib_qp *)
2795 le64_to_cpu(
2796 peek_req_hwcqe->qp_handle);
2797 peek_sq = &peek_qp->sq;
2798 peek_sq_cons_idx =
2799 ((le16_to_cpu(
2800 peek_req_hwcqe->sq_cons_idx)
2801 - 1) % sq->max_wqe);
2802 /* If the hwcqe's sq's wr_id matches */
2803 if (peek_sq == sq &&
2804 sq->swq[peek_sq_cons_idx].wr_id ==
2805 BNXT_QPLIB_FENCE_WRID) {
2806 /* Unbreak only if the phantom
2807 comes back */
2808 dev_dbg(&cq->hwq.pdev->dev,
2809 "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n",
2810 qp->id, swq_last,
2811 cqe_sq_cons);
2812 sq->condition = false;
2813 sq->single = true;
2814 sq->phantom_cqe_cnt++;
2815 dev_dbg(&cq->hwq.pdev->dev,
2816 "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n",
2817 peek_qp->id,
2818 peek_sw_cq_cons,
2819 peek_sq_cons_idx,
2820 sq->phantom_cqe_cnt);
2821 rc = 0;
2822 goto out;
2823 }
2824 break;
2825
2826 case CQ_BASE_CQE_TYPE_TERMINAL:
2827 /* In case the QP has gone into the
2828 error state */
2829 peek_term_hwcqe = (struct cq_terminal *)
2830 peek_hwcqe;
2831 peek_qp = (struct bnxt_qplib_qp *)
2832 le64_to_cpu(
2833 peek_term_hwcqe->qp_handle);
2834 if (peek_qp == qp) {
2835 sq->condition = false;
2836 rc = 0;
2837 goto out;
2838 }
2839 break;
2840 default:
2841 break;
2842 }
2843 /* Valid but not the phantom, so keep looping */
2844 } else {
2845 /* Not valid yet, just exit and wait */
2846 rc = -EINVAL;
2847 goto out;
2848 }
2849 bnxt_qplib_hwq_incr_cons(cq->hwq.depth,
2850 &peek_sw_cq_cons,
2851 1, &peek_flags);
2852 }
2853 dev_err(&cq->hwq.pdev->dev,
2854 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
2855 cq_cons, qp->id, swq_last, cqe_sq_cons);
2856 rc = -EINVAL;
2857 }
2858 out:
2859 return rc;
2860 }
2861
bnxt_qplib_cq_process_req(struct bnxt_qplib_cq * cq,struct cq_req * hwcqe,struct bnxt_qplib_cqe ** pcqe,int * budget,u32 cq_cons,struct bnxt_qplib_qp ** lib_qp)2862 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
2863 struct cq_req *hwcqe,
2864 struct bnxt_qplib_cqe **pcqe, int *budget,
2865 u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2866 {
2867 struct bnxt_qplib_qp *qp;
2868 struct bnxt_qplib_q *sq;
2869 struct bnxt_qplib_cqe *cqe;
2870 u32 cqe_sq_cons;
2871 struct bnxt_qplib_swq *swq;
2872 int rc = 0;
2873
2874 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
2875 dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp);
2876 if (!qp) {
2877 dev_err(&cq->hwq.pdev->dev,
2878 "QPLIB: FP: Process Req qp is NULL\n");
2879 return -EINVAL;
2880 }
2881 sq = &qp->sq;
2882
2883 cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
2884 if (qp->sq.flushed) {
2885 dev_dbg(&cq->hwq.pdev->dev,
2886 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2887 goto done;
2888 }
2889
2890 /* Require to walk the sq's swq to fabricate CQEs for all previously
2891 * signaled SWQEs due to CQE aggregation from the current sq cons
2892 * to the cqe_sq_cons
2893 */
2894 cqe = *pcqe;
2895 while (*budget) {
2896 if (sq->swq_last == cqe_sq_cons)
2897 /* Done */
2898 break;
2899
2900 swq = &sq->swq[sq->swq_last];
2901 memset(cqe, 0, sizeof(*cqe));
2902 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2903 cqe->qp_handle = (u64)qp;
2904 cqe->src_qp = qp->id;
2905 cqe->wr_id = swq->wr_id;
2906
2907 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
2908 goto skip;
2909
2910 cqe->type = swq->type;
2911
2912 /* For the last CQE, check for status. For errors, regardless
2913 * of the request being signaled or not, it must complete with
2914 * the hwcqe error status
2915 */
2916 if (swq->next_idx == cqe_sq_cons &&
2917 hwcqe->status != CQ_REQ_STATUS_OK) {
2918 cqe->status = hwcqe->status;
2919 dev_err(&cq->hwq.pdev->dev,
2920 "QPLIB: FP: CQ Processed Req \n");
2921 dev_err(&cq->hwq.pdev->dev,
2922 "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n",
2923 cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status);
2924 cqe++;
2925 (*budget)--;
2926 bnxt_qplib_mark_qp_error(qp);
2927 } else {
2928 /* Before we complete, do WA 9060 */
2929 if (!_is_chip_gen_p5_p7(qp->cctx)) {
2930 if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons,
2931 sq->swq_last,
2932 cqe_sq_cons)) {
2933 *lib_qp = qp;
2934 goto out;
2935 }
2936 }
2937 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2938
2939 dev_dbg(&cq->hwq.pdev->dev,
2940 "QPLIB: FP: CQ Processed Req \n");
2941 dev_dbg(&cq->hwq.pdev->dev,
2942 "QPLIB: wr_id[%d] = 0x%llx \n",
2943 sq->swq_last, cqe->wr_id);
2944 dev_dbg(&cq->hwq.pdev->dev,
2945 "QPLIB: with status 0x%x\n", cqe->status);
2946 cqe->status = CQ_REQ_STATUS_OK;
2947 cqe++;
2948 (*budget)--;
2949 }
2950 }
2951 skip:
2952 bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
2953 swq->slots, &sq->dbinfo.flags);
2954 sq->swq_last = swq->next_idx;
2955 if (sq->single == true)
2956 break;
2957 }
2958 out:
2959 *pcqe = cqe;
2960 if (sq->swq_last != cqe_sq_cons) {
2961 /* Out of budget */
2962 rc = -EAGAIN;
2963 goto done;
2964 }
2965 /* Back to normal completion mode only after it has completed all of
2966 the WC for this CQE */
2967 sq->single = false;
2968 done:
2969 return rc;
2970 }
2971
bnxt_qplib_release_srqe(struct bnxt_qplib_srq * srq,u32 tag)2972 static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
2973 {
2974 spin_lock(&srq->hwq.lock);
2975 srq->swq[srq->last_idx].next_idx = (int)tag;
2976 srq->last_idx = (int)tag;
2977 srq->swq[srq->last_idx].next_idx = -1;
2978 bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons,
2979 srq->dbinfo.max_slot, &srq->dbinfo.flags);
2980 spin_unlock(&srq->hwq.lock);
2981 }
2982
bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq * cq,struct cq_res_rc * hwcqe,struct bnxt_qplib_cqe ** pcqe,int * budget)2983 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
2984 struct cq_res_rc *hwcqe,
2985 struct bnxt_qplib_cqe **pcqe,
2986 int *budget)
2987 {
2988 struct bnxt_qplib_srq *srq;
2989 struct bnxt_qplib_cqe *cqe;
2990 struct bnxt_qplib_qp *qp;
2991 struct bnxt_qplib_q *rq;
2992 u32 wr_id_idx;
2993 int rc = 0;
2994
2995 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
2996 if (!qp) {
2997 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n");
2998 return -EINVAL;
2999 }
3000 if (qp->rq.flushed) {
3001 dev_dbg(&cq->hwq.pdev->dev,
3002 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
3003 goto done;
3004 }
3005
3006 cqe = *pcqe;
3007 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
3008 cqe->length = le32_to_cpu(hwcqe->length);
3009 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
3010 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
3011 cqe->flags = le16_to_cpu(hwcqe->flags);
3012 cqe->status = hwcqe->status;
3013 cqe->qp_handle = (u64)(unsigned long)qp;
3014
3015 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
3016 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
3017 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
3018 srq = qp->srq;
3019 if (!srq) {
3020 dev_err(&cq->hwq.pdev->dev,
3021 "QPLIB: FP: SRQ used but not defined??\n");
3022 return -EINVAL;
3023 }
3024 if (wr_id_idx > srq->hwq.depth - 1) {
3025 dev_err(&cq->hwq.pdev->dev,
3026 "QPLIB: FP: CQ Process RC \n");
3027 dev_err(&cq->hwq.pdev->dev,
3028 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
3029 wr_id_idx, srq->hwq.depth);
3030 return -EINVAL;
3031 }
3032 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
3033 bnxt_qplib_release_srqe(srq, wr_id_idx);
3034 dev_dbg(&srq->hwq.pdev->dev,
3035 "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n",
3036 wr_id_idx, cqe->wr_id);
3037 cqe++;
3038 (*budget)--;
3039 *pcqe = cqe;
3040 } else {
3041 rq = &qp->rq;
3042 if (wr_id_idx > (rq->max_wqe - 1)) {
3043 dev_err(&cq->hwq.pdev->dev,
3044 "QPLIB: FP: CQ Process RC \n");
3045 dev_err(&cq->hwq.pdev->dev,
3046 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
3047 wr_id_idx, rq->hwq.depth);
3048 return -EINVAL;
3049 }
3050 if (wr_id_idx != rq->swq_last)
3051 return -EINVAL;
3052 cqe->wr_id = rq->swq[rq->swq_last].wr_id;
3053 dev_dbg(&cq->hwq.pdev->dev,
3054 "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n",
3055 rq->swq_last, cqe->wr_id);
3056 cqe++;
3057 (*budget)--;
3058 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
3059 rq->swq[rq->swq_last].slots,
3060 &rq->dbinfo.flags);
3061 rq->swq_last = rq->swq[rq->swq_last].next_idx;
3062 *pcqe = cqe;
3063
3064 if (hwcqe->status != CQ_RES_RC_STATUS_OK)
3065 bnxt_qplib_mark_qp_error(qp);
3066 }
3067 done:
3068 return rc;
3069 }
3070
bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq * cq,struct cq_res_ud_v2 * hwcqe,struct bnxt_qplib_cqe ** pcqe,int * budget)3071 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
3072 struct cq_res_ud_v2 *hwcqe,
3073 struct bnxt_qplib_cqe **pcqe,
3074 int *budget)
3075 {
3076 struct bnxt_qplib_srq *srq;
3077 struct bnxt_qplib_cqe *cqe;
3078 struct bnxt_qplib_qp *qp;
3079 struct bnxt_qplib_q *rq;
3080 u32 wr_id_idx;
3081 int rc = 0;
3082 u16 *smac;
3083
3084 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
3085 if (!qp) {
3086 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n");
3087 return -EINVAL;
3088 }
3089 if (qp->rq.flushed) {
3090 dev_dbg(&cq->hwq.pdev->dev,
3091 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
3092 goto done;
3093 }
3094 cqe = *pcqe;
3095 cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK;
3096 cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK));
3097 cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0);
3098 /* V2 format has metadata1 */
3099 cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) &
3100 CQ_RES_UD_V2_CFA_METADATA1_MASK) >>
3101 CQ_RES_UD_V2_CFA_METADATA1_SFT) <<
3102 BNXT_QPLIB_META1_SHIFT);
3103 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
3104 cqe->flags = le16_to_cpu(hwcqe->flags);
3105 cqe->status = hwcqe->status;
3106 cqe->qp_handle = (u64)(unsigned long)qp;
3107 smac = (u16 *)cqe->smac;
3108 smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0]));
3109 smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1]));
3110 smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2]));
3111 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
3112 & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK;
3113 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
3114 ((le32_to_cpu(
3115 hwcqe->src_qp_high_srq_or_rq_wr_id) &
3116 CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8);
3117
3118 if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) {
3119 srq = qp->srq;
3120 if (!srq) {
3121 dev_err(&cq->hwq.pdev->dev,
3122 "QPLIB: FP: SRQ used but not defined??\n");
3123 return -EINVAL;
3124 }
3125 if (wr_id_idx > srq->hwq.depth - 1) {
3126 dev_err(&cq->hwq.pdev->dev,
3127 "QPLIB: FP: CQ Process UD \n");
3128 dev_err(&cq->hwq.pdev->dev,
3129 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
3130 wr_id_idx, srq->hwq.depth);
3131 return -EINVAL;
3132 }
3133 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
3134 bnxt_qplib_release_srqe(srq, wr_id_idx);
3135 dev_dbg(&cq->hwq.pdev->dev,
3136 "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n",
3137 wr_id_idx, cqe->wr_id);
3138 cqe++;
3139 (*budget)--;
3140 *pcqe = cqe;
3141 } else {
3142 rq = &qp->rq;
3143 if (wr_id_idx > (rq->max_wqe - 1)) {
3144 dev_err(&cq->hwq.pdev->dev,
3145 "QPLIB: FP: CQ Process UD \n");
3146 dev_err(&cq->hwq.pdev->dev,
3147 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
3148 wr_id_idx, rq->hwq.depth);
3149 return -EINVAL;
3150 }
3151 if (rq->swq_last != wr_id_idx)
3152 return -EINVAL;
3153
3154 cqe->wr_id = rq->swq[rq->swq_last].wr_id;
3155 dev_dbg(&cq->hwq.pdev->dev,
3156 "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n",
3157 rq->swq_last, cqe->wr_id);
3158 cqe++;
3159 (*budget)--;
3160 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
3161 rq->swq[rq->swq_last].slots,
3162 &rq->dbinfo.flags);
3163 rq->swq_last = rq->swq[rq->swq_last].next_idx;
3164 *pcqe = cqe;
3165
3166 if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK)
3167 bnxt_qplib_mark_qp_error(qp);
3168 }
3169 done:
3170 return rc;
3171 }
3172
bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq * cq)3173 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
3174 {
3175
3176 struct cq_base *hw_cqe;
3177 unsigned long flags;
3178 bool rc = true;
3179
3180 spin_lock_irqsave(&cq->hwq.lock, flags);
3181 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
3182
3183 /* Check for Valid bit. If the CQE is valid, return false */
3184 rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);
3185 spin_unlock_irqrestore(&cq->hwq.lock, flags);
3186 return rc;
3187 }
3188
bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq * cq,struct cq_res_raweth_qp1 * hwcqe,struct bnxt_qplib_cqe ** pcqe,int * budget)3189 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
3190 struct cq_res_raweth_qp1 *hwcqe,
3191 struct bnxt_qplib_cqe **pcqe,
3192 int *budget)
3193 {
3194 struct bnxt_qplib_qp *qp;
3195 struct bnxt_qplib_q *rq;
3196 struct bnxt_qplib_srq *srq;
3197 struct bnxt_qplib_cqe *cqe;
3198 u32 wr_id_idx;
3199 int rc = 0;
3200
3201 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
3202 if (!qp) {
3203 dev_err(&cq->hwq.pdev->dev,
3204 "QPLIB: process_cq Raw/QP1 qp is NULL\n");
3205 return -EINVAL;
3206 }
3207 if (qp->rq.flushed) {
3208 dev_dbg(&cq->hwq.pdev->dev,
3209 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
3210 goto done;
3211 }
3212 cqe = *pcqe;
3213 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
3214 cqe->flags = le16_to_cpu(hwcqe->flags);
3215 cqe->qp_handle = (u64)(unsigned long)qp;
3216
3217 wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
3218 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
3219 cqe->src_qp = qp->id;
3220 if (qp->id == 1 && !cqe->length) {
3221 /* Add workaround for the length misdetection */
3222 cqe->length = 296;
3223 } else {
3224 cqe->length = le16_to_cpu(hwcqe->length);
3225 }
3226 cqe->pkey_index = qp->pkey_index;
3227 memcpy(cqe->smac, qp->smac, 6);
3228
3229 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
3230 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
3231 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
3232
3233 dev_dbg(&cq->hwq.pdev->dev,
3234 "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n",
3235 cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2);
3236
3237 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
3238 srq = qp->srq;
3239 if (!srq) {
3240 dev_err(&cq->hwq.pdev->dev,
3241 "QPLIB: FP: SRQ used but not defined??\n");
3242 return -EINVAL;
3243 }
3244 if (wr_id_idx > srq->hwq.depth - 1) {
3245 dev_err(&cq->hwq.pdev->dev,
3246 "QPLIB: FP: CQ Process Raw/QP1 \n");
3247 dev_err(&cq->hwq.pdev->dev,
3248 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
3249 wr_id_idx, srq->hwq.depth);
3250 return -EINVAL;
3251 }
3252 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
3253 dev_dbg(&cq->hwq.pdev->dev,
3254 "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n");
3255 dev_dbg(&cq->hwq.pdev->dev,
3256 "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
3257 wr_id_idx, cqe->wr_id, hwcqe->status);
3258 cqe++;
3259 (*budget)--;
3260 srq->hwq.cons++;
3261 *pcqe = cqe;
3262 } else {
3263 rq = &qp->rq;
3264 if (wr_id_idx > (rq->max_wqe - 1)) {
3265 dev_err(&cq->hwq.pdev->dev,
3266 "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n");
3267 dev_err(&cq->hwq.pdev->dev,
3268 "QPLIB: ix 0x%x exceeded RQ max 0x%x\n",
3269 wr_id_idx, rq->max_wqe);
3270 return -EINVAL;
3271 }
3272 if (wr_id_idx != rq->swq_last)
3273 return -EINVAL;
3274 cqe->wr_id = rq->swq[rq->swq_last].wr_id;
3275 dev_dbg(&cq->hwq.pdev->dev,
3276 "QPLIB: FP: CQ Processed Raw/QP1 RQ \n");
3277 dev_dbg(&cq->hwq.pdev->dev,
3278 "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
3279 wr_id_idx, cqe->wr_id, hwcqe->status);
3280 cqe++;
3281 (*budget)--;
3282 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
3283 rq->swq[wr_id_idx].slots,
3284 &rq->dbinfo.flags);
3285 rq->swq_last = rq->swq[rq->swq_last].next_idx;
3286 *pcqe = cqe;
3287
3288 if (hwcqe->status != CQ_RES_RC_STATUS_OK)
3289 bnxt_qplib_mark_qp_error(qp);
3290 }
3291 done:
3292 return rc;
3293 }
3294
bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq * cq,struct cq_terminal * hwcqe,struct bnxt_qplib_cqe ** pcqe,int * budget)3295 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
3296 struct cq_terminal *hwcqe,
3297 struct bnxt_qplib_cqe **pcqe,
3298 int *budget)
3299 {
3300 struct bnxt_qplib_q *sq, *rq;
3301 struct bnxt_qplib_cqe *cqe;
3302 struct bnxt_qplib_qp *qp;
3303 u32 swq_last;
3304 u32 cqe_cons;
3305 int rc = 0;
3306
3307 /* Check the Status */
3308 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
3309 dev_warn(&cq->hwq.pdev->dev,
3310 "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n",
3311 hwcqe->status);
3312
3313 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
3314 if (!qp)
3315 return -EINVAL;
3316 dev_dbg(&cq->hwq.pdev->dev,
3317 "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id);
3318
3319 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
3320 * from the current rq->cons to the rq->prod regardless what the
3321 * rq->cons the terminal CQE indicates.
3322 */
3323 bnxt_qplib_mark_qp_error(qp);
3324
3325 sq = &qp->sq;
3326 rq = &qp->rq;
3327
3328 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
3329 if (cqe_cons == 0xFFFF)
3330 goto do_rq;
3331
3332 cqe_cons %= sq->max_wqe;
3333 if (qp->sq.flushed) {
3334 dev_dbg(&cq->hwq.pdev->dev,
3335 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
3336 goto sq_done;
3337 }
3338
3339 /* Terminal CQE can also include aggregated successful CQEs prior.
3340 So we must complete all CQEs from the current sq's cons to the
3341 cq_cons with status OK */
3342 cqe = *pcqe;
3343 while (*budget) {
3344 /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/
3345 swq_last = sq->swq_last;
3346 if (swq_last == cqe_cons)
3347 break;
3348 if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
3349 memset(cqe, 0, sizeof(*cqe));
3350 cqe->status = CQ_REQ_STATUS_OK;
3351 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
3352 cqe->qp_handle = (u64)qp;
3353 cqe->src_qp = qp->id;
3354 cqe->wr_id = sq->swq[swq_last].wr_id;
3355 cqe->type = sq->swq[swq_last].type;
3356 dev_dbg(&cq->hwq.pdev->dev,
3357 "QPLIB: FP: CQ Processed terminal Req \n");
3358 dev_dbg(&cq->hwq.pdev->dev,
3359 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
3360 swq_last, cqe->wr_id, cqe->status);
3361 cqe++;
3362 (*budget)--;
3363 }
3364 bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
3365 sq->swq[swq_last].slots,
3366 &sq->dbinfo.flags);
3367 sq->swq_last = sq->swq[swq_last].next_idx;
3368 }
3369 *pcqe = cqe;
3370 if (!*budget && swq_last != cqe_cons) {
3371 /* Out of budget */
3372 rc = -EAGAIN;
3373 goto sq_done;
3374 }
3375 sq_done:
3376 if (rc)
3377 return rc;
3378 do_rq:
3379 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
3380 if (cqe_cons == 0xFFFF) {
3381 goto done;
3382 } else if (cqe_cons > (rq->max_wqe - 1)) {
3383 dev_err(&cq->hwq.pdev->dev,
3384 "QPLIB: FP: CQ Processed terminal \n");
3385 dev_err(&cq->hwq.pdev->dev,
3386 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n",
3387 cqe_cons, rq->hwq.depth);
3388 goto done;
3389 }
3390 if (qp->rq.flushed) {
3391 dev_dbg(&cq->hwq.pdev->dev,
3392 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
3393 rc = 0;
3394 goto rq_done;
3395 }
3396
3397 rq_done:
3398 done:
3399 return rc;
3400 }
3401
bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq * cq,struct cq_cutoff * hwcqe)3402 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
3403 struct cq_cutoff *hwcqe)
3404 {
3405 /* Check the Status */
3406 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
3407 dev_err(&cq->hwq.pdev->dev,
3408 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n",
3409 hwcqe->status);
3410 return -EINVAL;
3411 }
3412 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
3413 wake_up_interruptible(&cq->waitq);
3414
3415 dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n");
3416 return 0;
3417 }
3418
bnxt_qplib_process_flush_list(struct bnxt_qplib_cq * cq,struct bnxt_qplib_cqe * cqe,int num_cqes)3419 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
3420 struct bnxt_qplib_cqe *cqe,
3421 int num_cqes)
3422 {
3423 struct bnxt_qplib_qp *qp = NULL;
3424 u32 budget = num_cqes;
3425 unsigned long flags;
3426
3427 spin_lock_irqsave(&cq->flush_lock, flags);
3428 list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
3429 dev_dbg(&cq->hwq.pdev->dev,
3430 "QPLIB: FP: Flushing SQ QP= %p\n",
3431 qp);
3432 __flush_sq(&qp->sq, qp, &cqe, &budget);
3433 }
3434
3435 list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
3436 dev_dbg(&cq->hwq.pdev->dev,
3437 "QPLIB: FP: Flushing RQ QP= %p\n",
3438 qp);
3439 __flush_rq(&qp->rq, qp, &cqe, &budget);
3440 }
3441 spin_unlock_irqrestore(&cq->flush_lock, flags);
3442
3443 return num_cqes - budget;
3444 }
3445
bnxt_qplib_poll_cq(struct bnxt_qplib_cq * cq,struct bnxt_qplib_cqe * cqe,int num_cqes,struct bnxt_qplib_qp ** lib_qp)3446 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
3447 int num_cqes, struct bnxt_qplib_qp **lib_qp)
3448 {
3449 struct cq_base *hw_cqe;
3450 u32 hw_polled = 0;
3451 int budget, rc = 0;
3452 u8 type;
3453
3454 budget = num_cqes;
3455
3456 while (budget) {
3457 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
3458
3459 /* Check for Valid bit */
3460 if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags))
3461 break;
3462
3463 /* The valid test of the entry must be done first before
3464 * reading any further.
3465 */
3466 dma_rmb();
3467 /* From the device's respective CQE format to qplib_wc*/
3468 type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
3469 switch (type) {
3470 case CQ_BASE_CQE_TYPE_REQ:
3471 rc = bnxt_qplib_cq_process_req(cq,
3472 (struct cq_req *)hw_cqe, &cqe, &budget,
3473 cq->hwq.cons, lib_qp);
3474 break;
3475 case CQ_BASE_CQE_TYPE_RES_RC:
3476 rc = bnxt_qplib_cq_process_res_rc(cq,
3477 (struct cq_res_rc *)hw_cqe, &cqe,
3478 &budget);
3479 break;
3480 case CQ_BASE_CQE_TYPE_RES_UD:
3481 rc = bnxt_qplib_cq_process_res_ud(cq,
3482 (struct cq_res_ud_v2 *)hw_cqe,
3483 &cqe, &budget);
3484 break;
3485 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3486 rc = bnxt_qplib_cq_process_res_raweth_qp1(cq,
3487 (struct cq_res_raweth_qp1 *)
3488 hw_cqe, &cqe, &budget);
3489 break;
3490 case CQ_BASE_CQE_TYPE_TERMINAL:
3491 rc = bnxt_qplib_cq_process_terminal(cq,
3492 (struct cq_terminal *)hw_cqe,
3493 &cqe, &budget);
3494 break;
3495 case CQ_BASE_CQE_TYPE_CUT_OFF:
3496 bnxt_qplib_cq_process_cutoff(cq,
3497 (struct cq_cutoff *)hw_cqe);
3498 /* Done processing this CQ */
3499 goto exit;
3500 default:
3501 dev_err(&cq->hwq.pdev->dev,
3502 "QPLIB: process_cq unknown type 0x%x\n",
3503 hw_cqe->cqe_type_toggle &
3504 CQ_BASE_CQE_TYPE_MASK);
3505 rc = -EINVAL;
3506 break;
3507 }
3508 if (rc < 0) {
3509 dev_dbg(&cq->hwq.pdev->dev,
3510 "QPLIB: process_cqe rc = 0x%x\n", rc);
3511 if (rc == -EAGAIN)
3512 break;
3513 /* Error while processing the CQE, just skip to the
3514 next one */
3515 if (type != CQ_BASE_CQE_TYPE_TERMINAL)
3516 dev_err(&cq->hwq.pdev->dev,
3517 "QPLIB: process_cqe error rc = 0x%x\n",
3518 rc);
3519 }
3520 hw_polled++;
3521 bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons,
3522 1, &cq->dbinfo.flags);
3523 }
3524 if (hw_polled)
3525 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
3526 exit:
3527 return num_cqes - budget;
3528 }
3529
bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq * cq,u32 arm_type)3530 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
3531 {
3532 cq->dbinfo.toggle = cq->toggle;
3533 if (arm_type)
3534 bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
3535 /* Using cq->arm_state variable to track whether to issue cq handler */
3536 atomic_set(&cq->arm_state, 1);
3537 }
3538
bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp * qp)3539 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
3540 {
3541 flush_workqueue(qp->scq->nq->cqn_wq);
3542 if (qp->scq != qp->rcq)
3543 flush_workqueue(qp->rcq->nq->cqn_wq);
3544 }
3545