1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: Fast Path Operators 29 */ 30 31 #include <linux/interrupt.h> 32 #include <linux/spinlock.h> 33 #include <linux/sched.h> 34 #include <linux/slab.h> 35 #include <linux/pci.h> 36 #include <linux/delay.h> 37 #include <linux/if_ether.h> 38 #include <linux/hardirq.h> 39 #include <rdma/ib_mad.h> 40 41 #include "hsi_struct_def.h" 42 #include "qplib_tlv.h" 43 #include "qplib_res.h" 44 #include "qplib_rcfw.h" 45 #include "qplib_sp.h" 46 #include "qplib_fp.h" 47 #include "ib_verbs.h" 48 49 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); 50 51 static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp) 52 { 53 qp->sq.condition = false; 54 qp->sq.legacy_send_phantom = false; 55 qp->sq.single = false; 56 } 57 58 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 59 { 60 struct bnxt_qplib_cq *scq, *rcq; 61 62 scq = qp->scq; 63 rcq = qp->rcq; 64 65 if (!qp->sq.flushed) { 66 dev_dbg(&scq->hwq.pdev->dev, 67 "QPLIB: FP: Adding to SQ Flush list = %p\n", 68 qp); 69 bnxt_re_legacy_cancel_phantom_processing(qp); 70 list_add_tail(&qp->sq_flush, &scq->sqf_head); 71 qp->sq.flushed = true; 72 } 73 if (!qp->srq) { 74 if (!qp->rq.flushed) { 75 dev_dbg(&rcq->hwq.pdev->dev, 76 "QPLIB: FP: Adding to RQ Flush list = %p\n", 77 qp); 78 list_add_tail(&qp->rq_flush, &rcq->rqf_head); 79 qp->rq.flushed = true; 80 } 81 } 82 } 83 84 static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp) 85 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) 86 { 87 /* Interrupts are already disabled in calling functions */ 88 spin_lock(&qp->scq->flush_lock); 89 if (qp->scq == qp->rcq) 90 __acquire(&qp->rcq->flush_lock); 91 else 92 spin_lock(&qp->rcq->flush_lock); 93 } 94 95 static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp) 96 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) 97 { 98 if (qp->scq == qp->rcq) 99 __release(&qp->rcq->flush_lock); 100 else 101 spin_unlock(&qp->rcq->flush_lock); 102 spin_unlock(&qp->scq->flush_lock); 103 } 104 105 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) 106 { 107 108 bnxt_qplib_acquire_cq_flush_locks(qp); 109 __bnxt_qplib_add_flush_qp(qp); 110 bnxt_qplib_release_cq_flush_locks(qp); 111 } 112 113 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) 114 { 115 if (qp->sq.flushed) { 116 qp->sq.flushed = false; 117 list_del(&qp->sq_flush); 118 } 119 if (!qp->srq) { 120 if (qp->rq.flushed) { 121 qp->rq.flushed = false; 122 list_del(&qp->rq_flush); 123 } 124 } 125 } 126 127 void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) 128 { 129 130 bnxt_qplib_acquire_cq_flush_locks(qp); 131 __clean_cq(qp->scq, (u64)(unsigned long)qp); 132 qp->sq.hwq.prod = 0; 133 qp->sq.hwq.cons = 0; 134 qp->sq.swq_start = 0; 135 qp->sq.swq_last = 0; 136 __clean_cq(qp->rcq, (u64)(unsigned long)qp); 137 qp->rq.hwq.prod = 0; 138 qp->rq.hwq.cons = 0; 139 qp->rq.swq_start = 0; 140 qp->rq.swq_last = 0; 141 142 __bnxt_qplib_del_flush_qp(qp); 143 bnxt_qplib_release_cq_flush_locks(qp); 144 } 145 146 static void bnxt_qpn_cqn_sched_task(struct work_struct *work) 147 { 148 struct bnxt_qplib_nq_work *nq_work = 149 container_of(work, struct bnxt_qplib_nq_work, work); 150 151 struct bnxt_qplib_cq *cq = nq_work->cq; 152 struct bnxt_qplib_nq *nq = nq_work->nq; 153 154 if (cq && nq) { 155 spin_lock_bh(&cq->compl_lock); 156 if (nq->cqn_handler) { 157 dev_dbg(&nq->res->pdev->dev, 158 "%s:Trigger cq = %p event nq = %p\n", 159 __func__, cq, nq); 160 nq->cqn_handler(nq, cq); 161 } 162 spin_unlock_bh(&cq->compl_lock); 163 } 164 kfree(nq_work); 165 } 166 167 static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev, 168 struct bnxt_qplib_hdrbuf *buf) 169 { 170 dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map); 171 kfree(buf); 172 } 173 174 static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt) 175 { 176 struct bnxt_qplib_hdrbuf *hdrbuf; 177 u32 len; 178 179 hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL); 180 if (!hdrbuf) 181 return NULL; 182 183 len = ALIGN((step * cnt), PAGE_SIZE); 184 hdrbuf->va = dma_alloc_coherent(&pdev->dev, len, 185 &hdrbuf->dma_map, GFP_KERNEL); 186 if (!hdrbuf->va) 187 goto out; 188 189 hdrbuf->len = len; 190 hdrbuf->step = step; 191 return hdrbuf; 192 out: 193 kfree(hdrbuf); 194 return NULL; 195 } 196 197 void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, 198 struct bnxt_qplib_qp *qp) 199 { 200 if (qp->rq_hdr_buf) { 201 bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf); 202 qp->rq_hdr_buf = NULL; 203 } 204 205 if (qp->sq_hdr_buf) { 206 bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf); 207 qp->sq_hdr_buf = NULL; 208 } 209 } 210 211 int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, 212 struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep) 213 { 214 struct pci_dev *pdev; 215 int rc = 0; 216 217 pdev = res->pdev; 218 if (sstep) { 219 qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep, 220 qp->sq.max_wqe); 221 if (!qp->sq_hdr_buf) { 222 dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n"); 223 return -ENOMEM; 224 } 225 } 226 227 if (rstep) { 228 qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep, 229 qp->rq.max_wqe); 230 if (!qp->rq_hdr_buf) { 231 rc = -ENOMEM; 232 dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n"); 233 goto fail; 234 } 235 } 236 237 return 0; 238 fail: 239 bnxt_qplib_free_hdr_buf(res, qp); 240 return rc; 241 } 242 243 /* 244 * clean_nq - Invalidate cqe from given nq. 245 * @cq - Completion queue 246 * 247 * Traverse whole notification queue and invalidate any completion 248 * associated cq handler provided by caller. 249 * Note - This function traverse the hardware queue but do not update 250 * consumer index. Invalidated cqe(marked from this function) will be 251 * ignored from actual completion of notification queue. 252 */ 253 static void clean_nq(struct bnxt_qplib_cq *cq) 254 { 255 struct bnxt_qplib_hwq *nq_hwq = NULL; 256 struct bnxt_qplib_nq *nq = NULL; 257 struct nq_base *hw_nqe = NULL; 258 struct nq_cn *nqcne = NULL; 259 u32 peek_flags, peek_cons; 260 u64 q_handle; 261 u32 type; 262 int i; 263 264 nq = cq->nq; 265 nq_hwq = &nq->hwq; 266 267 spin_lock_bh(&nq_hwq->lock); 268 peek_flags = nq->nq_db.dbinfo.flags; 269 peek_cons = nq_hwq->cons; 270 for (i = 0; i < nq_hwq->max_elements; i++) { 271 hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL); 272 if (!NQE_CMP_VALID(hw_nqe, peek_flags)) 273 break; 274 275 /* The valid test of the entry must be done first 276 * before reading any further. 277 */ 278 dma_rmb(); 279 type = le16_to_cpu(hw_nqe->info10_type) & 280 NQ_BASE_TYPE_MASK; 281 282 /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */ 283 if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) { 284 nqcne = (struct nq_cn *)hw_nqe; 285 286 q_handle = le32_to_cpu(nqcne->cq_handle_low); 287 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; 288 if (q_handle == (u64)cq) { 289 nqcne->cq_handle_low = 0; 290 nqcne->cq_handle_high = 0; 291 cq->cnq_events++; 292 } 293 } 294 bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons, 295 1, &peek_flags); 296 } 297 spin_unlock_bh(&nq_hwq->lock); 298 } 299 300 /* 301 * Wait for receiving all NQEs for this CQ. 302 * clean_nq is tried 100 times, each time clean_cq 303 * loops upto budget times. budget is based on the 304 * number of CQs shared by that NQ. So any NQE from 305 * CQ would be already in the NQ. 306 */ 307 static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) 308 { 309 u32 retry_cnt = 100; 310 u16 total_events; 311 312 if (!cnq_events) { 313 clean_nq(cq); 314 return; 315 } 316 while (retry_cnt--) { 317 total_events = cq->cnq_events; 318 319 /* Increment total_events by 1 if any CREQ event received with CQ notification */ 320 if (cq->is_cq_err_event) 321 total_events++; 322 323 if (cnq_events == total_events) { 324 dev_dbg(&cq->nq->res->pdev->dev, 325 "QPLIB: NQ cleanup - Received all NQ events\n"); 326 return; 327 } 328 msleep(1); 329 clean_nq(cq); 330 } 331 } 332 333 static void bnxt_qplib_service_nq(unsigned long data) 334 { 335 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; 336 struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; 337 int budget = nq->budget; 338 struct bnxt_qplib_res *res; 339 struct bnxt_qplib_cq *cq; 340 struct pci_dev *pdev; 341 struct nq_base *nqe; 342 u32 hw_polled = 0; 343 u64 q_handle; 344 u32 type; 345 346 res = nq->res; 347 pdev = res->pdev; 348 349 spin_lock_bh(&nq_hwq->lock); 350 /* Service the NQ until empty or budget expired */ 351 while (budget--) { 352 nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); 353 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) 354 break; 355 /* The valid test of the entry must be done first before 356 * reading any further. 357 */ 358 dma_rmb(); 359 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; 360 switch (type) { 361 case NQ_BASE_TYPE_CQ_NOTIFICATION: 362 { 363 struct nq_cn *nqcne = (struct nq_cn *)nqe; 364 365 q_handle = le32_to_cpu(nqcne->cq_handle_low); 366 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; 367 cq = (struct bnxt_qplib_cq *)q_handle; 368 if (!cq) 369 break; 370 cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT; 371 cq->dbinfo.toggle = cq->toggle; 372 bnxt_qplib_armen_db(&cq->dbinfo, 373 DBC_DBC_TYPE_CQ_ARMENA); 374 spin_lock_bh(&cq->compl_lock); 375 atomic_set(&cq->arm_state, 0) ; 376 if (!nq->cqn_handler(nq, (cq))) 377 nq->stats.num_cqne_processed++; 378 else 379 dev_warn(&pdev->dev, 380 "QPLIB: cqn - type 0x%x not handled\n", 381 type); 382 cq->cnq_events++; 383 spin_unlock_bh(&cq->compl_lock); 384 break; 385 } 386 case NQ_BASE_TYPE_SRQ_EVENT: 387 { 388 struct bnxt_qplib_srq *srq; 389 struct nq_srq_event *nqsrqe = 390 (struct nq_srq_event *)nqe; 391 392 q_handle = le32_to_cpu(nqsrqe->srq_handle_low); 393 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; 394 srq = (struct bnxt_qplib_srq *)q_handle; 395 bnxt_qplib_armen_db(&srq->dbinfo, 396 DBC_DBC_TYPE_SRQ_ARMENA); 397 if (!nq->srqn_handler(nq, 398 (struct bnxt_qplib_srq *)q_handle, 399 nqsrqe->event)) 400 nq->stats.num_srqne_processed++; 401 else 402 dev_warn(&pdev->dev, 403 "QPLIB: SRQ event 0x%x not handled\n", 404 nqsrqe->event); 405 break; 406 } 407 default: 408 dev_warn(&pdev->dev, 409 "QPLIB: nqe with opcode = 0x%x not handled\n", 410 type); 411 break; 412 } 413 hw_polled++; 414 bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons, 415 1, &nq->nq_db.dbinfo.flags); 416 } 417 nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); 418 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) { 419 nq->stats.num_nq_rearm++; 420 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); 421 } else if (nq->requested) { 422 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); 423 nq->stats.num_tasklet_resched++; 424 } 425 dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n"); 426 if (hw_polled >= 0) 427 dev_dbg(&pdev->dev, 428 "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n", 429 nq->stats.num_cqne_processed, nq->stats.num_srqne_processed, 430 nq->stats.num_dbqne_processed, budget, hw_polled); 431 dev_dbg(&pdev->dev, 432 "QPLIB: resched_cnt = %llu arm_count = %llu\n", 433 nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm); 434 spin_unlock_bh(&nq_hwq->lock); 435 } 436 437 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) 438 { 439 struct bnxt_qplib_nq *nq = dev_instance; 440 struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; 441 u32 sw_cons; 442 443 /* Prefetch the NQ element */ 444 sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq); 445 if (sw_cons >= 0) 446 prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL)); 447 448 bnxt_qplib_service_nq((unsigned long)nq); 449 450 return IRQ_HANDLED; 451 } 452 453 void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) 454 { 455 struct bnxt_qplib_res *res; 456 457 if (!nq->requested) 458 return; 459 460 nq->requested = false; 461 res = nq->res; 462 /* Mask h/w interrupt */ 463 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false); 464 /* Sync with last running IRQ handler */ 465 synchronize_irq(nq->msix_vec); 466 free_irq(nq->msix_vec, nq); 467 kfree(nq->name); 468 nq->name = NULL; 469 } 470 471 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) 472 { 473 if (nq->cqn_wq) { 474 destroy_workqueue(nq->cqn_wq); 475 nq->cqn_wq = NULL; 476 } 477 /* Make sure the HW is stopped! */ 478 bnxt_qplib_nq_stop_irq(nq, true); 479 480 nq->nq_db.reg.bar_reg = NULL; 481 nq->nq_db.db = NULL; 482 483 nq->cqn_handler = NULL; 484 nq->srqn_handler = NULL; 485 nq->msix_vec = 0; 486 } 487 488 int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, 489 int msix_vector, bool need_init) 490 { 491 struct bnxt_qplib_res *res; 492 int rc; 493 494 res = nq->res; 495 if (nq->requested) 496 return -EFAULT; 497 498 nq->msix_vec = msix_vector; 499 nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n", 500 nq_indx, pci_name(res->pdev)); 501 if (!nq->name) 502 return -ENOMEM; 503 rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); 504 if (rc) { 505 kfree(nq->name); 506 nq->name = NULL; 507 return rc; 508 } 509 nq->requested = true; 510 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); 511 512 return rc; 513 } 514 515 static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt) 516 { 517 struct bnxt_qplib_reg_desc *dbreg; 518 struct bnxt_qplib_nq_db *nq_db; 519 struct bnxt_qplib_res *res; 520 521 nq_db = &nq->nq_db; 522 res = nq->res; 523 dbreg = &res->dpi_tbl.ucreg; 524 525 nq_db->reg.bar_id = dbreg->bar_id; 526 nq_db->reg.bar_base = dbreg->bar_base; 527 nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; 528 nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : 529 sizeof(u32); 530 531 nq_db->dbinfo.db = nq_db->reg.bar_reg; 532 nq_db->dbinfo.hwq = &nq->hwq; 533 nq_db->dbinfo.xid = nq->ring_id; 534 nq_db->dbinfo.seed = nq->ring_id; 535 nq_db->dbinfo.flags = 0; 536 spin_lock_init(&nq_db->dbinfo.lock); 537 nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 538 nq_db->dbinfo.res = nq->res; 539 540 return; 541 } 542 543 int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, 544 int msix_vector, int bar_reg_offset, 545 cqn_handler_t cqn_handler, 546 srqn_handler_t srqn_handler) 547 { 548 struct pci_dev *pdev; 549 int rc; 550 551 pdev = nq->res->pdev; 552 nq->cqn_handler = cqn_handler; 553 nq->srqn_handler = srqn_handler; 554 nq->load = 0; 555 mutex_init(&nq->lock); 556 557 /* Have a task to schedule CQ notifiers in post send case */ 558 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n"); 559 if (!nq->cqn_wq) 560 return -ENOMEM; 561 562 bnxt_qplib_map_nq_db(nq, bar_reg_offset); 563 rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); 564 if (rc) { 565 dev_err(&pdev->dev, 566 "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx); 567 goto fail; 568 } 569 dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements); 570 571 return 0; 572 fail: 573 bnxt_qplib_disable_nq(nq); 574 return rc; 575 } 576 577 void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq) 578 { 579 if (nq->hwq.max_elements) { 580 bnxt_qplib_free_hwq(nq->res, &nq->hwq); 581 nq->hwq.max_elements = 0; 582 } 583 } 584 585 int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, 586 struct bnxt_qplib_nq *nq) 587 { 588 struct bnxt_qplib_hwq_attr hwq_attr = {}; 589 struct bnxt_qplib_sg_info sginfo = {}; 590 591 nq->res = res; 592 if (!nq->hwq.max_elements || 593 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) 594 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; 595 596 sginfo.pgsize = PAGE_SIZE; 597 sginfo.pgshft = PAGE_SHIFT; 598 hwq_attr.res = res; 599 hwq_attr.sginfo = &sginfo; 600 hwq_attr.depth = nq->hwq.max_elements; 601 hwq_attr.stride = sizeof(struct nq_base); 602 hwq_attr.type = _get_hwq_type(res); 603 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { 604 dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n"); 605 return -ENOMEM; 606 } 607 nq->budget = 8; 608 return 0; 609 } 610 611 /* SRQ */ 612 static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw, 613 struct bnxt_qplib_srq *srq) 614 { 615 struct creq_destroy_srq_resp resp = {}; 616 struct bnxt_qplib_cmdqmsg msg = {}; 617 struct cmdq_destroy_srq req = {}; 618 /* Configure the request */ 619 req.srq_cid = cpu_to_le32(srq->id); 620 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ, 621 sizeof(req)); 622 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 623 sizeof(resp), 0); 624 return bnxt_qplib_rcfw_send_message(rcfw, &msg); 625 } 626 627 int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, 628 struct bnxt_qplib_srq *srq) 629 { 630 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 631 int rc; 632 633 rc = __qplib_destroy_srq(rcfw, srq); 634 if (rc) 635 return rc; 636 bnxt_qplib_free_hwq(res, &srq->hwq); 637 kfree(srq->swq); 638 return 0; 639 } 640 641 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, 642 struct bnxt_qplib_srq *srq) 643 { 644 struct bnxt_qplib_hwq_attr hwq_attr = {}; 645 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 646 struct creq_create_srq_resp resp = {}; 647 struct bnxt_qplib_cmdqmsg msg = {}; 648 struct cmdq_create_srq req = {}; 649 u16 pg_sz_lvl = 0; 650 u16 srq_size; 651 int rc, idx; 652 653 hwq_attr.res = res; 654 hwq_attr.sginfo = &srq->sginfo; 655 hwq_attr.depth = srq->max_wqe; 656 hwq_attr.stride = srq->wqe_size; 657 hwq_attr.type = HWQ_TYPE_QUEUE; 658 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); 659 if (rc) 660 goto exit; 661 /* Configure the request */ 662 req.dpi = cpu_to_le32(srq->dpi->dpi); 663 req.srq_handle = cpu_to_le64((uintptr_t)srq); 664 srq_size = min_t(u32, srq->hwq.depth, U16_MAX); 665 req.srq_size = cpu_to_le16(srq_size); 666 pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) << 667 CMDQ_CREATE_SRQ_PG_SIZE_SFT); 668 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK); 669 req.pg_size_lvl = cpu_to_le16(pg_sz_lvl); 670 req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq)); 671 req.pd_id = cpu_to_le32(srq->pd->id); 672 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); 673 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ, 674 sizeof(req)); 675 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 676 sizeof(resp), 0); 677 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 678 if (rc) 679 goto fail; 680 if (!srq->is_user) { 681 srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq), 682 GFP_KERNEL); 683 if (!srq->swq) 684 goto srq_fail; 685 srq->start_idx = 0; 686 srq->last_idx = srq->hwq.depth - 1; 687 for (idx = 0; idx < srq->hwq.depth; idx++) 688 srq->swq[idx].next_idx = idx + 1; 689 srq->swq[srq->last_idx].next_idx = -1; 690 } 691 692 spin_lock_init(&srq->lock); 693 srq->id = le32_to_cpu(resp.xid); 694 srq->cctx = res->cctx; 695 srq->dbinfo.hwq = &srq->hwq; 696 srq->dbinfo.xid = srq->id; 697 srq->dbinfo.db = srq->dpi->dbr; 698 srq->dbinfo.max_slot = 1; 699 srq->dbinfo.priv_db = res->dpi_tbl.priv_db; 700 srq->dbinfo.flags = 0; 701 spin_lock_init(&srq->dbinfo.lock); 702 srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 703 srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; 704 srq->dbinfo.res = res; 705 srq->dbinfo.seed = srq->id; 706 if (srq->threshold) 707 bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); 708 srq->arm_req = false; 709 return 0; 710 srq_fail: 711 __qplib_destroy_srq(rcfw, srq); 712 fail: 713 bnxt_qplib_free_hwq(res, &srq->hwq); 714 exit: 715 return rc; 716 } 717 718 int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, 719 struct bnxt_qplib_srq *srq) 720 { 721 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; 722 u32 avail = 0; 723 724 avail = __bnxt_qplib_get_avail(srq_hwq); 725 if (avail <= srq->threshold) { 726 srq->arm_req = false; 727 bnxt_qplib_srq_arm_db(&srq->dbinfo); 728 } else { 729 /* Deferred arming */ 730 srq->arm_req = true; 731 } 732 return 0; 733 } 734 735 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, 736 struct bnxt_qplib_srq *srq) 737 { 738 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 739 struct creq_query_srq_resp resp = {}; 740 struct bnxt_qplib_cmdqmsg msg = {}; 741 struct creq_query_srq_resp_sb *sb; 742 struct bnxt_qplib_rcfw_sbuf sbuf; 743 struct cmdq_query_srq req = {}; 744 int rc = 0; 745 746 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ, 747 sizeof(req)); 748 sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); 749 sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, 750 &sbuf.dma_addr, GFP_KERNEL); 751 if (!sbuf.sb) 752 return -ENOMEM; 753 req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; 754 req.srq_cid = cpu_to_le32(srq->id); 755 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), 756 sizeof(resp), 0); 757 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 758 /* TODO: What to do with the query? */ 759 dma_free_coherent(&rcfw->pdev->dev, sbuf.size, 760 sbuf.sb, sbuf.dma_addr); 761 762 return rc; 763 } 764 765 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, 766 struct bnxt_qplib_swqe *wqe) 767 { 768 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; 769 struct sq_sge *hw_sge; 770 struct rq_wqe *srqe; 771 int i, rc = 0, next; 772 u32 avail; 773 774 spin_lock(&srq_hwq->lock); 775 if (srq->start_idx == srq->last_idx) { 776 dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n", 777 srq->id); 778 rc = -EINVAL; 779 spin_unlock(&srq_hwq->lock); 780 goto done; 781 } 782 next = srq->start_idx; 783 srq->start_idx = srq->swq[next].next_idx; 784 spin_unlock(&srq_hwq->lock); 785 786 srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL); 787 memset(srqe, 0, srq->wqe_size); 788 /* Calculate wqe_size and data_len */ 789 for (i = 0, hw_sge = (struct sq_sge *)srqe->data; 790 i < wqe->num_sge; i++, hw_sge++) { 791 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); 792 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); 793 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); 794 } 795 srqe->wqe_type = wqe->type; 796 srqe->flags = wqe->flags; 797 srqe->wqe_size = wqe->num_sge + 798 ((offsetof(typeof(*srqe), data) + 15) >> 4); 799 if (!wqe->num_sge) 800 srqe->wqe_size++; 801 srqe->wr_id |= cpu_to_le32((u32)next); 802 srq->swq[next].wr_id = wqe->wr_id; 803 bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); 804 /* retaining srq_hwq->cons for this logic actually the lock is only 805 * required to read srq_hwq->cons. 806 */ 807 spin_lock(&srq_hwq->lock); 808 avail = __bnxt_qplib_get_avail(srq_hwq); 809 spin_unlock(&srq_hwq->lock); 810 /* Ring DB */ 811 bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); 812 if (srq->arm_req && avail <= srq->threshold) { 813 srq->arm_req = false; 814 bnxt_qplib_srq_arm_db(&srq->dbinfo); 815 } 816 done: 817 return rc; 818 } 819 820 /* QP */ 821 static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw, 822 struct bnxt_qplib_qp *qp) 823 { 824 struct creq_destroy_qp_resp resp = {}; 825 struct bnxt_qplib_cmdqmsg msg = {}; 826 struct cmdq_destroy_qp req = {}; 827 828 req.qp_cid = cpu_to_le32(qp->id); 829 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP, 830 sizeof(req)); 831 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 832 sizeof(resp), 0); 833 return bnxt_qplib_rcfw_send_message(rcfw, &msg); 834 } 835 836 static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) 837 { 838 int rc = 0; 839 int indx; 840 841 que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); 842 if (!que->swq) { 843 rc = -ENOMEM; 844 goto out; 845 } 846 847 que->swq_start = 0; 848 que->swq_last = que->max_wqe - 1; 849 for (indx = 0; indx < que->max_wqe; indx++) 850 que->swq[indx].next_idx = indx + 1; 851 que->swq[que->swq_last].next_idx = 0; /* Make it circular */ 852 que->swq_last = 0; 853 out: 854 return rc; 855 } 856 857 static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, 858 u32 *swq_idx) 859 { 860 u32 idx; 861 862 idx = que->swq_start; 863 if (swq_idx) 864 *swq_idx = idx; 865 return &que->swq[idx]; 866 } 867 868 static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) 869 { 870 que->swq_start = que->swq[idx].next_idx; 871 } 872 873 static u32 bnxt_qplib_get_stride(void) 874 { 875 return sizeof(struct sq_sge); 876 } 877 878 static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) 879 { 880 u8 stride; 881 882 stride = bnxt_qplib_get_stride(); 883 return (que->wqe_size * que->max_wqe) / stride; 884 } 885 886 static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) 887 { 888 /* For Variable mode supply number of 16B slots */ 889 return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 890 que->max_wqe : bnxt_qplib_get_depth(que); 891 } 892 893 static u32 _set_sq_max_slot(u8 wqe_mode) 894 { 895 /* for static mode index divisor is 8 */ 896 return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 897 sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; 898 } 899 900 static u32 _set_rq_max_slot(struct bnxt_qplib_q *que) 901 { 902 return (que->wqe_size / sizeof(struct sq_sge)); 903 } 904 905 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 906 { 907 struct bnxt_qplib_hwq_attr hwq_attr = {}; 908 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 909 struct creq_create_qp1_resp resp = {}; 910 struct bnxt_qplib_cmdqmsg msg = {}; 911 struct bnxt_qplib_q *sq = &qp->sq; 912 struct bnxt_qplib_q *rq = &qp->rq; 913 struct cmdq_create_qp1 req = {}; 914 struct bnxt_qplib_reftbl *tbl; 915 unsigned long flag; 916 u8 pg_sz_lvl = 0; 917 u32 qp_flags = 0; 918 int rc; 919 920 /* General */ 921 req.type = qp->type; 922 req.dpi = cpu_to_le32(qp->dpi->dpi); 923 req.qp_handle = cpu_to_le64(qp->qp_handle); 924 /* SQ */ 925 hwq_attr.res = res; 926 hwq_attr.sginfo = &sq->sginfo; 927 hwq_attr.stride = bnxt_qplib_get_stride(); 928 hwq_attr.depth = bnxt_qplib_get_depth(sq); 929 hwq_attr.type = HWQ_TYPE_QUEUE; 930 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); 931 if (rc) 932 goto exit; 933 934 req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode)); 935 req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); 936 pg_sz_lvl = _get_base_pg_size(&sq->hwq) << 937 CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT; 938 pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) << 939 CMDQ_CREATE_QP1_SQ_LVL_SFT); 940 req.sq_pg_size_sq_lvl = pg_sz_lvl; 941 req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) & 942 CMDQ_CREATE_QP1_SQ_FWO_MASK) | 943 (sq->max_sge & 944 CMDQ_CREATE_QP1_SQ_SGE_MASK)); 945 req.scq_cid = cpu_to_le32(qp->scq->id); 946 947 /* RQ */ 948 if (!qp->srq) { 949 hwq_attr.res = res; 950 hwq_attr.sginfo = &rq->sginfo; 951 hwq_attr.stride = bnxt_qplib_get_stride(); 952 hwq_attr.depth = bnxt_qplib_get_depth(rq); 953 hwq_attr.type = HWQ_TYPE_QUEUE; 954 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 955 if (rc) 956 goto fail_sq; 957 req.rq_size = cpu_to_le32(rq->max_wqe); 958 req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); 959 pg_sz_lvl = _get_base_pg_size(&rq->hwq) << 960 CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT; 961 pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << 962 CMDQ_CREATE_QP1_RQ_LVL_SFT); 963 req.rq_pg_size_rq_lvl = pg_sz_lvl; 964 req.rq_fwo_rq_sge = 965 cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) & 966 CMDQ_CREATE_QP1_RQ_FWO_MASK) | 967 (rq->max_sge & 968 CMDQ_CREATE_QP1_RQ_SGE_MASK)); 969 } else { 970 /* SRQ */ 971 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED; 972 req.srq_cid = cpu_to_le32(qp->srq->id); 973 } 974 req.rcq_cid = cpu_to_le32(qp->rcq->id); 975 976 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; 977 req.qp_flags = cpu_to_le32(qp_flags); 978 req.pd_id = cpu_to_le32(qp->pd->id); 979 980 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1, 981 sizeof(req)); 982 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 983 sizeof(resp), 0); 984 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 985 if (rc) 986 goto fail_rq; 987 988 rc = bnxt_qplib_alloc_init_swq(sq); 989 if (rc) 990 goto sq_swq; 991 992 if (!qp->srq) { 993 rc = bnxt_qplib_alloc_init_swq(rq); 994 if (rc) 995 goto rq_swq; 996 } 997 998 qp->id = le32_to_cpu(resp.xid); 999 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 1000 qp->cctx = res->cctx; 1001 sq->dbinfo.hwq = &sq->hwq; 1002 sq->dbinfo.xid = qp->id; 1003 sq->dbinfo.db = qp->dpi->dbr; 1004 sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); 1005 sq->dbinfo.flags = 0; 1006 spin_lock_init(&sq->dbinfo.lock); 1007 sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 1008 sq->dbinfo.res = res; 1009 if (rq->max_wqe) { 1010 rq->dbinfo.hwq = &rq->hwq; 1011 rq->dbinfo.xid = qp->id; 1012 rq->dbinfo.db = qp->dpi->dbr; 1013 rq->dbinfo.max_slot = _set_rq_max_slot(rq); 1014 rq->dbinfo.flags = 0; 1015 spin_lock_init(&rq->dbinfo.lock); 1016 rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 1017 rq->dbinfo.res = res; 1018 } 1019 1020 tbl = &res->reftbl.qpref; 1021 spin_lock_irqsave(&tbl->lock, flag); 1022 tbl->rec[tbl->max].xid = qp->id; 1023 tbl->rec[tbl->max].handle = qp; 1024 spin_unlock_irqrestore(&tbl->lock, flag); 1025 1026 return 0; 1027 rq_swq: 1028 kfree(sq->swq); 1029 sq_swq: 1030 __qplib_destroy_qp(rcfw, qp); 1031 fail_rq: 1032 bnxt_qplib_free_hwq(res, &rq->hwq); 1033 fail_sq: 1034 bnxt_qplib_free_hwq(res, &sq->hwq); 1035 exit: 1036 return rc; 1037 } 1038 1039 static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) 1040 { 1041 struct bnxt_qplib_hwq *sq_hwq; 1042 struct bnxt_qplib_q *sq; 1043 u64 fpsne, psn_pg; 1044 u16 indx_pad = 0; 1045 1046 sq = &qp->sq; 1047 sq_hwq = &sq->hwq; 1048 /* First psn entry */ 1049 fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg); 1050 if (!IS_ALIGNED(fpsne, PAGE_SIZE)) 1051 indx_pad = (fpsne & ~PAGE_MASK) / size; 1052 sq_hwq->pad_pgofft = indx_pad; 1053 sq_hwq->pad_pg = (u64 *)psn_pg; 1054 sq_hwq->pad_stride = size; 1055 } 1056 1057 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1058 { 1059 struct bnxt_qplib_hwq_attr hwq_attr = {}; 1060 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1061 struct bnxt_qplib_sg_info sginfo = {}; 1062 struct creq_create_qp_resp resp = {}; 1063 struct bnxt_qplib_cmdqmsg msg = {}; 1064 struct bnxt_qplib_q *sq = &qp->sq; 1065 struct bnxt_qplib_q *rq = &qp->rq; 1066 struct cmdq_create_qp req = {}; 1067 struct bnxt_qplib_reftbl *tbl; 1068 struct bnxt_qplib_hwq *xrrq; 1069 int rc, req_size, psn_sz; 1070 unsigned long flag; 1071 u8 pg_sz_lvl = 0; 1072 u32 qp_flags = 0; 1073 u32 qp_idx; 1074 u16 nsge; 1075 u32 sqsz; 1076 1077 qp->cctx = res->cctx; 1078 if (res->dattr) 1079 qp->dev_cap_flags = res->dattr->dev_cap_flags; 1080 /* General */ 1081 req.type = qp->type; 1082 req.dpi = cpu_to_le32(qp->dpi->dpi); 1083 req.qp_handle = cpu_to_le64(qp->qp_handle); 1084 1085 /* SQ */ 1086 if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { 1087 psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? 1088 sizeof(struct sq_psn_search_ext) : 1089 sizeof(struct sq_psn_search); 1090 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { 1091 psn_sz = sizeof(struct sq_msn_search); 1092 qp->msn = 0; 1093 } 1094 } else { 1095 psn_sz = 0; 1096 } 1097 1098 hwq_attr.res = res; 1099 hwq_attr.sginfo = &sq->sginfo; 1100 hwq_attr.stride = bnxt_qplib_get_stride(); 1101 hwq_attr.depth = bnxt_qplib_get_depth(sq); 1102 hwq_attr.aux_stride = psn_sz; 1103 hwq_attr.aux_depth = (psn_sz) ? 1104 _set_sq_size(sq, qp->wqe_mode) : 0; 1105 /* Update msn tbl size */ 1106 if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { 1107 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1108 hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); 1109 else 1110 hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2; 1111 qp->msn_tbl_sz = hwq_attr.aux_depth; 1112 qp->msn = 0; 1113 } 1114 hwq_attr.type = HWQ_TYPE_QUEUE; 1115 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); 1116 if (rc) 1117 goto exit; 1118 1119 sqsz = _set_sq_size(sq, qp->wqe_mode); 1120 /* 0xffff is the max sq size hw limits to */ 1121 if (sqsz > BNXT_QPLIB_MAX_SQSZ) { 1122 pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz); 1123 goto fail_sq; 1124 } 1125 req.sq_size = cpu_to_le32(sqsz); 1126 req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); 1127 pg_sz_lvl = _get_base_pg_size(&sq->hwq) << 1128 CMDQ_CREATE_QP_SQ_PG_SIZE_SFT; 1129 pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) << 1130 CMDQ_CREATE_QP_SQ_LVL_SFT); 1131 req.sq_pg_size_sq_lvl = pg_sz_lvl; 1132 req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & 1133 CMDQ_CREATE_QP_SQ_FWO_MASK) | 1134 ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? 1135 BNXT_MSN_TBLE_SGE : sq->max_sge & 1136 CMDQ_CREATE_QP_SQ_SGE_MASK)); 1137 req.scq_cid = cpu_to_le32(qp->scq->id); 1138 1139 /* RQ/SRQ */ 1140 if (!qp->srq) { 1141 hwq_attr.res = res; 1142 hwq_attr.sginfo = &rq->sginfo; 1143 hwq_attr.stride = bnxt_qplib_get_stride(); 1144 hwq_attr.depth = bnxt_qplib_get_depth(rq); 1145 hwq_attr.aux_stride = 0; 1146 hwq_attr.aux_depth = 0; 1147 hwq_attr.type = HWQ_TYPE_QUEUE; 1148 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); 1149 if (rc) 1150 goto fail_sq; 1151 req.rq_size = cpu_to_le32(rq->max_wqe); 1152 req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); 1153 pg_sz_lvl = _get_base_pg_size(&rq->hwq) << 1154 CMDQ_CREATE_QP_RQ_PG_SIZE_SFT; 1155 pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << 1156 CMDQ_CREATE_QP_RQ_LVL_SFT); 1157 req.rq_pg_size_rq_lvl = pg_sz_lvl; 1158 nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1159 res->dattr->max_qp_sges : rq->max_sge; 1160 req.rq_fwo_rq_sge = 1161 cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) & 1162 CMDQ_CREATE_QP_RQ_FWO_MASK) | 1163 (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK)); 1164 } else { 1165 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; 1166 req.srq_cid = cpu_to_le32(qp->srq->id); 1167 } 1168 req.rcq_cid = cpu_to_le32(qp->rcq->id); 1169 1170 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; 1171 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; 1172 if (qp->sig_type) 1173 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; 1174 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) 1175 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; 1176 if (res->cctx->modes.te_bypass) 1177 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED; 1178 if (res->dattr && 1179 bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf)) 1180 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED; 1181 req.qp_flags = cpu_to_le32(qp_flags); 1182 1183 /* ORRQ and IRRQ */ 1184 if (psn_sz) { 1185 xrrq = &qp->orrq; 1186 xrrq->max_elements = 1187 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 1188 req_size = xrrq->max_elements * 1189 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; 1190 req_size &= ~(PAGE_SIZE - 1); 1191 sginfo.pgsize = req_size; 1192 sginfo.pgshft = PAGE_SHIFT; 1193 1194 hwq_attr.res = res; 1195 hwq_attr.sginfo = &sginfo; 1196 hwq_attr.depth = xrrq->max_elements; 1197 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; 1198 hwq_attr.aux_stride = 0; 1199 hwq_attr.aux_depth = 0; 1200 hwq_attr.type = HWQ_TYPE_CTX; 1201 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); 1202 if (rc) 1203 goto fail_rq; 1204 req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq)); 1205 1206 xrrq = &qp->irrq; 1207 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( 1208 qp->max_dest_rd_atomic); 1209 req_size = xrrq->max_elements * 1210 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; 1211 req_size &= ~(PAGE_SIZE - 1); 1212 sginfo.pgsize = req_size; 1213 hwq_attr.depth = xrrq->max_elements; 1214 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; 1215 rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); 1216 if (rc) 1217 goto fail_orrq; 1218 req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq)); 1219 } 1220 req.pd_id = cpu_to_le32(qp->pd->id); 1221 1222 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP, 1223 sizeof(req)); 1224 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 1225 sizeof(resp), 0); 1226 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 1227 if (rc) 1228 goto fail; 1229 1230 if (!qp->is_user) { 1231 rc = bnxt_qplib_alloc_init_swq(sq); 1232 if (rc) 1233 goto swq_sq; 1234 if (!qp->srq) { 1235 rc = bnxt_qplib_alloc_init_swq(rq); 1236 if (rc) 1237 goto swq_rq; 1238 } 1239 if (psn_sz) 1240 bnxt_qplib_init_psn_ptr(qp, psn_sz); 1241 } 1242 qp->id = le32_to_cpu(resp.xid); 1243 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; 1244 INIT_LIST_HEAD(&qp->sq_flush); 1245 INIT_LIST_HEAD(&qp->rq_flush); 1246 1247 sq->dbinfo.hwq = &sq->hwq; 1248 sq->dbinfo.xid = qp->id; 1249 sq->dbinfo.db = qp->dpi->dbr; 1250 sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); 1251 sq->dbinfo.flags = 0; 1252 spin_lock_init(&sq->dbinfo.lock); 1253 sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 1254 sq->dbinfo.res = res; 1255 sq->dbinfo.seed = qp->id; 1256 if (rq->max_wqe) { 1257 rq->dbinfo.hwq = &rq->hwq; 1258 rq->dbinfo.xid = qp->id; 1259 rq->dbinfo.db = qp->dpi->dbr; 1260 rq->dbinfo.max_slot = _set_rq_max_slot(rq); 1261 rq->dbinfo.flags = 0; 1262 spin_lock_init(&rq->dbinfo.lock); 1263 rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 1264 rq->dbinfo.res = res; 1265 rq->dbinfo.seed = qp->id; 1266 } 1267 1268 tbl = &res->reftbl.qpref; 1269 qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); 1270 spin_lock_irqsave(&tbl->lock, flag); 1271 tbl->rec[qp_idx].xid = qp->id; 1272 tbl->rec[qp_idx].handle = qp; 1273 spin_unlock_irqrestore(&tbl->lock, flag); 1274 1275 return 0; 1276 swq_rq: 1277 kfree(sq->swq); 1278 swq_sq: 1279 __qplib_destroy_qp(rcfw, qp); 1280 fail: 1281 bnxt_qplib_free_hwq(res, &qp->irrq); 1282 fail_orrq: 1283 bnxt_qplib_free_hwq(res, &qp->orrq); 1284 fail_rq: 1285 bnxt_qplib_free_hwq(res, &rq->hwq); 1286 fail_sq: 1287 bnxt_qplib_free_hwq(res, &sq->hwq); 1288 exit: 1289 return rc; 1290 } 1291 1292 static void __filter_modify_flags(struct bnxt_qplib_qp *qp) 1293 { 1294 switch (qp->cur_qp_state) { 1295 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1296 switch (qp->state) { 1297 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1298 break; 1299 default: 1300 break; 1301 } 1302 break; 1303 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1304 switch (qp->state) { 1305 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1306 if (!(qp->modify_flags & 1307 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { 1308 qp->modify_flags |= 1309 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1310 qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1311 } 1312 qp->modify_flags &= 1313 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1314 /* Bono FW requires the max_dest_rd_atomic to be >= 1 */ 1315 if (qp->max_dest_rd_atomic < 1) 1316 qp->max_dest_rd_atomic = 1; 1317 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; 1318 /* Bono FW 20.6.5 requires SGID_INDEX to be configured */ 1319 if (!(qp->modify_flags & 1320 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { 1321 qp->modify_flags |= 1322 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; 1323 qp->ah.sgid_index = 0; 1324 } 1325 break; 1326 default: 1327 break; 1328 } 1329 break; 1330 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1331 switch (qp->state) { 1332 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1333 /* Bono FW requires the max_rd_atomic to be >= 1 */ 1334 if (qp->max_rd_atomic < 1) 1335 qp->max_rd_atomic = 1; 1336 qp->modify_flags &= 1337 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | 1338 CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1339 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1340 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1341 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1342 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1343 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1344 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | 1345 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | 1346 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | 1347 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | 1348 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); 1349 break; 1350 default: 1351 break; 1352 } 1353 break; 1354 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1355 break; 1356 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1357 break; 1358 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1359 break; 1360 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1361 break; 1362 default: 1363 break; 1364 } 1365 } 1366 1367 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1368 { 1369 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1370 struct creq_modify_qp_resp resp = {}; 1371 struct bnxt_qplib_cmdqmsg msg = {}; 1372 struct cmdq_modify_qp req = {}; 1373 bool ppp_requested = false; 1374 u32 temp32[4]; 1375 u32 bmask; 1376 int rc; 1377 1378 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP, 1379 sizeof(req)); 1380 1381 /* Filter out the qp_attr_mask based on the state->new transition */ 1382 __filter_modify_flags(qp); 1383 bmask = qp->modify_flags; 1384 req.modify_mask = cpu_to_le32(qp->modify_flags); 1385 req.qp_cid = cpu_to_le32(qp->id); 1386 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { 1387 req.network_type_en_sqd_async_notify_new_state = 1388 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | 1389 (qp->en_sqd_async_notify == true ? 1390 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); 1391 if (__can_request_ppp(qp)) { 1392 req.path_mtu_pingpong_push_enable = 1393 CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE; 1394 req.pingpong_push_dpi = qp->ppp.dpi; 1395 ppp_requested = true; 1396 } 1397 } 1398 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; 1399 1400 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) { 1401 req.access = qp->access; 1402 } 1403 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) 1404 req.pkey = IB_DEFAULT_PKEY_FULL; 1405 1406 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) { 1407 req.qkey = cpu_to_le32(qp->qkey); 1408 } 1409 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { 1410 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); 1411 req.dgid[0] = cpu_to_le32(temp32[0]); 1412 req.dgid[1] = cpu_to_le32(temp32[1]); 1413 req.dgid[2] = cpu_to_le32(temp32[2]); 1414 req.dgid[3] = cpu_to_le32(temp32[3]); 1415 } 1416 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) { 1417 req.flow_label = cpu_to_le32(qp->ah.flow_label); 1418 } 1419 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) { 1420 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]); 1421 } 1422 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) { 1423 req.hop_limit = qp->ah.hop_limit; 1424 } 1425 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) { 1426 req.traffic_class = qp->ah.traffic_class; 1427 } 1428 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) { 1429 memcpy(req.dest_mac, qp->ah.dmac, 6); 1430 } 1431 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) { 1432 req.path_mtu_pingpong_push_enable = qp->path_mtu; 1433 } 1434 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) { 1435 req.timeout = qp->timeout; 1436 } 1437 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) { 1438 req.retry_cnt = qp->retry_cnt; 1439 } 1440 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) { 1441 req.rnr_retry = qp->rnr_retry; 1442 } 1443 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) { 1444 req.min_rnr_timer = qp->min_rnr_timer; 1445 } 1446 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) { 1447 req.rq_psn = cpu_to_le32(qp->rq.psn); 1448 } 1449 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) { 1450 req.sq_psn = cpu_to_le32(qp->sq.psn); 1451 } 1452 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) { 1453 req.max_rd_atomic = 1454 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); 1455 } 1456 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) { 1457 req.max_dest_rd_atomic = 1458 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); 1459 } 1460 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); 1461 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); 1462 req.sq_sge = cpu_to_le16(qp->sq.max_sge); 1463 req.rq_sge = cpu_to_le16(qp->rq.max_sge); 1464 req.max_inline_data = cpu_to_le32(qp->max_inline_data); 1465 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) 1466 req.dest_qp_id = cpu_to_le32(qp->dest_qpn); 1467 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC) 1468 req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC); 1469 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN) 1470 req.tos_dscp_tos_ecn = 1471 ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) & 1472 CMDQ_MODIFY_QP_TOS_ECN_MASK); 1473 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP) 1474 req.tos_dscp_tos_ecn |= 1475 ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) & 1476 CMDQ_MODIFY_QP_TOS_DSCP_MASK); 1477 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); 1478 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 1479 sizeof(resp), 0); 1480 msg.qp_state = qp->state; 1481 1482 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 1483 if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) { 1484 qp->cur_qp_state = qp->state; 1485 return 0; 1486 } else if (rc) { 1487 return rc; 1488 } 1489 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) 1490 qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac); 1491 1492 if (ppp_requested) 1493 qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled; 1494 1495 qp->cur_qp_state = qp->state; 1496 return 0; 1497 } 1498 1499 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) 1500 { 1501 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1502 struct creq_query_qp_resp resp = {}; 1503 struct bnxt_qplib_cmdqmsg msg = {}; 1504 struct bnxt_qplib_rcfw_sbuf sbuf; 1505 struct creq_query_qp_resp_sb *sb; 1506 struct cmdq_query_qp req = {}; 1507 u32 temp32[4]; 1508 int i, rc; 1509 1510 sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); 1511 sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, 1512 &sbuf.dma_addr, GFP_KERNEL); 1513 if (!sbuf.sb) 1514 return -ENOMEM; 1515 sb = sbuf.sb; 1516 1517 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP, 1518 sizeof(req)); 1519 req.qp_cid = cpu_to_le32(qp->id); 1520 req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; 1521 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), 1522 sizeof(resp), 0); 1523 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 1524 if (rc) 1525 goto bail; 1526 1527 /* Extract the context from the side buffer */ 1528 qp->state = sb->en_sqd_async_notify_state & 1529 CREQ_QUERY_QP_RESP_SB_STATE_MASK; 1530 qp->cur_qp_state = qp->state; 1531 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & 1532 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? 1533 true : false; 1534 qp->access = sb->access; 1535 qp->pkey_index = le16_to_cpu(sb->pkey); 1536 qp->qkey = le32_to_cpu(sb->qkey); 1537 1538 temp32[0] = le32_to_cpu(sb->dgid[0]); 1539 temp32[1] = le32_to_cpu(sb->dgid[1]); 1540 temp32[2] = le32_to_cpu(sb->dgid[2]); 1541 temp32[3] = le32_to_cpu(sb->dgid[3]); 1542 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); 1543 1544 qp->ah.flow_label = le32_to_cpu(sb->flow_label); 1545 1546 qp->ah.sgid_index = 0; 1547 for (i = 0; i < res->sgid_tbl.max; i++) { 1548 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { 1549 qp->ah.sgid_index = i; 1550 break; 1551 } 1552 } 1553 if (i == res->sgid_tbl.max) 1554 dev_dbg(&res->pdev->dev, 1555 "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n", 1556 qp->id, le16_to_cpu(sb->sgid_index)); 1557 1558 qp->ah.hop_limit = sb->hop_limit; 1559 qp->ah.traffic_class = sb->traffic_class; 1560 memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN); 1561 qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1562 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >> 1563 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; 1564 qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) & 1565 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK; 1566 qp->timeout = sb->timeout; 1567 qp->retry_cnt = sb->retry_cnt; 1568 qp->rnr_retry = sb->rnr_retry; 1569 qp->min_rnr_timer = sb->min_rnr_timer; 1570 qp->rq.psn = le32_to_cpu(sb->rq_psn); 1571 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); 1572 qp->sq.psn = le32_to_cpu(sb->sq_psn); 1573 qp->max_dest_rd_atomic = 1574 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); 1575 qp->sq.max_wqe = qp->sq.hwq.max_elements; 1576 qp->rq.max_wqe = qp->rq.hwq.max_elements; 1577 qp->sq.max_sge = le16_to_cpu(sb->sq_sge); 1578 qp->rq.max_sge = le16_to_cpu(sb->rq_sge); 1579 qp->max_inline_data = le32_to_cpu(sb->max_inline_data); 1580 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); 1581 memcpy(qp->smac, sb->src_mac, ETH_ALEN); 1582 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); 1583 qp->port_id = le16_to_cpu(sb->port_id); 1584 bail: 1585 dma_free_coherent(&rcfw->pdev->dev, sbuf.size, 1586 sbuf.sb, sbuf.dma_addr); 1587 return rc; 1588 } 1589 1590 1591 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) 1592 { 1593 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; 1594 u32 peek_flags, peek_cons; 1595 struct cq_base *hw_cqe; 1596 int i; 1597 1598 peek_flags = cq->dbinfo.flags; 1599 peek_cons = cq_hwq->cons; 1600 for (i = 0; i < cq_hwq->depth; i++) { 1601 hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL); 1602 if (CQE_CMP_VALID(hw_cqe, peek_flags)) { 1603 dma_rmb(); 1604 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { 1605 case CQ_BASE_CQE_TYPE_REQ: 1606 case CQ_BASE_CQE_TYPE_TERMINAL: 1607 { 1608 struct cq_req *cqe = (struct cq_req *)hw_cqe; 1609 1610 if (qp == le64_to_cpu(cqe->qp_handle)) 1611 cqe->qp_handle = 0; 1612 break; 1613 } 1614 case CQ_BASE_CQE_TYPE_RES_RC: 1615 case CQ_BASE_CQE_TYPE_RES_UD: 1616 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 1617 { 1618 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; 1619 1620 if (qp == le64_to_cpu(cqe->qp_handle)) 1621 cqe->qp_handle = 0; 1622 break; 1623 } 1624 default: 1625 break; 1626 } 1627 } 1628 bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons, 1629 1, &peek_flags); 1630 } 1631 } 1632 1633 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, 1634 struct bnxt_qplib_qp *qp) 1635 { 1636 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1637 struct bnxt_qplib_reftbl *tbl; 1638 unsigned long flags; 1639 u32 qp_idx; 1640 int rc; 1641 1642 tbl = &res->reftbl.qpref; 1643 qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); 1644 spin_lock_irqsave(&tbl->lock, flags); 1645 tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID; 1646 tbl->rec[qp_idx].handle = NULL; 1647 spin_unlock_irqrestore(&tbl->lock, flags); 1648 1649 rc = __qplib_destroy_qp(rcfw, qp); 1650 if (rc) { 1651 spin_lock_irqsave(&tbl->lock, flags); 1652 tbl->rec[qp_idx].xid = qp->id; 1653 tbl->rec[qp_idx].handle = qp; 1654 spin_unlock_irqrestore(&tbl->lock, flags); 1655 return rc; 1656 } 1657 1658 return 0; 1659 } 1660 1661 void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, 1662 struct bnxt_qplib_qp *qp) 1663 { 1664 if (qp->irrq.max_elements) 1665 bnxt_qplib_free_hwq(res, &qp->irrq); 1666 if (qp->orrq.max_elements) 1667 bnxt_qplib_free_hwq(res, &qp->orrq); 1668 1669 if (!qp->is_user) 1670 kfree(qp->rq.swq); 1671 bnxt_qplib_free_hwq(res, &qp->rq.hwq); 1672 1673 if (!qp->is_user) 1674 kfree(qp->sq.swq); 1675 bnxt_qplib_free_hwq(res, &qp->sq.hwq); 1676 } 1677 1678 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, 1679 struct bnxt_qplib_sge *sge) 1680 { 1681 struct bnxt_qplib_q *sq = &qp->sq; 1682 struct bnxt_qplib_hdrbuf *buf; 1683 u32 sw_prod; 1684 1685 memset(sge, 0, sizeof(*sge)); 1686 1687 buf = qp->sq_hdr_buf; 1688 if (buf) { 1689 sw_prod = sq->swq_start; 1690 sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); 1691 sge->lkey = 0xFFFFFFFF; 1692 sge->size = buf->step; 1693 return buf->va + sw_prod * sge->size; 1694 } 1695 return NULL; 1696 } 1697 1698 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) 1699 { 1700 struct bnxt_qplib_q *rq = &qp->rq; 1701 1702 return rq->swq_start; 1703 } 1704 1705 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, 1706 struct bnxt_qplib_sge *sge) 1707 { 1708 struct bnxt_qplib_q *rq = &qp->rq; 1709 struct bnxt_qplib_hdrbuf *buf; 1710 u32 sw_prod; 1711 1712 memset(sge, 0, sizeof(*sge)); 1713 1714 buf = qp->rq_hdr_buf; 1715 if (buf) { 1716 sw_prod = rq->swq_start; 1717 sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); 1718 sge->lkey = 0xFFFFFFFF; 1719 sge->size = buf->step; 1720 return buf->va + sw_prod * sge->size; 1721 } 1722 return NULL; 1723 } 1724 1725 /* Fil the MSN table into the next psn row */ 1726 static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp, 1727 struct bnxt_qplib_swqe *wqe, 1728 struct bnxt_qplib_swq *swq) 1729 { 1730 struct sq_msn_search *msns; 1731 u32 start_psn, next_psn; 1732 u16 start_idx; 1733 1734 msns = (struct sq_msn_search *)swq->psn_search; 1735 msns->start_idx_next_psn_start_psn = 0; 1736 1737 start_psn = swq->start_psn; 1738 next_psn = swq->next_psn; 1739 start_idx = swq->slot_idx; 1740 msns->start_idx_next_psn_start_psn |= 1741 bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn); 1742 pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n", 1743 qp->msn, 1744 (u16) 1745 cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)), 1746 (u32) 1747 cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)), 1748 (u32) 1749 cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn))); 1750 qp->msn++; 1751 qp->msn %= qp->msn_tbl_sz; 1752 } 1753 1754 static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, 1755 struct bnxt_qplib_swqe *wqe, 1756 struct bnxt_qplib_swq *swq) 1757 { 1758 struct sq_psn_search_ext *psns_ext; 1759 struct sq_psn_search *psns; 1760 u32 flg_npsn; 1761 u32 op_spsn; 1762 1763 if (!swq->psn_search) 1764 return; 1765 1766 /* Handle MSN differently on cap flags */ 1767 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { 1768 bnxt_qplib_fill_msn_search(qp, wqe, swq); 1769 return; 1770 } 1771 psns = (struct sq_psn_search *)swq->psn_search; 1772 psns_ext = (struct sq_psn_search_ext *)swq->psn_search; 1773 1774 op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & 1775 SQ_PSN_SEARCH_START_PSN_MASK); 1776 op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & 1777 SQ_PSN_SEARCH_OPCODE_MASK); 1778 flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & 1779 SQ_PSN_SEARCH_NEXT_PSN_MASK); 1780 1781 if (_is_chip_gen_p5_p7(qp->cctx)) { 1782 psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); 1783 psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); 1784 psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); 1785 } else { 1786 psns->opcode_start_psn = cpu_to_le32(op_spsn); 1787 psns->flags_next_psn = cpu_to_le32(flg_npsn); 1788 } 1789 } 1790 1791 static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe) 1792 { 1793 u16 size = 0; 1794 int indx; 1795 1796 for (indx = 0; indx < wqe->num_sge; indx++) 1797 size += wqe->sg_list[indx].size; 1798 return size; 1799 } 1800 1801 static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp, 1802 struct bnxt_qplib_swqe *wqe, 1803 u32 *sw_prod) 1804 { 1805 struct bnxt_qplib_hwq *sq_hwq; 1806 int len, t_len, offt = 0; 1807 int t_cplen = 0, cplen; 1808 bool pull_dst = true; 1809 void *il_dst = NULL; 1810 void *il_src = NULL; 1811 int indx; 1812 1813 sq_hwq = &qp->sq.hwq; 1814 t_len = 0; 1815 for (indx = 0; indx < wqe->num_sge; indx++) { 1816 len = wqe->sg_list[indx].size; 1817 il_src = (void *)wqe->sg_list[indx].addr; 1818 t_len += len; 1819 if (t_len > qp->max_inline_data) 1820 goto bad; 1821 while (len) { 1822 if (pull_dst) { 1823 pull_dst = false; 1824 il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % 1825 sq_hwq->depth), NULL); 1826 (*sw_prod)++; 1827 t_cplen = 0; 1828 offt = 0; 1829 } 1830 cplen = min_t(int, len, sizeof(struct sq_sge)); 1831 cplen = min_t(int, cplen, 1832 (sizeof(struct sq_sge) - offt)); 1833 memcpy(il_dst, il_src, cplen); 1834 t_cplen += cplen; 1835 il_src += cplen; 1836 il_dst += cplen; 1837 offt += cplen; 1838 len -= cplen; 1839 if (t_cplen == sizeof(struct sq_sge)) 1840 pull_dst = true; 1841 } 1842 } 1843 1844 return t_len; 1845 bad: 1846 return -ENOMEM; 1847 } 1848 1849 static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq, 1850 struct bnxt_qplib_sge *ssge, 1851 u32 nsge, u32 *sw_prod) 1852 { 1853 struct sq_sge *dsge; 1854 int indx, len = 0; 1855 1856 for (indx = 0; indx < nsge; indx++, (*sw_prod)++) { 1857 dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL); 1858 dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); 1859 dsge->l_key = cpu_to_le32(ssge[indx].lkey); 1860 dsge->size = cpu_to_le32(ssge[indx].size); 1861 len += ssge[indx].size; 1862 } 1863 return len; 1864 } 1865 1866 static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp, 1867 struct bnxt_qplib_swqe *wqe, u16 *wqe_byte) 1868 { 1869 u16 wqe_size; 1870 u32 ilsize; 1871 u16 nsge; 1872 1873 nsge = wqe->num_sge; 1874 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { 1875 ilsize = _calc_ilsize(wqe); 1876 wqe_size = (ilsize > qp->max_inline_data) ? 1877 qp->max_inline_data : ilsize; 1878 wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge)); 1879 } else { 1880 wqe_size = nsge * sizeof(struct sq_sge); 1881 } 1882 /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */ 1883 wqe_size += sizeof(struct sq_send_hdr); 1884 if (wqe_byte) 1885 *wqe_byte = wqe_size; 1886 return wqe_size / sizeof(struct sq_sge); 1887 } 1888 1889 static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes) 1890 { 1891 /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 1892 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. 1893 * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. 1894 * when 8916 is disabled. 1895 */ 1896 return (que->q_full_delta * wqe_bytes) / que->hwq.element_size; 1897 } 1898 1899 static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, 1900 struct bnxt_qplib_swq *swq, bool hw_retx) 1901 { 1902 struct bnxt_qplib_hwq *sq_hwq; 1903 u32 pg_num, pg_indx; 1904 void *buff; 1905 u32 tail; 1906 1907 sq_hwq = &sq->hwq; 1908 if (!sq_hwq->pad_pg) 1909 return; 1910 1911 tail = swq->slot_idx / sq->dbinfo.max_slot; 1912 if (hw_retx) 1913 tail %= qp->msn_tbl_sz; 1914 pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); 1915 pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); 1916 buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); 1917 /* the start ptr for buff is same ie after the SQ */ 1918 swq->psn_search = buff; 1919 } 1920 1921 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) 1922 { 1923 struct bnxt_qplib_q *sq = &qp->sq; 1924 1925 bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); 1926 } 1927 1928 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, 1929 struct bnxt_qplib_swqe *wqe) 1930 { 1931 struct bnxt_qplib_nq_work *nq_work = NULL; 1932 int i, rc = 0, data_len = 0, pkt_num = 0; 1933 struct bnxt_qplib_q *sq = &qp->sq; 1934 struct bnxt_qplib_hwq *sq_hwq; 1935 struct bnxt_qplib_swq *swq; 1936 bool sch_handler = false; 1937 u16 slots_needed; 1938 void *base_hdr; 1939 void *ext_hdr; 1940 __le32 temp32; 1941 u16 qfd_slots; 1942 u8 wqe_slots; 1943 u16 wqe_size; 1944 u32 sw_prod; 1945 u32 wqe_idx; 1946 1947 sq_hwq = &sq->hwq; 1948 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && 1949 qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1950 dev_err(&sq_hwq->pdev->dev, 1951 "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", 1952 qp->id, qp->state); 1953 rc = -EINVAL; 1954 goto done; 1955 } 1956 1957 wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size); 1958 slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1959 sq->dbinfo.max_slot : wqe_slots; 1960 qfd_slots = _translate_q_full_delta(sq, wqe_size); 1961 if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) { 1962 dev_err(&sq_hwq->pdev->dev, 1963 "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id); 1964 dev_err(&sq_hwq->pdev->dev, 1965 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n", 1966 HWQ_CMP(sq_hwq->prod, sq_hwq), 1967 HWQ_CMP(sq_hwq->cons, sq_hwq), 1968 sq_hwq->max_elements, qfd_slots, slots_needed); 1969 dev_err(&sq_hwq->pdev->dev, 1970 "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n", 1971 sq->phantom_wqe_cnt, sq->phantom_cqe_cnt); 1972 rc = -ENOMEM; 1973 goto done; 1974 } 1975 1976 sw_prod = sq_hwq->prod; 1977 swq = bnxt_qplib_get_swqe(sq, &wqe_idx); 1978 swq->slot_idx = sw_prod; 1979 bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); 1980 1981 swq->wr_id = wqe->wr_id; 1982 swq->type = wqe->type; 1983 swq->flags = wqe->flags; 1984 swq->slots = slots_needed; 1985 swq->start_psn = sq->psn & BTH_PSN_MASK; 1986 if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP) 1987 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; 1988 1989 dev_dbg(&sq_hwq->pdev->dev, 1990 "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n", 1991 qp->id, wqe_idx, swq->wr_id); 1992 if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1993 sch_handler = true; 1994 dev_dbg(&sq_hwq->pdev->dev, 1995 "%s Error QP. Scheduling for poll_cq\n", __func__); 1996 goto queue_err; 1997 } 1998 1999 base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL); 2000 sw_prod++; 2001 ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL); 2002 sw_prod++; 2003 memset(base_hdr, 0, sizeof(struct sq_sge)); 2004 memset(ext_hdr, 0, sizeof(struct sq_sge)); 2005 2006 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) 2007 data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod); 2008 else 2009 data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list, 2010 wqe->num_sge, &sw_prod); 2011 if (data_len < 0) 2012 goto queue_err; 2013 /* Specifics */ 2014 switch (wqe->type) { 2015 case BNXT_QPLIB_SWQE_TYPE_SEND: 2016 if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE || 2017 qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { 2018 /* Assemble info for Raw Ethertype QPs */ 2019 struct sq_send_raweth_qp1_hdr *sqe = base_hdr; 2020 struct sq_raw_ext_hdr *ext_sqe = ext_hdr; 2021 2022 sqe->wqe_type = wqe->type; 2023 sqe->flags = wqe->flags; 2024 sqe->wqe_size = wqe_slots; 2025 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); 2026 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); 2027 sqe->length = cpu_to_le32(data_len); 2028 ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & 2029 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << 2030 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); 2031 2032 dev_dbg(&sq_hwq->pdev->dev, 2033 "QPLIB: FP: RAW/QP1 Send WQE:\n" 2034 "\twqe_type = 0x%x\n" 2035 "\tflags = 0x%x\n" 2036 "\twqe_size = 0x%x\n" 2037 "\tlflags = 0x%x\n" 2038 "\tcfa_action = 0x%x\n" 2039 "\tlength = 0x%x\n" 2040 "\tcfa_meta = 0x%x\n", 2041 sqe->wqe_type, sqe->flags, sqe->wqe_size, 2042 sqe->lflags, sqe->cfa_action, 2043 sqe->length, ext_sqe->cfa_meta); 2044 break; 2045 } 2046 fallthrough; 2047 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 2048 fallthrough; 2049 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 2050 { 2051 struct sq_send_hdr *sqe = base_hdr; 2052 struct sq_ud_ext_hdr *ext_sqe = ext_hdr; 2053 2054 sqe->wqe_type = wqe->type; 2055 sqe->flags = wqe->flags; 2056 sqe->wqe_size = wqe_slots; 2057 sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); 2058 if (qp->type == CMDQ_CREATE_QP_TYPE_UD || 2059 qp->type == CMDQ_CREATE_QP_TYPE_GSI) { 2060 sqe->q_key = cpu_to_le32(wqe->send.q_key); 2061 sqe->length = cpu_to_le32(data_len); 2062 ext_sqe->dst_qp = cpu_to_le32( 2063 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); 2064 ext_sqe->avid = cpu_to_le32(wqe->send.avid & 2065 SQ_SEND_AVID_MASK); 2066 sq->psn = (sq->psn + 1) & BTH_PSN_MASK; 2067 } else { 2068 sqe->length = cpu_to_le32(data_len); 2069 if (qp->mtu) 2070 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 2071 if (!pkt_num) 2072 pkt_num = 1; 2073 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 2074 } 2075 dev_dbg(&sq_hwq->pdev->dev, 2076 "QPLIB: FP: Send WQE:\n" 2077 "\twqe_type = 0x%x\n" 2078 "\tflags = 0x%x\n" 2079 "\twqe_size = 0x%x\n" 2080 "\tinv_key/immdata = 0x%x\n" 2081 "\tq_key = 0x%x\n" 2082 "\tdst_qp = 0x%x\n" 2083 "\tlength = 0x%x\n" 2084 "\tavid = 0x%x\n", 2085 sqe->wqe_type, sqe->flags, sqe->wqe_size, 2086 sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp, 2087 sqe->length, ext_sqe->avid); 2088 break; 2089 } 2090 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 2091 /* fall-thru */ 2092 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 2093 /* fall-thru */ 2094 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 2095 { 2096 struct sq_rdma_hdr *sqe = base_hdr; 2097 struct sq_rdma_ext_hdr *ext_sqe = ext_hdr; 2098 2099 sqe->wqe_type = wqe->type; 2100 sqe->flags = wqe->flags; 2101 sqe->wqe_size = wqe_slots; 2102 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); 2103 sqe->length = cpu_to_le32((u32)data_len); 2104 ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); 2105 ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); 2106 if (qp->mtu) 2107 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 2108 if (!pkt_num) 2109 pkt_num = 1; 2110 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 2111 2112 dev_dbg(&sq_hwq->pdev->dev, 2113 "QPLIB: FP: RDMA WQE:\n" 2114 "\twqe_type = 0x%x\n" 2115 "\tflags = 0x%x\n" 2116 "\twqe_size = 0x%x\n" 2117 "\timmdata = 0x%x\n" 2118 "\tlength = 0x%x\n" 2119 "\tremote_va = 0x%llx\n" 2120 "\tremote_key = 0x%x\n", 2121 sqe->wqe_type, sqe->flags, sqe->wqe_size, 2122 sqe->imm_data, sqe->length, ext_sqe->remote_va, 2123 ext_sqe->remote_key); 2124 break; 2125 } 2126 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 2127 /* fall-thru */ 2128 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 2129 { 2130 struct sq_atomic_hdr *sqe = base_hdr; 2131 struct sq_atomic_ext_hdr *ext_sqe = ext_hdr; 2132 2133 sqe->wqe_type = wqe->type; 2134 sqe->flags = wqe->flags; 2135 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); 2136 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); 2137 ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); 2138 ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); 2139 if (qp->mtu) 2140 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; 2141 if (!pkt_num) 2142 pkt_num = 1; 2143 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; 2144 break; 2145 } 2146 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 2147 { 2148 struct sq_localinvalidate_hdr *sqe = base_hdr; 2149 2150 sqe->wqe_type = wqe->type; 2151 sqe->flags = wqe->flags; 2152 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); 2153 2154 dev_dbg(&sq_hwq->pdev->dev, 2155 "QPLIB: FP: LOCAL INV WQE:\n" 2156 "\twqe_type = 0x%x\n" 2157 "\tflags = 0x%x\n" 2158 "\tinv_l_key = 0x%x\n", 2159 sqe->wqe_type, sqe->flags, sqe->inv_l_key); 2160 break; 2161 } 2162 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: 2163 { 2164 struct sq_fr_pmr_hdr *sqe = base_hdr; 2165 struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr; 2166 2167 sqe->wqe_type = wqe->type; 2168 sqe->flags = wqe->flags; 2169 sqe->access_cntl = wqe->frmr.access_cntl | 2170 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2171 sqe->zero_based_page_size_log = 2172 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << 2173 SQ_FR_PMR_PAGE_SIZE_LOG_SFT | 2174 (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0); 2175 sqe->l_key = cpu_to_le32(wqe->frmr.l_key); 2176 /* TODO: OFED only provides length of MR up to 32-bits for FRMR */ 2177 temp32 = cpu_to_le32(wqe->frmr.length); 2178 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); 2179 sqe->numlevels_pbl_page_size_log = 2180 ((wqe->frmr.pbl_pg_sz_log << 2181 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & 2182 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | 2183 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & 2184 SQ_FR_PMR_NUMLEVELS_MASK); 2185 if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) { 2186 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]); 2187 } else { 2188 for (i = 0; i < wqe->frmr.page_list_len; i++) 2189 wqe->frmr.pbl_ptr[i] = cpu_to_le64( 2190 wqe->frmr.page_list[i] | 2191 PTU_PTE_VALID); 2192 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); 2193 } 2194 ext_sqe->va = cpu_to_le64(wqe->frmr.va); 2195 dev_dbg(&sq_hwq->pdev->dev, 2196 "QPLIB: FP: FRMR WQE:\n" 2197 "\twqe_type = 0x%x\n" 2198 "\tflags = 0x%x\n" 2199 "\taccess_cntl = 0x%x\n" 2200 "\tzero_based_page_size_log = 0x%x\n" 2201 "\tl_key = 0x%x\n" 2202 "\tlength = 0x%x\n" 2203 "\tnumlevels_pbl_page_size_log = 0x%x\n" 2204 "\tpblptr = 0x%llx\n" 2205 "\tva = 0x%llx\n", 2206 sqe->wqe_type, sqe->flags, sqe->access_cntl, 2207 sqe->zero_based_page_size_log, sqe->l_key, 2208 *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, 2209 ext_sqe->pblptr, ext_sqe->va); 2210 break; 2211 } 2212 case BNXT_QPLIB_SWQE_TYPE_BIND_MW: 2213 { 2214 struct sq_bind_hdr *sqe = base_hdr; 2215 struct sq_bind_ext_hdr *ext_sqe = ext_hdr; 2216 2217 sqe->wqe_type = wqe->type; 2218 sqe->flags = wqe->flags; 2219 sqe->access_cntl = wqe->bind.access_cntl; 2220 sqe->mw_type_zero_based = wqe->bind.mw_type | 2221 (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0); 2222 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); 2223 sqe->l_key = cpu_to_le32(wqe->bind.r_key); 2224 ext_sqe->va = cpu_to_le64(wqe->bind.va); 2225 ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); 2226 dev_dbg(&sq_hwq->pdev->dev, 2227 "QPLIB: FP: BIND WQE:\n" 2228 "\twqe_type = 0x%x\n" 2229 "\tflags = 0x%x\n" 2230 "\taccess_cntl = 0x%x\n" 2231 "\tmw_type_zero_based = 0x%x\n" 2232 "\tparent_l_key = 0x%x\n" 2233 "\tl_key = 0x%x\n" 2234 "\tva = 0x%llx\n" 2235 "\tlength = 0x%x\n", 2236 sqe->wqe_type, sqe->flags, sqe->access_cntl, 2237 sqe->mw_type_zero_based, sqe->parent_l_key, 2238 sqe->l_key, sqe->va, ext_sqe->length_lo); 2239 break; 2240 } 2241 default: 2242 /* Bad wqe, return error */ 2243 rc = -EINVAL; 2244 goto done; 2245 } 2246 swq->next_psn = sq->psn & BTH_PSN_MASK; 2247 bnxt_qplib_fill_psn_search(qp, wqe, swq); 2248 2249 queue_err: 2250 bnxt_qplib_swq_mod_start(sq, wqe_idx); 2251 bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots); 2252 qp->wqe_cnt++; 2253 done: 2254 if (sch_handler) { 2255 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 2256 if (nq_work) { 2257 nq_work->cq = qp->scq; 2258 nq_work->nq = qp->scq->nq; 2259 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 2260 queue_work(qp->scq->nq->cqn_wq, &nq_work->work); 2261 } else { 2262 dev_err(&sq->hwq.pdev->dev, 2263 "QPLIB: FP: Failed to allocate SQ nq_work!\n"); 2264 rc = -ENOMEM; 2265 } 2266 } 2267 return rc; 2268 } 2269 2270 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) 2271 { 2272 struct bnxt_qplib_q *rq = &qp->rq; 2273 2274 bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); 2275 } 2276 2277 void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq) 2278 { 2279 struct bnxt_qplib_nq *nq; 2280 2281 if (!(cq && cq->nq)) 2282 return; 2283 2284 nq = cq->nq; 2285 spin_lock_bh(&cq->compl_lock); 2286 if (nq->cqn_handler) { 2287 dev_dbg(&nq->res->pdev->dev, 2288 "%s:Trigger cq = %p event nq = %p\n", 2289 __func__, cq, nq); 2290 nq->cqn_handler(nq, cq); 2291 } 2292 spin_unlock_bh(&cq->compl_lock); 2293 } 2294 2295 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, 2296 struct bnxt_qplib_swqe *wqe) 2297 { 2298 struct bnxt_qplib_nq_work *nq_work = NULL; 2299 struct bnxt_qplib_q *rq = &qp->rq; 2300 struct bnxt_qplib_hwq *rq_hwq; 2301 struct bnxt_qplib_swq *swq; 2302 bool sch_handler = false; 2303 struct rq_wqe_hdr *base_hdr; 2304 struct rq_ext_hdr *ext_hdr; 2305 struct sq_sge *dsge; 2306 u8 wqe_slots; 2307 u32 wqe_idx; 2308 u32 sw_prod; 2309 int rc = 0; 2310 2311 rq_hwq = &rq->hwq; 2312 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 2313 dev_err(&rq_hwq->pdev->dev, 2314 "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", 2315 qp->id, qp->state); 2316 rc = -EINVAL; 2317 goto done; 2318 } 2319 2320 wqe_slots = _calculate_wqe_byte(qp, wqe, NULL); 2321 if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) { 2322 dev_err(&rq_hwq->pdev->dev, 2323 "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id); 2324 rc = -EINVAL; 2325 goto done; 2326 } 2327 2328 swq = bnxt_qplib_get_swqe(rq, &wqe_idx); 2329 swq->wr_id = wqe->wr_id; 2330 swq->slots = rq->dbinfo.max_slot; 2331 dev_dbg(&rq_hwq->pdev->dev, 2332 "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n", 2333 wqe_idx, swq->wr_id); 2334 if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 2335 sch_handler = true; 2336 dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n", 2337 __func__); 2338 goto queue_err; 2339 } 2340 2341 sw_prod = rq_hwq->prod; 2342 base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL); 2343 sw_prod++; 2344 ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); 2345 sw_prod++; 2346 memset(base_hdr, 0, sizeof(struct sq_sge)); 2347 memset(ext_hdr, 0, sizeof(struct sq_sge)); 2348 2349 if (!wqe->num_sge) { 2350 dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); 2351 dsge->size = 0; 2352 wqe_slots++; 2353 } else { 2354 bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod); 2355 } 2356 base_hdr->wqe_type = wqe->type; 2357 base_hdr->flags = wqe->flags; 2358 base_hdr->wqe_size = wqe_slots; 2359 base_hdr->wr_id |= cpu_to_le32(wqe_idx); 2360 queue_err: 2361 bnxt_qplib_swq_mod_start(rq, wqe_idx); 2362 bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots); 2363 done: 2364 if (sch_handler) { 2365 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); 2366 if (nq_work) { 2367 nq_work->cq = qp->rcq; 2368 nq_work->nq = qp->rcq->nq; 2369 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); 2370 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); 2371 } else { 2372 dev_err(&rq->hwq.pdev->dev, 2373 "QPLIB: FP: Failed to allocate RQ nq_work!\n"); 2374 rc = -ENOMEM; 2375 } 2376 } 2377 return rc; 2378 } 2379 2380 /* CQ */ 2381 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 2382 { 2383 struct bnxt_qplib_hwq_attr hwq_attr = {}; 2384 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 2385 struct creq_create_cq_resp resp = {}; 2386 struct bnxt_qplib_cmdqmsg msg = {}; 2387 struct cmdq_create_cq req = {}; 2388 struct bnxt_qplib_reftbl *tbl; 2389 unsigned long flag; 2390 u32 pg_sz_lvl = 0; 2391 int rc; 2392 2393 hwq_attr.res = res; 2394 hwq_attr.depth = cq->max_wqe; 2395 hwq_attr.stride = sizeof(struct cq_base); 2396 hwq_attr.type = HWQ_TYPE_QUEUE; 2397 hwq_attr.sginfo = &cq->sginfo; 2398 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); 2399 if (rc) 2400 goto exit; 2401 2402 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ, 2403 sizeof(req)); 2404 2405 if (!cq->dpi) { 2406 dev_err(&rcfw->pdev->dev, 2407 "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n"); 2408 return -EINVAL; 2409 } 2410 req.dpi = cpu_to_le32(cq->dpi->dpi); 2411 req.cq_handle = cpu_to_le64(cq->cq_handle); 2412 2413 req.cq_size = cpu_to_le32(cq->max_wqe); 2414 req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq)); 2415 pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT; 2416 pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << 2417 CMDQ_CREATE_CQ_LVL_SFT); 2418 req.pg_size_lvl = cpu_to_le32(pg_sz_lvl); 2419 2420 req.cq_fco_cnq_id = cpu_to_le32( 2421 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << 2422 CMDQ_CREATE_CQ_CNQ_ID_SFT); 2423 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 2424 sizeof(resp), 0); 2425 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 2426 if (rc) 2427 goto fail; 2428 cq->id = le32_to_cpu(resp.xid); 2429 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; 2430 init_waitqueue_head(&cq->waitq); 2431 INIT_LIST_HEAD(&cq->sqf_head); 2432 INIT_LIST_HEAD(&cq->rqf_head); 2433 spin_lock_init(&cq->flush_lock); 2434 spin_lock_init(&cq->compl_lock); 2435 2436 /* init dbinfo */ 2437 cq->cctx = res->cctx; 2438 cq->dbinfo.hwq = &cq->hwq; 2439 cq->dbinfo.xid = cq->id; 2440 cq->dbinfo.db = cq->dpi->dbr; 2441 cq->dbinfo.priv_db = res->dpi_tbl.priv_db; 2442 cq->dbinfo.flags = 0; 2443 cq->dbinfo.toggle = 0; 2444 cq->dbinfo.res = res; 2445 cq->dbinfo.seed = cq->id; 2446 spin_lock_init(&cq->dbinfo.lock); 2447 cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 2448 cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; 2449 2450 tbl = &res->reftbl.cqref; 2451 spin_lock_irqsave(&tbl->lock, flag); 2452 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id; 2453 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq; 2454 spin_unlock_irqrestore(&tbl->lock, flag); 2455 2456 bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); 2457 return 0; 2458 2459 fail: 2460 bnxt_qplib_free_hwq(res, &cq->hwq); 2461 exit: 2462 return rc; 2463 } 2464 2465 int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 2466 { 2467 /* TODO: Modify CQ threshold are passed to the HW via DBR */ 2468 return 0; 2469 } 2470 2471 void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, 2472 struct bnxt_qplib_cq *cq) 2473 { 2474 bnxt_qplib_free_hwq(res, &cq->hwq); 2475 memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq)); 2476 /* Reset only the cons bit in the flags */ 2477 cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT); 2478 2479 /* Tell HW to switch over to the new CQ */ 2480 if (!cq->resize_hwq.is_user) 2481 bnxt_qplib_cq_coffack_db(&cq->dbinfo); 2482 } 2483 2484 int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, 2485 int new_cqes) 2486 { 2487 struct bnxt_qplib_hwq_attr hwq_attr = {}; 2488 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 2489 struct creq_resize_cq_resp resp = {}; 2490 struct bnxt_qplib_cmdqmsg msg = {}; 2491 struct cmdq_resize_cq req = {}; 2492 u32 pgsz = 0, lvl = 0, nsz = 0; 2493 struct bnxt_qplib_pbl *pbl; 2494 u16 count = -1; 2495 int rc; 2496 2497 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ, 2498 sizeof(req)); 2499 2500 hwq_attr.sginfo = &cq->sginfo; 2501 hwq_attr.res = res; 2502 hwq_attr.depth = new_cqes; 2503 hwq_attr.stride = sizeof(struct cq_base); 2504 hwq_attr.type = HWQ_TYPE_QUEUE; 2505 rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr); 2506 if (rc) 2507 return rc; 2508 2509 dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__, 2510 cq->resize_hwq.level); 2511 req.cq_cid = cpu_to_le32(cq->id); 2512 pbl = &cq->resize_hwq.pbl[PBL_LVL_0]; 2513 pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K : 2514 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K : 2515 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K : 2516 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M : 2517 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M : 2518 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G : 2519 CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK); 2520 lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) & 2521 CMDQ_RESIZE_CQ_LVL_MASK; 2522 nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) & 2523 CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK; 2524 req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl); 2525 req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]); 2526 2527 if (!cq->resize_hwq.is_user) 2528 set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 2529 2530 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 2531 sizeof(resp), 0); 2532 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 2533 if (rc) 2534 goto fail; 2535 2536 if (!cq->resize_hwq.is_user) { 2537 wait: 2538 /* Wait here for the HW to switch the CQ over */ 2539 if (wait_event_interruptible_timeout(cq->waitq, 2540 !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags), 2541 msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) == 2542 -ERESTARTSYS && count--) 2543 goto wait; 2544 2545 if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) { 2546 dev_err(&rcfw->pdev->dev, 2547 "QPLIB: FP: RESIZE_CQ timed out\n"); 2548 rc = -ETIMEDOUT; 2549 goto fail; 2550 } 2551 2552 bnxt_qplib_resize_cq_complete(res, cq); 2553 } 2554 2555 return 0; 2556 fail: 2557 if (!cq->resize_hwq.is_user) { 2558 bnxt_qplib_free_hwq(res, &cq->resize_hwq); 2559 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 2560 } 2561 return rc; 2562 } 2563 2564 void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 2565 { 2566 bnxt_qplib_free_hwq(res, &cq->hwq); 2567 } 2568 2569 static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq) 2570 { 2571 struct bnxt_qplib_nq *nq = cq->nq; 2572 /* Flush any pending work and synchronize irq */ 2573 flush_workqueue(cq->nq->cqn_wq); 2574 mutex_lock(&nq->lock); 2575 if (nq->requested) 2576 synchronize_irq(nq->msix_vec); 2577 mutex_unlock(&nq->lock); 2578 } 2579 2580 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) 2581 { 2582 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 2583 struct creq_destroy_cq_resp resp = {}; 2584 struct bnxt_qplib_cmdqmsg msg = {}; 2585 struct cmdq_destroy_cq req = {}; 2586 struct bnxt_qplib_reftbl *tbl; 2587 u16 total_cnq_events; 2588 unsigned long flag; 2589 int rc; 2590 2591 tbl = &res->reftbl.cqref; 2592 spin_lock_irqsave(&tbl->lock, flag); 2593 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL; 2594 tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0; 2595 spin_unlock_irqrestore(&tbl->lock, flag); 2596 2597 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ, 2598 sizeof(req)); 2599 2600 req.cq_cid = cpu_to_le32(cq->id); 2601 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), 2602 sizeof(resp), 0); 2603 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 2604 if (rc) 2605 return rc; 2606 2607 total_cnq_events = le16_to_cpu(resp.total_cnq_events); 2608 if (total_cnq_events >= 0) 2609 dev_dbg(&rcfw->pdev->dev, 2610 "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n", 2611 __func__, cq->id, cq, total_cnq_events); 2612 __wait_for_all_nqes(cq, total_cnq_events); 2613 bnxt_qplib_sync_cq(cq); 2614 bnxt_qplib_free_hwq(res, &cq->hwq); 2615 return 0; 2616 } 2617 2618 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, 2619 struct bnxt_qplib_cqe **pcqe, int *budget) 2620 { 2621 struct bnxt_qplib_cqe *cqe; 2622 u32 start, last; 2623 int rc = 0; 2624 2625 /* Now complete all outstanding SQEs with FLUSHED_ERR */ 2626 start = sq->swq_start; 2627 cqe = *pcqe; 2628 while (*budget) { 2629 last = sq->swq_last; 2630 if (start == last) { 2631 break; 2632 } 2633 /* Skip the FENCE WQE completions */ 2634 if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { 2635 bnxt_re_legacy_cancel_phantom_processing(qp); 2636 goto skip_compl; 2637 } 2638 2639 memset(cqe, 0, sizeof(*cqe)); 2640 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; 2641 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2642 cqe->qp_handle = (u64)qp; 2643 cqe->wr_id = sq->swq[last].wr_id; 2644 cqe->src_qp = qp->id; 2645 cqe->type = sq->swq[last].type; 2646 dev_dbg(&sq->hwq.pdev->dev, 2647 "QPLIB: FP: CQ Processed terminal Req \n"); 2648 dev_dbg(&sq->hwq.pdev->dev, 2649 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", 2650 last, cqe->wr_id, cqe->status); 2651 cqe++; 2652 (*budget)--; 2653 skip_compl: 2654 bnxt_qplib_hwq_incr_cons(sq->hwq.depth, 2655 &sq->hwq.cons, 2656 sq->swq[last].slots, 2657 &sq->dbinfo.flags); 2658 sq->swq_last = sq->swq[last].next_idx; 2659 } 2660 *pcqe = cqe; 2661 if (!*budget && sq->swq_last != start) 2662 /* Out of budget */ 2663 rc = -EAGAIN; 2664 dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc); 2665 2666 return rc; 2667 } 2668 2669 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, 2670 struct bnxt_qplib_cqe **pcqe, int *budget) 2671 { 2672 struct bnxt_qplib_cqe *cqe; 2673 u32 start, last; 2674 int opcode = 0; 2675 int rc = 0; 2676 2677 switch (qp->type) { 2678 case CMDQ_CREATE_QP1_TYPE_GSI: 2679 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; 2680 break; 2681 case CMDQ_CREATE_QP_TYPE_RC: 2682 opcode = CQ_BASE_CQE_TYPE_RES_RC; 2683 break; 2684 case CMDQ_CREATE_QP_TYPE_UD: 2685 opcode = CQ_BASE_CQE_TYPE_RES_UD; 2686 break; 2687 } 2688 2689 /* Flush the rest of the RQ */ 2690 start = rq->swq_start; 2691 cqe = *pcqe; 2692 while (*budget) { 2693 last = rq->swq_last; 2694 if (last == start) 2695 break; 2696 memset(cqe, 0, sizeof(*cqe)); 2697 cqe->status = 2698 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; 2699 cqe->opcode = opcode; 2700 cqe->qp_handle = (u64)qp; 2701 cqe->wr_id = rq->swq[last].wr_id; 2702 dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n"); 2703 dev_dbg(&rq->hwq.pdev->dev, 2704 "QPLIB: rq[%d] = 0x%llx with status 0x%x\n", 2705 last, cqe->wr_id, cqe->status); 2706 cqe++; 2707 (*budget)--; 2708 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, 2709 &rq->hwq.cons, 2710 rq->swq[last].slots, 2711 &rq->dbinfo.flags); 2712 rq->swq_last = rq->swq[last].next_idx; 2713 } 2714 *pcqe = cqe; 2715 if (!*budget && rq->swq_last != start) 2716 /* Out of budget */ 2717 rc = -EAGAIN; 2718 2719 dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc); 2720 return rc; 2721 } 2722 2723 void bnxt_qplib_mark_qp_error(void *qp_handle) 2724 { 2725 struct bnxt_qplib_qp *qp = qp_handle; 2726 2727 if (!qp) 2728 return; 2729 2730 /* Must block new posting of SQ and RQ */ 2731 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR; 2732 qp->state = qp->cur_qp_state; 2733 2734 /* Add qp to flush list of the CQ */ 2735 if (!qp->is_user) 2736 bnxt_qplib_add_flush_qp(qp); 2737 } 2738 2739 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) 2740 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 2741 */ 2742 static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp, 2743 struct bnxt_qplib_cq *cq, 2744 u32 cq_cons, u32 swq_last, 2745 u32 cqe_sq_cons) 2746 { 2747 struct bnxt_qplib_q *sq = &qp->sq; 2748 struct bnxt_qplib_swq *swq; 2749 u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags; 2750 struct cq_terminal *peek_term_hwcqe; 2751 struct cq_req *peek_req_hwcqe; 2752 struct bnxt_qplib_qp *peek_qp; 2753 struct bnxt_qplib_q *peek_sq; 2754 struct cq_base *peek_hwcqe; 2755 int i, rc = 0; 2756 2757 /* Check for the psn_search marking before completing */ 2758 swq = &sq->swq[swq_last]; 2759 if (swq->psn_search && 2760 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { 2761 /* Unmark */ 2762 swq->psn_search->flags_next_psn = cpu_to_le32 2763 (le32_to_cpu(swq->psn_search->flags_next_psn) 2764 & ~0x80000000); 2765 dev_dbg(&cq->hwq.pdev->dev, 2766 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", 2767 cq_cons, qp->id, swq_last, cqe_sq_cons); 2768 sq->condition = true; 2769 sq->legacy_send_phantom = true; 2770 2771 /* TODO: Only ARM if the previous SQE is ARMALL */ 2772 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); 2773 2774 rc = -EAGAIN; 2775 goto out; 2776 } 2777 if (sq->condition == true) { 2778 /* Peek at the completions */ 2779 peek_flags = cq->dbinfo.flags; 2780 peek_sw_cq_cons = cq_cons; 2781 i = cq->hwq.depth; 2782 while (i--) { 2783 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, 2784 peek_sw_cq_cons, NULL); 2785 /* If the next hwcqe is VALID */ 2786 if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) { 2787 /* If the next hwcqe is a REQ */ 2788 dma_rmb(); 2789 switch (peek_hwcqe->cqe_type_toggle & 2790 CQ_BASE_CQE_TYPE_MASK) { 2791 case CQ_BASE_CQE_TYPE_REQ: 2792 peek_req_hwcqe = (struct cq_req *) 2793 peek_hwcqe; 2794 peek_qp = (struct bnxt_qplib_qp *) 2795 le64_to_cpu( 2796 peek_req_hwcqe->qp_handle); 2797 peek_sq = &peek_qp->sq; 2798 peek_sq_cons_idx = 2799 ((le16_to_cpu( 2800 peek_req_hwcqe->sq_cons_idx) 2801 - 1) % sq->max_wqe); 2802 /* If the hwcqe's sq's wr_id matches */ 2803 if (peek_sq == sq && 2804 sq->swq[peek_sq_cons_idx].wr_id == 2805 BNXT_QPLIB_FENCE_WRID) { 2806 /* Unbreak only if the phantom 2807 comes back */ 2808 dev_dbg(&cq->hwq.pdev->dev, 2809 "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n", 2810 qp->id, swq_last, 2811 cqe_sq_cons); 2812 sq->condition = false; 2813 sq->single = true; 2814 sq->phantom_cqe_cnt++; 2815 dev_dbg(&cq->hwq.pdev->dev, 2816 "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n", 2817 peek_qp->id, 2818 peek_sw_cq_cons, 2819 peek_sq_cons_idx, 2820 sq->phantom_cqe_cnt); 2821 rc = 0; 2822 goto out; 2823 } 2824 break; 2825 2826 case CQ_BASE_CQE_TYPE_TERMINAL: 2827 /* In case the QP has gone into the 2828 error state */ 2829 peek_term_hwcqe = (struct cq_terminal *) 2830 peek_hwcqe; 2831 peek_qp = (struct bnxt_qplib_qp *) 2832 le64_to_cpu( 2833 peek_term_hwcqe->qp_handle); 2834 if (peek_qp == qp) { 2835 sq->condition = false; 2836 rc = 0; 2837 goto out; 2838 } 2839 break; 2840 default: 2841 break; 2842 } 2843 /* Valid but not the phantom, so keep looping */ 2844 } else { 2845 /* Not valid yet, just exit and wait */ 2846 rc = -EINVAL; 2847 goto out; 2848 } 2849 bnxt_qplib_hwq_incr_cons(cq->hwq.depth, 2850 &peek_sw_cq_cons, 2851 1, &peek_flags); 2852 } 2853 dev_err(&cq->hwq.pdev->dev, 2854 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", 2855 cq_cons, qp->id, swq_last, cqe_sq_cons); 2856 rc = -EINVAL; 2857 } 2858 out: 2859 return rc; 2860 } 2861 2862 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, 2863 struct cq_req *hwcqe, 2864 struct bnxt_qplib_cqe **pcqe, int *budget, 2865 u32 cq_cons, struct bnxt_qplib_qp **lib_qp) 2866 { 2867 struct bnxt_qplib_qp *qp; 2868 struct bnxt_qplib_q *sq; 2869 struct bnxt_qplib_cqe *cqe; 2870 u32 cqe_sq_cons; 2871 struct bnxt_qplib_swq *swq; 2872 int rc = 0; 2873 2874 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); 2875 dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp); 2876 if (!qp) { 2877 dev_err(&cq->hwq.pdev->dev, 2878 "QPLIB: FP: Process Req qp is NULL\n"); 2879 return -EINVAL; 2880 } 2881 sq = &qp->sq; 2882 2883 cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; 2884 if (qp->sq.flushed) { 2885 dev_dbg(&cq->hwq.pdev->dev, 2886 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 2887 goto done; 2888 } 2889 2890 /* Require to walk the sq's swq to fabricate CQEs for all previously 2891 * signaled SWQEs due to CQE aggregation from the current sq cons 2892 * to the cqe_sq_cons 2893 */ 2894 cqe = *pcqe; 2895 while (*budget) { 2896 if (sq->swq_last == cqe_sq_cons) 2897 /* Done */ 2898 break; 2899 2900 swq = &sq->swq[sq->swq_last]; 2901 memset(cqe, 0, sizeof(*cqe)); 2902 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 2903 cqe->qp_handle = (u64)qp; 2904 cqe->src_qp = qp->id; 2905 cqe->wr_id = swq->wr_id; 2906 2907 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) 2908 goto skip; 2909 2910 cqe->type = swq->type; 2911 2912 /* For the last CQE, check for status. For errors, regardless 2913 * of the request being signaled or not, it must complete with 2914 * the hwcqe error status 2915 */ 2916 if (swq->next_idx == cqe_sq_cons && 2917 hwcqe->status != CQ_REQ_STATUS_OK) { 2918 cqe->status = hwcqe->status; 2919 dev_err(&cq->hwq.pdev->dev, 2920 "QPLIB: FP: CQ Processed Req \n"); 2921 dev_err(&cq->hwq.pdev->dev, 2922 "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n", 2923 cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status); 2924 cqe++; 2925 (*budget)--; 2926 bnxt_qplib_mark_qp_error(qp); 2927 } else { 2928 /* Before we complete, do WA 9060 */ 2929 if (!_is_chip_gen_p5_p7(qp->cctx)) { 2930 if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons, 2931 sq->swq_last, 2932 cqe_sq_cons)) { 2933 *lib_qp = qp; 2934 goto out; 2935 } 2936 } 2937 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 2938 2939 dev_dbg(&cq->hwq.pdev->dev, 2940 "QPLIB: FP: CQ Processed Req \n"); 2941 dev_dbg(&cq->hwq.pdev->dev, 2942 "QPLIB: wr_id[%d] = 0x%llx \n", 2943 sq->swq_last, cqe->wr_id); 2944 dev_dbg(&cq->hwq.pdev->dev, 2945 "QPLIB: with status 0x%x\n", cqe->status); 2946 cqe->status = CQ_REQ_STATUS_OK; 2947 cqe++; 2948 (*budget)--; 2949 } 2950 } 2951 skip: 2952 bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, 2953 swq->slots, &sq->dbinfo.flags); 2954 sq->swq_last = swq->next_idx; 2955 if (sq->single == true) 2956 break; 2957 } 2958 out: 2959 *pcqe = cqe; 2960 if (sq->swq_last != cqe_sq_cons) { 2961 /* Out of budget */ 2962 rc = -EAGAIN; 2963 goto done; 2964 } 2965 /* Back to normal completion mode only after it has completed all of 2966 the WC for this CQE */ 2967 sq->single = false; 2968 done: 2969 return rc; 2970 } 2971 2972 static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) 2973 { 2974 spin_lock(&srq->hwq.lock); 2975 srq->swq[srq->last_idx].next_idx = (int)tag; 2976 srq->last_idx = (int)tag; 2977 srq->swq[srq->last_idx].next_idx = -1; 2978 bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons, 2979 srq->dbinfo.max_slot, &srq->dbinfo.flags); 2980 spin_unlock(&srq->hwq.lock); 2981 } 2982 2983 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, 2984 struct cq_res_rc *hwcqe, 2985 struct bnxt_qplib_cqe **pcqe, 2986 int *budget) 2987 { 2988 struct bnxt_qplib_srq *srq; 2989 struct bnxt_qplib_cqe *cqe; 2990 struct bnxt_qplib_qp *qp; 2991 struct bnxt_qplib_q *rq; 2992 u32 wr_id_idx; 2993 int rc = 0; 2994 2995 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); 2996 if (!qp) { 2997 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n"); 2998 return -EINVAL; 2999 } 3000 if (qp->rq.flushed) { 3001 dev_dbg(&cq->hwq.pdev->dev, 3002 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 3003 goto done; 3004 } 3005 3006 cqe = *pcqe; 3007 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 3008 cqe->length = le32_to_cpu(hwcqe->length); 3009 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); 3010 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); 3011 cqe->flags = le16_to_cpu(hwcqe->flags); 3012 cqe->status = hwcqe->status; 3013 cqe->qp_handle = (u64)(unsigned long)qp; 3014 3015 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & 3016 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; 3017 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { 3018 srq = qp->srq; 3019 if (!srq) { 3020 dev_err(&cq->hwq.pdev->dev, 3021 "QPLIB: FP: SRQ used but not defined??\n"); 3022 return -EINVAL; 3023 } 3024 if (wr_id_idx > srq->hwq.depth - 1) { 3025 dev_err(&cq->hwq.pdev->dev, 3026 "QPLIB: FP: CQ Process RC \n"); 3027 dev_err(&cq->hwq.pdev->dev, 3028 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", 3029 wr_id_idx, srq->hwq.depth); 3030 return -EINVAL; 3031 } 3032 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 3033 bnxt_qplib_release_srqe(srq, wr_id_idx); 3034 dev_dbg(&srq->hwq.pdev->dev, 3035 "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n", 3036 wr_id_idx, cqe->wr_id); 3037 cqe++; 3038 (*budget)--; 3039 *pcqe = cqe; 3040 } else { 3041 rq = &qp->rq; 3042 if (wr_id_idx > (rq->max_wqe - 1)) { 3043 dev_err(&cq->hwq.pdev->dev, 3044 "QPLIB: FP: CQ Process RC \n"); 3045 dev_err(&cq->hwq.pdev->dev, 3046 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", 3047 wr_id_idx, rq->hwq.depth); 3048 return -EINVAL; 3049 } 3050 if (wr_id_idx != rq->swq_last) 3051 return -EINVAL; 3052 cqe->wr_id = rq->swq[rq->swq_last].wr_id; 3053 dev_dbg(&cq->hwq.pdev->dev, 3054 "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n", 3055 rq->swq_last, cqe->wr_id); 3056 cqe++; 3057 (*budget)--; 3058 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, 3059 rq->swq[rq->swq_last].slots, 3060 &rq->dbinfo.flags); 3061 rq->swq_last = rq->swq[rq->swq_last].next_idx; 3062 *pcqe = cqe; 3063 3064 if (hwcqe->status != CQ_RES_RC_STATUS_OK) 3065 bnxt_qplib_mark_qp_error(qp); 3066 } 3067 done: 3068 return rc; 3069 } 3070 3071 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, 3072 struct cq_res_ud_v2 *hwcqe, 3073 struct bnxt_qplib_cqe **pcqe, 3074 int *budget) 3075 { 3076 struct bnxt_qplib_srq *srq; 3077 struct bnxt_qplib_cqe *cqe; 3078 struct bnxt_qplib_qp *qp; 3079 struct bnxt_qplib_q *rq; 3080 u32 wr_id_idx; 3081 int rc = 0; 3082 u16 *smac; 3083 3084 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); 3085 if (!qp) { 3086 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n"); 3087 return -EINVAL; 3088 } 3089 if (qp->rq.flushed) { 3090 dev_dbg(&cq->hwq.pdev->dev, 3091 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 3092 goto done; 3093 } 3094 cqe = *pcqe; 3095 cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK; 3096 cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK)); 3097 cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0); 3098 /* V2 format has metadata1 */ 3099 cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & 3100 CQ_RES_UD_V2_CFA_METADATA1_MASK) >> 3101 CQ_RES_UD_V2_CFA_METADATA1_SFT) << 3102 BNXT_QPLIB_META1_SHIFT); 3103 cqe->invrkey = le32_to_cpu(hwcqe->imm_data); 3104 cqe->flags = le16_to_cpu(hwcqe->flags); 3105 cqe->status = hwcqe->status; 3106 cqe->qp_handle = (u64)(unsigned long)qp; 3107 smac = (u16 *)cqe->smac; 3108 smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0])); 3109 smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1])); 3110 smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2])); 3111 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) 3112 & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK; 3113 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | 3114 ((le32_to_cpu( 3115 hwcqe->src_qp_high_srq_or_rq_wr_id) & 3116 CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8); 3117 3118 if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) { 3119 srq = qp->srq; 3120 if (!srq) { 3121 dev_err(&cq->hwq.pdev->dev, 3122 "QPLIB: FP: SRQ used but not defined??\n"); 3123 return -EINVAL; 3124 } 3125 if (wr_id_idx > srq->hwq.depth - 1) { 3126 dev_err(&cq->hwq.pdev->dev, 3127 "QPLIB: FP: CQ Process UD \n"); 3128 dev_err(&cq->hwq.pdev->dev, 3129 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", 3130 wr_id_idx, srq->hwq.depth); 3131 return -EINVAL; 3132 } 3133 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 3134 bnxt_qplib_release_srqe(srq, wr_id_idx); 3135 dev_dbg(&cq->hwq.pdev->dev, 3136 "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n", 3137 wr_id_idx, cqe->wr_id); 3138 cqe++; 3139 (*budget)--; 3140 *pcqe = cqe; 3141 } else { 3142 rq = &qp->rq; 3143 if (wr_id_idx > (rq->max_wqe - 1)) { 3144 dev_err(&cq->hwq.pdev->dev, 3145 "QPLIB: FP: CQ Process UD \n"); 3146 dev_err(&cq->hwq.pdev->dev, 3147 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", 3148 wr_id_idx, rq->hwq.depth); 3149 return -EINVAL; 3150 } 3151 if (rq->swq_last != wr_id_idx) 3152 return -EINVAL; 3153 3154 cqe->wr_id = rq->swq[rq->swq_last].wr_id; 3155 dev_dbg(&cq->hwq.pdev->dev, 3156 "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n", 3157 rq->swq_last, cqe->wr_id); 3158 cqe++; 3159 (*budget)--; 3160 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, 3161 rq->swq[rq->swq_last].slots, 3162 &rq->dbinfo.flags); 3163 rq->swq_last = rq->swq[rq->swq_last].next_idx; 3164 *pcqe = cqe; 3165 3166 if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK) 3167 bnxt_qplib_mark_qp_error(qp); 3168 } 3169 done: 3170 return rc; 3171 } 3172 3173 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) 3174 { 3175 3176 struct cq_base *hw_cqe; 3177 unsigned long flags; 3178 bool rc = true; 3179 3180 spin_lock_irqsave(&cq->hwq.lock, flags); 3181 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); 3182 3183 /* Check for Valid bit. If the CQE is valid, return false */ 3184 rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags); 3185 spin_unlock_irqrestore(&cq->hwq.lock, flags); 3186 return rc; 3187 } 3188 3189 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, 3190 struct cq_res_raweth_qp1 *hwcqe, 3191 struct bnxt_qplib_cqe **pcqe, 3192 int *budget) 3193 { 3194 struct bnxt_qplib_qp *qp; 3195 struct bnxt_qplib_q *rq; 3196 struct bnxt_qplib_srq *srq; 3197 struct bnxt_qplib_cqe *cqe; 3198 u32 wr_id_idx; 3199 int rc = 0; 3200 3201 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); 3202 if (!qp) { 3203 dev_err(&cq->hwq.pdev->dev, 3204 "QPLIB: process_cq Raw/QP1 qp is NULL\n"); 3205 return -EINVAL; 3206 } 3207 if (qp->rq.flushed) { 3208 dev_dbg(&cq->hwq.pdev->dev, 3209 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 3210 goto done; 3211 } 3212 cqe = *pcqe; 3213 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 3214 cqe->flags = le16_to_cpu(hwcqe->flags); 3215 cqe->qp_handle = (u64)(unsigned long)qp; 3216 3217 wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) 3218 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; 3219 cqe->src_qp = qp->id; 3220 if (qp->id == 1 && !cqe->length) { 3221 /* Add workaround for the length misdetection */ 3222 cqe->length = 296; 3223 } else { 3224 cqe->length = le16_to_cpu(hwcqe->length); 3225 } 3226 cqe->pkey_index = qp->pkey_index; 3227 memcpy(cqe->smac, qp->smac, 6); 3228 3229 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); 3230 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); 3231 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); 3232 3233 dev_dbg(&cq->hwq.pdev->dev, 3234 "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n", 3235 cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2); 3236 3237 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { 3238 srq = qp->srq; 3239 if (!srq) { 3240 dev_err(&cq->hwq.pdev->dev, 3241 "QPLIB: FP: SRQ used but not defined??\n"); 3242 return -EINVAL; 3243 } 3244 if (wr_id_idx > srq->hwq.depth - 1) { 3245 dev_err(&cq->hwq.pdev->dev, 3246 "QPLIB: FP: CQ Process Raw/QP1 \n"); 3247 dev_err(&cq->hwq.pdev->dev, 3248 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", 3249 wr_id_idx, srq->hwq.depth); 3250 return -EINVAL; 3251 } 3252 cqe->wr_id = srq->swq[wr_id_idx].wr_id; 3253 dev_dbg(&cq->hwq.pdev->dev, 3254 "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n"); 3255 dev_dbg(&cq->hwq.pdev->dev, 3256 "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", 3257 wr_id_idx, cqe->wr_id, hwcqe->status); 3258 cqe++; 3259 (*budget)--; 3260 srq->hwq.cons++; 3261 *pcqe = cqe; 3262 } else { 3263 rq = &qp->rq; 3264 if (wr_id_idx > (rq->max_wqe - 1)) { 3265 dev_err(&cq->hwq.pdev->dev, 3266 "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n"); 3267 dev_err(&cq->hwq.pdev->dev, 3268 "QPLIB: ix 0x%x exceeded RQ max 0x%x\n", 3269 wr_id_idx, rq->max_wqe); 3270 return -EINVAL; 3271 } 3272 if (wr_id_idx != rq->swq_last) 3273 return -EINVAL; 3274 cqe->wr_id = rq->swq[rq->swq_last].wr_id; 3275 dev_dbg(&cq->hwq.pdev->dev, 3276 "QPLIB: FP: CQ Processed Raw/QP1 RQ \n"); 3277 dev_dbg(&cq->hwq.pdev->dev, 3278 "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", 3279 wr_id_idx, cqe->wr_id, hwcqe->status); 3280 cqe++; 3281 (*budget)--; 3282 bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, 3283 rq->swq[wr_id_idx].slots, 3284 &rq->dbinfo.flags); 3285 rq->swq_last = rq->swq[rq->swq_last].next_idx; 3286 *pcqe = cqe; 3287 3288 if (hwcqe->status != CQ_RES_RC_STATUS_OK) 3289 bnxt_qplib_mark_qp_error(qp); 3290 } 3291 done: 3292 return rc; 3293 } 3294 3295 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, 3296 struct cq_terminal *hwcqe, 3297 struct bnxt_qplib_cqe **pcqe, 3298 int *budget) 3299 { 3300 struct bnxt_qplib_q *sq, *rq; 3301 struct bnxt_qplib_cqe *cqe; 3302 struct bnxt_qplib_qp *qp; 3303 u32 swq_last; 3304 u32 cqe_cons; 3305 int rc = 0; 3306 3307 /* Check the Status */ 3308 if (hwcqe->status != CQ_TERMINAL_STATUS_OK) 3309 dev_warn(&cq->hwq.pdev->dev, 3310 "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n", 3311 hwcqe->status); 3312 3313 qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); 3314 if (!qp) 3315 return -EINVAL; 3316 dev_dbg(&cq->hwq.pdev->dev, 3317 "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id); 3318 3319 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR 3320 * from the current rq->cons to the rq->prod regardless what the 3321 * rq->cons the terminal CQE indicates. 3322 */ 3323 bnxt_qplib_mark_qp_error(qp); 3324 3325 sq = &qp->sq; 3326 rq = &qp->rq; 3327 3328 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); 3329 if (cqe_cons == 0xFFFF) 3330 goto do_rq; 3331 3332 cqe_cons %= sq->max_wqe; 3333 if (qp->sq.flushed) { 3334 dev_dbg(&cq->hwq.pdev->dev, 3335 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 3336 goto sq_done; 3337 } 3338 3339 /* Terminal CQE can also include aggregated successful CQEs prior. 3340 So we must complete all CQEs from the current sq's cons to the 3341 cq_cons with status OK */ 3342 cqe = *pcqe; 3343 while (*budget) { 3344 /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/ 3345 swq_last = sq->swq_last; 3346 if (swq_last == cqe_cons) 3347 break; 3348 if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { 3349 memset(cqe, 0, sizeof(*cqe)); 3350 cqe->status = CQ_REQ_STATUS_OK; 3351 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; 3352 cqe->qp_handle = (u64)qp; 3353 cqe->src_qp = qp->id; 3354 cqe->wr_id = sq->swq[swq_last].wr_id; 3355 cqe->type = sq->swq[swq_last].type; 3356 dev_dbg(&cq->hwq.pdev->dev, 3357 "QPLIB: FP: CQ Processed terminal Req \n"); 3358 dev_dbg(&cq->hwq.pdev->dev, 3359 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", 3360 swq_last, cqe->wr_id, cqe->status); 3361 cqe++; 3362 (*budget)--; 3363 } 3364 bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, 3365 sq->swq[swq_last].slots, 3366 &sq->dbinfo.flags); 3367 sq->swq_last = sq->swq[swq_last].next_idx; 3368 } 3369 *pcqe = cqe; 3370 if (!*budget && swq_last != cqe_cons) { 3371 /* Out of budget */ 3372 rc = -EAGAIN; 3373 goto sq_done; 3374 } 3375 sq_done: 3376 if (rc) 3377 return rc; 3378 do_rq: 3379 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); 3380 if (cqe_cons == 0xFFFF) { 3381 goto done; 3382 } else if (cqe_cons > (rq->max_wqe - 1)) { 3383 dev_err(&cq->hwq.pdev->dev, 3384 "QPLIB: FP: CQ Processed terminal \n"); 3385 dev_err(&cq->hwq.pdev->dev, 3386 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n", 3387 cqe_cons, rq->hwq.depth); 3388 goto done; 3389 } 3390 if (qp->rq.flushed) { 3391 dev_dbg(&cq->hwq.pdev->dev, 3392 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); 3393 rc = 0; 3394 goto rq_done; 3395 } 3396 3397 rq_done: 3398 done: 3399 return rc; 3400 } 3401 3402 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, 3403 struct cq_cutoff *hwcqe) 3404 { 3405 /* Check the Status */ 3406 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { 3407 dev_err(&cq->hwq.pdev->dev, 3408 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n", 3409 hwcqe->status); 3410 return -EINVAL; 3411 } 3412 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); 3413 wake_up_interruptible(&cq->waitq); 3414 3415 dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n"); 3416 return 0; 3417 } 3418 3419 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, 3420 struct bnxt_qplib_cqe *cqe, 3421 int num_cqes) 3422 { 3423 struct bnxt_qplib_qp *qp = NULL; 3424 u32 budget = num_cqes; 3425 unsigned long flags; 3426 3427 spin_lock_irqsave(&cq->flush_lock, flags); 3428 list_for_each_entry(qp, &cq->sqf_head, sq_flush) { 3429 dev_dbg(&cq->hwq.pdev->dev, 3430 "QPLIB: FP: Flushing SQ QP= %p\n", 3431 qp); 3432 __flush_sq(&qp->sq, qp, &cqe, &budget); 3433 } 3434 3435 list_for_each_entry(qp, &cq->rqf_head, rq_flush) { 3436 dev_dbg(&cq->hwq.pdev->dev, 3437 "QPLIB: FP: Flushing RQ QP= %p\n", 3438 qp); 3439 __flush_rq(&qp->rq, qp, &cqe, &budget); 3440 } 3441 spin_unlock_irqrestore(&cq->flush_lock, flags); 3442 3443 return num_cqes - budget; 3444 } 3445 3446 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, 3447 int num_cqes, struct bnxt_qplib_qp **lib_qp) 3448 { 3449 struct cq_base *hw_cqe; 3450 u32 hw_polled = 0; 3451 int budget, rc = 0; 3452 u8 type; 3453 3454 budget = num_cqes; 3455 3456 while (budget) { 3457 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); 3458 3459 /* Check for Valid bit */ 3460 if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags)) 3461 break; 3462 3463 /* The valid test of the entry must be done first before 3464 * reading any further. 3465 */ 3466 dma_rmb(); 3467 /* From the device's respective CQE format to qplib_wc*/ 3468 type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; 3469 switch (type) { 3470 case CQ_BASE_CQE_TYPE_REQ: 3471 rc = bnxt_qplib_cq_process_req(cq, 3472 (struct cq_req *)hw_cqe, &cqe, &budget, 3473 cq->hwq.cons, lib_qp); 3474 break; 3475 case CQ_BASE_CQE_TYPE_RES_RC: 3476 rc = bnxt_qplib_cq_process_res_rc(cq, 3477 (struct cq_res_rc *)hw_cqe, &cqe, 3478 &budget); 3479 break; 3480 case CQ_BASE_CQE_TYPE_RES_UD: 3481 rc = bnxt_qplib_cq_process_res_ud(cq, 3482 (struct cq_res_ud_v2 *)hw_cqe, 3483 &cqe, &budget); 3484 break; 3485 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3486 rc = bnxt_qplib_cq_process_res_raweth_qp1(cq, 3487 (struct cq_res_raweth_qp1 *) 3488 hw_cqe, &cqe, &budget); 3489 break; 3490 case CQ_BASE_CQE_TYPE_TERMINAL: 3491 rc = bnxt_qplib_cq_process_terminal(cq, 3492 (struct cq_terminal *)hw_cqe, 3493 &cqe, &budget); 3494 break; 3495 case CQ_BASE_CQE_TYPE_CUT_OFF: 3496 bnxt_qplib_cq_process_cutoff(cq, 3497 (struct cq_cutoff *)hw_cqe); 3498 /* Done processing this CQ */ 3499 goto exit; 3500 default: 3501 dev_err(&cq->hwq.pdev->dev, 3502 "QPLIB: process_cq unknown type 0x%x\n", 3503 hw_cqe->cqe_type_toggle & 3504 CQ_BASE_CQE_TYPE_MASK); 3505 rc = -EINVAL; 3506 break; 3507 } 3508 if (rc < 0) { 3509 dev_dbg(&cq->hwq.pdev->dev, 3510 "QPLIB: process_cqe rc = 0x%x\n", rc); 3511 if (rc == -EAGAIN) 3512 break; 3513 /* Error while processing the CQE, just skip to the 3514 next one */ 3515 if (type != CQ_BASE_CQE_TYPE_TERMINAL) 3516 dev_err(&cq->hwq.pdev->dev, 3517 "QPLIB: process_cqe error rc = 0x%x\n", 3518 rc); 3519 } 3520 hw_polled++; 3521 bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons, 3522 1, &cq->dbinfo.flags); 3523 } 3524 if (hw_polled) 3525 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); 3526 exit: 3527 return num_cqes - budget; 3528 } 3529 3530 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) 3531 { 3532 cq->dbinfo.toggle = cq->toggle; 3533 if (arm_type) 3534 bnxt_qplib_ring_db(&cq->dbinfo, arm_type); 3535 /* Using cq->arm_state variable to track whether to issue cq handler */ 3536 atomic_set(&cq->arm_state, 1); 3537 } 3538 3539 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) 3540 { 3541 flush_workqueue(qp->scq->nq->cqn_wq); 3542 if (qp->scq != qp->rcq) 3543 flush_workqueue(qp->rcq->nq->cqn_wq); 3544 } 3545