2  * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
53 	qp->sq.condition = false;  in bnxt_re_legacy_cancel_phantom_processing()
54 	qp->sq.legacy_send_phantom = false;  in bnxt_re_legacy_cancel_phantom_processing()
55 	qp->sq.single = false;  in bnxt_re_legacy_cancel_phantom_processing()
62 	scq = qp->scq;  in __bnxt_qplib_add_flush_qp()
63 	rcq = qp->rcq;  in __bnxt_qplib_add_flush_qp()
65 	if (!qp->sq.flushed) {  in __bnxt_qplib_add_flush_qp()
66 		dev_dbg(&scq->hwq.pdev->dev,  in __bnxt_qplib_add_flush_qp()
70 		list_add_tail(&qp->sq_flush, &scq->sqf_head);  in __bnxt_qplib_add_flush_qp()
71 		qp->sq.flushed = true;  in __bnxt_qplib_add_flush_qp()
73 	if (!qp->srq) {  in __bnxt_qplib_add_flush_qp()
74 		if (!qp->rq.flushed) {  in __bnxt_qplib_add_flush_qp()
75 			dev_dbg(&rcq->hwq.pdev->dev,  in __bnxt_qplib_add_flush_qp()
78 			list_add_tail(&qp->rq_flush, &rcq->rqf_head);  in __bnxt_qplib_add_flush_qp()
79 			qp->rq.flushed = true;  in __bnxt_qplib_add_flush_qp()
85 	__acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)  in bnxt_qplib_acquire_cq_flush_locks()
88 	spin_lock(&qp->scq->flush_lock);  in bnxt_qplib_acquire_cq_flush_locks()
89 	if (qp->scq == qp->rcq)  in bnxt_qplib_acquire_cq_flush_locks()
90 		__acquire(&qp->rcq->flush_lock);  in bnxt_qplib_acquire_cq_flush_locks()
92 		spin_lock(&qp->rcq->flush_lock);  in bnxt_qplib_acquire_cq_flush_locks()
96 	__releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)  in bnxt_qplib_release_cq_flush_locks()
98 	if (qp->scq == qp->rcq)  in bnxt_qplib_release_cq_flush_locks()
99 		__release(&qp->rcq->flush_lock);  in bnxt_qplib_release_cq_flush_locks()
101 		spin_unlock(&qp->rcq->flush_lock);  in bnxt_qplib_release_cq_flush_locks()
102 	spin_unlock(&qp->scq->flush_lock);  in bnxt_qplib_release_cq_flush_locks()
115 	if (qp->sq.flushed) {  in __bnxt_qplib_del_flush_qp()
116 		qp->sq.flushed = false;  in __bnxt_qplib_del_flush_qp()
117 		list_del(&qp->sq_flush);  in __bnxt_qplib_del_flush_qp()
119 	if (!qp->srq) {  in __bnxt_qplib_del_flush_qp()
120 		if (qp->rq.flushed) {  in __bnxt_qplib_del_flush_qp()
121 			qp->rq.flushed = false;  in __bnxt_qplib_del_flush_qp()
122 			list_del(&qp->rq_flush);  in __bnxt_qplib_del_flush_qp()
131 	__clean_cq(qp->scq, (u64)(unsigned long)qp);  in bnxt_qplib_clean_qp()
132 	qp->sq.hwq.prod = 0;  in bnxt_qplib_clean_qp()
133 	qp->sq.hwq.cons = 0;  in bnxt_qplib_clean_qp()
134 	qp->sq.swq_start = 0;  in bnxt_qplib_clean_qp()
135 	qp->sq.swq_last = 0;  in bnxt_qplib_clean_qp()
136 	__clean_cq(qp->rcq, (u64)(unsigned long)qp);  in bnxt_qplib_clean_qp()
137 	qp->rq.hwq.prod = 0;  in bnxt_qplib_clean_qp()
138 	qp->rq.hwq.cons = 0;  in bnxt_qplib_clean_qp()
139 	qp->rq.swq_start = 0;  in bnxt_qplib_clean_qp()
140 	qp->rq.swq_last = 0;  in bnxt_qplib_clean_qp()
151 	struct bnxt_qplib_cq *cq = nq_work->cq;  in bnxt_qpn_cqn_sched_task()
152 	struct bnxt_qplib_nq *nq = nq_work->nq;  in bnxt_qpn_cqn_sched_task()
155 		spin_lock_bh(&cq->compl_lock);  in bnxt_qpn_cqn_sched_task()
156 		if (nq->cqn_handler) {  in bnxt_qpn_cqn_sched_task()
157 			dev_dbg(&nq->res->pdev->dev,  in bnxt_qpn_cqn_sched_task()
160 			nq->cqn_handler(nq, cq);  in bnxt_qpn_cqn_sched_task()
162 		spin_unlock_bh(&cq->compl_lock);  in bnxt_qpn_cqn_sched_task()
170 	dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map);  in bnxt_qplib_put_hdr_buf()
184 	hdrbuf->va = dma_alloc_coherent(&pdev->dev, len,  in bnxt_qplib_get_hdr_buf()
185 					&hdrbuf->dma_map, GFP_KERNEL);  in bnxt_qplib_get_hdr_buf()
186 	if (!hdrbuf->va)  in bnxt_qplib_get_hdr_buf()
189 	hdrbuf->len = len;  in bnxt_qplib_get_hdr_buf()
190 	hdrbuf->step = step;  in bnxt_qplib_get_hdr_buf()
200 	if (qp->rq_hdr_buf) {  in bnxt_qplib_free_hdr_buf()
201 		bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf);  in bnxt_qplib_free_hdr_buf()
202 		qp->rq_hdr_buf = NULL;  in bnxt_qplib_free_hdr_buf()
205 	if (qp->sq_hdr_buf) {  in bnxt_qplib_free_hdr_buf()
206 		bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf);  in bnxt_qplib_free_hdr_buf()
207 		qp->sq_hdr_buf = NULL;  in bnxt_qplib_free_hdr_buf()
215 	int rc = 0;  in bnxt_qplib_alloc_hdr_buf()  local
217 	pdev = res->pdev;  in bnxt_qplib_alloc_hdr_buf()
219 		qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep,  in bnxt_qplib_alloc_hdr_buf()
220 							qp->sq.max_wqe);  in bnxt_qplib_alloc_hdr_buf()
221 		if (!qp->sq_hdr_buf) {  in bnxt_qplib_alloc_hdr_buf()
222 			dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n");  in bnxt_qplib_alloc_hdr_buf()
223 			return -ENOMEM;  in bnxt_qplib_alloc_hdr_buf()
228 		qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep,  in bnxt_qplib_alloc_hdr_buf()
229 							qp->rq.max_wqe);  in bnxt_qplib_alloc_hdr_buf()
230 		if (!qp->rq_hdr_buf) {  in bnxt_qplib_alloc_hdr_buf()
231 			rc = -ENOMEM;  in bnxt_qplib_alloc_hdr_buf()
232 			dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n");  in bnxt_qplib_alloc_hdr_buf()
240 	return rc;  in bnxt_qplib_alloc_hdr_buf()
244  * clean_nq -	Invalidate cqe from given nq.
245  * @cq      -	Completion queue
249  * Note - This function traverse the hardware queue but do not update
264 	nq = cq->nq;  in clean_nq()
265 	nq_hwq = &nq->hwq;  in clean_nq()
267 	spin_lock_bh(&nq_hwq->lock);  in clean_nq()
268 	peek_flags = nq->nq_db.dbinfo.flags;  in clean_nq()
269 	peek_cons = nq_hwq->cons;  in clean_nq()
270 	for (i = 0; i < nq_hwq->max_elements; i++) {  in clean_nq()
279 		type = le16_to_cpu(hw_nqe->info10_type) &  in clean_nq()
286 			q_handle = le32_to_cpu(nqcne->cq_handle_low);  in clean_nq()
287 			q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;  in clean_nq()
289 				nqcne->cq_handle_low = 0;  in clean_nq()
290 				nqcne->cq_handle_high = 0;  in clean_nq()
291 				cq->cnq_events++;  in clean_nq()
294 		bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons,  in clean_nq()
297 	spin_unlock_bh(&nq_hwq->lock);  in clean_nq()
316 	while (retry_cnt--) {  in __wait_for_all_nqes()
317 		total_events = cq->cnq_events;  in __wait_for_all_nqes()
320 		if (cq->is_cq_err_event)  in __wait_for_all_nqes()
324 			dev_dbg(&cq->nq->res->pdev->dev,  in __wait_for_all_nqes()
325 				"QPLIB: NQ cleanup - Received all NQ events\n");  in __wait_for_all_nqes()
336 	struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;  in bnxt_qplib_service_nq()
337 	int budget = nq->budget;  in bnxt_qplib_service_nq()
346 	res = nq->res;  in bnxt_qplib_service_nq()
347 	pdev = res->pdev;  in bnxt_qplib_service_nq()
349 	spin_lock_bh(&nq_hwq->lock);  in bnxt_qplib_service_nq()
351 	while (budget--) {  in bnxt_qplib_service_nq()
352 		nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);  in bnxt_qplib_service_nq()
353 		if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))  in bnxt_qplib_service_nq()
359 		type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;  in bnxt_qplib_service_nq()
365 			q_handle = le32_to_cpu(nqcne->cq_handle_low);  in bnxt_qplib_service_nq()
366 			q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;  in bnxt_qplib_service_nq()
370 			cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT;  in bnxt_qplib_service_nq()
371 			cq->dbinfo.toggle = cq->toggle;  in bnxt_qplib_service_nq()
372 			bnxt_qplib_armen_db(&cq->dbinfo,  in bnxt_qplib_service_nq()
374 			spin_lock_bh(&cq->compl_lock);  in bnxt_qplib_service_nq()
375 			atomic_set(&cq->arm_state, 0) ;  in bnxt_qplib_service_nq()
376 			if (!nq->cqn_handler(nq, (cq)))  in bnxt_qplib_service_nq()
377 				nq->stats.num_cqne_processed++;  in bnxt_qplib_service_nq()
379 				dev_warn(&pdev->dev,  in bnxt_qplib_service_nq()
380 					 "QPLIB: cqn - type 0x%x not handled\n",  in bnxt_qplib_service_nq()
382 			cq->cnq_events++;  in bnxt_qplib_service_nq()
383 			spin_unlock_bh(&cq->compl_lock);  in bnxt_qplib_service_nq()
392 			q_handle = le32_to_cpu(nqsrqe->srq_handle_low);  in bnxt_qplib_service_nq()
393 			q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32;  in bnxt_qplib_service_nq()
395 			bnxt_qplib_armen_db(&srq->dbinfo,  in bnxt_qplib_service_nq()
397 			if (!nq->srqn_handler(nq,  in bnxt_qplib_service_nq()
399 					      nqsrqe->event))  in bnxt_qplib_service_nq()
400 				nq->stats.num_srqne_processed++;  in bnxt_qplib_service_nq()
402 				dev_warn(&pdev->dev,  in bnxt_qplib_service_nq()
403 					 "QPLIB: SRQ event 0x%x not handled\n",  in bnxt_qplib_service_nq()
404 					 nqsrqe->event);  in bnxt_qplib_service_nq()
408 			dev_warn(&pdev->dev,  in bnxt_qplib_service_nq()
409 				 "QPLIB: nqe with opcode = 0x%x not handled\n",  in bnxt_qplib_service_nq()
414 		bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons,  in bnxt_qplib_service_nq()
415 					 1, &nq->nq_db.dbinfo.flags);  in bnxt_qplib_service_nq()
417 	nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);  in bnxt_qplib_service_nq()
418 	if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) {  in bnxt_qplib_service_nq()
419 		nq->stats.num_nq_rearm++;  in bnxt_qplib_service_nq()
420 		bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);  in bnxt_qplib_service_nq()
421 	} else if (nq->requested) {  in bnxt_qplib_service_nq()
422 		bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);  in bnxt_qplib_service_nq()
423 		nq->stats.num_tasklet_resched++;  in bnxt_qplib_service_nq()
425 	dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n");  in bnxt_qplib_service_nq()
427 		dev_dbg(&pdev->dev,  in bnxt_qplib_service_nq()
428 			"QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n",  in bnxt_qplib_service_nq()
429 			nq->stats.num_cqne_processed, nq->stats.num_srqne_processed,  in bnxt_qplib_service_nq()
430 			nq->stats.num_dbqne_processed, budget, hw_polled);  in bnxt_qplib_service_nq()
431 	dev_dbg(&pdev->dev,  in bnxt_qplib_service_nq()
433 		nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm);  in bnxt_qplib_service_nq()
434 	spin_unlock_bh(&nq_hwq->lock);  in bnxt_qplib_service_nq()
440 	struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;  in bnxt_qplib_nq_irq()
444 	sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq);  in bnxt_qplib_nq_irq()
457 	if (!nq->requested)  in bnxt_qplib_nq_stop_irq()
460 	nq->requested = false;  in bnxt_qplib_nq_stop_irq()
461 	res = nq->res;  in bnxt_qplib_nq_stop_irq()
463 	bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false);  in bnxt_qplib_nq_stop_irq()
465 	synchronize_irq(nq->msix_vec);  in bnxt_qplib_nq_stop_irq()
466 	free_irq(nq->msix_vec, nq);  in bnxt_qplib_nq_stop_irq()
467 	kfree(nq->name);  in bnxt_qplib_nq_stop_irq()
468 	nq->name = NULL;  in bnxt_qplib_nq_stop_irq()
473 	if (nq->cqn_wq) {  in bnxt_qplib_disable_nq()
474 		destroy_workqueue(nq->cqn_wq);  in bnxt_qplib_disable_nq()
475 		nq->cqn_wq = NULL;  in bnxt_qplib_disable_nq()
480 	nq->nq_db.reg.bar_reg = NULL;  in bnxt_qplib_disable_nq()
481 	nq->nq_db.db = NULL;  in bnxt_qplib_disable_nq()
483 	nq->cqn_handler = NULL;  in bnxt_qplib_disable_nq()
484 	nq->srqn_handler = NULL;  in bnxt_qplib_disable_nq()
485 	nq->msix_vec = 0;  in bnxt_qplib_disable_nq()
492 	int rc;  in bnxt_qplib_nq_start_irq()  local
494 	res = nq->res;  in bnxt_qplib_nq_start_irq()
495 	if (nq->requested)  in bnxt_qplib_nq_start_irq()
496 		return -EFAULT;  in bnxt_qplib_nq_start_irq()
498 	nq->msix_vec = msix_vector;  in bnxt_qplib_nq_start_irq()
499 	nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n",  in bnxt_qplib_nq_start_irq()
500 			     nq_indx, pci_name(res->pdev));  in bnxt_qplib_nq_start_irq()
501 	if (!nq->name)  in bnxt_qplib_nq_start_irq()
502 		return -ENOMEM;  in bnxt_qplib_nq_start_irq()
503 	rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);  in bnxt_qplib_nq_start_irq()
504 	if (rc) {  in bnxt_qplib_nq_start_irq()
505 		kfree(nq->name);  in bnxt_qplib_nq_start_irq()
506 		nq->name = NULL;  in bnxt_qplib_nq_start_irq()
507 		return rc;  in bnxt_qplib_nq_start_irq()
509 	nq->requested = true;  in bnxt_qplib_nq_start_irq()
510 	bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);  in bnxt_qplib_nq_start_irq()
512 	return rc;  in bnxt_qplib_nq_start_irq()
521 	nq_db = &nq->nq_db;  in bnxt_qplib_map_nq_db()
522 	res = nq->res;  in bnxt_qplib_map_nq_db()
523 	dbreg = &res->dpi_tbl.ucreg;  in bnxt_qplib_map_nq_db()
525 	nq_db->reg.bar_id = dbreg->bar_id;  in bnxt_qplib_map_nq_db()
526 	nq_db->reg.bar_base = dbreg->bar_base;  in bnxt_qplib_map_nq_db()
527 	nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;  in bnxt_qplib_map_nq_db()
528 	nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :  in bnxt_qplib_map_nq_db()
531 	nq_db->dbinfo.db = nq_db->reg.bar_reg;  in bnxt_qplib_map_nq_db()
532 	nq_db->dbinfo.hwq = &nq->hwq;  in bnxt_qplib_map_nq_db()
533 	nq_db->dbinfo.xid = nq->ring_id;  in bnxt_qplib_map_nq_db()
534 	nq_db->dbinfo.seed = nq->ring_id;  in bnxt_qplib_map_nq_db()
535 	nq_db->dbinfo.flags = 0;  in bnxt_qplib_map_nq_db()
536 	spin_lock_init(&nq_db->dbinfo.lock);  in bnxt_qplib_map_nq_db()
537 	nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_map_nq_db()
538 	nq_db->dbinfo.res = nq->res;  in bnxt_qplib_map_nq_db()
549 	int rc;  in bnxt_qplib_enable_nq()  local
551 	pdev = nq->res->pdev;  in bnxt_qplib_enable_nq()
552 	nq->cqn_handler = cqn_handler;  in bnxt_qplib_enable_nq()
553 	nq->srqn_handler = srqn_handler;  in bnxt_qplib_enable_nq()
554 	nq->load = 0;  in bnxt_qplib_enable_nq()
555 	mutex_init(&nq->lock);  in bnxt_qplib_enable_nq()
558 	nq->cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq\n");  in bnxt_qplib_enable_nq()
559 	if (!nq->cqn_wq)  in bnxt_qplib_enable_nq()
560 		return -ENOMEM;  in bnxt_qplib_enable_nq()
563 	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);  in bnxt_qplib_enable_nq()
564 	if (rc) {  in bnxt_qplib_enable_nq()
565 		dev_err(&pdev->dev,  in bnxt_qplib_enable_nq()
566 			"QPLIB: Failed to request irq for nq-idx %d\n", nq_idx);  in bnxt_qplib_enable_nq()
569 	dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements);  in bnxt_qplib_enable_nq()
574 	return rc;  in bnxt_qplib_enable_nq()
579 	if (nq->hwq.max_elements) {  in bnxt_qplib_free_nq_mem()
580 		bnxt_qplib_free_hwq(nq->res, &nq->hwq);  in bnxt_qplib_free_nq_mem()
581 		nq->hwq.max_elements = 0;  in bnxt_qplib_free_nq_mem()
591 	nq->res = res;  in bnxt_qplib_alloc_nq_mem()
592 	if (!nq->hwq.max_elements ||  in bnxt_qplib_alloc_nq_mem()
593 	    nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)  in bnxt_qplib_alloc_nq_mem()
594 		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;  in bnxt_qplib_alloc_nq_mem()
600 	hwq_attr.depth = nq->hwq.max_elements;  in bnxt_qplib_alloc_nq_mem()
603 	if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {  in bnxt_qplib_alloc_nq_mem()
604 		dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n");  in bnxt_qplib_alloc_nq_mem()
605 		return -ENOMEM;  in bnxt_qplib_alloc_nq_mem()
607 	nq->budget = 8;  in bnxt_qplib_alloc_nq_mem()
619 	req.srq_cid = cpu_to_le32(srq->id);  in __qplib_destroy_srq()
630 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_destroy_srq()
631 	int rc;  in bnxt_qplib_destroy_srq()  local
633 	rc = __qplib_destroy_srq(rcfw, srq);  in bnxt_qplib_destroy_srq()
634 	if (rc)  in bnxt_qplib_destroy_srq()
635 		return rc;  in bnxt_qplib_destroy_srq()
636 	bnxt_qplib_free_hwq(res, &srq->hwq);  in bnxt_qplib_destroy_srq()
637 	kfree(srq->swq);  in bnxt_qplib_destroy_srq()
645 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_create_srq()
651 	int rc, idx;  in bnxt_qplib_create_srq()  local
654 	hwq_attr.sginfo = &srq->sginfo;  in bnxt_qplib_create_srq()
655 	hwq_attr.depth = srq->max_wqe;  in bnxt_qplib_create_srq()
656 	hwq_attr.stride = srq->wqe_size;  in bnxt_qplib_create_srq()
658 	rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);  in bnxt_qplib_create_srq()
659 	if (rc)  in bnxt_qplib_create_srq()
662 	req.dpi = cpu_to_le32(srq->dpi->dpi);  in bnxt_qplib_create_srq()
664 	srq_size = min_t(u32, srq->hwq.depth, U16_MAX);  in bnxt_qplib_create_srq()
666 	pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) <<  in bnxt_qplib_create_srq()
668 	pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK);  in bnxt_qplib_create_srq()
670 	req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq));  in bnxt_qplib_create_srq()
671 	req.pd_id = cpu_to_le32(srq->pd->id);  in bnxt_qplib_create_srq()
672 	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);  in bnxt_qplib_create_srq()
677 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_create_srq()
678 	if (rc)  in bnxt_qplib_create_srq()
680 	if (!srq->is_user) {  in bnxt_qplib_create_srq()
681 		srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq),  in bnxt_qplib_create_srq()
683 		if (!srq->swq)  in bnxt_qplib_create_srq()
685 		srq->start_idx = 0;  in bnxt_qplib_create_srq()
686 		srq->last_idx = srq->hwq.depth - 1;  in bnxt_qplib_create_srq()
687 		for (idx = 0; idx < srq->hwq.depth; idx++)  in bnxt_qplib_create_srq()
688 			srq->swq[idx].next_idx = idx + 1;  in bnxt_qplib_create_srq()
689 		srq->swq[srq->last_idx].next_idx = -1;  in bnxt_qplib_create_srq()
692 	spin_lock_init(&srq->lock);  in bnxt_qplib_create_srq()
693 	srq->id = le32_to_cpu(resp.xid);  in bnxt_qplib_create_srq()
694 	srq->cctx = res->cctx;  in bnxt_qplib_create_srq()
695 	srq->dbinfo.hwq = &srq->hwq;  in bnxt_qplib_create_srq()
696 	srq->dbinfo.xid = srq->id;  in bnxt_qplib_create_srq()
697 	srq->dbinfo.db = srq->dpi->dbr;  in bnxt_qplib_create_srq()
698 	srq->dbinfo.max_slot = 1;  in bnxt_qplib_create_srq()
699 	srq->dbinfo.priv_db = res->dpi_tbl.priv_db;  in bnxt_qplib_create_srq()
700 	srq->dbinfo.flags = 0;  in bnxt_qplib_create_srq()
701 	spin_lock_init(&srq->dbinfo.lock);  in bnxt_qplib_create_srq()
702 	srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_srq()
703 	srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_srq()
704 	srq->dbinfo.res = res;  in bnxt_qplib_create_srq()
705 	srq->dbinfo.seed = srq->id;  in bnxt_qplib_create_srq()
706 	if (srq->threshold)  in bnxt_qplib_create_srq()
707 		bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);  in bnxt_qplib_create_srq()
708 	srq->arm_req = false;  in bnxt_qplib_create_srq()
713 	bnxt_qplib_free_hwq(res, &srq->hwq);  in bnxt_qplib_create_srq()
715 	return rc;  in bnxt_qplib_create_srq()
721 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;  in bnxt_qplib_modify_srq()
725 	if (avail <= srq->threshold) {  in bnxt_qplib_modify_srq()
726 		srq->arm_req = false;  in bnxt_qplib_modify_srq()
727 		bnxt_qplib_srq_arm_db(&srq->dbinfo);  in bnxt_qplib_modify_srq()
730 		srq->arm_req = true;  in bnxt_qplib_modify_srq()
738 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_query_srq()
744 	int rc = 0;  in bnxt_qplib_query_srq()  local
749 	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,  in bnxt_qplib_query_srq()
752 		return -ENOMEM;  in bnxt_qplib_query_srq()
754 	req.srq_cid = cpu_to_le32(srq->id);  in bnxt_qplib_query_srq()
757 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_query_srq()
759 	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,  in bnxt_qplib_query_srq()
762 	return rc;  in bnxt_qplib_query_srq()
768 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;  in bnxt_qplib_post_srq_recv()
771 	int i, rc = 0, next;  in bnxt_qplib_post_srq_recv()  local
774 	spin_lock(&srq_hwq->lock);  in bnxt_qplib_post_srq_recv()
775 	if (srq->start_idx == srq->last_idx) {  in bnxt_qplib_post_srq_recv()
776 		dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n",  in bnxt_qplib_post_srq_recv()
777 			srq->id);  in bnxt_qplib_post_srq_recv()
778 		rc = -EINVAL;  in bnxt_qplib_post_srq_recv()
779 		spin_unlock(&srq_hwq->lock);  in bnxt_qplib_post_srq_recv()
782 	next = srq->start_idx;  in bnxt_qplib_post_srq_recv()
783 	srq->start_idx = srq->swq[next].next_idx;  in bnxt_qplib_post_srq_recv()
784 	spin_unlock(&srq_hwq->lock);  in bnxt_qplib_post_srq_recv()
786 	srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL);  in bnxt_qplib_post_srq_recv()
787 	memset(srqe, 0, srq->wqe_size);  in bnxt_qplib_post_srq_recv()
789 	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;  in bnxt_qplib_post_srq_recv()
790 	     i < wqe->num_sge; i++, hw_sge++) {  in bnxt_qplib_post_srq_recv()
791 		hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);  in bnxt_qplib_post_srq_recv()
792 		hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);  in bnxt_qplib_post_srq_recv()
793 		hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);  in bnxt_qplib_post_srq_recv()
795 	srqe->wqe_type = wqe->type;  in bnxt_qplib_post_srq_recv()
796 	srqe->flags = wqe->flags;  in bnxt_qplib_post_srq_recv()
797 	srqe->wqe_size = wqe->num_sge +  in bnxt_qplib_post_srq_recv()
799 	if (!wqe->num_sge)  in bnxt_qplib_post_srq_recv()
800 		srqe->wqe_size++;  in bnxt_qplib_post_srq_recv()
801 	srqe->wr_id |= cpu_to_le32((u32)next);  in bnxt_qplib_post_srq_recv()
802 	srq->swq[next].wr_id = wqe->wr_id;  in bnxt_qplib_post_srq_recv()
803 	bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot);  in bnxt_qplib_post_srq_recv()
804 	/* retaining srq_hwq->cons for this logic actually the lock is only  in bnxt_qplib_post_srq_recv()
805 	 * required to read srq_hwq->cons.  in bnxt_qplib_post_srq_recv()
807 	spin_lock(&srq_hwq->lock);  in bnxt_qplib_post_srq_recv()
809 	spin_unlock(&srq_hwq->lock);  in bnxt_qplib_post_srq_recv()
811 	bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);  in bnxt_qplib_post_srq_recv()
812 	if (srq->arm_req && avail <= srq->threshold) {  in bnxt_qplib_post_srq_recv()
813 		srq->arm_req = false;  in bnxt_qplib_post_srq_recv()
814 		bnxt_qplib_srq_arm_db(&srq->dbinfo);  in bnxt_qplib_post_srq_recv()
817 	return rc;  in bnxt_qplib_post_srq_recv()
828 	req.qp_cid = cpu_to_le32(qp->id);  in __qplib_destroy_qp()
838 	int rc = 0;  in bnxt_qplib_alloc_init_swq()  local
841 	que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);  in bnxt_qplib_alloc_init_swq()
842 	if (!que->swq) {  in bnxt_qplib_alloc_init_swq()
843 		rc = -ENOMEM;  in bnxt_qplib_alloc_init_swq()
847 	que->swq_start = 0;  in bnxt_qplib_alloc_init_swq()
848 	que->swq_last = que->max_wqe - 1;  in bnxt_qplib_alloc_init_swq()
849 	for (indx = 0; indx < que->max_wqe; indx++)  in bnxt_qplib_alloc_init_swq()
850 		que->swq[indx].next_idx = indx + 1;  in bnxt_qplib_alloc_init_swq()
851 	que->swq[que->swq_last].next_idx = 0; /* Make it circular */  in bnxt_qplib_alloc_init_swq()
852 	que->swq_last = 0;  in bnxt_qplib_alloc_init_swq()
854 	return rc;  in bnxt_qplib_alloc_init_swq()
862 	idx = que->swq_start;  in bnxt_qplib_get_swqe()
865 	return &que->swq[idx];  in bnxt_qplib_get_swqe()
870 	que->swq_start = que->swq[idx].next_idx;  in bnxt_qplib_swq_mod_start()
883 	return (que->wqe_size * que->max_wqe) / stride;  in bnxt_qplib_get_depth()
890 		que->max_wqe : bnxt_qplib_get_depth(que);  in _set_sq_size()
902 	return (que->wqe_size / sizeof(struct sq_sge));  in _set_rq_max_slot()
908 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_create_qp1()
911 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_create_qp1()
912 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_create_qp1()
918 	int rc;  in bnxt_qplib_create_qp1()  local
921 	req.type = qp->type;  in bnxt_qplib_create_qp1()
922 	req.dpi = cpu_to_le32(qp->dpi->dpi);  in bnxt_qplib_create_qp1()
923 	req.qp_handle = cpu_to_le64(qp->qp_handle);  in bnxt_qplib_create_qp1()
926 	hwq_attr.sginfo = &sq->sginfo;  in bnxt_qplib_create_qp1()
930 	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);  in bnxt_qplib_create_qp1()
931 	if (rc)  in bnxt_qplib_create_qp1()
934 	req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode));  in bnxt_qplib_create_qp1()
935 	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));  in bnxt_qplib_create_qp1()
936 	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<  in bnxt_qplib_create_qp1()
938 	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) <<  in bnxt_qplib_create_qp1()
943 					 (sq->max_sge &  in bnxt_qplib_create_qp1()
945 	req.scq_cid = cpu_to_le32(qp->scq->id);  in bnxt_qplib_create_qp1()
948 	if (!qp->srq) {  in bnxt_qplib_create_qp1()
950 		hwq_attr.sginfo = &rq->sginfo;  in bnxt_qplib_create_qp1()
954 		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);  in bnxt_qplib_create_qp1()
955 		if (rc)  in bnxt_qplib_create_qp1()
957 		req.rq_size = cpu_to_le32(rq->max_wqe);  in bnxt_qplib_create_qp1()
958 		req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));  in bnxt_qplib_create_qp1()
959 		pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<  in bnxt_qplib_create_qp1()
961 		pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<  in bnxt_qplib_create_qp1()
967 				     (rq->max_sge &  in bnxt_qplib_create_qp1()
972 		req.srq_cid = cpu_to_le32(qp->srq->id);  in bnxt_qplib_create_qp1()
974 	req.rcq_cid = cpu_to_le32(qp->rcq->id);  in bnxt_qplib_create_qp1()
978 	req.pd_id = cpu_to_le32(qp->pd->id);  in bnxt_qplib_create_qp1()
984 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_create_qp1()
985 	if (rc)  in bnxt_qplib_create_qp1()
988 	rc = bnxt_qplib_alloc_init_swq(sq);  in bnxt_qplib_create_qp1()
989 	if (rc)  in bnxt_qplib_create_qp1()
992 	if (!qp->srq) {  in bnxt_qplib_create_qp1()
993 		rc = bnxt_qplib_alloc_init_swq(rq);  in bnxt_qplib_create_qp1()
994 		if (rc)  in bnxt_qplib_create_qp1()
998 	qp->id = le32_to_cpu(resp.xid);  in bnxt_qplib_create_qp1()
999 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;  in bnxt_qplib_create_qp1()
1000 	qp->cctx = res->cctx;  in bnxt_qplib_create_qp1()
1001 	sq->dbinfo.hwq = &sq->hwq;  in bnxt_qplib_create_qp1()
1002 	sq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp1()
1003 	sq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp1()
1004 	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);  in bnxt_qplib_create_qp1()
1005 	sq->dbinfo.flags = 0;  in bnxt_qplib_create_qp1()
1006 	spin_lock_init(&sq->dbinfo.lock);  in bnxt_qplib_create_qp1()
1007 	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp1()
1008 	sq->dbinfo.res = res;  in bnxt_qplib_create_qp1()
1009 	if (rq->max_wqe) {  in bnxt_qplib_create_qp1()
1010 		rq->dbinfo.hwq = &rq->hwq;  in bnxt_qplib_create_qp1()
1011 		rq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp1()
1012 		rq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp1()
1013 		rq->dbinfo.max_slot = _set_rq_max_slot(rq);  in bnxt_qplib_create_qp1()
1014 		rq->dbinfo.flags = 0;  in bnxt_qplib_create_qp1()
1015 		spin_lock_init(&rq->dbinfo.lock);  in bnxt_qplib_create_qp1()
1016 		rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp1()
1017 		rq->dbinfo.res = res;  in bnxt_qplib_create_qp1()
1020 	tbl = &res->reftbl.qpref;  in bnxt_qplib_create_qp1()
1021 	spin_lock_irqsave(&tbl->lock, flag);  in bnxt_qplib_create_qp1()
1022 	tbl->rec[tbl->max].xid = qp->id;  in bnxt_qplib_create_qp1()
1023 	tbl->rec[tbl->max].handle = qp;  in bnxt_qplib_create_qp1()
1024 	spin_unlock_irqrestore(&tbl->lock, flag);  in bnxt_qplib_create_qp1()
1028 	kfree(sq->swq);  in bnxt_qplib_create_qp1()
1032 	bnxt_qplib_free_hwq(res, &rq->hwq);  in bnxt_qplib_create_qp1()
1034 	bnxt_qplib_free_hwq(res, &sq->hwq);  in bnxt_qplib_create_qp1()
1036 	return rc;  in bnxt_qplib_create_qp1()
1046 	sq = &qp->sq;  in bnxt_qplib_init_psn_ptr()
1047 	sq_hwq = &sq->hwq;  in bnxt_qplib_init_psn_ptr()
1049 	fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg);  in bnxt_qplib_init_psn_ptr()
1052 	sq_hwq->pad_pgofft = indx_pad;  in bnxt_qplib_init_psn_ptr()
1053 	sq_hwq->pad_pg = (u64 *)psn_pg;  in bnxt_qplib_init_psn_ptr()
1054 	sq_hwq->pad_stride = size;  in bnxt_qplib_init_psn_ptr()
1060 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_create_qp()
1064 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_create_qp()
1065 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_create_qp()
1069 	int rc, req_size, psn_sz;  in bnxt_qplib_create_qp()  local
1077 	qp->cctx = res->cctx;  in bnxt_qplib_create_qp()
1078 	if (res->dattr)  in bnxt_qplib_create_qp()
1079 		qp->dev_cap_flags = res->dattr->dev_cap_flags;  in bnxt_qplib_create_qp()
1081 	req.type = qp->type;  in bnxt_qplib_create_qp()
1082 	req.dpi = cpu_to_le32(qp->dpi->dpi);  in bnxt_qplib_create_qp()
1083 	req.qp_handle = cpu_to_le64(qp->qp_handle);  in bnxt_qplib_create_qp()
1086 	if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {  in bnxt_qplib_create_qp()
1087 		psn_sz = _is_chip_gen_p5_p7(qp->cctx) ?  in bnxt_qplib_create_qp()
1090 		if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {  in bnxt_qplib_create_qp()
1092 			qp->msn = 0;  in bnxt_qplib_create_qp()
1099 	hwq_attr.sginfo = &sq->sginfo;  in bnxt_qplib_create_qp()
1104 				 _set_sq_size(sq, qp->wqe_mode) : 0;  in bnxt_qplib_create_qp()
1106 	if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) {  in bnxt_qplib_create_qp()
1107 		if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)  in bnxt_qplib_create_qp()
1108 			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode));  in bnxt_qplib_create_qp()
1110 			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2;  in bnxt_qplib_create_qp()
1111 		qp->msn_tbl_sz = hwq_attr.aux_depth;  in bnxt_qplib_create_qp()
1112 		qp->msn = 0;  in bnxt_qplib_create_qp()
1115 	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);  in bnxt_qplib_create_qp()
1116 	if (rc)  in bnxt_qplib_create_qp()
1119 	sqsz = _set_sq_size(sq, qp->wqe_mode);  in bnxt_qplib_create_qp()
1122 		pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz);  in bnxt_qplib_create_qp()
1126 	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));  in bnxt_qplib_create_qp()
1127 	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<  in bnxt_qplib_create_qp()
1129 	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<  in bnxt_qplib_create_qp()
1134 					 ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ?  in bnxt_qplib_create_qp()
1135 					  BNXT_MSN_TBLE_SGE : sq->max_sge &  in bnxt_qplib_create_qp()
1137 	req.scq_cid = cpu_to_le32(qp->scq->id);  in bnxt_qplib_create_qp()
1140 	if (!qp->srq) {  in bnxt_qplib_create_qp()
1142 		hwq_attr.sginfo = &rq->sginfo;  in bnxt_qplib_create_qp()
1148 		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);  in bnxt_qplib_create_qp()
1149 		if (rc)  in bnxt_qplib_create_qp()
1151 		req.rq_size = cpu_to_le32(rq->max_wqe);  in bnxt_qplib_create_qp()
1152 		req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));  in bnxt_qplib_create_qp()
1153 		pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<  in bnxt_qplib_create_qp()
1155 		pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<  in bnxt_qplib_create_qp()
1158 		nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?  in bnxt_qplib_create_qp()
1159 			res->dattr->max_qp_sges : rq->max_sge;  in bnxt_qplib_create_qp()
1166 		req.srq_cid = cpu_to_le32(qp->srq->id);  in bnxt_qplib_create_qp()
1168 	req.rcq_cid = cpu_to_le32(qp->rcq->id);  in bnxt_qplib_create_qp()
1172 	if (qp->sig_type)  in bnxt_qplib_create_qp()
1174 	if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)  in bnxt_qplib_create_qp()
1176 	if (res->cctx->modes.te_bypass)  in bnxt_qplib_create_qp()
1178 	if (res->dattr &&  in bnxt_qplib_create_qp()
1179 	    bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf))  in bnxt_qplib_create_qp()
1185 		xrrq = &qp->orrq;  in bnxt_qplib_create_qp()
1186 		xrrq->max_elements =  in bnxt_qplib_create_qp()
1187 			ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);  in bnxt_qplib_create_qp()
1188 		req_size = xrrq->max_elements *  in bnxt_qplib_create_qp()
1189 			   BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;  in bnxt_qplib_create_qp()
1190 		req_size &= ~(PAGE_SIZE - 1);  in bnxt_qplib_create_qp()
1196 		hwq_attr.depth = xrrq->max_elements;  in bnxt_qplib_create_qp()
1201 		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);  in bnxt_qplib_create_qp()
1202 		if (rc)  in bnxt_qplib_create_qp()
1206 		xrrq = &qp->irrq;  in bnxt_qplib_create_qp()
1207 		xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(  in bnxt_qplib_create_qp()
1208 						qp->max_dest_rd_atomic);  in bnxt_qplib_create_qp()
1209 		req_size = xrrq->max_elements *  in bnxt_qplib_create_qp()
1210 			   BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;  in bnxt_qplib_create_qp()
1211 		req_size &= ~(PAGE_SIZE - 1);  in bnxt_qplib_create_qp()
1213 		hwq_attr.depth =  xrrq->max_elements;  in bnxt_qplib_create_qp()
1215 		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);  in bnxt_qplib_create_qp()
1216 		if (rc)  in bnxt_qplib_create_qp()
1220 	req.pd_id = cpu_to_le32(qp->pd->id);  in bnxt_qplib_create_qp()
1226 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_create_qp()
1227 	if (rc)  in bnxt_qplib_create_qp()
1230 	if (!qp->is_user) {  in bnxt_qplib_create_qp()
1231 		rc = bnxt_qplib_alloc_init_swq(sq);  in bnxt_qplib_create_qp()
1232 		if (rc)  in bnxt_qplib_create_qp()
1234 		if (!qp->srq) {  in bnxt_qplib_create_qp()
1235 			rc = bnxt_qplib_alloc_init_swq(rq);  in bnxt_qplib_create_qp()
1236 			if (rc)  in bnxt_qplib_create_qp()
1242 	qp->id = le32_to_cpu(resp.xid);  in bnxt_qplib_create_qp()
1243 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;  in bnxt_qplib_create_qp()
1244 	INIT_LIST_HEAD(&qp->sq_flush);  in bnxt_qplib_create_qp()
1245 	INIT_LIST_HEAD(&qp->rq_flush);  in bnxt_qplib_create_qp()
1247 	sq->dbinfo.hwq = &sq->hwq;  in bnxt_qplib_create_qp()
1248 	sq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp()
1249 	sq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp()
1250 	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);  in bnxt_qplib_create_qp()
1251 	sq->dbinfo.flags = 0;  in bnxt_qplib_create_qp()
1252 	spin_lock_init(&sq->dbinfo.lock);  in bnxt_qplib_create_qp()
1253 	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp()
1254 	sq->dbinfo.res = res;  in bnxt_qplib_create_qp()
1255 	sq->dbinfo.seed = qp->id;  in bnxt_qplib_create_qp()
1256 	if (rq->max_wqe) {  in bnxt_qplib_create_qp()
1257 		rq->dbinfo.hwq = &rq->hwq;  in bnxt_qplib_create_qp()
1258 		rq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp()
1259 		rq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp()
1260 		rq->dbinfo.max_slot = _set_rq_max_slot(rq);  in bnxt_qplib_create_qp()
1261 		rq->dbinfo.flags = 0;  in bnxt_qplib_create_qp()
1262 		spin_lock_init(&rq->dbinfo.lock);  in bnxt_qplib_create_qp()
1263 		rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp()
1264 		rq->dbinfo.res = res;  in bnxt_qplib_create_qp()
1265 		rq->dbinfo.seed = qp->id;  in bnxt_qplib_create_qp()
1268 	tbl = &res->reftbl.qpref;  in bnxt_qplib_create_qp()
1269 	qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);  in bnxt_qplib_create_qp()
1270 	spin_lock_irqsave(&tbl->lock, flag);  in bnxt_qplib_create_qp()
1271 	tbl->rec[qp_idx].xid = qp->id;  in bnxt_qplib_create_qp()
1272 	tbl->rec[qp_idx].handle = qp;  in bnxt_qplib_create_qp()
1273 	spin_unlock_irqrestore(&tbl->lock, flag);  in bnxt_qplib_create_qp()
1277 	kfree(sq->swq);  in bnxt_qplib_create_qp()
1281 	bnxt_qplib_free_hwq(res, &qp->irrq);  in bnxt_qplib_create_qp()
1283 	bnxt_qplib_free_hwq(res, &qp->orrq);  in bnxt_qplib_create_qp()
1285 	bnxt_qplib_free_hwq(res, &rq->hwq);  in bnxt_qplib_create_qp()
1287 	bnxt_qplib_free_hwq(res, &sq->hwq);  in bnxt_qplib_create_qp()
1289 	return rc;  in bnxt_qplib_create_qp()
1294 	switch (qp->cur_qp_state) {  in __filter_modify_flags()
1296 		switch (qp->state) {  in __filter_modify_flags()
1304 		switch (qp->state) {  in __filter_modify_flags()
1306 			if (!(qp->modify_flags &  in __filter_modify_flags()
1308 				qp->modify_flags |=  in __filter_modify_flags()
1310 				qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;  in __filter_modify_flags()
1312 			qp->modify_flags &=  in __filter_modify_flags()
1315 			if (qp->max_dest_rd_atomic < 1)  in __filter_modify_flags()
1316 				qp->max_dest_rd_atomic = 1;  in __filter_modify_flags()
1317 			qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;  in __filter_modify_flags()
1319 			if (!(qp->modify_flags &  in __filter_modify_flags()
1321 				qp->modify_flags |=  in __filter_modify_flags()
1323 				qp->ah.sgid_index = 0;  in __filter_modify_flags()
1331 		switch (qp->state) {  in __filter_modify_flags()
1334 			if (qp->max_rd_atomic < 1)  in __filter_modify_flags()
1335 				qp->max_rd_atomic = 1;  in __filter_modify_flags()
1336 			qp->modify_flags &=  in __filter_modify_flags()
1369 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_modify_qp()
1376 	int rc;  in bnxt_qplib_modify_qp()  local
1381 	/* Filter out the qp_attr_mask based on the state->new transition */  in bnxt_qplib_modify_qp()
1383 	bmask = qp->modify_flags;  in bnxt_qplib_modify_qp()
1384 	req.modify_mask = cpu_to_le32(qp->modify_flags);  in bnxt_qplib_modify_qp()
1385 	req.qp_cid = cpu_to_le32(qp->id);  in bnxt_qplib_modify_qp()
1388 				(qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |  in bnxt_qplib_modify_qp()
1389 				(qp->en_sqd_async_notify == true ?  in bnxt_qplib_modify_qp()
1394 			req.pingpong_push_dpi = qp->ppp.dpi;  in bnxt_qplib_modify_qp()
1398 	req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;  in bnxt_qplib_modify_qp()
1401 		req.access = qp->access;  in bnxt_qplib_modify_qp()
1407 		req.qkey = cpu_to_le32(qp->qkey);  in bnxt_qplib_modify_qp()
1410 		memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));  in bnxt_qplib_modify_qp()
1417 		req.flow_label = cpu_to_le32(qp->ah.flow_label);  in bnxt_qplib_modify_qp()
1420 		req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]);  in bnxt_qplib_modify_qp()
1423 		req.hop_limit = qp->ah.hop_limit;  in bnxt_qplib_modify_qp()
1426 		req.traffic_class = qp->ah.traffic_class;  in bnxt_qplib_modify_qp()
1429 		memcpy(req.dest_mac, qp->ah.dmac, 6);  in bnxt_qplib_modify_qp()
1432 		req.path_mtu_pingpong_push_enable = qp->path_mtu;  in bnxt_qplib_modify_qp()
1435 		req.timeout = qp->timeout;  in bnxt_qplib_modify_qp()
1438 		req.retry_cnt = qp->retry_cnt;  in bnxt_qplib_modify_qp()
1441 		req.rnr_retry = qp->rnr_retry;  in bnxt_qplib_modify_qp()
1444 		req.min_rnr_timer = qp->min_rnr_timer;  in bnxt_qplib_modify_qp()
1447 		req.rq_psn = cpu_to_le32(qp->rq.psn);  in bnxt_qplib_modify_qp()
1450 		req.sq_psn = cpu_to_le32(qp->sq.psn);  in bnxt_qplib_modify_qp()
1454 			ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);  in bnxt_qplib_modify_qp()
1458 			IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);  in bnxt_qplib_modify_qp()
1460 	req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);  in bnxt_qplib_modify_qp()
1461 	req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);  in bnxt_qplib_modify_qp()
1462 	req.sq_sge = cpu_to_le16(qp->sq.max_sge);  in bnxt_qplib_modify_qp()
1463 	req.rq_sge = cpu_to_le16(qp->rq.max_sge);  in bnxt_qplib_modify_qp()
1464 	req.max_inline_data = cpu_to_le32(qp->max_inline_data);  in bnxt_qplib_modify_qp()
1466 		req.dest_qp_id = cpu_to_le32(qp->dest_qpn);  in bnxt_qplib_modify_qp()
1471 			((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) &  in bnxt_qplib_modify_qp()
1475 			((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) &  in bnxt_qplib_modify_qp()
1477 	req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);  in bnxt_qplib_modify_qp()
1480 	msg.qp_state = qp->state;  in bnxt_qplib_modify_qp()
1482 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_modify_qp()
1483 	if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) {  in bnxt_qplib_modify_qp()
1484 		qp->cur_qp_state = qp->state;  in bnxt_qplib_modify_qp()
1486 	} else if (rc) {  in bnxt_qplib_modify_qp()
1487 		return rc;  in bnxt_qplib_modify_qp()
1489 	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR)  in bnxt_qplib_modify_qp()
1490 		qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac);  in bnxt_qplib_modify_qp()
1493 		qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled;  in bnxt_qplib_modify_qp()
1495 	qp->cur_qp_state = qp->state;  in bnxt_qplib_modify_qp()
1501 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_query_qp()
1508 	int i, rc;  in bnxt_qplib_query_qp()  local
1511 	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,  in bnxt_qplib_query_qp()
1514 		return -ENOMEM;  in bnxt_qplib_query_qp()
1519 	req.qp_cid = cpu_to_le32(qp->id);  in bnxt_qplib_query_qp()
1523 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_query_qp()
1524 	if (rc)  in bnxt_qplib_query_qp()
1528 	qp->state = sb->en_sqd_async_notify_state &  in bnxt_qplib_query_qp()
1530 	qp->cur_qp_state = qp->state;  in bnxt_qplib_query_qp()
1531 	qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &  in bnxt_qplib_query_qp()
1534 	qp->access = sb->access;  in bnxt_qplib_query_qp()
1535 	qp->pkey_index = le16_to_cpu(sb->pkey);  in bnxt_qplib_query_qp()
1536 	qp->qkey = le32_to_cpu(sb->qkey);  in bnxt_qplib_query_qp()
1538 	temp32[0] = le32_to_cpu(sb->dgid[0]);  in bnxt_qplib_query_qp()
1539 	temp32[1] = le32_to_cpu(sb->dgid[1]);  in bnxt_qplib_query_qp()
1540 	temp32[2] = le32_to_cpu(sb->dgid[2]);  in bnxt_qplib_query_qp()
1541 	temp32[3] = le32_to_cpu(sb->dgid[3]);  in bnxt_qplib_query_qp()
1542 	memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));  in bnxt_qplib_query_qp()
1544 	qp->ah.flow_label = le32_to_cpu(sb->flow_label);  in bnxt_qplib_query_qp()
1546 	qp->ah.sgid_index = 0;  in bnxt_qplib_query_qp()
1547 	for (i = 0; i < res->sgid_tbl.max; i++) {  in bnxt_qplib_query_qp()
1548 		if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {  in bnxt_qplib_query_qp()
1549 			qp->ah.sgid_index = i;  in bnxt_qplib_query_qp()
1553 	if (i == res->sgid_tbl.max)  in bnxt_qplib_query_qp()
1554 		dev_dbg(&res->pdev->dev,  in bnxt_qplib_query_qp()
1555 			"QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n",  in bnxt_qplib_query_qp()
1556 			qp->id, le16_to_cpu(sb->sgid_index));  in bnxt_qplib_query_qp()
1558 	qp->ah.hop_limit = sb->hop_limit;  in bnxt_qplib_query_qp()
1559 	qp->ah.traffic_class = sb->traffic_class;  in bnxt_qplib_query_qp()
1560 	memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN);  in bnxt_qplib_query_qp()
1561 	qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) &  in bnxt_qplib_query_qp()
1564 	qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) &  in bnxt_qplib_query_qp()
1566 	qp->timeout = sb->timeout;  in bnxt_qplib_query_qp()
1567 	qp->retry_cnt = sb->retry_cnt;  in bnxt_qplib_query_qp()
1568 	qp->rnr_retry = sb->rnr_retry;  in bnxt_qplib_query_qp()
1569 	qp->min_rnr_timer = sb->min_rnr_timer;  in bnxt_qplib_query_qp()
1570 	qp->rq.psn = le32_to_cpu(sb->rq_psn);  in bnxt_qplib_query_qp()
1571 	qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);  in bnxt_qplib_query_qp()
1572 	qp->sq.psn = le32_to_cpu(sb->sq_psn);  in bnxt_qplib_query_qp()
1573 	qp->max_dest_rd_atomic =  in bnxt_qplib_query_qp()
1574 			IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);  in bnxt_qplib_query_qp()
1575 	qp->sq.max_wqe = qp->sq.hwq.max_elements;  in bnxt_qplib_query_qp()
1576 	qp->rq.max_wqe = qp->rq.hwq.max_elements;  in bnxt_qplib_query_qp()
1577 	qp->sq.max_sge = le16_to_cpu(sb->sq_sge);  in bnxt_qplib_query_qp()
1578 	qp->rq.max_sge = le16_to_cpu(sb->rq_sge);  in bnxt_qplib_query_qp()
1579 	qp->max_inline_data = le32_to_cpu(sb->max_inline_data);  in bnxt_qplib_query_qp()
1580 	qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);  in bnxt_qplib_query_qp()
1581 	memcpy(qp->smac, sb->src_mac, ETH_ALEN);  in bnxt_qplib_query_qp()
1582 	qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);  in bnxt_qplib_query_qp()
1583 	qp->port_id = le16_to_cpu(sb->port_id);  in bnxt_qplib_query_qp()
1585 	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,  in bnxt_qplib_query_qp()
1587 	return rc;  in bnxt_qplib_query_qp()
1593 	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;  in __clean_cq()
1598 	peek_flags = cq->dbinfo.flags;  in __clean_cq()
1599 	peek_cons = cq_hwq->cons;  in __clean_cq()
1600 	for (i = 0; i < cq_hwq->depth; i++) {  in __clean_cq()
1604 			switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {  in __clean_cq()
1610 				if (qp == le64_to_cpu(cqe->qp_handle))  in __clean_cq()
1611 					cqe->qp_handle = 0;  in __clean_cq()
1620 				if (qp == le64_to_cpu(cqe->qp_handle))  in __clean_cq()
1621 					cqe->qp_handle = 0;  in __clean_cq()
1628 		bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons,  in __clean_cq()
1636 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_destroy_qp()
1640 	int rc;  in bnxt_qplib_destroy_qp()  local
1642 	tbl = &res->reftbl.qpref;  in bnxt_qplib_destroy_qp()
1643 	qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);  in bnxt_qplib_destroy_qp()
1644 	spin_lock_irqsave(&tbl->lock, flags);  in bnxt_qplib_destroy_qp()
1645 	tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID;  in bnxt_qplib_destroy_qp()
1646 	tbl->rec[qp_idx].handle = NULL;  in bnxt_qplib_destroy_qp()
1647 	spin_unlock_irqrestore(&tbl->lock, flags);  in bnxt_qplib_destroy_qp()
1649 	rc = __qplib_destroy_qp(rcfw, qp);  in bnxt_qplib_destroy_qp()
1650 	if (rc) {  in bnxt_qplib_destroy_qp()
1651 		spin_lock_irqsave(&tbl->lock, flags);  in bnxt_qplib_destroy_qp()
1652 		tbl->rec[qp_idx].xid = qp->id;  in bnxt_qplib_destroy_qp()
1653 		tbl->rec[qp_idx].handle = qp;  in bnxt_qplib_destroy_qp()
1654 		spin_unlock_irqrestore(&tbl->lock, flags);  in bnxt_qplib_destroy_qp()
1655 		return rc;  in bnxt_qplib_destroy_qp()
1664 	if (qp->irrq.max_elements)  in bnxt_qplib_free_qp_res()
1665 		bnxt_qplib_free_hwq(res, &qp->irrq);  in bnxt_qplib_free_qp_res()
1666 	if (qp->orrq.max_elements)  in bnxt_qplib_free_qp_res()
1667 		bnxt_qplib_free_hwq(res, &qp->orrq);  in bnxt_qplib_free_qp_res()
1669 	if (!qp->is_user)  in bnxt_qplib_free_qp_res()
1670 		kfree(qp->rq.swq);  in bnxt_qplib_free_qp_res()
1671 	bnxt_qplib_free_hwq(res, &qp->rq.hwq);  in bnxt_qplib_free_qp_res()
1673 	if (!qp->is_user)  in bnxt_qplib_free_qp_res()
1674 		kfree(qp->sq.swq);  in bnxt_qplib_free_qp_res()
1675 	bnxt_qplib_free_hwq(res, &qp->sq.hwq);  in bnxt_qplib_free_qp_res()
1681 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_get_qp1_sq_buf()
1687 	buf = qp->sq_hdr_buf;  in bnxt_qplib_get_qp1_sq_buf()
1689 		sw_prod = sq->swq_start;  in bnxt_qplib_get_qp1_sq_buf()
1690 		sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);  in bnxt_qplib_get_qp1_sq_buf()
1691 		sge->lkey = 0xFFFFFFFF;  in bnxt_qplib_get_qp1_sq_buf()
1692 		sge->size = buf->step;  in bnxt_qplib_get_qp1_sq_buf()
1693 		return buf->va + sw_prod * sge->size;  in bnxt_qplib_get_qp1_sq_buf()
1700 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_get_rq_prod_index()
1702 	return rq->swq_start;  in bnxt_qplib_get_rq_prod_index()
1708 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_get_qp1_rq_buf()
1714 	buf = qp->rq_hdr_buf;  in bnxt_qplib_get_qp1_rq_buf()
1716 		sw_prod = rq->swq_start;  in bnxt_qplib_get_qp1_rq_buf()
1717 		sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);  in bnxt_qplib_get_qp1_rq_buf()
1718 		sge->lkey = 0xFFFFFFFF;  in bnxt_qplib_get_qp1_rq_buf()
1719 		sge->size = buf->step;  in bnxt_qplib_get_qp1_rq_buf()
1720 		return buf->va + sw_prod * sge->size;  in bnxt_qplib_get_qp1_rq_buf()
1734 	msns = (struct sq_msn_search *)swq->psn_search;  in bnxt_qplib_fill_msn_search()
1735 	msns->start_idx_next_psn_start_psn = 0;  in bnxt_qplib_fill_msn_search()
1737 	start_psn = swq->start_psn;  in bnxt_qplib_fill_msn_search()
1738 	next_psn = swq->next_psn;  in bnxt_qplib_fill_msn_search()
1739 	start_idx = swq->slot_idx;  in bnxt_qplib_fill_msn_search()
1740 	msns->start_idx_next_psn_start_psn |=  in bnxt_qplib_fill_msn_search()
1743 		 qp->msn,  in bnxt_qplib_fill_msn_search()
1745 		 cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)),  in bnxt_qplib_fill_msn_search()
1747 		 cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)),  in bnxt_qplib_fill_msn_search()
1749 		 cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn)));  in bnxt_qplib_fill_msn_search()
1750 	qp->msn++;  in bnxt_qplib_fill_msn_search()
1751 	qp->msn %= qp->msn_tbl_sz;  in bnxt_qplib_fill_msn_search()
1763 	if (!swq->psn_search)  in bnxt_qplib_fill_psn_search()
1767 	if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {  in bnxt_qplib_fill_psn_search()
1771 	psns = (struct sq_psn_search *)swq->psn_search;  in bnxt_qplib_fill_psn_search()
1772 	psns_ext = (struct sq_psn_search_ext *)swq->psn_search;  in bnxt_qplib_fill_psn_search()
1774 	op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &  in bnxt_qplib_fill_psn_search()
1776 	op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &  in bnxt_qplib_fill_psn_search()
1778 	flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &  in bnxt_qplib_fill_psn_search()
1781 	if (_is_chip_gen_p5_p7(qp->cctx)) {  in bnxt_qplib_fill_psn_search()
1782 		psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);  in bnxt_qplib_fill_psn_search()
1783 		psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);  in bnxt_qplib_fill_psn_search()
1784 		psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);  in bnxt_qplib_fill_psn_search()
1786 		psns->opcode_start_psn = cpu_to_le32(op_spsn);  in bnxt_qplib_fill_psn_search()
1787 		psns->flags_next_psn = cpu_to_le32(flg_npsn);  in bnxt_qplib_fill_psn_search()
1796 	for (indx = 0; indx < wqe->num_sge; indx++)  in _calc_ilsize()
1797 		size += wqe->sg_list[indx].size;  in _calc_ilsize()
1813 	sq_hwq = &qp->sq.hwq;  in bnxt_qplib_put_inline()
1815 	for (indx = 0; indx < wqe->num_sge; indx++) {  in bnxt_qplib_put_inline()
1816 		len = wqe->sg_list[indx].size;  in bnxt_qplib_put_inline()
1817 		il_src = (void *)wqe->sg_list[indx].addr;  in bnxt_qplib_put_inline()
1819 		if (t_len > qp->max_inline_data)  in bnxt_qplib_put_inline()
1825 							   sq_hwq->depth), NULL);  in bnxt_qplib_put_inline()
1832 				      (sizeof(struct sq_sge) - offt));  in bnxt_qplib_put_inline()
1838 			len -= cplen;  in bnxt_qplib_put_inline()
1846 	return -ENOMEM;  in bnxt_qplib_put_inline()
1857 		dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL);  in bnxt_qplib_put_sges()
1858 		dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);  in bnxt_qplib_put_sges()
1859 		dsge->l_key = cpu_to_le32(ssge[indx].lkey);  in bnxt_qplib_put_sges()
1860 		dsge->size = cpu_to_le32(ssge[indx].size);  in bnxt_qplib_put_sges()
1873 	nsge = wqe->num_sge;  in _calculate_wqe_byte()
1874 	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {  in _calculate_wqe_byte()
1876 		wqe_size = (ilsize > qp->max_inline_data) ?  in _calculate_wqe_byte()
1877 			    qp->max_inline_data : ilsize;  in _calculate_wqe_byte()
1892 	 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.  in _translate_q_full_delta()
1893 	 * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.  in _translate_q_full_delta()
1896 	return (que->q_full_delta * wqe_bytes) / que->hwq.element_size;  in _translate_q_full_delta()
1907 	sq_hwq = &sq->hwq;  in bnxt_qplib_pull_psn_buff()
1908 	if (!sq_hwq->pad_pg)  in bnxt_qplib_pull_psn_buff()
1911 	tail = swq->slot_idx / sq->dbinfo.max_slot;  in bnxt_qplib_pull_psn_buff()
1913 		tail %= qp->msn_tbl_sz;  in bnxt_qplib_pull_psn_buff()
1914 	pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride);  in bnxt_qplib_pull_psn_buff()
1915 	pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride);  in bnxt_qplib_pull_psn_buff()
1916 	buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride);  in bnxt_qplib_pull_psn_buff()
1918 	swq->psn_search = buff;  in bnxt_qplib_pull_psn_buff()
1923 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_post_send_db()
1925 	bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);  in bnxt_qplib_post_send_db()
1932 	int i, rc = 0, data_len = 0, pkt_num = 0;  in bnxt_qplib_post_send()  local
1933 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_post_send()
1947 	sq_hwq = &sq->hwq;  in bnxt_qplib_post_send()
1948 	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&  in bnxt_qplib_post_send()
1949 	    qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {  in bnxt_qplib_post_send()
1950 		dev_err(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
1951 			"QPLIB: FP: QP (0x%x) is in the 0x%x state\n",  in bnxt_qplib_post_send()
1952 			qp->id, qp->state);  in bnxt_qplib_post_send()
1953 		rc = -EINVAL;  in bnxt_qplib_post_send()
1958 	slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?  in bnxt_qplib_post_send()
1959 			sq->dbinfo.max_slot : wqe_slots;  in bnxt_qplib_post_send()
1962 		dev_err(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
1963 			"QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id);  in bnxt_qplib_post_send()
1964 		dev_err(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
1965 			"QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n",  in bnxt_qplib_post_send()
1966 			HWQ_CMP(sq_hwq->prod, sq_hwq),  in bnxt_qplib_post_send()
1967 			HWQ_CMP(sq_hwq->cons, sq_hwq),  in bnxt_qplib_post_send()
1968 			sq_hwq->max_elements, qfd_slots, slots_needed);  in bnxt_qplib_post_send()
1969 		dev_err(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
1971 			sq->phantom_wqe_cnt, sq->phantom_cqe_cnt);  in bnxt_qplib_post_send()
1972 		rc = -ENOMEM;  in bnxt_qplib_post_send()
1976 	sw_prod = sq_hwq->prod;  in bnxt_qplib_post_send()
1978 	swq->slot_idx = sw_prod;  in bnxt_qplib_post_send()
1979 	bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags));  in bnxt_qplib_post_send()
1981 	swq->wr_id = wqe->wr_id;  in bnxt_qplib_post_send()
1982 	swq->type = wqe->type;  in bnxt_qplib_post_send()
1983 	swq->flags = wqe->flags;  in bnxt_qplib_post_send()
1984 	swq->slots = slots_needed;  in bnxt_qplib_post_send()
1985 	swq->start_psn = sq->psn & BTH_PSN_MASK;  in bnxt_qplib_post_send()
1986 	if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP)  in bnxt_qplib_post_send()
1987 		swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;  in bnxt_qplib_post_send()
1989 	dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
1990 		"QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_post_send()
1991 		qp->id, wqe_idx, swq->wr_id);  in bnxt_qplib_post_send()
1992 	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {  in bnxt_qplib_post_send()
1994 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2001 	ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL);  in bnxt_qplib_post_send()
2006 	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)  in bnxt_qplib_post_send()
2009 		data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list,  in bnxt_qplib_post_send()
2010 					       wqe->num_sge, &sw_prod);  in bnxt_qplib_post_send()
2014 	switch (wqe->type) {  in bnxt_qplib_post_send()
2016 		if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE ||  in bnxt_qplib_post_send()
2017 		    qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {  in bnxt_qplib_post_send()
2022 			sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2023 			sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2024 			sqe->wqe_size = wqe_slots;  in bnxt_qplib_post_send()
2025 			sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);  in bnxt_qplib_post_send()
2026 			sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);  in bnxt_qplib_post_send()
2027 			sqe->length = cpu_to_le32(data_len);  in bnxt_qplib_post_send()
2028 			ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &  in bnxt_qplib_post_send()
2032 			dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2034 				"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2035 				"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2036 				"\twqe_size = 0x%x\n"  in bnxt_qplib_post_send()
2037 				"\tlflags = 0x%x\n"  in bnxt_qplib_post_send()
2038 				"\tcfa_action = 0x%x\n"  in bnxt_qplib_post_send()
2039 				"\tlength = 0x%x\n"  in bnxt_qplib_post_send()
2040 				"\tcfa_meta = 0x%x\n",  in bnxt_qplib_post_send()
2041 				sqe->wqe_type, sqe->flags, sqe->wqe_size,  in bnxt_qplib_post_send()
2042 				sqe->lflags, sqe->cfa_action,  in bnxt_qplib_post_send()
2043 				sqe->length, ext_sqe->cfa_meta);  in bnxt_qplib_post_send()
2054 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2055 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2056 		sqe->wqe_size = wqe_slots;  in bnxt_qplib_post_send()
2057 		sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);  in bnxt_qplib_post_send()
2058 		if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||  in bnxt_qplib_post_send()
2059 		    qp->type == CMDQ_CREATE_QP_TYPE_GSI) {  in bnxt_qplib_post_send()
2060 			sqe->q_key = cpu_to_le32(wqe->send.q_key);  in bnxt_qplib_post_send()
2061 			sqe->length = cpu_to_le32(data_len);  in bnxt_qplib_post_send()
2062 			ext_sqe->dst_qp = cpu_to_le32(  in bnxt_qplib_post_send()
2063 					wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);  in bnxt_qplib_post_send()
2064 			ext_sqe->avid = cpu_to_le32(wqe->send.avid &  in bnxt_qplib_post_send()
2066 			sq->psn = (sq->psn + 1) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2068 			sqe->length = cpu_to_le32(data_len);  in bnxt_qplib_post_send()
2069 			if (qp->mtu)  in bnxt_qplib_post_send()
2070 				pkt_num = (data_len + qp->mtu - 1) / qp->mtu;  in bnxt_qplib_post_send()
2073 			sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2075 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2077 			"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2078 			"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2079 			"\twqe_size = 0x%x\n"  in bnxt_qplib_post_send()
2080 			"\tinv_key/immdata = 0x%x\n"  in bnxt_qplib_post_send()
2081 			"\tq_key = 0x%x\n"  in bnxt_qplib_post_send()
2082 			"\tdst_qp = 0x%x\n"  in bnxt_qplib_post_send()
2083 			"\tlength = 0x%x\n"  in bnxt_qplib_post_send()
2084 			"\tavid = 0x%x\n",  in bnxt_qplib_post_send()
2085 			sqe->wqe_type, sqe->flags, sqe->wqe_size,  in bnxt_qplib_post_send()
2086 			sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp,  in bnxt_qplib_post_send()
2087 			sqe->length, ext_sqe->avid);  in bnxt_qplib_post_send()
2091 		/* fall-thru */  in bnxt_qplib_post_send()
2093 		/* fall-thru */  in bnxt_qplib_post_send()
2099 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2100 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2101 		sqe->wqe_size = wqe_slots;  in bnxt_qplib_post_send()
2102 		sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);  in bnxt_qplib_post_send()
2103 		sqe->length = cpu_to_le32((u32)data_len);  in bnxt_qplib_post_send()
2104 		ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);  in bnxt_qplib_post_send()
2105 		ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);  in bnxt_qplib_post_send()
2106 		if (qp->mtu)  in bnxt_qplib_post_send()
2107 			pkt_num = (data_len + qp->mtu - 1) / qp->mtu;  in bnxt_qplib_post_send()
2110 		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2112 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2114 			"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2115 			"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2116 			"\twqe_size = 0x%x\n"  in bnxt_qplib_post_send()
2117 			"\timmdata = 0x%x\n"  in bnxt_qplib_post_send()
2118 			"\tlength = 0x%x\n"  in bnxt_qplib_post_send()
2119 			"\tremote_va = 0x%llx\n"  in bnxt_qplib_post_send()
2120 			"\tremote_key = 0x%x\n",  in bnxt_qplib_post_send()
2121 			sqe->wqe_type, sqe->flags, sqe->wqe_size,  in bnxt_qplib_post_send()
2122 			sqe->imm_data, sqe->length, ext_sqe->remote_va,  in bnxt_qplib_post_send()
2123 			ext_sqe->remote_key);  in bnxt_qplib_post_send()
2127 		/* fall-thru */  in bnxt_qplib_post_send()
2133 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2134 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2135 		sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);  in bnxt_qplib_post_send()
2136 		sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);  in bnxt_qplib_post_send()
2137 		ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);  in bnxt_qplib_post_send()
2138 		ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);  in bnxt_qplib_post_send()
2139 		if (qp->mtu)  in bnxt_qplib_post_send()
2140 			pkt_num = (data_len + qp->mtu - 1) / qp->mtu;  in bnxt_qplib_post_send()
2143 		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2150 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2151 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2152 		sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);  in bnxt_qplib_post_send()
2154 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2156 			"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2157 			"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2158 			"\tinv_l_key = 0x%x\n",  in bnxt_qplib_post_send()
2159 			sqe->wqe_type, sqe->flags, sqe->inv_l_key);  in bnxt_qplib_post_send()
2167 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2168 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2169 		sqe->access_cntl = wqe->frmr.access_cntl |  in bnxt_qplib_post_send()
2171 		sqe->zero_based_page_size_log =  in bnxt_qplib_post_send()
2172 			(wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<  in bnxt_qplib_post_send()
2174 			(wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0);  in bnxt_qplib_post_send()
2175 		sqe->l_key = cpu_to_le32(wqe->frmr.l_key);  in bnxt_qplib_post_send()
2176 		/* TODO: OFED only provides length of MR up to 32-bits for FRMR */  in bnxt_qplib_post_send()
2177 		temp32 = cpu_to_le32(wqe->frmr.length);  in bnxt_qplib_post_send()
2178 		memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));  in bnxt_qplib_post_send()
2179 		sqe->numlevels_pbl_page_size_log =  in bnxt_qplib_post_send()
2180 			((wqe->frmr.pbl_pg_sz_log <<  in bnxt_qplib_post_send()
2183 			((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &  in bnxt_qplib_post_send()
2185 		if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) {  in bnxt_qplib_post_send()
2186 			ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]);  in bnxt_qplib_post_send()
2188 			for (i = 0; i < wqe->frmr.page_list_len; i++)  in bnxt_qplib_post_send()
2189 				wqe->frmr.pbl_ptr[i] = cpu_to_le64(  in bnxt_qplib_post_send()
2190 						wqe->frmr.page_list[i] |  in bnxt_qplib_post_send()
2192 			ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);  in bnxt_qplib_post_send()
2194 		ext_sqe->va = cpu_to_le64(wqe->frmr.va);  in bnxt_qplib_post_send()
2195 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2197 			"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2198 			"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2199 			"\taccess_cntl = 0x%x\n"  in bnxt_qplib_post_send()
2200 			"\tzero_based_page_size_log = 0x%x\n"  in bnxt_qplib_post_send()
2201 			"\tl_key = 0x%x\n"  in bnxt_qplib_post_send()
2202 			"\tlength = 0x%x\n"  in bnxt_qplib_post_send()
2203 			"\tnumlevels_pbl_page_size_log = 0x%x\n"  in bnxt_qplib_post_send()
2204 			"\tpblptr = 0x%llx\n"  in bnxt_qplib_post_send()
2205 			"\tva = 0x%llx\n",  in bnxt_qplib_post_send()
2206 			sqe->wqe_type, sqe->flags, sqe->access_cntl,  in bnxt_qplib_post_send()
2207 			sqe->zero_based_page_size_log, sqe->l_key,  in bnxt_qplib_post_send()
2208 			*(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log,  in bnxt_qplib_post_send()
2209 			ext_sqe->pblptr, ext_sqe->va);  in bnxt_qplib_post_send()
2217 		sqe->wqe_type = wqe->type;  in bnxt_qplib_post_send()
2218 		sqe->flags = wqe->flags;  in bnxt_qplib_post_send()
2219 		sqe->access_cntl = wqe->bind.access_cntl;  in bnxt_qplib_post_send()
2220 		sqe->mw_type_zero_based = wqe->bind.mw_type |  in bnxt_qplib_post_send()
2221 			(wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0);  in bnxt_qplib_post_send()
2222 		sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);  in bnxt_qplib_post_send()
2223 		sqe->l_key = cpu_to_le32(wqe->bind.r_key);  in bnxt_qplib_post_send()
2224 		ext_sqe->va = cpu_to_le64(wqe->bind.va);  in bnxt_qplib_post_send()
2225 		ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);  in bnxt_qplib_post_send()
2226 		dev_dbg(&sq_hwq->pdev->dev,  in bnxt_qplib_post_send()
2228 			"\twqe_type = 0x%x\n"  in bnxt_qplib_post_send()
2229 			"\tflags = 0x%x\n"  in bnxt_qplib_post_send()
2230 			"\taccess_cntl = 0x%x\n"  in bnxt_qplib_post_send()
2231 			"\tmw_type_zero_based = 0x%x\n"  in bnxt_qplib_post_send()
2232 			"\tparent_l_key = 0x%x\n"  in bnxt_qplib_post_send()
2233 			"\tl_key = 0x%x\n"  in bnxt_qplib_post_send()
2234 			"\tva = 0x%llx\n"  in bnxt_qplib_post_send()
2235 			"\tlength = 0x%x\n",  in bnxt_qplib_post_send()
2236 			sqe->wqe_type, sqe->flags, sqe->access_cntl,  in bnxt_qplib_post_send()
2237 			sqe->mw_type_zero_based, sqe->parent_l_key,  in bnxt_qplib_post_send()
2238 			sqe->l_key, sqe->va, ext_sqe->length_lo);  in bnxt_qplib_post_send()
2243 		rc = -EINVAL;  in bnxt_qplib_post_send()
2246 	swq->next_psn = sq->psn & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2251 	bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots);  in bnxt_qplib_post_send()
2252 	qp->wqe_cnt++;  in bnxt_qplib_post_send()
2257 			nq_work->cq = qp->scq;  in bnxt_qplib_post_send()
2258 			nq_work->nq = qp->scq->nq;  in bnxt_qplib_post_send()
2259 			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);  in bnxt_qplib_post_send()
2260 			queue_work(qp->scq->nq->cqn_wq, &nq_work->work);  in bnxt_qplib_post_send()
2262 			dev_err(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
2264 			rc = -ENOMEM;  in bnxt_qplib_post_send()
2267 	return rc;  in bnxt_qplib_post_send()
2272 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_post_recv_db()
2274 	bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);  in bnxt_qplib_post_recv_db()
2281 	if (!(cq && cq->nq))  in bnxt_re_handle_cqn()
2284 	nq = cq->nq;  in bnxt_re_handle_cqn()
2285 	spin_lock_bh(&cq->compl_lock);  in bnxt_re_handle_cqn()
2286 	if (nq->cqn_handler) {  in bnxt_re_handle_cqn()
2287 		dev_dbg(&nq->res->pdev->dev,  in bnxt_re_handle_cqn()
2290 		nq->cqn_handler(nq, cq);  in bnxt_re_handle_cqn()
2292 	spin_unlock_bh(&cq->compl_lock);  in bnxt_re_handle_cqn()
2299 	struct bnxt_qplib_q *rq = &qp->rq;  in bnxt_qplib_post_recv()
2309 	int rc = 0;  in bnxt_qplib_post_recv()  local
2311 	rq_hwq = &rq->hwq;  in bnxt_qplib_post_recv()
2312 	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {  in bnxt_qplib_post_recv()
2313 		dev_err(&rq_hwq->pdev->dev,  in bnxt_qplib_post_recv()
2314 			"QPLIB: FP: QP (0x%x) is in the 0x%x state\n",  in bnxt_qplib_post_recv()
2315 			qp->id, qp->state);  in bnxt_qplib_post_recv()
2316 		rc = -EINVAL;  in bnxt_qplib_post_recv()
2321 	if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) {  in bnxt_qplib_post_recv()
2322 		dev_err(&rq_hwq->pdev->dev,  in bnxt_qplib_post_recv()
2323 			"QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id);  in bnxt_qplib_post_recv()
2324 		rc = -EINVAL;  in bnxt_qplib_post_recv()
2329 	swq->wr_id = wqe->wr_id;  in bnxt_qplib_post_recv()
2330 	swq->slots = rq->dbinfo.max_slot;  in bnxt_qplib_post_recv()
2331 	dev_dbg(&rq_hwq->pdev->dev,  in bnxt_qplib_post_recv()
2332 		"QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_post_recv()
2333 		wqe_idx, swq->wr_id);  in bnxt_qplib_post_recv()
2334 	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {  in bnxt_qplib_post_recv()
2336 		dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n",  in bnxt_qplib_post_recv()
2341 	sw_prod = rq_hwq->prod;  in bnxt_qplib_post_recv()
2344 	ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);  in bnxt_qplib_post_recv()
2349 	if (!wqe->num_sge) {  in bnxt_qplib_post_recv()
2350 		dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);  in bnxt_qplib_post_recv()
2351 		dsge->size = 0;  in bnxt_qplib_post_recv()
2354 		bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod);  in bnxt_qplib_post_recv()
2356 	base_hdr->wqe_type = wqe->type;  in bnxt_qplib_post_recv()
2357 	base_hdr->flags = wqe->flags;  in bnxt_qplib_post_recv()
2358 	base_hdr->wqe_size = wqe_slots;  in bnxt_qplib_post_recv()
2359 	base_hdr->wr_id |= cpu_to_le32(wqe_idx);  in bnxt_qplib_post_recv()
2362 	bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots);  in bnxt_qplib_post_recv()
2367 			nq_work->cq = qp->rcq;  in bnxt_qplib_post_recv()
2368 			nq_work->nq = qp->rcq->nq;  in bnxt_qplib_post_recv()
2369 			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);  in bnxt_qplib_post_recv()
2370 			queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);  in bnxt_qplib_post_recv()
2372 			dev_err(&rq->hwq.pdev->dev,  in bnxt_qplib_post_recv()
2374 			rc = -ENOMEM;  in bnxt_qplib_post_recv()
2377 	return rc;  in bnxt_qplib_post_recv()
2384 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_create_cq()
2391 	int rc;  in bnxt_qplib_create_cq()  local
2394 	hwq_attr.depth = cq->max_wqe;  in bnxt_qplib_create_cq()
2397 	hwq_attr.sginfo = &cq->sginfo;  in bnxt_qplib_create_cq()
2398 	rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);  in bnxt_qplib_create_cq()
2399 	if (rc)  in bnxt_qplib_create_cq()
2405 	if (!cq->dpi) {  in bnxt_qplib_create_cq()
2406 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_create_cq()
2408 		return -EINVAL;  in bnxt_qplib_create_cq()
2410 	req.dpi = cpu_to_le32(cq->dpi->dpi);  in bnxt_qplib_create_cq()
2411 	req.cq_handle = cpu_to_le64(cq->cq_handle);  in bnxt_qplib_create_cq()
2413 	req.cq_size = cpu_to_le32(cq->max_wqe);  in bnxt_qplib_create_cq()
2414 	req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq));  in bnxt_qplib_create_cq()
2415 	pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT;  in bnxt_qplib_create_cq()
2416 	pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<  in bnxt_qplib_create_cq()
2421 			(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<  in bnxt_qplib_create_cq()
2425 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_create_cq()
2426 	if (rc)  in bnxt_qplib_create_cq()
2428 	cq->id = le32_to_cpu(resp.xid);  in bnxt_qplib_create_cq()
2429 	cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;  in bnxt_qplib_create_cq()
2430 	init_waitqueue_head(&cq->waitq);  in bnxt_qplib_create_cq()
2431 	INIT_LIST_HEAD(&cq->sqf_head);  in bnxt_qplib_create_cq()
2432 	INIT_LIST_HEAD(&cq->rqf_head);  in bnxt_qplib_create_cq()
2433 	spin_lock_init(&cq->flush_lock);  in bnxt_qplib_create_cq()
2434 	spin_lock_init(&cq->compl_lock);  in bnxt_qplib_create_cq()
2437 	cq->cctx = res->cctx;  in bnxt_qplib_create_cq()
2438 	cq->dbinfo.hwq = &cq->hwq;  in bnxt_qplib_create_cq()
2439 	cq->dbinfo.xid = cq->id;  in bnxt_qplib_create_cq()
2440 	cq->dbinfo.db = cq->dpi->dbr;  in bnxt_qplib_create_cq()
2441 	cq->dbinfo.priv_db = res->dpi_tbl.priv_db;  in bnxt_qplib_create_cq()
2442 	cq->dbinfo.flags = 0;  in bnxt_qplib_create_cq()
2443 	cq->dbinfo.toggle = 0;  in bnxt_qplib_create_cq()
2444 	cq->dbinfo.res = res;  in bnxt_qplib_create_cq()
2445 	cq->dbinfo.seed = cq->id;  in bnxt_qplib_create_cq()
2446 	spin_lock_init(&cq->dbinfo.lock);  in bnxt_qplib_create_cq()
2447 	cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_cq()
2448 	cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_cq()
2450 	tbl = &res->reftbl.cqref;  in bnxt_qplib_create_cq()
2451 	spin_lock_irqsave(&tbl->lock, flag);  in bnxt_qplib_create_cq()
2452 	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id;  in bnxt_qplib_create_cq()
2453 	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq;  in bnxt_qplib_create_cq()
2454 	spin_unlock_irqrestore(&tbl->lock, flag);  in bnxt_qplib_create_cq()
2456 	bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);  in bnxt_qplib_create_cq()
2460 	bnxt_qplib_free_hwq(res, &cq->hwq);  in bnxt_qplib_create_cq()
2462 	return rc;  in bnxt_qplib_create_cq()
2474 	bnxt_qplib_free_hwq(res, &cq->hwq);  in bnxt_qplib_resize_cq_complete()
2475 	memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));  in bnxt_qplib_resize_cq_complete()
2477 	cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);  in bnxt_qplib_resize_cq_complete()
2480 	if (!cq->resize_hwq.is_user)  in bnxt_qplib_resize_cq_complete()
2481 		bnxt_qplib_cq_coffack_db(&cq->dbinfo);  in bnxt_qplib_resize_cq_complete()
2488 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_resize_cq()
2494 	u16 count = -1;  in bnxt_qplib_resize_cq()
2495 	int rc;  in bnxt_qplib_resize_cq()  local
2500 	hwq_attr.sginfo = &cq->sginfo;  in bnxt_qplib_resize_cq()
2505 	rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);  in bnxt_qplib_resize_cq()
2506 	if (rc)  in bnxt_qplib_resize_cq()
2507 		return rc;  in bnxt_qplib_resize_cq()
2509 	dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__,  in bnxt_qplib_resize_cq()
2510 		cq->resize_hwq.level);  in bnxt_qplib_resize_cq()
2511 	req.cq_cid = cpu_to_le32(cq->id);  in bnxt_qplib_resize_cq()
2512 	pbl = &cq->resize_hwq.pbl[PBL_LVL_0];  in bnxt_qplib_resize_cq()
2513 	pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K :  in bnxt_qplib_resize_cq()
2514 		pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K :  in bnxt_qplib_resize_cq()
2515 		pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K :  in bnxt_qplib_resize_cq()
2516 		pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M :  in bnxt_qplib_resize_cq()
2517 		pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M :  in bnxt_qplib_resize_cq()
2518 		pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G :  in bnxt_qplib_resize_cq()
2520 	lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &  in bnxt_qplib_resize_cq()
2525 	req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);  in bnxt_qplib_resize_cq()
2527 	if (!cq->resize_hwq.is_user)  in bnxt_qplib_resize_cq()
2528 		set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);  in bnxt_qplib_resize_cq()
2532 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_resize_cq()
2533 	if (rc)  in bnxt_qplib_resize_cq()
2536 	if (!cq->resize_hwq.is_user) {  in bnxt_qplib_resize_cq()
2539 		if (wait_event_interruptible_timeout(cq->waitq,  in bnxt_qplib_resize_cq()
2540 		    !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags),  in bnxt_qplib_resize_cq()
2542 		    -ERESTARTSYS && count--)  in bnxt_qplib_resize_cq()
2545 		if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) {  in bnxt_qplib_resize_cq()
2546 			dev_err(&rcfw->pdev->dev,  in bnxt_qplib_resize_cq()
2548 			rc = -ETIMEDOUT;  in bnxt_qplib_resize_cq()
2557 	if (!cq->resize_hwq.is_user) {  in bnxt_qplib_resize_cq()
2558 		bnxt_qplib_free_hwq(res, &cq->resize_hwq);  in bnxt_qplib_resize_cq()
2559 		clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);  in bnxt_qplib_resize_cq()
2561 	return rc;  in bnxt_qplib_resize_cq()
2566 	bnxt_qplib_free_hwq(res, &cq->hwq);  in bnxt_qplib_free_cq()
2571 	struct bnxt_qplib_nq *nq = cq->nq;  in bnxt_qplib_sync_cq()
2573 	flush_workqueue(cq->nq->cqn_wq);  in bnxt_qplib_sync_cq()
2574 	mutex_lock(&nq->lock);  in bnxt_qplib_sync_cq()
2575 	if (nq->requested)  in bnxt_qplib_sync_cq()
2576 		synchronize_irq(nq->msix_vec);  in bnxt_qplib_sync_cq()
2577 	mutex_unlock(&nq->lock);  in bnxt_qplib_sync_cq()
2582 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_destroy_cq()
2589 	int rc;  in bnxt_qplib_destroy_cq()  local
2591 	tbl = &res->reftbl.cqref;  in bnxt_qplib_destroy_cq()
2592 	spin_lock_irqsave(&tbl->lock, flag);  in bnxt_qplib_destroy_cq()
2593 	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL;  in bnxt_qplib_destroy_cq()
2594 	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0;  in bnxt_qplib_destroy_cq()
2595 	spin_unlock_irqrestore(&tbl->lock, flag);  in bnxt_qplib_destroy_cq()
2600 	req.cq_cid = cpu_to_le32(cq->id);  in bnxt_qplib_destroy_cq()
2603 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_destroy_cq()
2604 	if (rc)  in bnxt_qplib_destroy_cq()
2605 		return rc;  in bnxt_qplib_destroy_cq()
2609 		dev_dbg(&rcfw->pdev->dev,  in bnxt_qplib_destroy_cq()
2610 			"%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n",  in bnxt_qplib_destroy_cq()
2611 			__func__, cq->id, cq, total_cnq_events);  in bnxt_qplib_destroy_cq()
2614 	bnxt_qplib_free_hwq(res, &cq->hwq);  in bnxt_qplib_destroy_cq()
2623 	int rc = 0;  in __flush_sq()  local
2626 	start = sq->swq_start;  in __flush_sq()
2629 		last = sq->swq_last;  in __flush_sq()
2634 		if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {  in __flush_sq()
2640 		cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;  in __flush_sq()
2641 		cqe->opcode = CQ_BASE_CQE_TYPE_REQ;  in __flush_sq()
2642 		cqe->qp_handle = (u64)qp;  in __flush_sq()
2643 		cqe->wr_id = sq->swq[last].wr_id;  in __flush_sq()
2644 		cqe->src_qp = qp->id;  in __flush_sq()
2645 		cqe->type = sq->swq[last].type;  in __flush_sq()
2646 		dev_dbg(&sq->hwq.pdev->dev,  in __flush_sq()
2648 		dev_dbg(&sq->hwq.pdev->dev,  in __flush_sq()
2649 			"QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",  in __flush_sq()
2650 			last, cqe->wr_id, cqe->status);  in __flush_sq()
2652 		(*budget)--;  in __flush_sq()
2654 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth,  in __flush_sq()
2655 					 &sq->hwq.cons,  in __flush_sq()
2656 					 sq->swq[last].slots,  in __flush_sq()
2657 					 &sq->dbinfo.flags);  in __flush_sq()
2658 		sq->swq_last = sq->swq[last].next_idx;  in __flush_sq()
2661 	if (!*budget && sq->swq_last != start)  in __flush_sq()
2663 		rc = -EAGAIN;  in __flush_sq()
2664 	dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc);  in __flush_sq()
2666 	return rc;  in __flush_sq()
2675 	int rc = 0;  in __flush_rq()  local
2677 	switch (qp->type) {  in __flush_rq()
2690 	start = rq->swq_start;  in __flush_rq()
2693 		last = rq->swq_last;  in __flush_rq()
2697 		cqe->status =  in __flush_rq()
2699 		cqe->opcode = opcode;  in __flush_rq()
2700 		cqe->qp_handle = (u64)qp;  in __flush_rq()
2701 		cqe->wr_id = rq->swq[last].wr_id;  in __flush_rq()
2702 		dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n");  in __flush_rq()
2703 		dev_dbg(&rq->hwq.pdev->dev,  in __flush_rq()
2704 			"QPLIB: rq[%d] = 0x%llx with status 0x%x\n",  in __flush_rq()
2705 			last, cqe->wr_id, cqe->status);  in __flush_rq()
2707 		(*budget)--;  in __flush_rq()
2708 		bnxt_qplib_hwq_incr_cons(rq->hwq.depth,  in __flush_rq()
2709 					 &rq->hwq.cons,  in __flush_rq()
2710 					 rq->swq[last].slots,  in __flush_rq()
2711 					 &rq->dbinfo.flags);  in __flush_rq()
2712 		rq->swq_last = rq->swq[last].next_idx;  in __flush_rq()
2715 	if (!*budget && rq->swq_last != start)  in __flush_rq()
2717 		rc = -EAGAIN;  in __flush_rq()
2719 	dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc);  in __flush_rq()
2720 	return rc;  in __flush_rq()
2731 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR;  in bnxt_qplib_mark_qp_error()
2732 	qp->state = qp->cur_qp_state;  in bnxt_qplib_mark_qp_error()
2735 	if (!qp->is_user)  in bnxt_qplib_mark_qp_error()
2747 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_re_legacy_do_wa9060()
2755 	int i, rc = 0;  in bnxt_re_legacy_do_wa9060()  local
2758 	swq = &sq->swq[swq_last];  in bnxt_re_legacy_do_wa9060()
2759 	if (swq->psn_search &&  in bnxt_re_legacy_do_wa9060()
2760 	    le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {  in bnxt_re_legacy_do_wa9060()
2762 		swq->psn_search->flags_next_psn = cpu_to_le32  in bnxt_re_legacy_do_wa9060()
2763 				(le32_to_cpu(swq->psn_search->flags_next_psn)  in bnxt_re_legacy_do_wa9060()
2765 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_re_legacy_do_wa9060()
2766 			"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",  in bnxt_re_legacy_do_wa9060()
2767 			cq_cons, qp->id, swq_last, cqe_sq_cons);  in bnxt_re_legacy_do_wa9060()
2768 		sq->condition = true;  in bnxt_re_legacy_do_wa9060()
2769 		sq->legacy_send_phantom = true;  in bnxt_re_legacy_do_wa9060()
2772 		bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);  in bnxt_re_legacy_do_wa9060()
2774 		rc = -EAGAIN;  in bnxt_re_legacy_do_wa9060()
2777 	if (sq->condition == true) {  in bnxt_re_legacy_do_wa9060()
2779 		peek_flags = cq->dbinfo.flags;  in bnxt_re_legacy_do_wa9060()
2781 		i = cq->hwq.depth;  in bnxt_re_legacy_do_wa9060()
2782 		while (i--) {  in bnxt_re_legacy_do_wa9060()
2783 			peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,  in bnxt_re_legacy_do_wa9060()
2789 				switch (peek_hwcqe->cqe_type_toggle &  in bnxt_re_legacy_do_wa9060()
2796 						peek_req_hwcqe->qp_handle);  in bnxt_re_legacy_do_wa9060()
2797 					peek_sq = &peek_qp->sq;  in bnxt_re_legacy_do_wa9060()
2800 						  peek_req_hwcqe->sq_cons_idx)  in bnxt_re_legacy_do_wa9060()
2801 						  - 1) % sq->max_wqe);  in bnxt_re_legacy_do_wa9060()
2804 					    sq->swq[peek_sq_cons_idx].wr_id ==  in bnxt_re_legacy_do_wa9060()
2808 						dev_dbg(&cq->hwq.pdev->dev,  in bnxt_re_legacy_do_wa9060()
2809 							"FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n",  in bnxt_re_legacy_do_wa9060()
2810 							qp->id, swq_last,  in bnxt_re_legacy_do_wa9060()
2812 						sq->condition = false;  in bnxt_re_legacy_do_wa9060()
2813 						sq->single = true;  in bnxt_re_legacy_do_wa9060()
2814 						sq->phantom_cqe_cnt++;  in bnxt_re_legacy_do_wa9060()
2815 						dev_dbg(&cq->hwq.pdev->dev,  in bnxt_re_legacy_do_wa9060()
2816 …					"qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n",  in bnxt_re_legacy_do_wa9060()
2817 							peek_qp->id,  in bnxt_re_legacy_do_wa9060()
2820 							sq->phantom_cqe_cnt);  in bnxt_re_legacy_do_wa9060()
2821 						rc = 0;  in bnxt_re_legacy_do_wa9060()
2833 						peek_term_hwcqe->qp_handle);  in bnxt_re_legacy_do_wa9060()
2835 						sq->condition = false;  in bnxt_re_legacy_do_wa9060()
2836 						rc = 0;  in bnxt_re_legacy_do_wa9060()
2846 				rc = -EINVAL;  in bnxt_re_legacy_do_wa9060()
2849 			bnxt_qplib_hwq_incr_cons(cq->hwq.depth,  in bnxt_re_legacy_do_wa9060()
2853 		dev_err(&cq->hwq.pdev->dev,  in bnxt_re_legacy_do_wa9060()
2854 			"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",  in bnxt_re_legacy_do_wa9060()
2855 			cq_cons, qp->id, swq_last, cqe_sq_cons);  in bnxt_re_legacy_do_wa9060()
2856 		rc = -EINVAL;  in bnxt_re_legacy_do_wa9060()
2859 	return rc;  in bnxt_re_legacy_do_wa9060()
2872 	int rc = 0;  in bnxt_qplib_cq_process_req()  local
2874 	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);  in bnxt_qplib_cq_process_req()
2875 	dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp);  in bnxt_qplib_cq_process_req()
2877 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2879 		return -EINVAL;  in bnxt_qplib_cq_process_req()
2881 	sq = &qp->sq;  in bnxt_qplib_cq_process_req()
2883 	cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;  in bnxt_qplib_cq_process_req()
2884 	if (qp->sq.flushed) {  in bnxt_qplib_cq_process_req()
2885 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2896 		if (sq->swq_last == cqe_sq_cons)  in bnxt_qplib_cq_process_req()
2900 		swq = &sq->swq[sq->swq_last];  in bnxt_qplib_cq_process_req()
2902 		cqe->opcode = CQ_BASE_CQE_TYPE_REQ;  in bnxt_qplib_cq_process_req()
2903 		cqe->qp_handle = (u64)qp;  in bnxt_qplib_cq_process_req()
2904 		cqe->src_qp = qp->id;  in bnxt_qplib_cq_process_req()
2905 		cqe->wr_id = swq->wr_id;  in bnxt_qplib_cq_process_req()
2907 		if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)  in bnxt_qplib_cq_process_req()
2910 		cqe->type = swq->type;  in bnxt_qplib_cq_process_req()
2916 		if (swq->next_idx == cqe_sq_cons &&  in bnxt_qplib_cq_process_req()
2917 		    hwcqe->status != CQ_REQ_STATUS_OK) {  in bnxt_qplib_cq_process_req()
2918 			cqe->status = hwcqe->status;  in bnxt_qplib_cq_process_req()
2919 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2921 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2922 				"QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n",  in bnxt_qplib_cq_process_req()
2923 				cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status);  in bnxt_qplib_cq_process_req()
2925 			(*budget)--;  in bnxt_qplib_cq_process_req()
2929 			if (!_is_chip_gen_p5_p7(qp->cctx)) {  in bnxt_qplib_cq_process_req()
2931 					      sq->swq_last,  in bnxt_qplib_cq_process_req()
2937 			if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {  in bnxt_qplib_cq_process_req()
2939 				dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2941 				dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2942 					"QPLIB: wr_id[%d] = 0x%llx \n",  in bnxt_qplib_cq_process_req()
2943 					sq->swq_last, cqe->wr_id);  in bnxt_qplib_cq_process_req()
2944 				dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_req()
2945 					"QPLIB: with status 0x%x\n", cqe->status);  in bnxt_qplib_cq_process_req()
2946 				cqe->status = CQ_REQ_STATUS_OK;  in bnxt_qplib_cq_process_req()
2948 				(*budget)--;  in bnxt_qplib_cq_process_req()
2952 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,  in bnxt_qplib_cq_process_req()
2953 					 swq->slots, &sq->dbinfo.flags);  in bnxt_qplib_cq_process_req()
2954 		sq->swq_last = swq->next_idx;  in bnxt_qplib_cq_process_req()
2955 		if (sq->single == true)  in bnxt_qplib_cq_process_req()
2960 	if (sq->swq_last != cqe_sq_cons) {  in bnxt_qplib_cq_process_req()
2962 		rc = -EAGAIN;  in bnxt_qplib_cq_process_req()
2967 	sq->single = false;  in bnxt_qplib_cq_process_req()
2969 	return rc;  in bnxt_qplib_cq_process_req()
2974 	spin_lock(&srq->hwq.lock);  in bnxt_qplib_release_srqe()
2975 	srq->swq[srq->last_idx].next_idx = (int)tag;  in bnxt_qplib_release_srqe()
2976 	srq->last_idx = (int)tag;  in bnxt_qplib_release_srqe()
2977 	srq->swq[srq->last_idx].next_idx = -1;  in bnxt_qplib_release_srqe()
2978 	bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons,  in bnxt_qplib_release_srqe()
2979 				 srq->dbinfo.max_slot, &srq->dbinfo.flags);  in bnxt_qplib_release_srqe()
2980 	spin_unlock(&srq->hwq.lock);  in bnxt_qplib_release_srqe()
2993 	int rc = 0;  in bnxt_qplib_cq_process_res_rc()  local
2995 	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);  in bnxt_qplib_cq_process_res_rc()
2997 		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n");  in bnxt_qplib_cq_process_res_rc()
2998 		return -EINVAL;  in bnxt_qplib_cq_process_res_rc()
3000 	if (qp->rq.flushed) {  in bnxt_qplib_cq_process_res_rc()
3001 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3007 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;  in bnxt_qplib_cq_process_res_rc()
3008 	cqe->length = le32_to_cpu(hwcqe->length);  in bnxt_qplib_cq_process_res_rc()
3009 	cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);  in bnxt_qplib_cq_process_res_rc()
3010 	cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);  in bnxt_qplib_cq_process_res_rc()
3011 	cqe->flags = le16_to_cpu(hwcqe->flags);  in bnxt_qplib_cq_process_res_rc()
3012 	cqe->status = hwcqe->status;  in bnxt_qplib_cq_process_res_rc()
3013 	cqe->qp_handle = (u64)(unsigned long)qp;  in bnxt_qplib_cq_process_res_rc()
3015 	wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &  in bnxt_qplib_cq_process_res_rc()
3017 	if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {  in bnxt_qplib_cq_process_res_rc()
3018 		srq = qp->srq;  in bnxt_qplib_cq_process_res_rc()
3020 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3022 			return -EINVAL;  in bnxt_qplib_cq_process_res_rc()
3024 		if (wr_id_idx > srq->hwq.depth - 1) {  in bnxt_qplib_cq_process_res_rc()
3025 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3026 				"QPLIB: FP: CQ Process RC \n");  in bnxt_qplib_cq_process_res_rc()
3027 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3028 				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",  in bnxt_qplib_cq_process_res_rc()
3029 				wr_id_idx, srq->hwq.depth);  in bnxt_qplib_cq_process_res_rc()
3030 			return -EINVAL;  in bnxt_qplib_cq_process_res_rc()
3032 		cqe->wr_id = srq->swq[wr_id_idx].wr_id;  in bnxt_qplib_cq_process_res_rc()
3034 		dev_dbg(&srq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3035 			"QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_cq_process_res_rc()
3036 			wr_id_idx, cqe->wr_id);  in bnxt_qplib_cq_process_res_rc()
3038 		(*budget)--;  in bnxt_qplib_cq_process_res_rc()
3041 		rq = &qp->rq;  in bnxt_qplib_cq_process_res_rc()
3042 		if (wr_id_idx > (rq->max_wqe - 1)) {  in bnxt_qplib_cq_process_res_rc()
3043 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3044 				"QPLIB: FP: CQ Process RC \n");  in bnxt_qplib_cq_process_res_rc()
3045 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3046 				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",  in bnxt_qplib_cq_process_res_rc()
3047 				wr_id_idx, rq->hwq.depth);  in bnxt_qplib_cq_process_res_rc()
3048 			return -EINVAL;  in bnxt_qplib_cq_process_res_rc()
3050 		if (wr_id_idx != rq->swq_last)  in bnxt_qplib_cq_process_res_rc()
3051 			return -EINVAL;  in bnxt_qplib_cq_process_res_rc()
3052 		cqe->wr_id = rq->swq[rq->swq_last].wr_id;  in bnxt_qplib_cq_process_res_rc()
3053 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_rc()
3054 			"QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_cq_process_res_rc()
3055 			rq->swq_last, cqe->wr_id);  in bnxt_qplib_cq_process_res_rc()
3057 		(*budget)--;  in bnxt_qplib_cq_process_res_rc()
3058 		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,  in bnxt_qplib_cq_process_res_rc()
3059 					 rq->swq[rq->swq_last].slots,  in bnxt_qplib_cq_process_res_rc()
3060 					 &rq->dbinfo.flags);  in bnxt_qplib_cq_process_res_rc()
3061 		rq->swq_last = rq->swq[rq->swq_last].next_idx;  in bnxt_qplib_cq_process_res_rc()
3064 		if (hwcqe->status != CQ_RES_RC_STATUS_OK)  in bnxt_qplib_cq_process_res_rc()
3068 	return rc;  in bnxt_qplib_cq_process_res_rc()
3081 	int rc = 0;  in bnxt_qplib_cq_process_res_ud()  local
3084 	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);  in bnxt_qplib_cq_process_res_ud()
3086 		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n");  in bnxt_qplib_cq_process_res_ud()
3087 		return -EINVAL;  in bnxt_qplib_cq_process_res_ud()
3089 	if (qp->rq.flushed) {  in bnxt_qplib_cq_process_res_ud()
3090 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3095 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK;  in bnxt_qplib_cq_process_res_ud()
3096 	cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK));  in bnxt_qplib_cq_process_res_ud()
3097 	cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0);  in bnxt_qplib_cq_process_res_ud()
3099 	cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) &  in bnxt_qplib_cq_process_res_ud()
3103 	cqe->invrkey = le32_to_cpu(hwcqe->imm_data);  in bnxt_qplib_cq_process_res_ud()
3104 	cqe->flags = le16_to_cpu(hwcqe->flags);  in bnxt_qplib_cq_process_res_ud()
3105 	cqe->status = hwcqe->status;  in bnxt_qplib_cq_process_res_ud()
3106 	cqe->qp_handle = (u64)(unsigned long)qp;  in bnxt_qplib_cq_process_res_ud()
3107 	smac = (u16 *)cqe->smac;  in bnxt_qplib_cq_process_res_ud()
3108 	smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0]));  in bnxt_qplib_cq_process_res_ud()
3109 	smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1]));  in bnxt_qplib_cq_process_res_ud()
3110 	smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2]));  in bnxt_qplib_cq_process_res_ud()
3111 	wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)  in bnxt_qplib_cq_process_res_ud()
3113 	cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |  in bnxt_qplib_cq_process_res_ud()
3115 				    hwcqe->src_qp_high_srq_or_rq_wr_id) &  in bnxt_qplib_cq_process_res_ud()
3118 	if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) {  in bnxt_qplib_cq_process_res_ud()
3119 		srq = qp->srq;  in bnxt_qplib_cq_process_res_ud()
3121 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3123 			return -EINVAL;  in bnxt_qplib_cq_process_res_ud()
3125 		if (wr_id_idx > srq->hwq.depth - 1) {  in bnxt_qplib_cq_process_res_ud()
3126 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3128 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3129 				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",  in bnxt_qplib_cq_process_res_ud()
3130 				wr_id_idx, srq->hwq.depth);  in bnxt_qplib_cq_process_res_ud()
3131 			return -EINVAL;  in bnxt_qplib_cq_process_res_ud()
3133 		cqe->wr_id = srq->swq[wr_id_idx].wr_id;  in bnxt_qplib_cq_process_res_ud()
3135 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3136 			"QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_cq_process_res_ud()
3137 			wr_id_idx, cqe->wr_id);  in bnxt_qplib_cq_process_res_ud()
3139 		(*budget)--;  in bnxt_qplib_cq_process_res_ud()
3142 		rq = &qp->rq;  in bnxt_qplib_cq_process_res_ud()
3143 		if (wr_id_idx > (rq->max_wqe - 1)) {  in bnxt_qplib_cq_process_res_ud()
3144 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3146 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3147 				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",  in bnxt_qplib_cq_process_res_ud()
3148 				wr_id_idx, rq->hwq.depth);  in bnxt_qplib_cq_process_res_ud()
3149 			return -EINVAL;  in bnxt_qplib_cq_process_res_ud()
3151 		if (rq->swq_last != wr_id_idx)  in bnxt_qplib_cq_process_res_ud()
3152 			return -EINVAL;  in bnxt_qplib_cq_process_res_ud()
3154 		cqe->wr_id = rq->swq[rq->swq_last].wr_id;  in bnxt_qplib_cq_process_res_ud()
3155 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_ud()
3156 			"QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n",  in bnxt_qplib_cq_process_res_ud()
3157 			 rq->swq_last, cqe->wr_id);  in bnxt_qplib_cq_process_res_ud()
3159 		(*budget)--;  in bnxt_qplib_cq_process_res_ud()
3160 		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,  in bnxt_qplib_cq_process_res_ud()
3161 					 rq->swq[rq->swq_last].slots,  in bnxt_qplib_cq_process_res_ud()
3162 					 &rq->dbinfo.flags);  in bnxt_qplib_cq_process_res_ud()
3163 		rq->swq_last = rq->swq[rq->swq_last].next_idx;  in bnxt_qplib_cq_process_res_ud()
3166 		if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK)  in bnxt_qplib_cq_process_res_ud()
3170 	return rc;  in bnxt_qplib_cq_process_res_ud()
3178 	bool rc = true;  in bnxt_qplib_is_cq_empty()  local
3180 	spin_lock_irqsave(&cq->hwq.lock, flags);  in bnxt_qplib_is_cq_empty()
3181 	hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);  in bnxt_qplib_is_cq_empty()
3184 	rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);  in bnxt_qplib_is_cq_empty()
3185 	spin_unlock_irqrestore(&cq->hwq.lock, flags);  in bnxt_qplib_is_cq_empty()
3186 	return rc;  in bnxt_qplib_is_cq_empty()
3199 	int rc = 0;  in bnxt_qplib_cq_process_res_raweth_qp1()  local
3201 	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);  in bnxt_qplib_cq_process_res_raweth_qp1()
3203 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3205 		return -EINVAL;  in bnxt_qplib_cq_process_res_raweth_qp1()
3207 	if (qp->rq.flushed) {  in bnxt_qplib_cq_process_res_raweth_qp1()
3208 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3213 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;  in bnxt_qplib_cq_process_res_raweth_qp1()
3214 	cqe->flags = le16_to_cpu(hwcqe->flags);  in bnxt_qplib_cq_process_res_raweth_qp1()
3215 	cqe->qp_handle = (u64)(unsigned long)qp;  in bnxt_qplib_cq_process_res_raweth_qp1()
3217 	wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)  in bnxt_qplib_cq_process_res_raweth_qp1()
3219 	cqe->src_qp = qp->id;  in bnxt_qplib_cq_process_res_raweth_qp1()
3220 	if (qp->id == 1 && !cqe->length) {  in bnxt_qplib_cq_process_res_raweth_qp1()
3222 		cqe->length = 296;  in bnxt_qplib_cq_process_res_raweth_qp1()
3224 		cqe->length = le16_to_cpu(hwcqe->length);  in bnxt_qplib_cq_process_res_raweth_qp1()
3226 	cqe->pkey_index = qp->pkey_index;  in bnxt_qplib_cq_process_res_raweth_qp1()
3227 	memcpy(cqe->smac, qp->smac, 6);  in bnxt_qplib_cq_process_res_raweth_qp1()
3229 	cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);  in bnxt_qplib_cq_process_res_raweth_qp1()
3230 	cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);  in bnxt_qplib_cq_process_res_raweth_qp1()
3231 	cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);  in bnxt_qplib_cq_process_res_raweth_qp1()
3233 	dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3234 		 "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n",  in bnxt_qplib_cq_process_res_raweth_qp1()
3235 		 cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2);  in bnxt_qplib_cq_process_res_raweth_qp1()
3237 	if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {  in bnxt_qplib_cq_process_res_raweth_qp1()
3238 		srq = qp->srq;  in bnxt_qplib_cq_process_res_raweth_qp1()
3240 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3242 			return -EINVAL;  in bnxt_qplib_cq_process_res_raweth_qp1()
3244 		if (wr_id_idx > srq->hwq.depth - 1) {  in bnxt_qplib_cq_process_res_raweth_qp1()
3245 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3247 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3248 				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",  in bnxt_qplib_cq_process_res_raweth_qp1()
3249 				wr_id_idx, srq->hwq.depth);  in bnxt_qplib_cq_process_res_raweth_qp1()
3250 			return -EINVAL;  in bnxt_qplib_cq_process_res_raweth_qp1()
3252 		cqe->wr_id = srq->swq[wr_id_idx].wr_id;  in bnxt_qplib_cq_process_res_raweth_qp1()
3253 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3255 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3256 			"QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",  in bnxt_qplib_cq_process_res_raweth_qp1()
3257 			wr_id_idx, cqe->wr_id, hwcqe->status);  in bnxt_qplib_cq_process_res_raweth_qp1()
3259 		(*budget)--;  in bnxt_qplib_cq_process_res_raweth_qp1()
3260 		srq->hwq.cons++;  in bnxt_qplib_cq_process_res_raweth_qp1()
3263 		rq = &qp->rq;  in bnxt_qplib_cq_process_res_raweth_qp1()
3264 		if (wr_id_idx > (rq->max_wqe - 1)) {  in bnxt_qplib_cq_process_res_raweth_qp1()
3265 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3267 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3268 				"QPLIB: ix 0x%x exceeded RQ max 0x%x\n",  in bnxt_qplib_cq_process_res_raweth_qp1()
3269 				wr_id_idx, rq->max_wqe);  in bnxt_qplib_cq_process_res_raweth_qp1()
3270 			return -EINVAL;  in bnxt_qplib_cq_process_res_raweth_qp1()
3272 		if (wr_id_idx != rq->swq_last)  in bnxt_qplib_cq_process_res_raweth_qp1()
3273 			return -EINVAL;  in bnxt_qplib_cq_process_res_raweth_qp1()
3274 		cqe->wr_id = rq->swq[rq->swq_last].wr_id;  in bnxt_qplib_cq_process_res_raweth_qp1()
3275 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3277 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_res_raweth_qp1()
3278 			"QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",  in bnxt_qplib_cq_process_res_raweth_qp1()
3279 			wr_id_idx, cqe->wr_id, hwcqe->status);  in bnxt_qplib_cq_process_res_raweth_qp1()
3281 		(*budget)--;  in bnxt_qplib_cq_process_res_raweth_qp1()
3282 		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,  in bnxt_qplib_cq_process_res_raweth_qp1()
3283 					 rq->swq[wr_id_idx].slots,  in bnxt_qplib_cq_process_res_raweth_qp1()
3284 					 &rq->dbinfo.flags);  in bnxt_qplib_cq_process_res_raweth_qp1()
3285 		rq->swq_last = rq->swq[rq->swq_last].next_idx;  in bnxt_qplib_cq_process_res_raweth_qp1()
3288 		if (hwcqe->status != CQ_RES_RC_STATUS_OK)  in bnxt_qplib_cq_process_res_raweth_qp1()
3292 	return rc;  in bnxt_qplib_cq_process_res_raweth_qp1()
3305 	int rc = 0;  in bnxt_qplib_cq_process_terminal()  local
3308 	if (hwcqe->status != CQ_TERMINAL_STATUS_OK)  in bnxt_qplib_cq_process_terminal()
3309 		dev_warn(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3310 			 "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n",  in bnxt_qplib_cq_process_terminal()
3311 			 hwcqe->status);  in bnxt_qplib_cq_process_terminal()
3313 	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);  in bnxt_qplib_cq_process_terminal()
3315 		return -EINVAL;  in bnxt_qplib_cq_process_terminal()
3316 	dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3317 		"QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id);  in bnxt_qplib_cq_process_terminal()
3320 	 * from the current rq->cons to the rq->prod regardless what the  in bnxt_qplib_cq_process_terminal()
3321 	 * rq->cons the terminal CQE indicates.  in bnxt_qplib_cq_process_terminal()
3325 	sq = &qp->sq;  in bnxt_qplib_cq_process_terminal()
3326 	rq = &qp->rq;  in bnxt_qplib_cq_process_terminal()
3328 	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);  in bnxt_qplib_cq_process_terminal()
3332 	cqe_cons %= sq->max_wqe;  in bnxt_qplib_cq_process_terminal()
3333 	if (qp->sq.flushed) {  in bnxt_qplib_cq_process_terminal()
3334 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3344 		/*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/  in bnxt_qplib_cq_process_terminal()
3345 		swq_last = sq->swq_last;  in bnxt_qplib_cq_process_terminal()
3348 		if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {  in bnxt_qplib_cq_process_terminal()
3350 			cqe->status = CQ_REQ_STATUS_OK;  in bnxt_qplib_cq_process_terminal()
3351 			cqe->opcode = CQ_BASE_CQE_TYPE_REQ;  in bnxt_qplib_cq_process_terminal()
3352 			cqe->qp_handle = (u64)qp;  in bnxt_qplib_cq_process_terminal()
3353 			cqe->src_qp = qp->id;  in bnxt_qplib_cq_process_terminal()
3354 			cqe->wr_id = sq->swq[swq_last].wr_id;  in bnxt_qplib_cq_process_terminal()
3355 			cqe->type = sq->swq[swq_last].type;  in bnxt_qplib_cq_process_terminal()
3356 			dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3358 			dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3359 				"QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",  in bnxt_qplib_cq_process_terminal()
3360 				swq_last, cqe->wr_id, cqe->status);  in bnxt_qplib_cq_process_terminal()
3362 			(*budget)--;  in bnxt_qplib_cq_process_terminal()
3364 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,  in bnxt_qplib_cq_process_terminal()
3365 					 sq->swq[swq_last].slots,  in bnxt_qplib_cq_process_terminal()
3366 					 &sq->dbinfo.flags);  in bnxt_qplib_cq_process_terminal()
3367 		sq->swq_last = sq->swq[swq_last].next_idx;  in bnxt_qplib_cq_process_terminal()
3372 		rc = -EAGAIN;  in bnxt_qplib_cq_process_terminal()
3376 	if (rc)  in bnxt_qplib_cq_process_terminal()
3377 		return rc;  in bnxt_qplib_cq_process_terminal()
3379 	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);  in bnxt_qplib_cq_process_terminal()
3382 	} else if (cqe_cons > (rq->max_wqe - 1)) {  in bnxt_qplib_cq_process_terminal()
3383 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3385 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3386 			"QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n",  in bnxt_qplib_cq_process_terminal()
3387 			cqe_cons, rq->hwq.depth);  in bnxt_qplib_cq_process_terminal()
3390 	if (qp->rq.flushed) {  in bnxt_qplib_cq_process_terminal()
3391 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_terminal()
3393 		rc = 0;  in bnxt_qplib_cq_process_terminal()
3399 	return rc;  in bnxt_qplib_cq_process_terminal()
3406 	if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {  in bnxt_qplib_cq_process_cutoff()
3407 		dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_cq_process_cutoff()
3408 			"QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n",  in bnxt_qplib_cq_process_cutoff()
3409 			hwcqe->status);  in bnxt_qplib_cq_process_cutoff()
3410 		return -EINVAL;  in bnxt_qplib_cq_process_cutoff()
3412 	clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);  in bnxt_qplib_cq_process_cutoff()
3413 	wake_up_interruptible(&cq->waitq);  in bnxt_qplib_cq_process_cutoff()
3415 	dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n");  in bnxt_qplib_cq_process_cutoff()
3427 	spin_lock_irqsave(&cq->flush_lock, flags);  in bnxt_qplib_process_flush_list()
3428 	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {  in bnxt_qplib_process_flush_list()
3429 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_process_flush_list()
3432 		__flush_sq(&qp->sq, qp, &cqe, &budget);  in bnxt_qplib_process_flush_list()
3435 	list_for_each_entry(qp, &cq->rqf_head, rq_flush) {  in bnxt_qplib_process_flush_list()
3436 		dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_process_flush_list()
3439 		__flush_rq(&qp->rq, qp, &cqe, &budget);  in bnxt_qplib_process_flush_list()
3441 	spin_unlock_irqrestore(&cq->flush_lock, flags);  in bnxt_qplib_process_flush_list()
3443 	return num_cqes - budget;  in bnxt_qplib_process_flush_list()
3451 	int budget, rc = 0;  in bnxt_qplib_poll_cq()  local
3457 		hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);  in bnxt_qplib_poll_cq()
3460 		if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags))  in bnxt_qplib_poll_cq()
3468 		type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;  in bnxt_qplib_poll_cq()
3471 			rc = bnxt_qplib_cq_process_req(cq,  in bnxt_qplib_poll_cq()
3473 					cq->hwq.cons, lib_qp);  in bnxt_qplib_poll_cq()
3476 			rc = bnxt_qplib_cq_process_res_rc(cq,  in bnxt_qplib_poll_cq()
3481 			rc = bnxt_qplib_cq_process_res_ud(cq,  in bnxt_qplib_poll_cq()
3486 			rc = bnxt_qplib_cq_process_res_raweth_qp1(cq,  in bnxt_qplib_poll_cq()
3491 			rc = bnxt_qplib_cq_process_terminal(cq,  in bnxt_qplib_poll_cq()
3501 			dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_poll_cq()
3502 				"QPLIB: process_cq unknown type 0x%x\n",  in bnxt_qplib_poll_cq()
3503 				hw_cqe->cqe_type_toggle &  in bnxt_qplib_poll_cq()
3505 			rc = -EINVAL;  in bnxt_qplib_poll_cq()
3508 		if (rc < 0) {  in bnxt_qplib_poll_cq()
3509 			dev_dbg(&cq->hwq.pdev->dev,  in bnxt_qplib_poll_cq()
3510 				"QPLIB: process_cqe rc = 0x%x\n", rc);  in bnxt_qplib_poll_cq()
3511 			if (rc == -EAGAIN)  in bnxt_qplib_poll_cq()
3516 				dev_err(&cq->hwq.pdev->dev,  in bnxt_qplib_poll_cq()
3517 					"QPLIB: process_cqe error rc = 0x%x\n",  in bnxt_qplib_poll_cq()
3518 					rc);  in bnxt_qplib_poll_cq()
3521 		bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons,  in bnxt_qplib_poll_cq()
3522 					 1, &cq->dbinfo.flags);  in bnxt_qplib_poll_cq()
3525 		bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);  in bnxt_qplib_poll_cq()
3527 	return num_cqes - budget;  in bnxt_qplib_poll_cq()
3532 	cq->dbinfo.toggle = cq->toggle;  in bnxt_qplib_req_notify_cq()
3534 		bnxt_qplib_ring_db(&cq->dbinfo, arm_type);  in bnxt_qplib_req_notify_cq()
3535 	/* Using cq->arm_state variable to track whether to issue cq handler */  in bnxt_qplib_req_notify_cq()
3536 	atomic_set(&cq->arm_state, 1);  in bnxt_qplib_req_notify_cq()
3541 	flush_workqueue(qp->scq->nq->cqn_wq);  in bnxt_qplib_flush_cqn_wq()
3542 	if (qp->scq != qp->rcq)  in bnxt_qplib_flush_cqn_wq()
3543 		flush_workqueue(qp->rcq->nq->cqn_wq);  in bnxt_qplib_flush_cqn_wq()