53 	qp->sq.condition = false;  in bnxt_re_legacy_cancel_phantom_processing()
54 	qp->sq.legacy_send_phantom = false;  in bnxt_re_legacy_cancel_phantom_processing()
55 	qp->sq.single = false;  in bnxt_re_legacy_cancel_phantom_processing()
65 	if (!qp->sq.flushed) {  in __bnxt_qplib_add_flush_qp()
71 		qp->sq.flushed = true;  in __bnxt_qplib_add_flush_qp()
115 	if (qp->sq.flushed) {  in __bnxt_qplib_del_flush_qp()
116 		qp->sq.flushed = false;  in __bnxt_qplib_del_flush_qp()
132 	qp->sq.hwq.prod = 0;  in bnxt_qplib_clean_qp()
133 	qp->sq.hwq.cons = 0;  in bnxt_qplib_clean_qp()
134 	qp->sq.swq_start = 0;  in bnxt_qplib_clean_qp()
135 	qp->sq.swq_last = 0;  in bnxt_qplib_clean_qp()
220 							qp->sq.max_wqe);  in bnxt_qplib_alloc_hdr_buf()
911 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_create_qp1()  local
926 	hwq_attr.sginfo = &sq->sginfo;  in bnxt_qplib_create_qp1()
928 	hwq_attr.depth = bnxt_qplib_get_depth(sq);  in bnxt_qplib_create_qp1()
930 	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);  in bnxt_qplib_create_qp1()
934 	req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode));  in bnxt_qplib_create_qp1()
935 	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));  in bnxt_qplib_create_qp1()
936 	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<  in bnxt_qplib_create_qp1()
938 	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) <<  in bnxt_qplib_create_qp1()
943 					 (sq->max_sge &  in bnxt_qplib_create_qp1()
988 	rc = bnxt_qplib_alloc_init_swq(sq);  in bnxt_qplib_create_qp1()
1001 	sq->dbinfo.hwq = &sq->hwq;  in bnxt_qplib_create_qp1()
1002 	sq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp1()
1003 	sq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp1()
1004 	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);  in bnxt_qplib_create_qp1()
1005 	sq->dbinfo.flags = 0;  in bnxt_qplib_create_qp1()
1006 	spin_lock_init(&sq->dbinfo.lock);  in bnxt_qplib_create_qp1()
1007 	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp1()
1008 	sq->dbinfo.res = res;  in bnxt_qplib_create_qp1()
1028 	kfree(sq->swq);  in bnxt_qplib_create_qp1()
1034 	bnxt_qplib_free_hwq(res, &sq->hwq);  in bnxt_qplib_create_qp1()
1042 	struct bnxt_qplib_q *sq;  in bnxt_qplib_init_psn_ptr()  local
1046 	sq = &qp->sq;  in bnxt_qplib_init_psn_ptr()
1047 	sq_hwq = &sq->hwq;  in bnxt_qplib_init_psn_ptr()
1064 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_create_qp()  local
1099 	hwq_attr.sginfo = &sq->sginfo;  in bnxt_qplib_create_qp()
1101 	hwq_attr.depth = bnxt_qplib_get_depth(sq);  in bnxt_qplib_create_qp()
1104 				 _set_sq_size(sq, qp->wqe_mode) : 0;  in bnxt_qplib_create_qp()
1108 			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode));  in bnxt_qplib_create_qp()
1110 			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2;  in bnxt_qplib_create_qp()
1115 	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);  in bnxt_qplib_create_qp()
1119 	sqsz = _set_sq_size(sq, qp->wqe_mode);  in bnxt_qplib_create_qp()
1126 	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));  in bnxt_qplib_create_qp()
1127 	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<  in bnxt_qplib_create_qp()
1129 	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<  in bnxt_qplib_create_qp()
1135 					  BNXT_MSN_TBLE_SGE : sq->max_sge &  in bnxt_qplib_create_qp()
1231 		rc = bnxt_qplib_alloc_init_swq(sq);  in bnxt_qplib_create_qp()
1247 	sq->dbinfo.hwq = &sq->hwq;  in bnxt_qplib_create_qp()
1248 	sq->dbinfo.xid = qp->id;  in bnxt_qplib_create_qp()
1249 	sq->dbinfo.db = qp->dpi->dbr;  in bnxt_qplib_create_qp()
1250 	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);  in bnxt_qplib_create_qp()
1251 	sq->dbinfo.flags = 0;  in bnxt_qplib_create_qp()
1252 	spin_lock_init(&sq->dbinfo.lock);  in bnxt_qplib_create_qp()
1253 	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;  in bnxt_qplib_create_qp()
1254 	sq->dbinfo.res = res;  in bnxt_qplib_create_qp()
1255 	sq->dbinfo.seed = qp->id;  in bnxt_qplib_create_qp()
1277 	kfree(sq->swq);  in bnxt_qplib_create_qp()
1287 	bnxt_qplib_free_hwq(res, &sq->hwq);  in bnxt_qplib_create_qp()
1450 		req.sq_psn = cpu_to_le32(qp->sq.psn);  in bnxt_qplib_modify_qp()
1460 	req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);  in bnxt_qplib_modify_qp()
1462 	req.sq_sge = cpu_to_le16(qp->sq.max_sge);  in bnxt_qplib_modify_qp()
1572 	qp->sq.psn = le32_to_cpu(sb->sq_psn);  in bnxt_qplib_query_qp()
1575 	qp->sq.max_wqe = qp->sq.hwq.max_elements;  in bnxt_qplib_query_qp()
1577 	qp->sq.max_sge = le16_to_cpu(sb->sq_sge);  in bnxt_qplib_query_qp()
1674 		kfree(qp->sq.swq);  in bnxt_qplib_free_qp_res()
1675 	bnxt_qplib_free_hwq(res, &qp->sq.hwq);  in bnxt_qplib_free_qp_res()
1681 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_get_qp1_sq_buf()  local
1689 		sw_prod = sq->swq_start;  in bnxt_qplib_get_qp1_sq_buf()
1813 	sq_hwq = &qp->sq.hwq;  in bnxt_qplib_put_inline()
1899 static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq,  in bnxt_qplib_pull_psn_buff()  argument
1907 	sq_hwq = &sq->hwq;  in bnxt_qplib_pull_psn_buff()
1911 	tail = swq->slot_idx / sq->dbinfo.max_slot;  in bnxt_qplib_pull_psn_buff()
1923 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_post_send_db()  local
1925 	bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);  in bnxt_qplib_post_send_db()
1933 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_qplib_post_send()  local
1947 	sq_hwq = &sq->hwq;  in bnxt_qplib_post_send()
1959 			sq->dbinfo.max_slot : wqe_slots;  in bnxt_qplib_post_send()
1960 	qfd_slots = _translate_q_full_delta(sq, wqe_size);  in bnxt_qplib_post_send()
1971 			sq->phantom_wqe_cnt, sq->phantom_cqe_cnt);  in bnxt_qplib_post_send()
1977 	swq = bnxt_qplib_get_swqe(sq, &wqe_idx);  in bnxt_qplib_post_send()
1979 	bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags));  in bnxt_qplib_post_send()
1985 	swq->start_psn = sq->psn & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2066 			sq->psn = (sq->psn + 1) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2073 			sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2110 		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2143 		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2246 	swq->next_psn = sq->psn & BTH_PSN_MASK;  in bnxt_qplib_post_send()
2250 	bnxt_qplib_swq_mod_start(sq, wqe_idx);  in bnxt_qplib_post_send()
2251 	bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots);  in bnxt_qplib_post_send()
2262 			dev_err(&sq->hwq.pdev->dev,  in bnxt_qplib_post_send()
2618 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,  in __flush_sq()  argument
2626 	start = sq->swq_start;  in __flush_sq()
2629 		last = sq->swq_last;  in __flush_sq()
2634 		if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {  in __flush_sq()
2643 		cqe->wr_id = sq->swq[last].wr_id;  in __flush_sq()
2645 		cqe->type = sq->swq[last].type;  in __flush_sq()
2646 		dev_dbg(&sq->hwq.pdev->dev,  in __flush_sq()
2648 		dev_dbg(&sq->hwq.pdev->dev,  in __flush_sq()
2654 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth,  in __flush_sq()
2655 					 &sq->hwq.cons,  in __flush_sq()
2656 					 sq->swq[last].slots,  in __flush_sq()
2657 					 &sq->dbinfo.flags);  in __flush_sq()
2658 		sq->swq_last = sq->swq[last].next_idx;  in __flush_sq()
2661 	if (!*budget && sq->swq_last != start)  in __flush_sq()
2664 	dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc);  in __flush_sq()
2747 	struct bnxt_qplib_q *sq = &qp->sq;  in bnxt_re_legacy_do_wa9060()  local
2758 	swq = &sq->swq[swq_last];  in bnxt_re_legacy_do_wa9060()
2768 		sq->condition = true;  in bnxt_re_legacy_do_wa9060()
2769 		sq->legacy_send_phantom = true;  in bnxt_re_legacy_do_wa9060()
2777 	if (sq->condition == true) {  in bnxt_re_legacy_do_wa9060()
2797 					peek_sq = &peek_qp->sq;  in bnxt_re_legacy_do_wa9060()
2801 						  - 1) % sq->max_wqe);  in bnxt_re_legacy_do_wa9060()
2803 					if (peek_sq == sq &&  in bnxt_re_legacy_do_wa9060()
2804 					    sq->swq[peek_sq_cons_idx].wr_id ==  in bnxt_re_legacy_do_wa9060()
2812 						sq->condition = false;  in bnxt_re_legacy_do_wa9060()
2813 						sq->single = true;  in bnxt_re_legacy_do_wa9060()
2814 						sq->phantom_cqe_cnt++;  in bnxt_re_legacy_do_wa9060()
2820 							sq->phantom_cqe_cnt);  in bnxt_re_legacy_do_wa9060()
2835 						sq->condition = false;  in bnxt_re_legacy_do_wa9060()
2868 	struct bnxt_qplib_q *sq;  in bnxt_qplib_cq_process_req()  local
2881 	sq = &qp->sq;  in bnxt_qplib_cq_process_req()
2883 	cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;  in bnxt_qplib_cq_process_req()
2884 	if (qp->sq.flushed) {  in bnxt_qplib_cq_process_req()
2896 		if (sq->swq_last == cqe_sq_cons)  in bnxt_qplib_cq_process_req()
2900 		swq = &sq->swq[sq->swq_last];  in bnxt_qplib_cq_process_req()
2923 				cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status);  in bnxt_qplib_cq_process_req()
2931 					      sq->swq_last,  in bnxt_qplib_cq_process_req()
2943 					sq->swq_last, cqe->wr_id);  in bnxt_qplib_cq_process_req()
2952 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,  in bnxt_qplib_cq_process_req()
2953 					 swq->slots, &sq->dbinfo.flags);  in bnxt_qplib_cq_process_req()
2954 		sq->swq_last = swq->next_idx;  in bnxt_qplib_cq_process_req()
2955 		if (sq->single == true)  in bnxt_qplib_cq_process_req()
2960 	if (sq->swq_last != cqe_sq_cons) {  in bnxt_qplib_cq_process_req()
2967 	sq->single = false;  in bnxt_qplib_cq_process_req()
3300 	struct bnxt_qplib_q *sq, *rq;  in bnxt_qplib_cq_process_terminal()  local
3325 	sq = &qp->sq;  in bnxt_qplib_cq_process_terminal()
3332 	cqe_cons %= sq->max_wqe;  in bnxt_qplib_cq_process_terminal()
3333 	if (qp->sq.flushed) {  in bnxt_qplib_cq_process_terminal()
3345 		swq_last = sq->swq_last;  in bnxt_qplib_cq_process_terminal()
3348 		if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {  in bnxt_qplib_cq_process_terminal()
3354 			cqe->wr_id = sq->swq[swq_last].wr_id;  in bnxt_qplib_cq_process_terminal()
3355 			cqe->type = sq->swq[swq_last].type;  in bnxt_qplib_cq_process_terminal()
3364 		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,  in bnxt_qplib_cq_process_terminal()
3365 					 sq->swq[swq_last].slots,  in bnxt_qplib_cq_process_terminal()
3366 					 &sq->dbinfo.flags);  in bnxt_qplib_cq_process_terminal()
3367 		sq->swq_last = sq->swq[swq_last].next_idx;  in bnxt_qplib_cq_process_terminal()
3432 		__flush_sq(&qp->sq, qp, &cqe, &budget);  in bnxt_qplib_process_flush_list()