Lines Matching refs:qp

92 static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){  in irdma_nop_hdr()  argument
95 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_nop_hdr()
103 irdma_nop_1(struct irdma_qp_uk *qp) in irdma_nop_1() argument
108 if (!qp->sq_ring.head) in irdma_nop_1()
111 wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_nop_1()
112 wqe = qp->sq_base[wqe_idx].elem; in irdma_nop_1()
114 qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA; in irdma_nop_1()
123 set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp)); in irdma_nop_1()
134 irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx) in irdma_clr_wqes() argument
140 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size; in irdma_clr_wqes()
141 wqe = qp->sq_base[wqe_idx].elem; in irdma_clr_wqes()
143 memset(wqe, qp->swqe_polarity ? 0 : 0xFF, 0x1000); in irdma_clr_wqes()
145 memset(wqe, qp->swqe_polarity ? 0xFF : 0, 0x1000); in irdma_clr_wqes()
154 irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) in irdma_uk_qp_post_wr() argument
164 get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp); in irdma_uk_qp_post_wr()
167 sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_uk_qp_post_wr()
168 if (sw_sq_head != qp->initial_ring.head) { in irdma_uk_qp_post_wr()
169 if (qp->push_dropped) { in irdma_uk_qp_post_wr()
170 db_wr32(qp->qp_id, qp->wqe_alloc_db); in irdma_uk_qp_post_wr()
171 qp->push_dropped = false; in irdma_uk_qp_post_wr()
173 if (sw_sq_head > qp->initial_ring.head) { in irdma_uk_qp_post_wr()
174 if (hw_sq_tail >= qp->initial_ring.head && in irdma_uk_qp_post_wr()
176 db_wr32(qp->qp_id, qp->wqe_alloc_db); in irdma_uk_qp_post_wr()
178 if (hw_sq_tail >= qp->initial_ring.head || in irdma_uk_qp_post_wr()
180 db_wr32(qp->qp_id, qp->wqe_alloc_db); in irdma_uk_qp_post_wr()
185 qp->initial_ring.head = qp->sq_ring.head; in irdma_uk_qp_post_wr()
194 irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx) in irdma_qp_ring_push_db() argument
196 set_32bit_val(qp->push_db, 0, in irdma_qp_ring_push_db()
197 FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id); in irdma_qp_ring_push_db()
198 qp->initial_ring.head = qp->sq_ring.head; in irdma_qp_ring_push_db()
199 qp->push_mode = true; in irdma_qp_ring_push_db()
200 qp->push_dropped = false; in irdma_qp_ring_push_db()
204 irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta, in irdma_qp_push_wqe() argument
209 if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) != in irdma_qp_push_wqe()
210 IRDMA_RING_CURRENT_TAIL(qp->sq_ring) && in irdma_qp_push_wqe()
211 !qp->push_mode) { in irdma_qp_push_wqe()
212 irdma_uk_qp_post_wr(qp); in irdma_qp_push_wqe()
214 push = (__le64 *) ((uintptr_t)qp->push_wqe + in irdma_qp_push_wqe()
217 irdma_qp_ring_push_db(qp, wqe_idx); in irdma_qp_push_wqe()
230 irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, in irdma_qp_get_next_send_wqe() argument
240 avail_quanta = qp->uk_attrs->max_hw_sq_chunk - in irdma_qp_get_next_send_wqe()
241 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % in irdma_qp_get_next_send_wqe()
242 qp->uk_attrs->max_hw_sq_chunk); in irdma_qp_get_next_send_wqe()
246 if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) in irdma_qp_get_next_send_wqe()
251 IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) in irdma_qp_get_next_send_wqe()
254 nop_wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_qp_get_next_send_wqe()
256 irdma_nop_1(qp); in irdma_qp_get_next_send_wqe()
257 IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); in irdma_qp_get_next_send_wqe()
259 if (qp->push_db && info->push_wqe) in irdma_qp_get_next_send_wqe()
260 irdma_qp_push_wqe(qp, qp->sq_base[nop_wqe_idx].elem, in irdma_qp_get_next_send_wqe()
264 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); in irdma_qp_get_next_send_wqe()
266 qp->swqe_polarity = !qp->swqe_polarity; in irdma_qp_get_next_send_wqe()
268 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta); in irdma_qp_get_next_send_wqe()
270 irdma_clr_wqes(qp, *wqe_idx); in irdma_qp_get_next_send_wqe()
272 wqe = qp->sq_base[*wqe_idx].elem; in irdma_qp_get_next_send_wqe()
273 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && in irdma_qp_get_next_send_wqe()
274 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { in irdma_qp_get_next_send_wqe()
275 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; in irdma_qp_get_next_send_wqe()
277 qp->swqe_polarity ? 0 : 1)); in irdma_qp_get_next_send_wqe()
279 qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; in irdma_qp_get_next_send_wqe()
280 qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; in irdma_qp_get_next_send_wqe()
281 qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta; in irdma_qp_get_next_send_wqe()
282 qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled; in irdma_qp_get_next_send_wqe()
293 irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx) in irdma_qp_get_next_recv_wqe() argument
298 if (IRDMA_RING_FULL_ERR(qp->rq_ring)) in irdma_qp_get_next_recv_wqe()
301 IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); in irdma_qp_get_next_recv_wqe()
306 qp->rwqe_polarity = !qp->rwqe_polarity; in irdma_qp_get_next_recv_wqe()
308 wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem; in irdma_qp_get_next_recv_wqe()
320 irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_rdma_write() argument
333 info->push_wqe = qp->push_db ? true : false; in irdma_uk_rdma_write()
336 if (op_info->num_lo_sges > qp->max_sq_frag_cnt) in irdma_uk_rdma_write()
353 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); in irdma_uk_rdma_write()
357 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; in irdma_uk_rdma_write()
366 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, in irdma_uk_rdma_write()
368 qp->swqe_polarity); in irdma_uk_rdma_write()
373 qp->wqe_ops.iw_set_fragment(wqe, byte_off, in irdma_uk_rdma_write()
375 qp->swqe_polarity); in irdma_uk_rdma_write()
380 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && in irdma_uk_rdma_write()
382 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_rdma_write()
383 qp->swqe_polarity); in irdma_uk_rdma_write()
384 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_rdma_write()
397 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_rdma_write()
403 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_rdma_write()
405 irdma_uk_qp_post_wr(qp); in irdma_uk_rdma_write()
418 irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_rdma_read() argument
432 info->push_wqe = qp->push_db ? true : false; in irdma_uk_rdma_read()
435 if (qp->max_sq_frag_cnt < op_info->num_lo_sges) in irdma_uk_rdma_read()
445 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); in irdma_uk_rdma_read()
449 if (qp->rd_fence_rate && (qp->ord_cnt++ == qp->rd_fence_rate)) { in irdma_uk_rdma_read()
451 qp->ord_cnt = 0; in irdma_uk_rdma_read()
454 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; in irdma_uk_rdma_read()
459 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->lo_sg_list, in irdma_uk_rdma_read()
460 qp->swqe_polarity); in irdma_uk_rdma_read()
462 qp->wqe_ops.iw_set_fragment(wqe, byte_off, in irdma_uk_rdma_read()
464 qp->swqe_polarity); in irdma_uk_rdma_read()
469 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && in irdma_uk_rdma_read()
471 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_rdma_read()
472 qp->swqe_polarity); in irdma_uk_rdma_read()
473 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_rdma_read()
488 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_rdma_read()
494 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_rdma_read()
496 irdma_uk_qp_post_wr(qp); in irdma_uk_rdma_read()
508 irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, in irdma_uk_send() argument
520 info->push_wqe = qp->push_db ? true : false; in irdma_uk_send()
523 if (qp->max_sq_frag_cnt < op_info->num_sges) in irdma_uk_send()
537 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); in irdma_uk_send()
548 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, in irdma_uk_send()
550 qp->swqe_polarity); in irdma_uk_send()
555 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i], in irdma_uk_send()
556 qp->swqe_polarity); in irdma_uk_send()
561 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && in irdma_uk_send()
563 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_send()
564 qp->swqe_polarity); in irdma_uk_send()
565 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_send()
585 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_send()
591 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_send()
593 irdma_uk_qp_post_wr(qp); in irdma_uk_send()
727 irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, in irdma_uk_inline_rdma_write() argument
738 info->push_wqe = qp->push_db ? true : false; in irdma_uk_inline_rdma_write()
741 if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges)) in irdma_uk_inline_rdma_write()
747 if (unlikely(total_size > qp->max_inline_data)) in irdma_uk_inline_rdma_write()
750 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); in irdma_uk_inline_rdma_write()
751 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); in irdma_uk_inline_rdma_write()
755 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; in irdma_uk_inline_rdma_write()
770 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_inline_rdma_write()
776 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list, in irdma_uk_inline_rdma_write()
777 op_info->num_lo_sges, qp->swqe_polarity); in irdma_uk_inline_rdma_write()
784 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_inline_rdma_write()
786 irdma_uk_qp_post_wr(qp); in irdma_uk_inline_rdma_write()
798 irdma_uk_inline_send(struct irdma_qp_uk *qp, in irdma_uk_inline_send() argument
809 info->push_wqe = qp->push_db ? true : false; in irdma_uk_inline_send()
812 if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges)) in irdma_uk_inline_send()
818 if (unlikely(total_size > qp->max_inline_data)) in irdma_uk_inline_send()
821 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); in irdma_uk_inline_send()
822 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); in irdma_uk_inline_send()
845 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_inline_send()
850 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list, in irdma_uk_inline_send()
851 op_info->num_sges, qp->swqe_polarity); in irdma_uk_inline_send()
858 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_inline_send()
860 irdma_uk_qp_post_wr(qp); in irdma_uk_inline_send()
872 irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, in irdma_uk_stag_local_invalidate() argument
884 info->push_wqe = qp->push_db ? true : false; in irdma_uk_stag_local_invalidate()
888 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info); in irdma_uk_stag_local_invalidate()
893 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0); in irdma_uk_stag_local_invalidate()
902 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); in irdma_uk_stag_local_invalidate()
909 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); in irdma_uk_stag_local_invalidate()
911 irdma_uk_qp_post_wr(qp); in irdma_uk_stag_local_invalidate()
922 irdma_uk_post_receive(struct irdma_qp_uk *qp, in irdma_uk_post_receive() argument
930 if (qp->max_rq_frag_cnt < info->num_sges) in irdma_uk_post_receive()
933 wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx); in irdma_uk_post_receive()
937 qp->rq_wrid_array[wqe_idx] = info->wr_id; in irdma_uk_post_receive()
939 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, info->sg_list, in irdma_uk_post_receive()
940 qp->rwqe_polarity); in irdma_uk_post_receive()
943 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i], in irdma_uk_post_receive()
944 qp->rwqe_polarity); in irdma_uk_post_receive()
949 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) && in irdma_uk_post_receive()
951 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, in irdma_uk_post_receive()
952 qp->rwqe_polarity); in irdma_uk_post_receive()
953 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) in irdma_uk_post_receive()
959 FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity); in irdma_uk_post_receive()
1050 irdma_check_rq_cqe(struct irdma_qp_uk *qp, u32 *array_idx) in irdma_check_rq_cqe() argument
1052 u32 exp_idx = (qp->last_rx_cmpl_idx + 1) % qp->rq_size; in irdma_check_rq_cqe()
1057 qp->last_rx_cmpl_idx = exp_idx; in irdma_check_rq_cqe()
1062 qp->last_rx_cmpl_idx = *array_idx; in irdma_check_rq_cqe()
1099 struct irdma_qp_uk *qp, in irdma_detect_unsignaled_cmpls() argument
1108 if (qp->sq_wrtrk_array[wqe_idx].signaled == 0) { in irdma_detect_unsignaled_cmpls()
1122 qp->sq_wrtrk_array[widx].wrid, qp->sq_wrtrk_array[widx].wr_len, in irdma_detect_unsignaled_cmpls()
1123 qp->sq_wrtrk_array[widx].quanta, qp->sq_wrtrk_array[widx].signaled); in irdma_detect_unsignaled_cmpls()
1124 wqe = qp->sq_base[widx].elem; in irdma_detect_unsignaled_cmpls()
1153 struct irdma_qp_uk *qp; in irdma_uk_cq_poll_cmpl() local
1231 qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; in irdma_uk_cq_poll_cmpl()
1261 if (!qp || qp->destroy_pending) { in irdma_uk_cq_poll_cmpl()
1266 info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp; in irdma_uk_cq_poll_cmpl()
1272 ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring, in irdma_uk_cq_poll_cmpl()
1273 qp->rq_flush_seen, in irdma_uk_cq_poll_cmpl()
1279 array_idx = wqe_idx / qp->rq_wqe_size_multiplier; in irdma_uk_cq_poll_cmpl()
1283 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) { in irdma_uk_cq_poll_cmpl()
1288 info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; in irdma_uk_cq_poll_cmpl()
1290 array_idx = qp->rq_ring.tail; in irdma_uk_cq_poll_cmpl()
1292 info->wr_id = qp->rq_wrid_array[array_idx]; in irdma_uk_cq_poll_cmpl()
1294 if (irdma_check_rq_cqe(qp, &array_idx)) { in irdma_uk_cq_poll_cmpl()
1295 info->wr_id = qp->rq_wrid_array[array_idx]; in irdma_uk_cq_poll_cmpl()
1297 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1); in irdma_uk_cq_poll_cmpl()
1310 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1); in irdma_uk_cq_poll_cmpl()
1312 qp->rq_flush_seen = true; in irdma_uk_cq_poll_cmpl()
1313 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) in irdma_uk_cq_poll_cmpl()
1314 qp->rq_flush_complete = true; in irdma_uk_cq_poll_cmpl()
1318 pring = &qp->rq_ring; in irdma_uk_cq_poll_cmpl()
1320 if (qp->first_sq_wq) { in irdma_uk_cq_poll_cmpl()
1321 if (wqe_idx + 1 >= qp->conn_wqes) in irdma_uk_cq_poll_cmpl()
1322 qp->first_sq_wq = false; in irdma_uk_cq_poll_cmpl()
1324 if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) { in irdma_uk_cq_poll_cmpl()
1335 qp->push_mode = false; in irdma_uk_cq_poll_cmpl()
1336 qp->push_dropped = true; in irdma_uk_cq_poll_cmpl()
1338 ret_code = irdma_skip_duplicate_flush_cmpl(qp->sq_ring, in irdma_uk_cq_poll_cmpl()
1339 qp->sq_flush_seen, in irdma_uk_cq_poll_cmpl()
1345 info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; in irdma_uk_cq_poll_cmpl()
1346 info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled; in irdma_uk_cq_poll_cmpl()
1348 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; in irdma_uk_cq_poll_cmpl()
1349 ret_code = irdma_detect_unsignaled_cmpls(cq, qp, info, wqe_idx); in irdma_uk_cq_poll_cmpl()
1353 IRDMA_RING_SET_TAIL(qp->sq_ring, in irdma_uk_cq_poll_cmpl()
1354 wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); in irdma_uk_cq_poll_cmpl()
1358 spin_lock_irqsave(qp->lock, flags); in irdma_uk_cq_poll_cmpl()
1359 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { in irdma_uk_cq_poll_cmpl()
1360 spin_unlock_irqrestore(qp->lock, flags); in irdma_uk_cq_poll_cmpl()
1370 tail = qp->sq_ring.tail; in irdma_uk_cq_poll_cmpl()
1371 sw_wqe = qp->sq_base[tail].elem; in irdma_uk_cq_poll_cmpl()
1376 IRDMA_RING_SET_TAIL(qp->sq_ring, in irdma_uk_cq_poll_cmpl()
1377 tail + qp->sq_wrtrk_array[tail].quanta); in irdma_uk_cq_poll_cmpl()
1379 info->wr_id = qp->sq_wrtrk_array[tail].wrid; in irdma_uk_cq_poll_cmpl()
1380 info->signaled = qp->sq_wrtrk_array[tail].signaled; in irdma_uk_cq_poll_cmpl()
1381 info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; in irdma_uk_cq_poll_cmpl()
1389 qp->sq_flush_seen = true; in irdma_uk_cq_poll_cmpl()
1390 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) in irdma_uk_cq_poll_cmpl()
1391 qp->sq_flush_complete = true; in irdma_uk_cq_poll_cmpl()
1392 spin_unlock_irqrestore(qp->lock, flags); in irdma_uk_cq_poll_cmpl()
1394 pring = &qp->sq_ring; in irdma_uk_cq_poll_cmpl()
1535 irdma_setup_connection_wqes(struct irdma_qp_uk *qp, in irdma_setup_connection_wqes() argument
1542 else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) in irdma_setup_connection_wqes()
1544 qp->conn_wqes = move_cnt; in irdma_setup_connection_wqes()
1545 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); in irdma_setup_connection_wqes()
1546 IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); in irdma_setup_connection_wqes()
1547 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); in irdma_setup_connection_wqes()
1636 irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) in irdma_uk_qp_init() argument
1641 qp->uk_attrs = info->uk_attrs; in irdma_uk_qp_init()
1642 if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || in irdma_uk_qp_init()
1643 info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) in irdma_uk_qp_init()
1646 qp->qp_caps = info->qp_caps; in irdma_uk_qp_init()
1647 qp->sq_base = info->sq; in irdma_uk_qp_init()
1648 qp->rq_base = info->rq; in irdma_uk_qp_init()
1649 qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP; in irdma_uk_qp_init()
1650 qp->shadow_area = info->shadow_area; in irdma_uk_qp_init()
1651 qp->sq_wrtrk_array = info->sq_wrtrk_array; in irdma_uk_qp_init()
1653 qp->rq_wrid_array = info->rq_wrid_array; in irdma_uk_qp_init()
1654 qp->wqe_alloc_db = info->wqe_alloc_db; in irdma_uk_qp_init()
1655 qp->last_rx_cmpl_idx = 0xffffffff; in irdma_uk_qp_init()
1656 qp->rd_fence_rate = info->rd_fence_rate; in irdma_uk_qp_init()
1657 qp->qp_id = info->qp_id; in irdma_uk_qp_init()
1658 qp->sq_size = info->sq_size; in irdma_uk_qp_init()
1659 qp->push_mode = false; in irdma_uk_qp_init()
1660 qp->max_sq_frag_cnt = info->max_sq_frag_cnt; in irdma_uk_qp_init()
1661 sq_ring_size = qp->sq_size << info->sq_shift; in irdma_uk_qp_init()
1662 IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); in irdma_uk_qp_init()
1663 IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); in irdma_uk_qp_init()
1665 irdma_setup_connection_wqes(qp, info); in irdma_uk_qp_init()
1666 qp->swqe_polarity = 1; in irdma_uk_qp_init()
1667 qp->first_sq_wq = true; in irdma_uk_qp_init()
1669 qp->swqe_polarity = 0; in irdma_uk_qp_init()
1671 qp->swqe_polarity_deferred = 1; in irdma_uk_qp_init()
1672 qp->rwqe_polarity = 0; in irdma_uk_qp_init()
1673 qp->rq_size = info->rq_size; in irdma_uk_qp_init()
1674 qp->max_rq_frag_cnt = info->max_rq_frag_cnt; in irdma_uk_qp_init()
1675 qp->max_inline_data = info->max_inline_data; in irdma_uk_qp_init()
1676 qp->rq_wqe_size = info->rq_shift; in irdma_uk_qp_init()
1677 IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); in irdma_uk_qp_init()
1678 qp->rq_wqe_size_multiplier = 1 << info->rq_shift; in irdma_uk_qp_init()
1679 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) in irdma_uk_qp_init()
1680 qp->wqe_ops = iw_wqe_uk_ops_gen_1; in irdma_uk_qp_init()
1682 qp->wqe_ops = iw_wqe_uk_ops; in irdma_uk_qp_init()
1683 qp->start_wqe_idx = info->start_wqe_idx; in irdma_uk_qp_init()