49 	CQ_OK					=  0,
55 	MLX5_CQ_MODIFY_RESEIZE = 0,
68 	return (cqe->l4_hdr_type_etc >> 2) & 0x3;  in get_cqe_l3_hdr_type()
103 	cq->dbrec[MLX5_CQ_SET_CI] = htobe32(cq->cons_index & 0xffffff);  in update_cons_index()
191 	int err = 0;  in handle_responder()
251 	wc->sl		   = (be32toh(cqe->flags_rqpn) >> 24) & 0xf;  in handle_responder()
252 	wc->src_qp	   = be32toh(cqe->flags_rqpn) & 0xffffff;  in handle_responder()
253 	wc->dlid_path_bits = cqe->ml_path & 0x7f;  in handle_responder()
255 	wc->wc_flags |= g ? IBV_WC_GRH : 0;  in handle_responder()
256 	wc->pkey_index     = be32toh(cqe->imm_inval_pkey) & 0xffff;  in handle_responder()
266 	for (i = 0; i < 16; i += 4)  in dump_cqe()
320 	for (i = 0; i < mlx5_stall_num_loop; i++)  in mlx5_stall_poll_cq()
497 			mlx5_dbg(fp, MLX5_DBG_CQ_CQE, "dump cqe for cqn 0x%x:\n", cq->cqn);  in mlx5_get_next_cqe()
531 	int err = 0;  in mlx5_parse_cqe()
534 	uint8_t is_srq = 0;  in mlx5_parse_cqe()
537 	qpn = be32toh(cqe64->sop_drop_qpn) & 0xffffff;  in mlx5_parse_cqe()
542 		wc->wc_flags = 0;  in mlx5_parse_cqe()
551 				      (cqe_ver ? (be32toh(cqe64->srqn_uidx) & 0xffffff) : qpn),  in mlx5_parse_cqe()
606 		srqn_uidx = be32toh(cqe64->srqn_uidx) & 0xffffff;  in mlx5_parse_cqe()
624 		srqn_uidx = be32toh(cqe64->srqn_uidx) & 0xffffff;  in mlx5_parse_cqe()
726 	return mlx5_parse_cqe(cq, cqe64, cqe, cur_rsc, cur_srq, wc, cqe_ver, 0);  in mlx5_poll_one()
746 			cq->stall_next_poll = 0;  in poll_cq()
753 	for (npolled = 0; npolled < ne; ++npolled) {  in poll_cq()
765 			if (npolled == 0) {  in poll_cq()
776 				cq->stall_last_count = 0;  in poll_cq()
818 				cq->stall_last_count = 0;  in _mlx5_end_poll()
847 			cq->stall_next_poll = 0;  in mlx5_start_poll()
887 			cq->stall_last_count = 0;  in mlx5_start_poll()
921 	return mlx5_next_poll(ibcq, POLLING_MODE_STALL_ADAPTIVE, 0);  in mlx5_next_poll_adaptive_v0()
931 	return mlx5_next_poll(ibcq, 0, 0);  in mlx5_next_poll_v0()
936 	return mlx5_next_poll(ibcq, 0, 1);  in mlx5_next_poll_v1()
942 	return mlx5_start_poll(ibcq, attr, 0, 0, 0);  in mlx5_start_poll_v0()
948 	return mlx5_start_poll(ibcq, attr, 0, 0, 1);  in mlx5_start_poll_v1()
954 	return mlx5_start_poll(ibcq, attr, 1, 0, 0);  in mlx5_start_poll_v0_lock()
960 	return mlx5_start_poll(ibcq, attr, 1, 0, 1);  in mlx5_start_poll_v1_lock()
966 	return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL_ADAPTIVE, 0);  in mlx5_start_poll_adaptive_stall_v0_lock()
972 	return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL, 0);  in mlx5_start_poll_stall_v0_lock()
990 	return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL, 0);  in mlx5_start_poll_stall_v0()
996 	return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL_ADAPTIVE, 0);  in mlx5_start_poll_adaptive_stall_v0()
1002 	return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL_ADAPTIVE, 1);  in mlx5_start_poll_adaptive_stall_v1()
1008 	return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL, 1);  in mlx5_start_poll_stall_v1()
1023 	_mlx5_end_poll(ibcq, 0, POLLING_MODE_STALL_ADAPTIVE);  in mlx5_end_poll_adaptive_stall()
1028 	_mlx5_end_poll(ibcq, 0, POLLING_MODE_STALL);  in mlx5_end_poll_stall()
1033 	_mlx5_end_poll(ibcq, 0, 0);  in mlx5_end_poll()
1038 	_mlx5_end_poll(ibcq, 1, 0);  in mlx5_end_poll_lock()
1043 	return poll_cq(ibcq, ne, wc, 0);  in mlx5_poll_cq()
1091 	return 0;  in mlx5_cq_read_wc_opcode()
1098 	return be32toh(cq->cqe64->sop_drop_qpn) & 0xffffff;  in mlx5_cq_read_wc_qp_num()
1104 	int wc_flags = 0;  in mlx5_cq_read_wc_flags()
1123 	wc_flags |= ((be32toh(cq->cqe64->flags_rqpn) >> 28) & 3) ? IBV_WC_GRH : 0;  in mlx5_cq_read_wc_flags()
1165 	return (be32toh(cq->cqe64->flags_rqpn) >> 24) & 0xf;  in mlx5_cq_read_wc_sl()
1172 	return be32toh(cq->cqe64->flags_rqpn) & 0xffffff;  in mlx5_cq_read_wc_src_qp()
1179 	return cq->cqe64->ml_path & 0x7f;  in mlx5_cq_read_wc_dlid_path_bits()
1205 #define SINGLE_THREADED BIT(0)
1230 	[0] =  POLL_FN_ENTRY(_v0, _lock, , ),
1246 	const struct op *poll_ops = &ops[((cq->stall_enable && cq->stall_adaptive_enable) ? ADAPTIVE : 0) |  in mlx5_cq_fill_pfns()
1247 					 (mctx->cqe_version ? V1 : 0) |  in mlx5_cq_fill_pfns()
1249 						      SINGLE_THREADED : 0) |  in mlx5_cq_fill_pfns()
1250 					 (cq->stall_enable ? STALL : 0)];  in mlx5_cq_fill_pfns()
1291 	ci  = cq->cons_index & 0xffffff;  in mlx5_arm_cq()
1302 	doorbell[0] = htobe32(sn << 28 | cmd | ci);  in mlx5_arm_cq()
1305 	mlx5_write64(doorbell, ctx->uar[0] + MLX5_CQ_DOORBELL, &ctx->lock32);  in mlx5_arm_cq()
1309 	return 0;  in mlx5_arm_cq()
1319 	return rsn == (be32toh(cqe64->sop_drop_qpn) & 0xffffff);  in is_equal_rsn()
1324 	return uidx == (be32toh(cqe64->srqn_uidx) & 0xffffff);  in is_equal_uidx()
1338 	return 0;  in is_responder()
1353 			if (srq && (be32toh(cqe64->srqn_uidx) & 0xffffff))  in free_res_cqe()
1360 	return 0;  in free_res_cqe()
1366 	int nfreed = 0;  in __mlx5_cq_clean()
1391 	while ((int) --prod_index - (int) cq->cons_index >= 0) {  in __mlx5_cq_clean()
1426 	return (n & nent) ? 1 : 0;  in sw_ownership_bit()
1505 	memset(buf->buf, 0, nent * cqe_sz);  in mlx5_alloc_cq_buf()
1507 	for (i = 0; i < nent; ++i) {  in mlx5_alloc_cq_buf()
1509 		cqe += cqe_sz == 128 ? 1 : 0;  in mlx5_alloc_cq_buf()
1513 	return 0;  in mlx5_alloc_cq_buf()