3063 	void *seg = eseg;  in set_eth_seg()  local
3071 	seg += sizeof(struct mlx5_wqe_eth_seg);  in set_eth_seg()
3092 		memcpy(seg - size_of_inl_hdr_start, pdata, copysz);  in set_eth_seg()
3095 			seg += ALIGN(copysz - size_of_inl_hdr_start, 16);  in set_eth_seg()
3100 			seg = mlx5_get_send_wqe(qp, 0);  in set_eth_seg()
3103 			memcpy(seg, pdata, left);  in set_eth_seg()
3104 			seg += ALIGN(left, 16);  in set_eth_seg()
3109 	return seg;  in set_eth_seg()
3315 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,  in set_reg_mkey_seg()  argument
3321 	memset(seg, 0, sizeof(*seg));  in set_reg_mkey_seg()
3324 		seg->log2_page_size = ilog2(mr->ibmr.page_size);  in set_reg_mkey_seg()
3329 	seg->flags = get_umr_flags(access) | mr->access_mode;  in set_reg_mkey_seg()
3330 	seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);  in set_reg_mkey_seg()
3331 	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);  in set_reg_mkey_seg()
3332 	seg->start_addr = cpu_to_be64(mr->ibmr.iova);  in set_reg_mkey_seg()
3333 	seg->len = cpu_to_be64(mr->ibmr.length);  in set_reg_mkey_seg()
3334 	seg->xlt_oct_size = cpu_to_be32(ndescs);  in set_reg_mkey_seg()
3337 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)  in set_linv_mkey_seg()  argument
3339 	memset(seg, 0, sizeof(*seg));  in set_linv_mkey_seg()
3340 	seg->status = MLX5_MKEY_STATUS_FREE;  in set_linv_mkey_seg()
3343 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, const struct ib_send_wr *wr)  in set_reg_mkey_segment()  argument
3347 	memset(seg, 0, sizeof(*seg));  in set_reg_mkey_segment()
3349 		seg->status = MLX5_MKEY_STATUS_FREE;  in set_reg_mkey_segment()
3353 	seg->flags = convert_access(umrwr->access_flags);  in set_reg_mkey_segment()
3356 			seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);  in set_reg_mkey_segment()
3357 		seg->start_addr = cpu_to_be64(umrwr->target.virt_addr);  in set_reg_mkey_segment()
3359 	seg->len = cpu_to_be64(umrwr->length);  in set_reg_mkey_segment()
3360 	seg->log2_page_size = umrwr->page_shift;  in set_reg_mkey_segment()
3361 	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |  in set_reg_mkey_segment()
3411 	struct mlx5_wqe_inline_seg *seg;  in set_data_inl_seg()  local
3419 	seg = wqe;  in set_data_inl_seg()
3420 	wqe += sizeof(*seg);  in set_data_inl_seg()
3440 	seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);  in set_data_inl_seg()
3442 	*sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;  in set_data_inl_seg()
3555 				struct mlx5_ib_qp *qp, void **seg, int *size)  in set_sig_data_segment()  argument
3580 		struct mlx5_klm *data_klm = *seg;  in set_sig_data_segment()
3608 		sblock_ctrl = *seg;  in set_sig_data_segment()
3637 	*seg += wqe_size;  in set_sig_data_segment()
3639 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_data_segment()
3640 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_data_segment()
3642 	bsf = *seg;  in set_sig_data_segment()
3647 	*seg += sizeof(*bsf);  in set_sig_data_segment()
3649 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_data_segment()
3650 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_data_segment()
3655 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,  in set_sig_mkey_segment()  argument
3663 	memset(seg, 0, sizeof(*seg));  in set_sig_mkey_segment()
3665 	seg->flags = get_umr_flags(wr->access_flags) |  in set_sig_mkey_segment()
3667 	seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);  in set_sig_mkey_segment()
3668 	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |  in set_sig_mkey_segment()
3670 	seg->len = cpu_to_be64(length);  in set_sig_mkey_segment()
3671 	seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements)));  in set_sig_mkey_segment()
3672 	seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);  in set_sig_mkey_segment()
3688 			  void **seg, int *size)  in set_sig_umr_wr()  argument
3717 	set_sig_umr_segment(*seg, klm_oct_size);  in set_sig_umr_wr()
3718 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in set_sig_umr_wr()
3720 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_umr_wr()
3721 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_umr_wr()
3723 	set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);  in set_sig_umr_wr()
3724 	*seg += sizeof(struct mlx5_mkey_seg);  in set_sig_umr_wr()
3726 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_umr_wr()
3727 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_umr_wr()
3729 	ret = set_sig_data_segment(wr, qp, seg, size);  in set_sig_umr_wr()
3738 		      u32 psv_idx, void **seg, int *size)  in set_psv_wr()  argument
3740 	struct mlx5_seg_set_psv *psv_seg = *seg;  in set_psv_wr()
3757 	*seg += sizeof(*psv_seg);  in set_psv_wr()
3765 		      void **seg, int *size)  in set_reg_wr()  argument
3776 	set_reg_umr_seg(*seg, mr);  in set_reg_wr()
3777 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in set_reg_wr()
3779 	if (unlikely((*seg == qp->sq.qend)))  in set_reg_wr()
3780 		*seg = mlx5_get_send_wqe(qp, 0);  in set_reg_wr()
3782 	set_reg_mkey_seg(*seg, mr, wr->key, wr->access);  in set_reg_wr()
3783 	*seg += sizeof(struct mlx5_mkey_seg);  in set_reg_wr()
3785 	if (unlikely((*seg == qp->sq.qend)))  in set_reg_wr()
3786 		*seg = mlx5_get_send_wqe(qp, 0);  in set_reg_wr()
3788 	set_reg_data_seg(*seg, mr, pd);  in set_reg_wr()
3789 	*seg += sizeof(struct mlx5_wqe_data_seg);  in set_reg_wr()
3795 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size)  in set_linv_wr()  argument
3797 	set_linv_umr_seg(*seg);  in set_linv_wr()
3798 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in set_linv_wr()
3800 	if (unlikely((*seg == qp->sq.qend)))  in set_linv_wr()
3801 		*seg = mlx5_get_send_wqe(qp, 0);  in set_linv_wr()
3802 	set_linv_mkey_seg(*seg);  in set_linv_wr()
3803 	*seg += sizeof(struct mlx5_mkey_seg);  in set_linv_wr()
3805 	if (unlikely((*seg == qp->sq.qend)))  in set_linv_wr()
3806 		*seg = mlx5_get_send_wqe(qp, 0);  in set_linv_wr()
3847 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,  in begin_wqe()  argument
3856 	*seg = mlx5_get_send_wqe(qp, *idx);  in begin_wqe()
3857 	*ctrl = *seg;  in begin_wqe()
3858 	*(uint32_t *)(*seg + 8) = 0;  in begin_wqe()
3866 	*seg += sizeof(**ctrl);  in begin_wqe()
3913 	void *seg;  in mlx5_ib_post_send()  local
3952 		err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq, wr->send_flags);  in mlx5_ib_post_send()
3962 			xrc = seg;  in mlx5_ib_post_send()
3963 			seg += sizeof(*xrc);  in mlx5_ib_post_send()
3971 				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,  in mlx5_ib_post_send()
3973 				seg += sizeof(struct mlx5_wqe_raddr_seg);  in mlx5_ib_post_send()
3989 				set_linv_wr(qp, &seg, &size);  in mlx5_ib_post_send()
3997 				err = set_reg_wr(qp, reg_wr(wr), &seg, &size);  in mlx5_ib_post_send()
4010 				err = set_sig_umr_wr(wr, qp, &seg, &size);  in mlx5_ib_post_send()
4024 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
4034 						 mr->sig->psv_memory.psv_idx, &seg,  in mlx5_ib_post_send()
4045 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
4056 						 mr->sig->psv_wire.psv_idx, &seg,  in mlx5_ib_post_send()
4079 				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,  in mlx5_ib_post_send()
4081 				seg  += sizeof(struct mlx5_wqe_raddr_seg);  in mlx5_ib_post_send()
4092 			set_datagram_seg(seg, wr);  in mlx5_ib_post_send()
4093 			seg += sizeof(struct mlx5_wqe_datagram_seg);  in mlx5_ib_post_send()
4095 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
4096 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4099 			set_datagram_seg(seg, wr);  in mlx5_ib_post_send()
4100 			seg += sizeof(struct mlx5_wqe_datagram_seg);  in mlx5_ib_post_send()
4103 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
4104 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4110 				pad = seg;  in mlx5_ib_post_send()
4112 				seg += sizeof(struct mlx5_wqe_eth_pad);  in mlx5_ib_post_send()
4115 				seg = set_eth_seg(seg, wr, qend, qp, &size);  in mlx5_ib_post_send()
4117 				if (unlikely((seg == qend)))  in mlx5_ib_post_send()
4118 					seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4129 			set_reg_umr_segment(seg, wr);  in mlx5_ib_post_send()
4130 			seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in mlx5_ib_post_send()
4132 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
4133 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4134 			set_reg_mkey_segment(seg, wr);  in mlx5_ib_post_send()
4135 			seg += sizeof(struct mlx5_mkey_seg);  in mlx5_ib_post_send()
4137 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
4138 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4148 			err = set_data_inl_seg(qp, wr, seg, &sz);  in mlx5_ib_post_send()
4156 			dpseg = seg;  in mlx5_ib_post_send()
4159 					seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
4160 					dpseg = seg;  in mlx5_ib_post_send()