xref: /illumos-gate/usr/src/lib/udapl/udapl_tavor/tavor/dapl_tavor_ibtf_qp.c (revision 1ed53a3f65abecaadc1b967e341970ad0f6b2aeb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include "dapl.h"
28 #include "dapl_adapter_util.h"
29 #include "dapl_evd_util.h"
30 #include "dapl_cr_util.h"
31 #include "dapl_lmr_util.h"
32 #include "dapl_rmr_util.h"
33 #include "dapl_cookie.h"
34 #include "dapl_tavor_ibtf_impl.h"
35 #include "dapl_hash.h"
36 
37 /* Function prototypes */
38 extern DAT_RETURN dapls_tavor_wrid_init(ib_qp_handle_t);
39 extern DAT_RETURN dapls_tavor_srq_wrid_init(ib_srq_handle_t);
40 extern void dapls_tavor_srq_wrid_free(ib_srq_handle_t);
41 extern DAT_BOOLEAN dapls_tavor_srq_wrid_resize(ib_srq_handle_t, uint32_t);
42 
43 static DAT_RETURN dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr,
44     IN uint32_t qpnum, IN DAPL_EP *ep_ptr);
45 static void dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr,
46     IN uint32_t qpnum);
47 static DAT_RETURN dapli_ib_srq_resize_internal(IN DAPL_SRQ *srq_ptr,
48     IN DAT_COUNT srqlen);
49 /*
50  * dapli_get_dto_cq
51  *
52  * Obtain the cq_handle for a DTO EVD. If the EVD is NULL, use the
53  * null_ib_cq_handle. If it hasn't been created yet, create it now in
54  * the HCA structure. It will be cleaned up in dapls_ib_cqd_destroy().
55  *
56  * This is strictly internal to IB. DAPL allows a NULL DTO EVD handle,
57  * but IB does not. So we create a CQ under the hood and make sure
58  * an error is generated if the user every tries to post, by
59  * setting the WQ length to 0 in ep_create and/or ep_modify.
60  *
61  * Returns
62  *	A valid CQ handle
63  */
64 static ib_cq_handle_t
dapli_get_dto_cq(IN DAPL_IA * ia_ptr,IN DAPL_EVD * evd_ptr)65 dapli_get_dto_cq(
66 	IN  DAPL_IA	*ia_ptr,
67 	IN  DAPL_EVD	*evd_ptr)
68 {
69 	dapl_evd_create_t	create_msg;
70 	ib_cq_handle_t		cq_handle;
71 	int			ia_fd;
72 	int			retval;
73 	mlnx_umap_cq_data_out_t	*mcq;
74 
75 	if (evd_ptr != DAT_HANDLE_NULL) {
76 		cq_handle = evd_ptr->ib_cq_handle;
77 	} else if (ia_ptr->hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) {
78 		cq_handle = ia_ptr->hca_ptr->null_ib_cq_handle;
79 	} else {
80 		cq_handle = (ib_cq_handle_t)
81 		    dapl_os_alloc(sizeof (struct dapls_ib_cq_handle));
82 		if (cq_handle == NULL) {
83 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
84 			    "dapli_get_dto_cq: cq malloc failed\n");
85 			ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
86 			return (IB_INVALID_HANDLE);
87 		}
88 
89 		/*
90 		 * create a fake a CQ, we don't bother to mmap this CQ
91 		 * since nobody know about it to reap events from it.
92 		 */
93 		(void) dapl_os_memzero(&create_msg, sizeof (create_msg));
94 		create_msg.evd_flags = DAT_EVD_DTO_FLAG;
95 		mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out;
96 
97 		ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
98 
99 		/* call into driver to allocate cq */
100 		retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg);
101 		if (retval != 0) {
102 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
103 			    "dapli_get_dto_cq: DAPL_EVD_CREATE failed\n");
104 			dapl_os_free(cq_handle,
105 			    sizeof (struct dapls_ib_cq_handle));
106 			ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
107 			return (IB_INVALID_HANDLE);
108 		}
109 
110 		(void) dapl_os_memzero(cq_handle,
111 		    sizeof (struct dapls_ib_cq_handle));
112 		dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock);
113 		cq_handle->evd_hkey = create_msg.evd_hkey;
114 		cq_handle->cq_addr = NULL;
115 		cq_handle->cq_map_offset = mcq->mcq_mapoffset;
116 		cq_handle->cq_map_len = mcq->mcq_maplen;
117 		cq_handle->cq_num = mcq->mcq_cqnum;
118 		cq_handle->cq_size = create_msg.evd_cq_real_size;
119 		cq_handle->cq_cqesz = mcq->mcq_cqesz;
120 		cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
121 
122 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
123 		    "dapli_get_dto_cq: cq 0x%p created, hkey 0x%016llx\n",
124 		    cq_handle, create_msg.evd_hkey);
125 
126 		/* save this dummy CQ handle into the hca */
127 		ia_ptr->hca_ptr->null_ib_cq_handle = cq_handle;
128 	}
129 	return (cq_handle);
130 }
131 
132 
133 /*
134  * dapl_ib_qp_alloc
135  *
136  * Alloc a QP
137  *
138  * Input:
139  *        *ep_ptr                pointer to EP INFO
140  *        ib_hca_handle          provider HCA handle
141  *        ib_pd_handle           provider protection domain handle
142  *        cq_recv                provider recv CQ handle
143  *        cq_send                provider send CQ handle
144  *
145  * Output:
146  *        none
147  *
148  * Returns:
149  *        DAT_SUCCESS
150  *        DAT_INSUFFICIENT_RESOURCES
151  *
152  */
153 DAT_RETURN
dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr,IN DAPL_EP * ep_ctx_ptr)154 dapls_ib_qp_alloc(
155 	IN DAPL_IA *ia_ptr,
156 	IN DAPL_EP *ep_ptr,
157 	IN DAPL_EP *ep_ctx_ptr)
158 {
159 	dapl_ep_create_t	ep_args;
160 	dapl_ep_free_t		epf_args;
161 	ib_qp_handle_t		qp_p;
162 	DAPL_SRQ		*srq_p;
163 	ib_cq_handle_t		cq_recv;
164 	ib_cq_handle_t		cq_send;
165 	DAPL_PZ			*pz_handle;
166 	DAPL_EVD		*evd_handle;
167 	uint32_t		mpt_mask;
168 	size_t			premev_size;
169 	uint32_t		i;
170 	int			ia_fd;
171 	int			hca_fd;
172 	DAT_RETURN		dat_status;
173 	int			retval;
174 	mlnx_umap_qp_data_out_t *mqp;
175 
176 	/* check parameters */
177 	if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
178 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
179 		    "qp_alloc: hca_handle == NULL\n");
180 		return (DAT_INVALID_PARAMETER);
181 	}
182 
183 	ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
184 	hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
185 	dapl_os_assert(ep_ptr->param.pz_handle != NULL);
186 	dapl_os_assert(ep_ptr->param.connect_evd_handle != NULL);
187 
188 	/* fill in args for ep_create */
189 	(void) dapl_os_memzero(&ep_args, sizeof (ep_args));
190 	mqp = (mlnx_umap_qp_data_out_t *)ep_args.ep_qp_data_out;
191 	pz_handle = (DAPL_PZ *)ep_ptr->param.pz_handle;
192 	ep_args.ep_pd_hkey = pz_handle->pd_handle->pd_hkey;
193 
194 	cq_recv = dapli_get_dto_cq(ia_ptr,
195 	    (DAPL_EVD *)ep_ptr->param.recv_evd_handle);
196 	ep_args.ep_rcv_evd_hkey = cq_recv->evd_hkey;
197 
198 	cq_send = dapli_get_dto_cq(ia_ptr,
199 	    (DAPL_EVD *)ep_ptr->param.request_evd_handle);
200 	ep_args.ep_snd_evd_hkey = cq_send->evd_hkey;
201 
202 	evd_handle = (DAPL_EVD *)ep_ptr->param.connect_evd_handle;
203 	ep_args.ep_conn_evd_hkey = evd_handle->ib_cq_handle->evd_hkey;
204 
205 	ep_args.ep_ch_sizes.dcs_sq = ep_ptr->param.ep_attr.max_request_dtos;
206 	ep_args.ep_ch_sizes.dcs_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
207 
208 	qp_p = (ib_qp_handle_t)dapl_os_alloc(
209 	    sizeof (struct dapls_ib_qp_handle));
210 	if (qp_p == NULL) {
211 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
212 		    "qp_alloc: os_alloc failed\n");
213 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
214 		    DAT_RESOURCE_MEMORY));
215 	}
216 
217 	(void) dapl_os_memzero(qp_p, sizeof (*qp_p));
218 
219 	if (ep_ptr->param.srq_handle == NULL) {
220 		premev_size = ep_ptr->param.ep_attr.max_recv_dtos *
221 		    sizeof (ib_work_completion_t);
222 		if (premev_size != 0) {
223 			qp_p->qp_premature_events = (ib_work_completion_t *)
224 			    dapl_os_alloc(premev_size);
225 			if (qp_p->qp_premature_events == NULL) {
226 				dapl_dbg_log(DAPL_DBG_TYPE_EP,
227 				    "qp_alloc:alloc premature_events failed\n");
228 				dapl_os_free(qp_p, sizeof (*qp_p));
229 				return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
230 				    DAT_RESOURCE_MEMORY));
231 			}
232 		}
233 		qp_p->qp_num_premature_events = 0;
234 		ep_args.ep_srq_hkey = 0;
235 		ep_args.ep_srq_attached = 0;
236 		ep_args.ep_ch_sizes.dcs_rq =
237 		    ep_ptr->param.ep_attr.max_recv_dtos;
238 		ep_args.ep_ch_sizes.dcs_rq_sgl =
239 		    ep_ptr->param.ep_attr.max_recv_iov;
240 	} else {
241 		premev_size = 0;
242 		srq_p = (DAPL_SRQ *)ep_ptr->param.srq_handle;
243 		/* premature events for EPs with SRQ sit on the SRQ */
244 		qp_p->qp_premature_events = srq_p->srq_handle->
245 		    srq_premature_events;
246 		qp_p->qp_num_premature_events = 0;
247 		ep_args.ep_srq_hkey = srq_p->srq_handle->srq_hkey;
248 		ep_args.ep_srq_attached = 1;
249 		ep_args.ep_ch_sizes.dcs_rq = 0;
250 		ep_args.ep_ch_sizes.dcs_rq_sgl = 0;
251 	}
252 
253 	/*
254 	 * there are cases when ep_ptr is a dummy container ep, and the orig
255 	 * ep pointer is passed in ep_ctx_ptr. eg - dapl_ep_modify does this.
256 	 * ep_cookie should be the actual ep pointer, not the dummy container
257 	 * ep since the kernel returns this via events and the CM callback
258 	 * routines
259 	 */
260 	ep_args.ep_cookie = (uintptr_t)ep_ctx_ptr;
261 
262 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
263 	    "qp_alloc: ep_ptr 0x%p, pz 0x%p (0x%llx), rcv_evd 0x%p (0x%llx)\n"
264 	    "          snd_evd 0x%p (0x%llx), conn_evd 0x%p (0x%llx)\n"
265 	    "          srq_hdl 0x%p (0x%llx)\n"
266 	    "          sq_sz %d, rq_sz %d, sq_sgl_sz %d, rq_sgl_sz %d\n",
267 	    ep_ptr, pz_handle, ep_args.ep_pd_hkey,
268 	    ep_ptr->param.recv_evd_handle, ep_args.ep_rcv_evd_hkey,
269 	    ep_ptr->param.request_evd_handle, ep_args.ep_snd_evd_hkey,
270 	    ep_ptr->param.connect_evd_handle, ep_args.ep_conn_evd_hkey,
271 	    ep_ptr->param.srq_handle, ep_args.ep_srq_hkey,
272 	    ep_args.ep_ch_sizes.dcs_sq, ep_args.ep_ch_sizes.dcs_rq,
273 	    ep_args.ep_ch_sizes.dcs_sq_sgl, ep_args.ep_ch_sizes.dcs_rq_sgl);
274 
275 	/* The next line is only needed for backward compatibility */
276 	mqp->mqp_rev = MLNX_UMAP_IF_VERSION;
277 	retval = ioctl(ia_fd, DAPL_EP_CREATE, &ep_args);
278 	if (retval != 0 || mqp->mqp_rev != MLNX_UMAP_IF_VERSION) {
279 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
280 		    "qp_alloc: ep_create failed errno %d, retval %d\n",
281 		    errno, retval);
282 		if (premev_size != 0) {
283 			dapl_os_free(qp_p->qp_premature_events, premev_size);
284 		}
285 		dapl_os_free(qp_p, sizeof (*qp_p));
286 		return (dapls_convert_error(errno, retval));
287 	}
288 
289 	/* In the case of Arbel or Hermon */
290 	if (mqp->mqp_sdbr_mapoffset != 0 || mqp->mqp_sdbr_maplen != 0)
291 		qp_p->qp_sq_dbp = dapls_ib_get_dbp(mqp->mqp_sdbr_maplen,
292 		    hca_fd, mqp->mqp_sdbr_mapoffset, mqp->mqp_sdbr_offset);
293 	if (mqp->mqp_rdbr_mapoffset != 0 || mqp->mqp_rdbr_maplen != 0)
294 		qp_p->qp_rq_dbp = dapls_ib_get_dbp(mqp->mqp_rdbr_maplen,
295 		    hca_fd, mqp->mqp_rdbr_mapoffset, mqp->mqp_rdbr_offset);
296 
297 	qp_p->qp_addr = mmap64((void *)0, mqp->mqp_maplen,
298 	    (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd,
299 	    mqp->mqp_mapoffset);
300 
301 	if (qp_p->qp_addr == MAP_FAILED ||
302 	    qp_p->qp_sq_dbp == MAP_FAILED ||
303 	    qp_p->qp_rq_dbp == MAP_FAILED) {
304 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
305 		    "qp_alloc: mmap failed(%d)\n", errno);
306 		epf_args.epf_hkey = ep_args.ep_hkey;
307 		retval = ioctl(ia_fd, DAPL_EP_FREE, &epf_args);
308 		if (retval != 0) {
309 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
310 			    "qp_alloc: EP_FREE err:%d\n", errno);
311 		}
312 		if (premev_size != 0) {
313 			dapl_os_free(qp_p->qp_premature_events, premev_size);
314 		}
315 		dapl_os_free(qp_p, sizeof (*qp_p));
316 		return (dapls_convert_error(errno, 0));
317 	}
318 
319 	qp_p->qp_map_len = mqp->mqp_maplen;
320 	qp_p->qp_num = mqp->mqp_qpnum;
321 	qp_p->qp_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
322 	qp_p->qp_ia_bf = ia_ptr->hca_ptr->ib_hca_handle->ia_bf;
323 	qp_p->qp_ia_bf_toggle = ia_ptr->hca_ptr->ib_hca_handle->ia_bf_toggle;
324 
325 	evd_handle = (DAPL_EVD *)ep_ptr->param.request_evd_handle;
326 	qp_p->qp_sq_cqhdl = evd_handle->ib_cq_handle;
327 	qp_p->qp_sq_lastwqeaddr = NULL;
328 	qp_p->qp_sq_wqhdr = NULL;
329 	qp_p->qp_sq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_sq_off);
330 	qp_p->qp_sq_desc_addr = mqp->mqp_sq_desc_addr;
331 	qp_p->qp_sq_numwqe = mqp->mqp_sq_numwqe;
332 	qp_p->qp_sq_wqesz = mqp->mqp_sq_wqesz;
333 	qp_p->qp_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
334 	qp_p->qp_sq_inline = ia_ptr->hca_ptr->max_inline_send;
335 	qp_p->qp_sq_headroom = mqp->mqp_sq_headroomwqes;
336 
337 	evd_handle = (DAPL_EVD *)ep_ptr->param.recv_evd_handle;
338 	qp_p->qp_rq_cqhdl = evd_handle->ib_cq_handle;
339 	qp_p->qp_rq_lastwqeaddr = NULL;
340 	qp_p->qp_rq_wqhdr = NULL;
341 	qp_p->qp_rq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_rq_off);
342 	qp_p->qp_rq_desc_addr = mqp->mqp_rq_desc_addr;
343 	qp_p->qp_rq_numwqe = mqp->mqp_rq_numwqe;
344 	qp_p->qp_rq_wqesz = mqp->mqp_rq_wqesz;
345 	qp_p->qp_rq_sgl = ep_ptr->param.ep_attr.max_recv_iov;
346 
347 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
348 	    "qp_alloc: created, qp_sq_buf %p, qp_rq_buf %p\n",
349 	    qp_p->qp_sq_buf, qp_p->qp_rq_buf);
350 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
351 	    "qp_alloc: created, sq numwqe %x wqesz %x, rq numwqe %x wqesz %x\n",
352 	    qp_p->qp_sq_numwqe, qp_p->qp_sq_wqesz,
353 	    qp_p->qp_rq_numwqe, qp_p->qp_rq_wqesz);
354 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
355 	    "qp_alloc: created, qp_sq_desc_addr %x, qp_rq_desc_addr %x\n",
356 	    mqp->mqp_sq_desc_addr, mqp->mqp_rq_desc_addr);
357 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
358 	    "qp_alloc: created, ep_ptr 0x%p, ep_hkey 0x%016llx\n\n",
359 	    ep_ptr, ep_args.ep_hkey);
360 
361 	qp_p->ep_hkey = ep_args.ep_hkey;
362 
363 	/*
364 	 * Calculate the number of bits in max_rmrs - this is indirectly
365 	 * the max number of entried in the MPT table (defaults to 512K
366 	 * but is configurable). This value is used while creating new
367 	 * rkeys in bind processing (see dapl_tavor_hw.c).
368 	 * Stash this value in the qp handle, don't want to do this math
369 	 * for every bind
370 	 */
371 	mpt_mask = (uint32_t)ia_ptr->hca_ptr->ia_attr.max_rmrs - 1;
372 	for (i = 0; mpt_mask > 0; mpt_mask = (mpt_mask >> 1), i++)
373 		;
374 	qp_p->qp_num_mpt_shift = (uint32_t)i;
375 
376 	ep_ptr->qpn = qp_p->qp_num;
377 	/* update the qp handle in the ep ptr */
378 	ep_ptr->qp_handle = qp_p;
379 	/*
380 	 * ibt_alloc_rc_channel transitions the qp state to INIT.
381 	 * hence we directly transition from UNATTACHED to INIT
382 	 */
383 	ep_ptr->qp_state = IBT_STATE_INIT;
384 
385 	if (ep_ptr->param.srq_handle) {
386 		/* insert ep into the SRQ's ep_table */
387 		dat_status = dapli_ib_srq_add_ep(srq_p->srq_handle,
388 		    qp_p->qp_num, ep_ptr);
389 		if (dat_status != DAT_SUCCESS) {
390 			dapl_dbg_log(DAPL_DBG_TYPE_EP,
391 			    "qp_alloc: srq_add_ep failed ep_ptr 0x%p, 0x%x\n",
392 			    ep_ptr, dat_status);
393 			(void) dapls_ib_qp_free(ia_ptr, ep_ptr);
394 			return (DAT_INVALID_PARAMETER);
395 		}
396 		qp_p->qp_srq_enabled = 1;
397 		qp_p->qp_srq = srq_p->srq_handle;
398 	} else {
399 		qp_p->qp_srq_enabled = 0;
400 		qp_p->qp_srq = NULL;
401 	}
402 	DAPL_INIT_QP(ia_ptr)(qp_p);
403 
404 	if (dapls_tavor_wrid_init(qp_p) != DAT_SUCCESS) {
405 		(void) dapls_ib_qp_free(ia_ptr, ep_ptr);
406 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
407 		    DAT_RESOURCE_MEMORY));
408 	}
409 
410 	return (DAT_SUCCESS);
411 }
412 
413 
414 /*
415  * dapls_ib_qp_free
416  *
417  * Free a QP
418  *
419  * Input:
420  *        *ep_ptr                pointer to EP INFO
421  *        ib_hca_handle          provider HCA handle
422  *
423  * Output:
424  *        none
425  *
426  * Returns:
427  *        none
428  *
429  */
430 DAT_RETURN
dapls_ib_qp_free(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr)431 dapls_ib_qp_free(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr)
432 {
433 	ib_qp_handle_t	qp_p = ep_ptr->qp_handle;
434 	ib_hca_handle_t	ib_hca_handle = ia_ptr->hca_ptr->ib_hca_handle;
435 	dapl_ep_free_t	args;
436 	int		retval;
437 
438 	if ((ep_ptr->qp_handle != IB_INVALID_HANDLE) &&
439 	    (ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED)) {
440 		if (munmap((void *)qp_p->qp_addr, qp_p->qp_map_len) < 0) {
441 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
442 			    "qp_free: munmap failed(%d)\n", errno);
443 		}
444 		args.epf_hkey = qp_p->ep_hkey;
445 		retval = ioctl(ib_hca_handle->ia_fd, DAPL_EP_FREE, &args);
446 		if (retval != 0) {
447 			dapl_dbg_log(DAPL_DBG_TYPE_EP,
448 			    "qp_free: ioctl errno = %d, retval = %d\n",
449 			    errno, retval);
450 		}
451 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
452 		    "qp_free: freed, ep_ptr 0x%p, ep_hkey 0x%016llx\n",
453 		    ep_ptr, qp_p->ep_hkey);
454 
455 		if (qp_p->qp_srq) {
456 			dapli_ib_srq_remove_ep(qp_p->qp_srq, qp_p->qp_num);
457 		} else {
458 			if (qp_p->qp_premature_events) {
459 				dapl_os_free(qp_p->qp_premature_events,
460 				    ep_ptr->param.ep_attr.max_recv_dtos *
461 				    sizeof (ib_work_completion_t));
462 			}
463 		}
464 		dapl_os_free(qp_p, sizeof (*qp_p));
465 		ep_ptr->qp_handle = NULL;
466 	}
467 	return (DAT_SUCCESS);
468 }
469 
470 
471 /*
472  * dapl_ib_qp_modify
473  *
474  * Set the QP to the parameters specified in an EP_PARAM
475  *
476  * We can't be sure what state the QP is in so we first obtain the state
477  * from the driver. The EP_PARAM structure that is provided has been
478  * sanitized such that only non-zero values are valid.
479  *
480  * Input:
481  *        ib_hca_handle          HCA handle
482  *        qp_handle              QP handle
483  *        ep_attr                Sanitized EP Params
484  *
485  * Output:
486  *        none
487  *
488  * Returns:
489  *        DAT_SUCCESS
490  *        DAT_INSUFFICIENT_RESOURCES
491  *        DAT_INVALID_PARAMETER
492  *
493  */
494 DAT_RETURN
dapls_ib_qp_modify(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr,IN DAT_EP_ATTR * ep_attr)495 dapls_ib_qp_modify(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr,
496     IN DAT_EP_ATTR *ep_attr)
497 {
498 	dapl_ep_modify_t 	epm_args;
499 	boolean_t		epm_needed;
500 	int	ia_fd;
501 	int	retval;
502 
503 
504 	if (ep_ptr->qp_handle == NULL) {
505 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
506 		    "qp_modify: qp_handle == NULL\n");
507 		return (DAT_INVALID_PARAMETER);
508 	}
509 	if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
510 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
511 		    "qp_modify: hca_handle == NULL\n");
512 		return (DAT_INVALID_PARAMETER);
513 	}
514 
515 	epm_needed = B_FALSE;
516 
517 	/*
518 	 * NOTE: ep_attr->max_mtu_size  indicates the maximum message
519 	 * size, which is always 2GB for IB. Nothing to do with the IB
520 	 * implementation, nothing to set up.
521 	 */
522 
523 	if (ep_attr->max_rdma_size > 0) {
524 		if (ep_attr->max_rdma_size > DAPL_IB_MAX_MESSAGE_SIZE) {
525 			return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
526 		}
527 	}
528 
529 	(void) memset((void *)&epm_args, 0, sizeof (epm_args));
530 	/*
531 	 * The following parameters are dealt by creating a new qp
532 	 * in dapl_ep_modify.
533 	 *	- max_recv_dtos
534 	 *	- max_request_dtos
535 	 *	- max_recv_iov
536 	 *	- max_request_iov
537 	 */
538 
539 	if (ep_attr->max_rdma_read_in > 0) {
540 		epm_args.epm_flags |= IBT_CEP_SET_RDMARA_IN;
541 		epm_args.epm_rdma_ra_in = ep_attr->max_rdma_read_in;
542 		epm_needed = B_TRUE;
543 	}
544 	if (ep_attr->max_rdma_read_out > 0) {
545 		epm_args.epm_flags |= IBT_CEP_SET_RDMARA_OUT;
546 		epm_args.epm_rdma_ra_out = ep_attr->max_rdma_read_out;
547 		epm_needed = B_TRUE;
548 	}
549 
550 	if (!epm_needed) {
551 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
552 		    "qp_modify: ep_hkey = %016llx nothing to do\n",
553 		    ep_ptr->qp_handle->ep_hkey);
554 		return (DAT_SUCCESS);
555 	}
556 
557 	epm_args.epm_hkey = ep_ptr->qp_handle->ep_hkey;
558 
559 	ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
560 
561 	retval = ioctl(ia_fd, DAPL_EP_MODIFY, &epm_args);
562 	if (retval != 0) {
563 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
564 		    "qp_modify: ioctl failed errno %d, retval %d\n",
565 		    errno, retval);
566 		return (dapls_convert_error(errno, retval));
567 	}
568 
569 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
570 	    "qp_modify: ep_hkey = %016llx\n", ep_ptr->qp_handle->ep_hkey);
571 	return (DAT_SUCCESS);
572 }
573 
574 /*
575  * Allocate the srq data structure as well as the kernel resource
576  * corresponding to it.
577  */
578 DAT_RETURN
dapls_ib_srq_alloc(IN DAPL_IA * ia_ptr,IN DAPL_SRQ * srq_ptr)579 dapls_ib_srq_alloc(IN DAPL_IA *ia_ptr, IN DAPL_SRQ *srq_ptr)
580 {
581 	dapl_srq_create_t	srqc_args;
582 	dapl_srq_free_t		srqf_args;
583 	ib_srq_handle_t		ibsrq_p;
584 	DAPL_PZ			*pz_handle;
585 	uint32_t		i;
586 	size_t			premev_size;
587 	size_t			freeev_size;
588 	int			ia_fd;
589 	int			hca_fd;
590 	int			retval;
591 	mlnx_umap_srq_data_out_t *msrq;
592 
593 	/* check parameters */
594 	if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
595 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
596 		    "srq_alloc: hca_handle == NULL\n");
597 		return (DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1));
598 	}
599 
600 	ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
601 	hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
602 	dapl_os_assert(srq_ptr->param.pz_handle != NULL);
603 
604 	/* fill in args for srq_create */
605 	pz_handle = (DAPL_PZ *)srq_ptr->param.pz_handle;
606 
607 	ibsrq_p = (ib_srq_handle_t)dapl_os_alloc(sizeof (*ibsrq_p));
608 	if (ibsrq_p == NULL) {
609 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
610 		    "srq_alloc: os_alloc failed\n");
611 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
612 		    DAT_RESOURCE_MEMORY));
613 	}
614 	(void) dapl_os_memzero(ibsrq_p, sizeof (*ibsrq_p));
615 
616 	(void) dapl_os_memzero(&srqc_args, sizeof (srqc_args));
617 	msrq = (mlnx_umap_srq_data_out_t *)srqc_args.srqc_data_out;
618 	srqc_args.srqc_pd_hkey = pz_handle->pd_handle->pd_hkey;
619 	srqc_args.srqc_sizes.srqs_sz = srq_ptr->param.max_recv_dtos;
620 	srqc_args.srqc_sizes.srqs_sgl = srq_ptr->param.max_recv_iov;
621 
622 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
623 	    "srq_alloc: srq_ptr 0x%p, pz 0x%p (0x%llx), srq_sz %d"
624 	    " srq_sgl %d\n",
625 	    srq_ptr, pz_handle, srqc_args.srqc_pd_hkey,
626 	    srqc_args.srqc_sizes.srqs_sz, srqc_args.srqc_sizes.srqs_sgl);
627 
628 	/* The next line is only needed for backward compatibility */
629 	msrq->msrq_rev = MLNX_UMAP_IF_VERSION;
630 	retval = ioctl(ia_fd, DAPL_SRQ_CREATE, &srqc_args);
631 	if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
632 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
633 		    "srq_alloc: srq_create failed errno %d, retval %d\n",
634 		    errno, retval);
635 		dapl_os_free(ibsrq_p, sizeof (*ibsrq_p));
636 		return (dapls_convert_error(errno, retval));
637 	}
638 
639 	/* In the case of Arbel or Hermon */
640 	if (msrq->msrq_rdbr_mapoffset != 0 || msrq->msrq_rdbr_maplen != 0)
641 		ibsrq_p->srq_dbp = dapls_ib_get_dbp(
642 		    msrq->msrq_rdbr_maplen, hca_fd,
643 		    msrq->msrq_rdbr_mapoffset, msrq->msrq_rdbr_offset);
644 
645 	ibsrq_p->srq_addr = mmap64((void *)0,
646 	    msrq->msrq_maplen, (PROT_READ | PROT_WRITE),
647 	    MAP_SHARED, hca_fd, msrq->msrq_mapoffset);
648 
649 	if (ibsrq_p->srq_addr == MAP_FAILED ||
650 	    ibsrq_p->srq_dbp == MAP_FAILED) {
651 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
652 		    "srq_alloc: mmap failed(%d)\n", errno);
653 		srqf_args.srqf_hkey = srqc_args.srqc_hkey;
654 		retval = ioctl(ia_fd, DAPL_SRQ_FREE, &srqf_args);
655 		if (retval != 0) {
656 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
657 			    "srq_alloc: SRQ_FREE err:%d\n", errno);
658 		}
659 		dapl_os_free(ibsrq_p, sizeof (*ibsrq_p));
660 		return (dapls_convert_error(errno, 0));
661 	}
662 
663 	ibsrq_p->srq_hkey = srqc_args.srqc_hkey;
664 	ibsrq_p->srq_map_len = msrq->msrq_maplen;
665 	ibsrq_p->srq_map_offset = msrq->msrq_mapoffset;
666 	ibsrq_p->srq_num = msrq->msrq_srqnum;
667 	ibsrq_p->srq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
668 	/* since 0 is a valid index, -1 indicates invalid value */
669 	ibsrq_p->srq_wq_lastwqeindex = -1;
670 	ibsrq_p->srq_wq_desc_addr = msrq->msrq_desc_addr;
671 	ibsrq_p->srq_wq_numwqe = msrq->msrq_numwqe;
672 	ibsrq_p->srq_wq_wqesz = msrq->msrq_wqesz;
673 	ibsrq_p->srq_wq_sgl = srqc_args.srqc_real_sizes.srqs_sgl;
674 
675 	/*
676 	 * update the srq handle in the srq ptr, this is needed since from
677 	 * here on cleanup is done by calling dapls_ib_srq_free()
678 	 */
679 	srq_ptr->srq_handle = ibsrq_p;
680 
681 	premev_size = ibsrq_p->srq_wq_numwqe * sizeof (ib_work_completion_t);
682 	ibsrq_p->srq_premature_events = (ib_work_completion_t *)
683 	    dapl_os_alloc(premev_size);
684 	if (ibsrq_p->srq_premature_events == NULL) {
685 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
686 		    "srq_alloc: os_alloc premature_events failed\n");
687 		dapls_ib_srq_free(ia_ptr, srq_ptr);
688 		srq_ptr->srq_handle = NULL;
689 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
690 		    DAT_RESOURCE_MEMORY));
691 	}
692 
693 	freeev_size = ibsrq_p->srq_wq_numwqe * sizeof (uint32_t);
694 	ibsrq_p->srq_freepr_events = (uint32_t *)dapl_os_alloc(freeev_size);
695 	if (ibsrq_p->srq_freepr_events == NULL) {
696 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
697 		    "srq_alloc: os_alloc freepr_events failed\n");
698 		dapls_ib_srq_free(ia_ptr, srq_ptr);
699 		srq_ptr->srq_handle = NULL;
700 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
701 		    DAT_RESOURCE_MEMORY));
702 	}
703 	ibsrq_p->srq_freepr_head = 0;
704 	ibsrq_p->srq_freepr_tail = 0;
705 	ibsrq_p->srq_freepr_num_events = ibsrq_p->srq_wq_numwqe;
706 
707 	/* initialize the free list of premature events */
708 	for (i = 0; i < ibsrq_p->srq_freepr_num_events; i++) {
709 		ibsrq_p->srq_freepr_events[i] = i;
710 		/*
711 		 * wc_res_hash field is used to mark entries in the premature
712 		 * events list
713 		 */
714 		DAPL_SET_CQE_INVALID(&(ibsrq_p->srq_premature_events[i]));
715 	}
716 
717 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
718 	    "srq_alloc: created, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
719 	    srq_ptr, srqc_args.srqc_hkey);
720 
721 	DAPL_INIT_SRQ(ia_ptr)(ibsrq_p);
722 
723 	if (dapls_tavor_srq_wrid_init(ibsrq_p) != DAT_SUCCESS) {
724 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
725 		    "srq_alloc: wridlist alloc failed\n");
726 		dapls_ib_srq_free(ia_ptr, srq_ptr);
727 		srq_ptr->srq_handle = NULL;
728 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
729 		    DAT_RESOURCE_MEMORY));
730 	}
731 	ibsrq_p->srq_ep_table = NULL;
732 	/* allocate a hash table to to store EPs */
733 	retval = dapls_hash_create(DAPL_HASH_TABLE_DEFAULT_CAPACITY,
734 	    DAT_FALSE, &ibsrq_p->srq_ep_table);
735 	if (retval != DAT_SUCCESS) {
736 		dapl_dbg_log(DAPL_DBG_TYPE_ERR, "dapls_ib_srq_alloc hash "
737 		    "create failed %d\n", retval);
738 		dapls_ib_srq_free(ia_ptr, srq_ptr);
739 		srq_ptr->srq_handle = NULL;
740 		return (retval);
741 	}
742 
743 	return (DAT_SUCCESS);
744 }
745 
746 
747 /*
748  * SRQ Free routine
749  */
750 void
dapls_ib_srq_free(IN DAPL_IA * ia_handle,IN DAPL_SRQ * srq_ptr)751 dapls_ib_srq_free(IN DAPL_IA *ia_handle, IN DAPL_SRQ *srq_ptr)
752 {
753 	ib_srq_handle_t	srq_handle = srq_ptr->srq_handle;
754 	ib_hca_handle_t	ib_hca_handle = ia_handle->hca_ptr->ib_hca_handle;
755 	dapl_srq_free_t	srqf_args;
756 	int		retval;
757 
758 	if (srq_handle == IB_INVALID_HANDLE) {
759 		return; /* nothing to do */
760 	}
761 
762 	if (munmap((void *)srq_handle->srq_addr, srq_handle->srq_map_len) < 0) {
763 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
764 		    "srq_free: munmap failed(%d)\n", errno);
765 	}
766 	srqf_args.srqf_hkey = srq_handle->srq_hkey;
767 	retval = ioctl(ib_hca_handle->ia_fd, DAPL_SRQ_FREE, &srqf_args);
768 	if (retval != 0) {
769 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
770 		    "srq_free: ioctl errno = %d, retval = %d\n", errno, retval);
771 	}
772 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
773 	    "srq_free: freed, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
774 	    srq_ptr, srq_handle->srq_hkey);
775 	if (srq_handle->srq_ep_table) {
776 		(void) dapls_hash_free(srq_handle->srq_ep_table);
777 	}
778 	if (srq_handle->srq_wridlist) {
779 		dapls_tavor_srq_wrid_free(srq_handle);
780 	}
781 	if (srq_handle->srq_freepr_events) {
782 		dapl_os_free(srq_handle->srq_freepr_events,
783 		    srq_handle->srq_wq_numwqe * sizeof (ib_work_completion_t));
784 	}
785 	if (srq_handle->srq_premature_events) {
786 		dapl_os_free(srq_handle->srq_premature_events,
787 		    srq_handle->srq_wq_numwqe * sizeof (uint32_t));
788 	}
789 	dapl_os_free(srq_handle, sizeof (*srq_handle));
790 	srq_ptr->srq_handle = NULL;
791 }
792 
793 /*
794  * Adds EP to a hashtable in SRQ
795  */
796 static DAT_RETURN
dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr,IN uint32_t qp_num,IN DAPL_EP * ep_ptr)797 dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num,
798     IN DAPL_EP *ep_ptr)
799 {
800 	DAPL_HASH_TABLE	*htable;
801 	DAPL_HASH_KEY	key;
802 
803 	dapl_os_assert(srq_ptr);
804 
805 	htable = srq_ptr->srq_ep_table;
806 	key = qp_num;
807 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
808 	    "srq_insert_ep:%p %p %llx\n", srq_ptr, htable, key);
809 	return (dapls_hash_insert(htable, key, ep_ptr));
810 }
811 
812 /*
813  * Removes an EP from the hashtable in SRQ
814  */
815 static void
dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr,IN uint32_t qp_num)816 dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num)
817 {
818 	DAPL_HASH_TABLE	*htable;
819 	DAPL_HASH_KEY	key;
820 	DAPL_EP		*epp;
821 	DAT_RETURN	retval;
822 
823 	dapl_os_assert(srq_ptr);
824 
825 	htable = srq_ptr->srq_ep_table;
826 	key = qp_num;
827 
828 	retval = dapls_hash_remove(htable, key, (DAPL_HASH_DATA *)&epp);
829 	if (retval != DAT_SUCCESS) {
830 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
831 		    "srq_remove_ep(%d): %p %llx\n", retval, htable, key);
832 	}
833 }
834 
835 /*
836  * Lookup an EP from the hashtable in SRQ
837  */
838 DAPL_EP *
dapls_ib_srq_lookup_ep(IN DAPL_SRQ * srq_ptr,IN ib_work_completion_t * cqe_ptr)839 dapls_ib_srq_lookup_ep(IN DAPL_SRQ *srq_ptr, IN ib_work_completion_t *cqe_ptr)
840 {
841 	DAPL_HASH_TABLE	*htable;
842 	DAPL_HASH_KEY	key;
843 	DAPL_EP		*epp;
844 	DAT_RETURN	retval;
845 
846 	dapl_os_assert(srq_ptr && srq_ptr->srq_handle);
847 
848 	htable = srq_ptr->srq_handle->srq_ep_table;
849 	key = DAPL_GET_CQE_QPN(cqe_ptr);
850 	epp = NULL;
851 
852 	retval = dapls_hash_search(htable, key, (DAPL_HASH_DATA *)&epp);
853 	if (retval != DAT_SUCCESS) {
854 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
855 		    "srq_lookup_ep(%x): %p %llx\n", retval, htable, key);
856 	}
857 	return (epp);
858 }
859 
860 
861 /*
862  * dapl_ib_srq_resize
863  *
864  * Resize an SRQ
865  *
866  * Input:
867  *	srq_ptr			pointer to SRQ struct
868  *	srqlen			new length of the SRQ
869  * Output:
870  *	none
871  *
872  * Returns:
873  *	DAT_SUCCESS
874  *	DAT_INVALID_HANDLE
875  *	DAT_INTERNAL_ERROR
876  *	DAT_INSUFFICIENT_RESOURCES
877  *
878  */
879 DAT_RETURN
dapls_ib_srq_resize(IN DAPL_SRQ * srq_ptr,IN DAT_COUNT srqlen)880 dapls_ib_srq_resize(
881 	IN  DAPL_SRQ		*srq_ptr,
882 	IN  DAT_COUNT		srqlen)
883 {
884 	ib_srq_handle_t	srq_handle;
885 	DAT_RETURN	dat_status;
886 
887 	dat_status = dapli_ib_srq_resize_internal(srq_ptr, srqlen);
888 	if (DAT_INSUFFICIENT_RESOURCES == DAT_GET_TYPE(dat_status)) {
889 		srq_handle = srq_ptr->srq_handle;
890 		/* attempt to resize back to the current size */
891 		dat_status = dapli_ib_srq_resize_internal(srq_ptr,
892 		    srq_handle->srq_wq_numwqe);
893 		if (DAT_SUCCESS != dat_status) {
894 			/*
895 			 * XXX this is catastrophic need to post an event
896 			 * to the async evd
897 			 */
898 			return (DAT_INTERNAL_ERROR);
899 		}
900 	}
901 
902 	return (dat_status);
903 }
904 
905 /*
906  * dapli_ib_srq_resize_internal
907  *
908  * An internal routine to resize a SRQ.
909  *
910  * Input:
911  *	srq_ptr			pointer to SRQ struct
912  *	srqlen			new length of the srq
913  * Output:
914  *	none
915  *
916  * Returns:
917  *	DAT_SUCCESS
918  *	DAT_INVALID_HANDLE
919  *	DAT_INSUFFICIENT_RESOURCES
920  *
921  */
922 static DAT_RETURN
dapli_ib_srq_resize_internal(IN DAPL_SRQ * srq_ptr,IN DAT_COUNT srqlen)923 dapli_ib_srq_resize_internal(
924 	IN  DAPL_SRQ		*srq_ptr,
925 	IN  DAT_COUNT		srqlen)
926 {
927 	ib_srq_handle_t		srq_handle;
928 	dapl_srq_resize_t	resize_msg;
929 	int			ia_fd;
930 	int			hca_fd;
931 	ib_work_completion_t	*new_premature_events;
932 	ib_work_completion_t	*old_premature_events;
933 	uint32_t		*new_freepr_events;
934 	uint32_t		*old_freepr_events;
935 	size_t			old_premature_size;
936 	size_t			old_freepr_size;
937 	size_t			new_premature_size;
938 	size_t			new_freepr_size;
939 	int			idx, i;
940 	int			retval;
941 	mlnx_umap_srq_data_out_t *msrq;
942 
943 	dapl_dbg_log(DAPL_DBG_TYPE_EP,
944 	    "dapls_ib_srq_resize: srq 0x%p srq_hdl 0x%p "
945 	    "srq_hkey 0x%016llx srqlen %d\n",
946 	    srq_ptr, (void *)srq_ptr->srq_handle,
947 	    srq_ptr->srq_handle->srq_hkey, srqlen);
948 
949 	srq_handle = srq_ptr->srq_handle;
950 	/*
951 	 * Since SRQs are created in powers of 2 its possible that the
952 	 * previously allocated SRQ has sufficient entries. If the current
953 	 * SRQ is big enough and it is mapped we are done.
954 	 */
955 	if ((srqlen < srq_handle->srq_wq_numwqe) && (srq_handle->srq_addr)) {
956 		return (DAT_SUCCESS);
957 	}
958 
959 	/* unmap the SRQ before resizing it */
960 	if ((srq_handle->srq_addr) && (munmap((char *)srq_handle->srq_addr,
961 	    srq_handle->srq_map_len) < 0)) {
962 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
963 		    "srq_resize: munmap(%p:0x%llx) failed(%d)\n",
964 		    srq_handle->srq_addr, srq_handle->srq_map_len, errno);
965 		return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ));
966 	}
967 	/* srq_addr is unmapped and no longer valid */
968 	srq_handle->srq_addr = NULL;
969 
970 	ia_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;
971 	hca_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->hca_fd;
972 
973 	(void) dapl_os_memzero(&resize_msg, sizeof (resize_msg));
974 	resize_msg.srqr_hkey = srq_handle->srq_hkey;
975 	resize_msg.srqr_new_size = srqlen;
976 	msrq = (mlnx_umap_srq_data_out_t *)resize_msg.srqr_data_out;
977 
978 	/* The next line is only needed for backward compatibility */
979 	msrq->msrq_rev = MLNX_UMAP_IF_VERSION;
980 	retval = ioctl(ia_fd, DAPL_SRQ_RESIZE, &resize_msg);
981 	if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
982 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
983 		    "dapls_ib_srq_resize: srq 0x%p, err: %s\n",
984 		    srq_ptr, strerror(errno));
985 		if (errno == EINVAL) { /* Couldn't find this srq */
986 			return (DAT_ERROR(DAT_INVALID_HANDLE,
987 			    DAT_INVALID_HANDLE_SRQ));
988 		} else { /* Need to retry resize with a smaller qlen */
989 			return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
990 			    DAT_RESOURCE_SRQ));
991 		}
992 	}
993 
994 	dapl_os_assert(srq_handle->srq_num == msrq->msrq_srqnum);
995 
996 	/* In the case of Arbel or Hermon */
997 	if (msrq->msrq_rdbr_mapoffset != 0 ||
998 	    msrq->msrq_rdbr_maplen != 0)
999 		srq_handle->srq_dbp = dapls_ib_get_dbp(
1000 		    msrq->msrq_rdbr_maplen,
1001 		    hca_fd, msrq->msrq_rdbr_mapoffset,
1002 		    msrq->msrq_rdbr_offset);
1003 
1004 	srq_handle->srq_addr = mmap64((void *)0,
1005 	    msrq->msrq_maplen, (PROT_READ | PROT_WRITE),
1006 	    MAP_SHARED, hca_fd, msrq->msrq_mapoffset);
1007 
1008 	if (srq_handle->srq_addr == MAP_FAILED ||
1009 	    srq_handle->srq_dbp == MAP_FAILED) {
1010 		srq_handle->srq_addr = NULL;
1011 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
1012 		    "srq_resize: mmap failed(%d)\n", errno);
1013 		/* Need to retry resize with a smaller qlen */
1014 		return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
1015 		    DAT_RESOURCE_MEMORY));
1016 	}
1017 
1018 	old_freepr_size = srq_handle->srq_wq_numwqe * sizeof (uint32_t);
1019 	old_premature_size = srq_handle->srq_wq_numwqe *
1020 	    sizeof (ib_work_completion_t);
1021 
1022 	old_freepr_events = srq_handle->srq_freepr_events;
1023 	old_premature_events = srq_handle->srq_premature_events;
1024 
1025 	new_freepr_size = resize_msg.srqr_real_size * sizeof (uint32_t);
1026 	new_premature_size = resize_msg.srqr_real_size *
1027 	    sizeof (ib_work_completion_t);
1028 
1029 	new_freepr_events = (uint32_t *)dapl_os_alloc(new_freepr_size);
1030 	if (new_freepr_events == NULL) {
1031 		goto bail;
1032 	}
1033 	new_premature_events = (ib_work_completion_t *)dapl_os_alloc(
1034 	    new_premature_size);
1035 	if (new_premature_events == NULL) {
1036 		goto bail;
1037 	}
1038 	if (!dapls_tavor_srq_wrid_resize(srq_handle,
1039 	    resize_msg.srqr_real_size)) {
1040 		goto bail;
1041 	}
1042 	idx = 0;
1043 	/* copy valid premature events  */
1044 	for (i = 0; i < srq_handle->srq_wq_numwqe; i++) {
1045 		if (!DAPL_CQE_IS_VALID(&old_premature_events[i])) {
1046 			continue;
1047 		}
1048 		(void) dapl_os_memcpy(&new_premature_events[idx],
1049 		    &old_premature_events[i], sizeof (ib_work_completion_t));
1050 		idx++;
1051 	}
1052 	dapl_os_assert(srq_handle->srq_wq_numwqe - idx ==
1053 	    srq_handle->srq_freepr_num_events);
1054 
1055 	/* Initialize free events lists */
1056 	for (i = 0; i < resize_msg.srqr_real_size - idx; i++) {
1057 		new_freepr_events[i] = idx + i;
1058 	}
1059 
1060 	srq_handle->srq_freepr_events = new_freepr_events;
1061 	srq_handle->srq_premature_events = new_premature_events;
1062 	srq_handle->srq_freepr_num_events = resize_msg.srqr_real_size - idx;
1063 	srq_handle->srq_freepr_head = 0;
1064 	/* a full freepr list has tail at 0 */
1065 	if (idx == 0) {
1066 		srq_handle->srq_freepr_tail = 0;
1067 	} else {
1068 		srq_handle->srq_freepr_tail = srq_handle->srq_freepr_num_events;
1069 	}
1070 
1071 	if (old_freepr_events) {
1072 		old_freepr_size = old_freepr_size; /* pacify lint */
1073 		dapl_os_free(old_freepr_events, old_freepr_size);
1074 	}
1075 	if (old_premature_events) {
1076 		old_premature_size = old_premature_size; /* pacify lint */
1077 		dapl_os_free(old_premature_events, old_premature_size);
1078 	}
1079 
1080 	/*
1081 	 * update the srq fields,
1082 	 * note: the srq_wq_lastwqeindex doesn't change since the old
1083 	 * work queue is copied as a whole into the new work queue.
1084 	 */
1085 	srq_handle->srq_map_offset = msrq->msrq_mapoffset;
1086 	srq_handle->srq_map_len = msrq->msrq_maplen;
1087 	srq_handle->srq_wq_desc_addr = msrq->msrq_desc_addr;
1088 	srq_handle->srq_wq_numwqe = msrq->msrq_numwqe;
1089 	srq_handle->srq_wq_wqesz = msrq->msrq_wqesz;
1090 
1091 	return (DAT_SUCCESS);
1092 bail:
1093 	if (new_freepr_events) {
1094 		dapl_os_free(new_freepr_events, new_freepr_size);
1095 	}
1096 	if (new_premature_events) {
1097 		dapl_os_free(new_premature_events, new_premature_size);
1098 	}
1099 	return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY));
1100 }
1101