1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include "dapl.h"
28 #include "dapl_adapter_util.h"
29 #include "dapl_evd_util.h"
30 #include "dapl_cr_util.h"
31 #include "dapl_lmr_util.h"
32 #include "dapl_rmr_util.h"
33 #include "dapl_cookie.h"
34 #include "dapl_tavor_ibtf_impl.h"
35 #include "dapl_hash.h"
36
37 /* Function prototypes */
38 extern DAT_RETURN dapls_tavor_wrid_init(ib_qp_handle_t);
39 extern DAT_RETURN dapls_tavor_srq_wrid_init(ib_srq_handle_t);
40 extern void dapls_tavor_srq_wrid_free(ib_srq_handle_t);
41 extern DAT_BOOLEAN dapls_tavor_srq_wrid_resize(ib_srq_handle_t, uint32_t);
42
43 static DAT_RETURN dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr,
44 IN uint32_t qpnum, IN DAPL_EP *ep_ptr);
45 static void dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr,
46 IN uint32_t qpnum);
47 static DAT_RETURN dapli_ib_srq_resize_internal(IN DAPL_SRQ *srq_ptr,
48 IN DAT_COUNT srqlen);
49 /*
50 * dapli_get_dto_cq
51 *
52 * Obtain the cq_handle for a DTO EVD. If the EVD is NULL, use the
53 * null_ib_cq_handle. If it hasn't been created yet, create it now in
54 * the HCA structure. It will be cleaned up in dapls_ib_cqd_destroy().
55 *
56 * This is strictly internal to IB. DAPL allows a NULL DTO EVD handle,
57 * but IB does not. So we create a CQ under the hood and make sure
58 * an error is generated if the user every tries to post, by
59 * setting the WQ length to 0 in ep_create and/or ep_modify.
60 *
61 * Returns
62 * A valid CQ handle
63 */
64 static ib_cq_handle_t
dapli_get_dto_cq(IN DAPL_IA * ia_ptr,IN DAPL_EVD * evd_ptr)65 dapli_get_dto_cq(
66 IN DAPL_IA *ia_ptr,
67 IN DAPL_EVD *evd_ptr)
68 {
69 dapl_evd_create_t create_msg;
70 ib_cq_handle_t cq_handle;
71 int ia_fd;
72 int retval;
73 mlnx_umap_cq_data_out_t *mcq;
74
75 if (evd_ptr != DAT_HANDLE_NULL) {
76 cq_handle = evd_ptr->ib_cq_handle;
77 } else if (ia_ptr->hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) {
78 cq_handle = ia_ptr->hca_ptr->null_ib_cq_handle;
79 } else {
80 cq_handle = (ib_cq_handle_t)
81 dapl_os_alloc(sizeof (struct dapls_ib_cq_handle));
82 if (cq_handle == NULL) {
83 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
84 "dapli_get_dto_cq: cq malloc failed\n");
85 ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
86 return (IB_INVALID_HANDLE);
87 }
88
89 /*
90 * create a fake a CQ, we don't bother to mmap this CQ
91 * since nobody know about it to reap events from it.
92 */
93 (void) dapl_os_memzero(&create_msg, sizeof (create_msg));
94 create_msg.evd_flags = DAT_EVD_DTO_FLAG;
95 mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out;
96
97 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
98
99 /* call into driver to allocate cq */
100 retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg);
101 if (retval != 0) {
102 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
103 "dapli_get_dto_cq: DAPL_EVD_CREATE failed\n");
104 dapl_os_free(cq_handle,
105 sizeof (struct dapls_ib_cq_handle));
106 ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
107 return (IB_INVALID_HANDLE);
108 }
109
110 (void) dapl_os_memzero(cq_handle,
111 sizeof (struct dapls_ib_cq_handle));
112 dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock);
113 cq_handle->evd_hkey = create_msg.evd_hkey;
114 cq_handle->cq_addr = NULL;
115 cq_handle->cq_map_offset = mcq->mcq_mapoffset;
116 cq_handle->cq_map_len = mcq->mcq_maplen;
117 cq_handle->cq_num = mcq->mcq_cqnum;
118 cq_handle->cq_size = create_msg.evd_cq_real_size;
119 cq_handle->cq_cqesz = mcq->mcq_cqesz;
120 cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
121
122 dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
123 "dapli_get_dto_cq: cq 0x%p created, hkey 0x%016llx\n",
124 cq_handle, create_msg.evd_hkey);
125
126 /* save this dummy CQ handle into the hca */
127 ia_ptr->hca_ptr->null_ib_cq_handle = cq_handle;
128 }
129 return (cq_handle);
130 }
131
132
133 /*
134 * dapl_ib_qp_alloc
135 *
136 * Alloc a QP
137 *
138 * Input:
139 * *ep_ptr pointer to EP INFO
140 * ib_hca_handle provider HCA handle
141 * ib_pd_handle provider protection domain handle
142 * cq_recv provider recv CQ handle
143 * cq_send provider send CQ handle
144 *
145 * Output:
146 * none
147 *
148 * Returns:
149 * DAT_SUCCESS
150 * DAT_INSUFFICIENT_RESOURCES
151 *
152 */
153 DAT_RETURN
dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr,IN DAPL_EP * ep_ctx_ptr)154 dapls_ib_qp_alloc(
155 IN DAPL_IA *ia_ptr,
156 IN DAPL_EP *ep_ptr,
157 IN DAPL_EP *ep_ctx_ptr)
158 {
159 dapl_ep_create_t ep_args;
160 dapl_ep_free_t epf_args;
161 ib_qp_handle_t qp_p;
162 DAPL_SRQ *srq_p;
163 ib_cq_handle_t cq_recv;
164 ib_cq_handle_t cq_send;
165 DAPL_PZ *pz_handle;
166 DAPL_EVD *evd_handle;
167 uint32_t mpt_mask;
168 size_t premev_size;
169 uint32_t i;
170 int ia_fd;
171 int hca_fd;
172 DAT_RETURN dat_status;
173 int retval;
174 mlnx_umap_qp_data_out_t *mqp;
175
176 /* check parameters */
177 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
178 dapl_dbg_log(DAPL_DBG_TYPE_EP,
179 "qp_alloc: hca_handle == NULL\n");
180 return (DAT_INVALID_PARAMETER);
181 }
182
183 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
184 hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
185 dapl_os_assert(ep_ptr->param.pz_handle != NULL);
186 dapl_os_assert(ep_ptr->param.connect_evd_handle != NULL);
187
188 /* fill in args for ep_create */
189 (void) dapl_os_memzero(&ep_args, sizeof (ep_args));
190 mqp = (mlnx_umap_qp_data_out_t *)ep_args.ep_qp_data_out;
191 pz_handle = (DAPL_PZ *)ep_ptr->param.pz_handle;
192 ep_args.ep_pd_hkey = pz_handle->pd_handle->pd_hkey;
193
194 cq_recv = dapli_get_dto_cq(ia_ptr,
195 (DAPL_EVD *)ep_ptr->param.recv_evd_handle);
196 ep_args.ep_rcv_evd_hkey = cq_recv->evd_hkey;
197
198 cq_send = dapli_get_dto_cq(ia_ptr,
199 (DAPL_EVD *)ep_ptr->param.request_evd_handle);
200 ep_args.ep_snd_evd_hkey = cq_send->evd_hkey;
201
202 evd_handle = (DAPL_EVD *)ep_ptr->param.connect_evd_handle;
203 ep_args.ep_conn_evd_hkey = evd_handle->ib_cq_handle->evd_hkey;
204
205 ep_args.ep_ch_sizes.dcs_sq = ep_ptr->param.ep_attr.max_request_dtos;
206 ep_args.ep_ch_sizes.dcs_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
207
208 qp_p = (ib_qp_handle_t)dapl_os_alloc(
209 sizeof (struct dapls_ib_qp_handle));
210 if (qp_p == NULL) {
211 dapl_dbg_log(DAPL_DBG_TYPE_EP,
212 "qp_alloc: os_alloc failed\n");
213 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
214 DAT_RESOURCE_MEMORY));
215 }
216
217 (void) dapl_os_memzero(qp_p, sizeof (*qp_p));
218
219 if (ep_ptr->param.srq_handle == NULL) {
220 premev_size = ep_ptr->param.ep_attr.max_recv_dtos *
221 sizeof (ib_work_completion_t);
222 if (premev_size != 0) {
223 qp_p->qp_premature_events = (ib_work_completion_t *)
224 dapl_os_alloc(premev_size);
225 if (qp_p->qp_premature_events == NULL) {
226 dapl_dbg_log(DAPL_DBG_TYPE_EP,
227 "qp_alloc:alloc premature_events failed\n");
228 dapl_os_free(qp_p, sizeof (*qp_p));
229 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
230 DAT_RESOURCE_MEMORY));
231 }
232 }
233 qp_p->qp_num_premature_events = 0;
234 ep_args.ep_srq_hkey = 0;
235 ep_args.ep_srq_attached = 0;
236 ep_args.ep_ch_sizes.dcs_rq =
237 ep_ptr->param.ep_attr.max_recv_dtos;
238 ep_args.ep_ch_sizes.dcs_rq_sgl =
239 ep_ptr->param.ep_attr.max_recv_iov;
240 } else {
241 premev_size = 0;
242 srq_p = (DAPL_SRQ *)ep_ptr->param.srq_handle;
243 /* premature events for EPs with SRQ sit on the SRQ */
244 qp_p->qp_premature_events = srq_p->srq_handle->
245 srq_premature_events;
246 qp_p->qp_num_premature_events = 0;
247 ep_args.ep_srq_hkey = srq_p->srq_handle->srq_hkey;
248 ep_args.ep_srq_attached = 1;
249 ep_args.ep_ch_sizes.dcs_rq = 0;
250 ep_args.ep_ch_sizes.dcs_rq_sgl = 0;
251 }
252
253 /*
254 * there are cases when ep_ptr is a dummy container ep, and the orig
255 * ep pointer is passed in ep_ctx_ptr. eg - dapl_ep_modify does this.
256 * ep_cookie should be the actual ep pointer, not the dummy container
257 * ep since the kernel returns this via events and the CM callback
258 * routines
259 */
260 ep_args.ep_cookie = (uintptr_t)ep_ctx_ptr;
261
262 dapl_dbg_log(DAPL_DBG_TYPE_EP,
263 "qp_alloc: ep_ptr 0x%p, pz 0x%p (0x%llx), rcv_evd 0x%p (0x%llx)\n"
264 " snd_evd 0x%p (0x%llx), conn_evd 0x%p (0x%llx)\n"
265 " srq_hdl 0x%p (0x%llx)\n"
266 " sq_sz %d, rq_sz %d, sq_sgl_sz %d, rq_sgl_sz %d\n",
267 ep_ptr, pz_handle, ep_args.ep_pd_hkey,
268 ep_ptr->param.recv_evd_handle, ep_args.ep_rcv_evd_hkey,
269 ep_ptr->param.request_evd_handle, ep_args.ep_snd_evd_hkey,
270 ep_ptr->param.connect_evd_handle, ep_args.ep_conn_evd_hkey,
271 ep_ptr->param.srq_handle, ep_args.ep_srq_hkey,
272 ep_args.ep_ch_sizes.dcs_sq, ep_args.ep_ch_sizes.dcs_rq,
273 ep_args.ep_ch_sizes.dcs_sq_sgl, ep_args.ep_ch_sizes.dcs_rq_sgl);
274
275 /* The next line is only needed for backward compatibility */
276 mqp->mqp_rev = MLNX_UMAP_IF_VERSION;
277 retval = ioctl(ia_fd, DAPL_EP_CREATE, &ep_args);
278 if (retval != 0 || mqp->mqp_rev != MLNX_UMAP_IF_VERSION) {
279 dapl_dbg_log(DAPL_DBG_TYPE_EP,
280 "qp_alloc: ep_create failed errno %d, retval %d\n",
281 errno, retval);
282 if (premev_size != 0) {
283 dapl_os_free(qp_p->qp_premature_events, premev_size);
284 }
285 dapl_os_free(qp_p, sizeof (*qp_p));
286 return (dapls_convert_error(errno, retval));
287 }
288
289 /* In the case of Arbel or Hermon */
290 if (mqp->mqp_sdbr_mapoffset != 0 || mqp->mqp_sdbr_maplen != 0)
291 qp_p->qp_sq_dbp = dapls_ib_get_dbp(mqp->mqp_sdbr_maplen,
292 hca_fd, mqp->mqp_sdbr_mapoffset, mqp->mqp_sdbr_offset);
293 if (mqp->mqp_rdbr_mapoffset != 0 || mqp->mqp_rdbr_maplen != 0)
294 qp_p->qp_rq_dbp = dapls_ib_get_dbp(mqp->mqp_rdbr_maplen,
295 hca_fd, mqp->mqp_rdbr_mapoffset, mqp->mqp_rdbr_offset);
296
297 qp_p->qp_addr = mmap64((void *)0, mqp->mqp_maplen,
298 (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd,
299 mqp->mqp_mapoffset);
300
301 if (qp_p->qp_addr == MAP_FAILED ||
302 qp_p->qp_sq_dbp == MAP_FAILED ||
303 qp_p->qp_rq_dbp == MAP_FAILED) {
304 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
305 "qp_alloc: mmap failed(%d)\n", errno);
306 epf_args.epf_hkey = ep_args.ep_hkey;
307 retval = ioctl(ia_fd, DAPL_EP_FREE, &epf_args);
308 if (retval != 0) {
309 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
310 "qp_alloc: EP_FREE err:%d\n", errno);
311 }
312 if (premev_size != 0) {
313 dapl_os_free(qp_p->qp_premature_events, premev_size);
314 }
315 dapl_os_free(qp_p, sizeof (*qp_p));
316 return (dapls_convert_error(errno, 0));
317 }
318
319 qp_p->qp_map_len = mqp->mqp_maplen;
320 qp_p->qp_num = mqp->mqp_qpnum;
321 qp_p->qp_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
322 qp_p->qp_ia_bf = ia_ptr->hca_ptr->ib_hca_handle->ia_bf;
323 qp_p->qp_ia_bf_toggle = ia_ptr->hca_ptr->ib_hca_handle->ia_bf_toggle;
324
325 evd_handle = (DAPL_EVD *)ep_ptr->param.request_evd_handle;
326 qp_p->qp_sq_cqhdl = evd_handle->ib_cq_handle;
327 qp_p->qp_sq_lastwqeaddr = NULL;
328 qp_p->qp_sq_wqhdr = NULL;
329 qp_p->qp_sq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_sq_off);
330 qp_p->qp_sq_desc_addr = mqp->mqp_sq_desc_addr;
331 qp_p->qp_sq_numwqe = mqp->mqp_sq_numwqe;
332 qp_p->qp_sq_wqesz = mqp->mqp_sq_wqesz;
333 qp_p->qp_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
334 qp_p->qp_sq_inline = ia_ptr->hca_ptr->max_inline_send;
335 qp_p->qp_sq_headroom = mqp->mqp_sq_headroomwqes;
336
337 evd_handle = (DAPL_EVD *)ep_ptr->param.recv_evd_handle;
338 qp_p->qp_rq_cqhdl = evd_handle->ib_cq_handle;
339 qp_p->qp_rq_lastwqeaddr = NULL;
340 qp_p->qp_rq_wqhdr = NULL;
341 qp_p->qp_rq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_rq_off);
342 qp_p->qp_rq_desc_addr = mqp->mqp_rq_desc_addr;
343 qp_p->qp_rq_numwqe = mqp->mqp_rq_numwqe;
344 qp_p->qp_rq_wqesz = mqp->mqp_rq_wqesz;
345 qp_p->qp_rq_sgl = ep_ptr->param.ep_attr.max_recv_iov;
346
347 dapl_dbg_log(DAPL_DBG_TYPE_EP,
348 "qp_alloc: created, qp_sq_buf %p, qp_rq_buf %p\n",
349 qp_p->qp_sq_buf, qp_p->qp_rq_buf);
350 dapl_dbg_log(DAPL_DBG_TYPE_EP,
351 "qp_alloc: created, sq numwqe %x wqesz %x, rq numwqe %x wqesz %x\n",
352 qp_p->qp_sq_numwqe, qp_p->qp_sq_wqesz,
353 qp_p->qp_rq_numwqe, qp_p->qp_rq_wqesz);
354 dapl_dbg_log(DAPL_DBG_TYPE_EP,
355 "qp_alloc: created, qp_sq_desc_addr %x, qp_rq_desc_addr %x\n",
356 mqp->mqp_sq_desc_addr, mqp->mqp_rq_desc_addr);
357 dapl_dbg_log(DAPL_DBG_TYPE_EP,
358 "qp_alloc: created, ep_ptr 0x%p, ep_hkey 0x%016llx\n\n",
359 ep_ptr, ep_args.ep_hkey);
360
361 qp_p->ep_hkey = ep_args.ep_hkey;
362
363 /*
364 * Calculate the number of bits in max_rmrs - this is indirectly
365 * the max number of entried in the MPT table (defaults to 512K
366 * but is configurable). This value is used while creating new
367 * rkeys in bind processing (see dapl_tavor_hw.c).
368 * Stash this value in the qp handle, don't want to do this math
369 * for every bind
370 */
371 mpt_mask = (uint32_t)ia_ptr->hca_ptr->ia_attr.max_rmrs - 1;
372 for (i = 0; mpt_mask > 0; mpt_mask = (mpt_mask >> 1), i++)
373 ;
374 qp_p->qp_num_mpt_shift = (uint32_t)i;
375
376 ep_ptr->qpn = qp_p->qp_num;
377 /* update the qp handle in the ep ptr */
378 ep_ptr->qp_handle = qp_p;
379 /*
380 * ibt_alloc_rc_channel transitions the qp state to INIT.
381 * hence we directly transition from UNATTACHED to INIT
382 */
383 ep_ptr->qp_state = IBT_STATE_INIT;
384
385 if (ep_ptr->param.srq_handle) {
386 /* insert ep into the SRQ's ep_table */
387 dat_status = dapli_ib_srq_add_ep(srq_p->srq_handle,
388 qp_p->qp_num, ep_ptr);
389 if (dat_status != DAT_SUCCESS) {
390 dapl_dbg_log(DAPL_DBG_TYPE_EP,
391 "qp_alloc: srq_add_ep failed ep_ptr 0x%p, 0x%x\n",
392 ep_ptr, dat_status);
393 (void) dapls_ib_qp_free(ia_ptr, ep_ptr);
394 return (DAT_INVALID_PARAMETER);
395 }
396 qp_p->qp_srq_enabled = 1;
397 qp_p->qp_srq = srq_p->srq_handle;
398 } else {
399 qp_p->qp_srq_enabled = 0;
400 qp_p->qp_srq = NULL;
401 }
402 DAPL_INIT_QP(ia_ptr)(qp_p);
403
404 if (dapls_tavor_wrid_init(qp_p) != DAT_SUCCESS) {
405 (void) dapls_ib_qp_free(ia_ptr, ep_ptr);
406 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
407 DAT_RESOURCE_MEMORY));
408 }
409
410 return (DAT_SUCCESS);
411 }
412
413
414 /*
415 * dapls_ib_qp_free
416 *
417 * Free a QP
418 *
419 * Input:
420 * *ep_ptr pointer to EP INFO
421 * ib_hca_handle provider HCA handle
422 *
423 * Output:
424 * none
425 *
426 * Returns:
427 * none
428 *
429 */
430 DAT_RETURN
dapls_ib_qp_free(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr)431 dapls_ib_qp_free(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr)
432 {
433 ib_qp_handle_t qp_p = ep_ptr->qp_handle;
434 ib_hca_handle_t ib_hca_handle = ia_ptr->hca_ptr->ib_hca_handle;
435 dapl_ep_free_t args;
436 int retval;
437
438 if ((ep_ptr->qp_handle != IB_INVALID_HANDLE) &&
439 (ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED)) {
440 if (munmap((void *)qp_p->qp_addr, qp_p->qp_map_len) < 0) {
441 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
442 "qp_free: munmap failed(%d)\n", errno);
443 }
444 args.epf_hkey = qp_p->ep_hkey;
445 retval = ioctl(ib_hca_handle->ia_fd, DAPL_EP_FREE, &args);
446 if (retval != 0) {
447 dapl_dbg_log(DAPL_DBG_TYPE_EP,
448 "qp_free: ioctl errno = %d, retval = %d\n",
449 errno, retval);
450 }
451 dapl_dbg_log(DAPL_DBG_TYPE_EP,
452 "qp_free: freed, ep_ptr 0x%p, ep_hkey 0x%016llx\n",
453 ep_ptr, qp_p->ep_hkey);
454
455 if (qp_p->qp_srq) {
456 dapli_ib_srq_remove_ep(qp_p->qp_srq, qp_p->qp_num);
457 } else {
458 if (qp_p->qp_premature_events) {
459 dapl_os_free(qp_p->qp_premature_events,
460 ep_ptr->param.ep_attr.max_recv_dtos *
461 sizeof (ib_work_completion_t));
462 }
463 }
464 dapl_os_free(qp_p, sizeof (*qp_p));
465 ep_ptr->qp_handle = NULL;
466 }
467 return (DAT_SUCCESS);
468 }
469
470
471 /*
472 * dapl_ib_qp_modify
473 *
474 * Set the QP to the parameters specified in an EP_PARAM
475 *
476 * We can't be sure what state the QP is in so we first obtain the state
477 * from the driver. The EP_PARAM structure that is provided has been
478 * sanitized such that only non-zero values are valid.
479 *
480 * Input:
481 * ib_hca_handle HCA handle
482 * qp_handle QP handle
483 * ep_attr Sanitized EP Params
484 *
485 * Output:
486 * none
487 *
488 * Returns:
489 * DAT_SUCCESS
490 * DAT_INSUFFICIENT_RESOURCES
491 * DAT_INVALID_PARAMETER
492 *
493 */
494 DAT_RETURN
dapls_ib_qp_modify(IN DAPL_IA * ia_ptr,IN DAPL_EP * ep_ptr,IN DAT_EP_ATTR * ep_attr)495 dapls_ib_qp_modify(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr,
496 IN DAT_EP_ATTR *ep_attr)
497 {
498 dapl_ep_modify_t epm_args;
499 boolean_t epm_needed;
500 int ia_fd;
501 int retval;
502
503
504 if (ep_ptr->qp_handle == NULL) {
505 dapl_dbg_log(DAPL_DBG_TYPE_EP,
506 "qp_modify: qp_handle == NULL\n");
507 return (DAT_INVALID_PARAMETER);
508 }
509 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
510 dapl_dbg_log(DAPL_DBG_TYPE_EP,
511 "qp_modify: hca_handle == NULL\n");
512 return (DAT_INVALID_PARAMETER);
513 }
514
515 epm_needed = B_FALSE;
516
517 /*
518 * NOTE: ep_attr->max_mtu_size indicates the maximum message
519 * size, which is always 2GB for IB. Nothing to do with the IB
520 * implementation, nothing to set up.
521 */
522
523 if (ep_attr->max_rdma_size > 0) {
524 if (ep_attr->max_rdma_size > DAPL_IB_MAX_MESSAGE_SIZE) {
525 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
526 }
527 }
528
529 (void) memset((void *)&epm_args, 0, sizeof (epm_args));
530 /*
531 * The following parameters are dealt by creating a new qp
532 * in dapl_ep_modify.
533 * - max_recv_dtos
534 * - max_request_dtos
535 * - max_recv_iov
536 * - max_request_iov
537 */
538
539 if (ep_attr->max_rdma_read_in > 0) {
540 epm_args.epm_flags |= IBT_CEP_SET_RDMARA_IN;
541 epm_args.epm_rdma_ra_in = ep_attr->max_rdma_read_in;
542 epm_needed = B_TRUE;
543 }
544 if (ep_attr->max_rdma_read_out > 0) {
545 epm_args.epm_flags |= IBT_CEP_SET_RDMARA_OUT;
546 epm_args.epm_rdma_ra_out = ep_attr->max_rdma_read_out;
547 epm_needed = B_TRUE;
548 }
549
550 if (!epm_needed) {
551 dapl_dbg_log(DAPL_DBG_TYPE_EP,
552 "qp_modify: ep_hkey = %016llx nothing to do\n",
553 ep_ptr->qp_handle->ep_hkey);
554 return (DAT_SUCCESS);
555 }
556
557 epm_args.epm_hkey = ep_ptr->qp_handle->ep_hkey;
558
559 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
560
561 retval = ioctl(ia_fd, DAPL_EP_MODIFY, &epm_args);
562 if (retval != 0) {
563 dapl_dbg_log(DAPL_DBG_TYPE_EP,
564 "qp_modify: ioctl failed errno %d, retval %d\n",
565 errno, retval);
566 return (dapls_convert_error(errno, retval));
567 }
568
569 dapl_dbg_log(DAPL_DBG_TYPE_EP,
570 "qp_modify: ep_hkey = %016llx\n", ep_ptr->qp_handle->ep_hkey);
571 return (DAT_SUCCESS);
572 }
573
574 /*
575 * Allocate the srq data structure as well as the kernel resource
576 * corresponding to it.
577 */
578 DAT_RETURN
dapls_ib_srq_alloc(IN DAPL_IA * ia_ptr,IN DAPL_SRQ * srq_ptr)579 dapls_ib_srq_alloc(IN DAPL_IA *ia_ptr, IN DAPL_SRQ *srq_ptr)
580 {
581 dapl_srq_create_t srqc_args;
582 dapl_srq_free_t srqf_args;
583 ib_srq_handle_t ibsrq_p;
584 DAPL_PZ *pz_handle;
585 uint32_t i;
586 size_t premev_size;
587 size_t freeev_size;
588 int ia_fd;
589 int hca_fd;
590 int retval;
591 mlnx_umap_srq_data_out_t *msrq;
592
593 /* check parameters */
594 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) {
595 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
596 "srq_alloc: hca_handle == NULL\n");
597 return (DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1));
598 }
599
600 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
601 hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
602 dapl_os_assert(srq_ptr->param.pz_handle != NULL);
603
604 /* fill in args for srq_create */
605 pz_handle = (DAPL_PZ *)srq_ptr->param.pz_handle;
606
607 ibsrq_p = (ib_srq_handle_t)dapl_os_alloc(sizeof (*ibsrq_p));
608 if (ibsrq_p == NULL) {
609 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
610 "srq_alloc: os_alloc failed\n");
611 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
612 DAT_RESOURCE_MEMORY));
613 }
614 (void) dapl_os_memzero(ibsrq_p, sizeof (*ibsrq_p));
615
616 (void) dapl_os_memzero(&srqc_args, sizeof (srqc_args));
617 msrq = (mlnx_umap_srq_data_out_t *)srqc_args.srqc_data_out;
618 srqc_args.srqc_pd_hkey = pz_handle->pd_handle->pd_hkey;
619 srqc_args.srqc_sizes.srqs_sz = srq_ptr->param.max_recv_dtos;
620 srqc_args.srqc_sizes.srqs_sgl = srq_ptr->param.max_recv_iov;
621
622 dapl_dbg_log(DAPL_DBG_TYPE_EP,
623 "srq_alloc: srq_ptr 0x%p, pz 0x%p (0x%llx), srq_sz %d"
624 " srq_sgl %d\n",
625 srq_ptr, pz_handle, srqc_args.srqc_pd_hkey,
626 srqc_args.srqc_sizes.srqs_sz, srqc_args.srqc_sizes.srqs_sgl);
627
628 /* The next line is only needed for backward compatibility */
629 msrq->msrq_rev = MLNX_UMAP_IF_VERSION;
630 retval = ioctl(ia_fd, DAPL_SRQ_CREATE, &srqc_args);
631 if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
632 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
633 "srq_alloc: srq_create failed errno %d, retval %d\n",
634 errno, retval);
635 dapl_os_free(ibsrq_p, sizeof (*ibsrq_p));
636 return (dapls_convert_error(errno, retval));
637 }
638
639 /* In the case of Arbel or Hermon */
640 if (msrq->msrq_rdbr_mapoffset != 0 || msrq->msrq_rdbr_maplen != 0)
641 ibsrq_p->srq_dbp = dapls_ib_get_dbp(
642 msrq->msrq_rdbr_maplen, hca_fd,
643 msrq->msrq_rdbr_mapoffset, msrq->msrq_rdbr_offset);
644
645 ibsrq_p->srq_addr = mmap64((void *)0,
646 msrq->msrq_maplen, (PROT_READ | PROT_WRITE),
647 MAP_SHARED, hca_fd, msrq->msrq_mapoffset);
648
649 if (ibsrq_p->srq_addr == MAP_FAILED ||
650 ibsrq_p->srq_dbp == MAP_FAILED) {
651 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
652 "srq_alloc: mmap failed(%d)\n", errno);
653 srqf_args.srqf_hkey = srqc_args.srqc_hkey;
654 retval = ioctl(ia_fd, DAPL_SRQ_FREE, &srqf_args);
655 if (retval != 0) {
656 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
657 "srq_alloc: SRQ_FREE err:%d\n", errno);
658 }
659 dapl_os_free(ibsrq_p, sizeof (*ibsrq_p));
660 return (dapls_convert_error(errno, 0));
661 }
662
663 ibsrq_p->srq_hkey = srqc_args.srqc_hkey;
664 ibsrq_p->srq_map_len = msrq->msrq_maplen;
665 ibsrq_p->srq_map_offset = msrq->msrq_mapoffset;
666 ibsrq_p->srq_num = msrq->msrq_srqnum;
667 ibsrq_p->srq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
668 /* since 0 is a valid index, -1 indicates invalid value */
669 ibsrq_p->srq_wq_lastwqeindex = -1;
670 ibsrq_p->srq_wq_desc_addr = msrq->msrq_desc_addr;
671 ibsrq_p->srq_wq_numwqe = msrq->msrq_numwqe;
672 ibsrq_p->srq_wq_wqesz = msrq->msrq_wqesz;
673 ibsrq_p->srq_wq_sgl = srqc_args.srqc_real_sizes.srqs_sgl;
674
675 /*
676 * update the srq handle in the srq ptr, this is needed since from
677 * here on cleanup is done by calling dapls_ib_srq_free()
678 */
679 srq_ptr->srq_handle = ibsrq_p;
680
681 premev_size = ibsrq_p->srq_wq_numwqe * sizeof (ib_work_completion_t);
682 ibsrq_p->srq_premature_events = (ib_work_completion_t *)
683 dapl_os_alloc(premev_size);
684 if (ibsrq_p->srq_premature_events == NULL) {
685 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
686 "srq_alloc: os_alloc premature_events failed\n");
687 dapls_ib_srq_free(ia_ptr, srq_ptr);
688 srq_ptr->srq_handle = NULL;
689 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
690 DAT_RESOURCE_MEMORY));
691 }
692
693 freeev_size = ibsrq_p->srq_wq_numwqe * sizeof (uint32_t);
694 ibsrq_p->srq_freepr_events = (uint32_t *)dapl_os_alloc(freeev_size);
695 if (ibsrq_p->srq_freepr_events == NULL) {
696 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
697 "srq_alloc: os_alloc freepr_events failed\n");
698 dapls_ib_srq_free(ia_ptr, srq_ptr);
699 srq_ptr->srq_handle = NULL;
700 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
701 DAT_RESOURCE_MEMORY));
702 }
703 ibsrq_p->srq_freepr_head = 0;
704 ibsrq_p->srq_freepr_tail = 0;
705 ibsrq_p->srq_freepr_num_events = ibsrq_p->srq_wq_numwqe;
706
707 /* initialize the free list of premature events */
708 for (i = 0; i < ibsrq_p->srq_freepr_num_events; i++) {
709 ibsrq_p->srq_freepr_events[i] = i;
710 /*
711 * wc_res_hash field is used to mark entries in the premature
712 * events list
713 */
714 DAPL_SET_CQE_INVALID(&(ibsrq_p->srq_premature_events[i]));
715 }
716
717 dapl_dbg_log(DAPL_DBG_TYPE_EP,
718 "srq_alloc: created, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
719 srq_ptr, srqc_args.srqc_hkey);
720
721 DAPL_INIT_SRQ(ia_ptr)(ibsrq_p);
722
723 if (dapls_tavor_srq_wrid_init(ibsrq_p) != DAT_SUCCESS) {
724 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
725 "srq_alloc: wridlist alloc failed\n");
726 dapls_ib_srq_free(ia_ptr, srq_ptr);
727 srq_ptr->srq_handle = NULL;
728 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
729 DAT_RESOURCE_MEMORY));
730 }
731 ibsrq_p->srq_ep_table = NULL;
732 /* allocate a hash table to to store EPs */
733 retval = dapls_hash_create(DAPL_HASH_TABLE_DEFAULT_CAPACITY,
734 DAT_FALSE, &ibsrq_p->srq_ep_table);
735 if (retval != DAT_SUCCESS) {
736 dapl_dbg_log(DAPL_DBG_TYPE_ERR, "dapls_ib_srq_alloc hash "
737 "create failed %d\n", retval);
738 dapls_ib_srq_free(ia_ptr, srq_ptr);
739 srq_ptr->srq_handle = NULL;
740 return (retval);
741 }
742
743 return (DAT_SUCCESS);
744 }
745
746
747 /*
748 * SRQ Free routine
749 */
750 void
dapls_ib_srq_free(IN DAPL_IA * ia_handle,IN DAPL_SRQ * srq_ptr)751 dapls_ib_srq_free(IN DAPL_IA *ia_handle, IN DAPL_SRQ *srq_ptr)
752 {
753 ib_srq_handle_t srq_handle = srq_ptr->srq_handle;
754 ib_hca_handle_t ib_hca_handle = ia_handle->hca_ptr->ib_hca_handle;
755 dapl_srq_free_t srqf_args;
756 int retval;
757
758 if (srq_handle == IB_INVALID_HANDLE) {
759 return; /* nothing to do */
760 }
761
762 if (munmap((void *)srq_handle->srq_addr, srq_handle->srq_map_len) < 0) {
763 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
764 "srq_free: munmap failed(%d)\n", errno);
765 }
766 srqf_args.srqf_hkey = srq_handle->srq_hkey;
767 retval = ioctl(ib_hca_handle->ia_fd, DAPL_SRQ_FREE, &srqf_args);
768 if (retval != 0) {
769 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
770 "srq_free: ioctl errno = %d, retval = %d\n", errno, retval);
771 }
772 dapl_dbg_log(DAPL_DBG_TYPE_EP,
773 "srq_free: freed, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
774 srq_ptr, srq_handle->srq_hkey);
775 if (srq_handle->srq_ep_table) {
776 (void) dapls_hash_free(srq_handle->srq_ep_table);
777 }
778 if (srq_handle->srq_wridlist) {
779 dapls_tavor_srq_wrid_free(srq_handle);
780 }
781 if (srq_handle->srq_freepr_events) {
782 dapl_os_free(srq_handle->srq_freepr_events,
783 srq_handle->srq_wq_numwqe * sizeof (ib_work_completion_t));
784 }
785 if (srq_handle->srq_premature_events) {
786 dapl_os_free(srq_handle->srq_premature_events,
787 srq_handle->srq_wq_numwqe * sizeof (uint32_t));
788 }
789 dapl_os_free(srq_handle, sizeof (*srq_handle));
790 srq_ptr->srq_handle = NULL;
791 }
792
793 /*
794 * Adds EP to a hashtable in SRQ
795 */
796 static DAT_RETURN
dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr,IN uint32_t qp_num,IN DAPL_EP * ep_ptr)797 dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num,
798 IN DAPL_EP *ep_ptr)
799 {
800 DAPL_HASH_TABLE *htable;
801 DAPL_HASH_KEY key;
802
803 dapl_os_assert(srq_ptr);
804
805 htable = srq_ptr->srq_ep_table;
806 key = qp_num;
807 dapl_dbg_log(DAPL_DBG_TYPE_EP,
808 "srq_insert_ep:%p %p %llx\n", srq_ptr, htable, key);
809 return (dapls_hash_insert(htable, key, ep_ptr));
810 }
811
812 /*
813 * Removes an EP from the hashtable in SRQ
814 */
815 static void
dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr,IN uint32_t qp_num)816 dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num)
817 {
818 DAPL_HASH_TABLE *htable;
819 DAPL_HASH_KEY key;
820 DAPL_EP *epp;
821 DAT_RETURN retval;
822
823 dapl_os_assert(srq_ptr);
824
825 htable = srq_ptr->srq_ep_table;
826 key = qp_num;
827
828 retval = dapls_hash_remove(htable, key, (DAPL_HASH_DATA *)&epp);
829 if (retval != DAT_SUCCESS) {
830 dapl_dbg_log(DAPL_DBG_TYPE_EP,
831 "srq_remove_ep(%d): %p %llx\n", retval, htable, key);
832 }
833 }
834
835 /*
836 * Lookup an EP from the hashtable in SRQ
837 */
838 DAPL_EP *
dapls_ib_srq_lookup_ep(IN DAPL_SRQ * srq_ptr,IN ib_work_completion_t * cqe_ptr)839 dapls_ib_srq_lookup_ep(IN DAPL_SRQ *srq_ptr, IN ib_work_completion_t *cqe_ptr)
840 {
841 DAPL_HASH_TABLE *htable;
842 DAPL_HASH_KEY key;
843 DAPL_EP *epp;
844 DAT_RETURN retval;
845
846 dapl_os_assert(srq_ptr && srq_ptr->srq_handle);
847
848 htable = srq_ptr->srq_handle->srq_ep_table;
849 key = DAPL_GET_CQE_QPN(cqe_ptr);
850 epp = NULL;
851
852 retval = dapls_hash_search(htable, key, (DAPL_HASH_DATA *)&epp);
853 if (retval != DAT_SUCCESS) {
854 dapl_dbg_log(DAPL_DBG_TYPE_EP,
855 "srq_lookup_ep(%x): %p %llx\n", retval, htable, key);
856 }
857 return (epp);
858 }
859
860
861 /*
862 * dapl_ib_srq_resize
863 *
864 * Resize an SRQ
865 *
866 * Input:
867 * srq_ptr pointer to SRQ struct
868 * srqlen new length of the SRQ
869 * Output:
870 * none
871 *
872 * Returns:
873 * DAT_SUCCESS
874 * DAT_INVALID_HANDLE
875 * DAT_INTERNAL_ERROR
876 * DAT_INSUFFICIENT_RESOURCES
877 *
878 */
879 DAT_RETURN
dapls_ib_srq_resize(IN DAPL_SRQ * srq_ptr,IN DAT_COUNT srqlen)880 dapls_ib_srq_resize(
881 IN DAPL_SRQ *srq_ptr,
882 IN DAT_COUNT srqlen)
883 {
884 ib_srq_handle_t srq_handle;
885 DAT_RETURN dat_status;
886
887 dat_status = dapli_ib_srq_resize_internal(srq_ptr, srqlen);
888 if (DAT_INSUFFICIENT_RESOURCES == DAT_GET_TYPE(dat_status)) {
889 srq_handle = srq_ptr->srq_handle;
890 /* attempt to resize back to the current size */
891 dat_status = dapli_ib_srq_resize_internal(srq_ptr,
892 srq_handle->srq_wq_numwqe);
893 if (DAT_SUCCESS != dat_status) {
894 /*
895 * XXX this is catastrophic need to post an event
896 * to the async evd
897 */
898 return (DAT_INTERNAL_ERROR);
899 }
900 }
901
902 return (dat_status);
903 }
904
905 /*
906 * dapli_ib_srq_resize_internal
907 *
908 * An internal routine to resize a SRQ.
909 *
910 * Input:
911 * srq_ptr pointer to SRQ struct
912 * srqlen new length of the srq
913 * Output:
914 * none
915 *
916 * Returns:
917 * DAT_SUCCESS
918 * DAT_INVALID_HANDLE
919 * DAT_INSUFFICIENT_RESOURCES
920 *
921 */
922 static DAT_RETURN
dapli_ib_srq_resize_internal(IN DAPL_SRQ * srq_ptr,IN DAT_COUNT srqlen)923 dapli_ib_srq_resize_internal(
924 IN DAPL_SRQ *srq_ptr,
925 IN DAT_COUNT srqlen)
926 {
927 ib_srq_handle_t srq_handle;
928 dapl_srq_resize_t resize_msg;
929 int ia_fd;
930 int hca_fd;
931 ib_work_completion_t *new_premature_events;
932 ib_work_completion_t *old_premature_events;
933 uint32_t *new_freepr_events;
934 uint32_t *old_freepr_events;
935 size_t old_premature_size;
936 size_t old_freepr_size;
937 size_t new_premature_size;
938 size_t new_freepr_size;
939 int idx, i;
940 int retval;
941 mlnx_umap_srq_data_out_t *msrq;
942
943 dapl_dbg_log(DAPL_DBG_TYPE_EP,
944 "dapls_ib_srq_resize: srq 0x%p srq_hdl 0x%p "
945 "srq_hkey 0x%016llx srqlen %d\n",
946 srq_ptr, (void *)srq_ptr->srq_handle,
947 srq_ptr->srq_handle->srq_hkey, srqlen);
948
949 srq_handle = srq_ptr->srq_handle;
950 /*
951 * Since SRQs are created in powers of 2 its possible that the
952 * previously allocated SRQ has sufficient entries. If the current
953 * SRQ is big enough and it is mapped we are done.
954 */
955 if ((srqlen < srq_handle->srq_wq_numwqe) && (srq_handle->srq_addr)) {
956 return (DAT_SUCCESS);
957 }
958
959 /* unmap the SRQ before resizing it */
960 if ((srq_handle->srq_addr) && (munmap((char *)srq_handle->srq_addr,
961 srq_handle->srq_map_len) < 0)) {
962 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
963 "srq_resize: munmap(%p:0x%llx) failed(%d)\n",
964 srq_handle->srq_addr, srq_handle->srq_map_len, errno);
965 return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ));
966 }
967 /* srq_addr is unmapped and no longer valid */
968 srq_handle->srq_addr = NULL;
969
970 ia_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;
971 hca_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->hca_fd;
972
973 (void) dapl_os_memzero(&resize_msg, sizeof (resize_msg));
974 resize_msg.srqr_hkey = srq_handle->srq_hkey;
975 resize_msg.srqr_new_size = srqlen;
976 msrq = (mlnx_umap_srq_data_out_t *)resize_msg.srqr_data_out;
977
978 /* The next line is only needed for backward compatibility */
979 msrq->msrq_rev = MLNX_UMAP_IF_VERSION;
980 retval = ioctl(ia_fd, DAPL_SRQ_RESIZE, &resize_msg);
981 if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
982 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
983 "dapls_ib_srq_resize: srq 0x%p, err: %s\n",
984 srq_ptr, strerror(errno));
985 if (errno == EINVAL) { /* Couldn't find this srq */
986 return (DAT_ERROR(DAT_INVALID_HANDLE,
987 DAT_INVALID_HANDLE_SRQ));
988 } else { /* Need to retry resize with a smaller qlen */
989 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
990 DAT_RESOURCE_SRQ));
991 }
992 }
993
994 dapl_os_assert(srq_handle->srq_num == msrq->msrq_srqnum);
995
996 /* In the case of Arbel or Hermon */
997 if (msrq->msrq_rdbr_mapoffset != 0 ||
998 msrq->msrq_rdbr_maplen != 0)
999 srq_handle->srq_dbp = dapls_ib_get_dbp(
1000 msrq->msrq_rdbr_maplen,
1001 hca_fd, msrq->msrq_rdbr_mapoffset,
1002 msrq->msrq_rdbr_offset);
1003
1004 srq_handle->srq_addr = mmap64((void *)0,
1005 msrq->msrq_maplen, (PROT_READ | PROT_WRITE),
1006 MAP_SHARED, hca_fd, msrq->msrq_mapoffset);
1007
1008 if (srq_handle->srq_addr == MAP_FAILED ||
1009 srq_handle->srq_dbp == MAP_FAILED) {
1010 srq_handle->srq_addr = NULL;
1011 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
1012 "srq_resize: mmap failed(%d)\n", errno);
1013 /* Need to retry resize with a smaller qlen */
1014 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
1015 DAT_RESOURCE_MEMORY));
1016 }
1017
1018 old_freepr_size = srq_handle->srq_wq_numwqe * sizeof (uint32_t);
1019 old_premature_size = srq_handle->srq_wq_numwqe *
1020 sizeof (ib_work_completion_t);
1021
1022 old_freepr_events = srq_handle->srq_freepr_events;
1023 old_premature_events = srq_handle->srq_premature_events;
1024
1025 new_freepr_size = resize_msg.srqr_real_size * sizeof (uint32_t);
1026 new_premature_size = resize_msg.srqr_real_size *
1027 sizeof (ib_work_completion_t);
1028
1029 new_freepr_events = (uint32_t *)dapl_os_alloc(new_freepr_size);
1030 if (new_freepr_events == NULL) {
1031 goto bail;
1032 }
1033 new_premature_events = (ib_work_completion_t *)dapl_os_alloc(
1034 new_premature_size);
1035 if (new_premature_events == NULL) {
1036 goto bail;
1037 }
1038 if (!dapls_tavor_srq_wrid_resize(srq_handle,
1039 resize_msg.srqr_real_size)) {
1040 goto bail;
1041 }
1042 idx = 0;
1043 /* copy valid premature events */
1044 for (i = 0; i < srq_handle->srq_wq_numwqe; i++) {
1045 if (!DAPL_CQE_IS_VALID(&old_premature_events[i])) {
1046 continue;
1047 }
1048 (void) dapl_os_memcpy(&new_premature_events[idx],
1049 &old_premature_events[i], sizeof (ib_work_completion_t));
1050 idx++;
1051 }
1052 dapl_os_assert(srq_handle->srq_wq_numwqe - idx ==
1053 srq_handle->srq_freepr_num_events);
1054
1055 /* Initialize free events lists */
1056 for (i = 0; i < resize_msg.srqr_real_size - idx; i++) {
1057 new_freepr_events[i] = idx + i;
1058 }
1059
1060 srq_handle->srq_freepr_events = new_freepr_events;
1061 srq_handle->srq_premature_events = new_premature_events;
1062 srq_handle->srq_freepr_num_events = resize_msg.srqr_real_size - idx;
1063 srq_handle->srq_freepr_head = 0;
1064 /* a full freepr list has tail at 0 */
1065 if (idx == 0) {
1066 srq_handle->srq_freepr_tail = 0;
1067 } else {
1068 srq_handle->srq_freepr_tail = srq_handle->srq_freepr_num_events;
1069 }
1070
1071 if (old_freepr_events) {
1072 old_freepr_size = old_freepr_size; /* pacify lint */
1073 dapl_os_free(old_freepr_events, old_freepr_size);
1074 }
1075 if (old_premature_events) {
1076 old_premature_size = old_premature_size; /* pacify lint */
1077 dapl_os_free(old_premature_events, old_premature_size);
1078 }
1079
1080 /*
1081 * update the srq fields,
1082 * note: the srq_wq_lastwqeindex doesn't change since the old
1083 * work queue is copied as a whole into the new work queue.
1084 */
1085 srq_handle->srq_map_offset = msrq->msrq_mapoffset;
1086 srq_handle->srq_map_len = msrq->msrq_maplen;
1087 srq_handle->srq_wq_desc_addr = msrq->msrq_desc_addr;
1088 srq_handle->srq_wq_numwqe = msrq->msrq_numwqe;
1089 srq_handle->srq_wq_wqesz = msrq->msrq_wqesz;
1090
1091 return (DAT_SUCCESS);
1092 bail:
1093 if (new_freepr_events) {
1094 dapl_os_free(new_freepr_events, new_freepr_size);
1095 }
1096 if (new_premature_events) {
1097 dapl_os_free(new_premature_events, new_premature_size);
1098 }
1099 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY));
1100 }
1101