1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include "dapl.h" 28 #include "dapl_adapter_util.h" 29 #include "dapl_evd_util.h" 30 #include "dapl_cr_util.h" 31 #include "dapl_lmr_util.h" 32 #include "dapl_rmr_util.h" 33 #include "dapl_cookie.h" 34 #include "dapl_tavor_ibtf_impl.h" 35 #include "dapl_hash.h" 36 37 /* Function prototypes */ 38 extern DAT_RETURN dapls_tavor_wrid_init(ib_qp_handle_t); 39 extern DAT_RETURN dapls_tavor_srq_wrid_init(ib_srq_handle_t); 40 extern void dapls_tavor_srq_wrid_free(ib_srq_handle_t); 41 extern DAT_BOOLEAN dapls_tavor_srq_wrid_resize(ib_srq_handle_t, uint32_t); 42 43 static DAT_RETURN dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr, 44 IN uint32_t qpnum, IN DAPL_EP *ep_ptr); 45 static void dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr, 46 IN uint32_t qpnum); 47 static DAT_RETURN dapli_ib_srq_resize_internal(IN DAPL_SRQ *srq_ptr, 48 IN DAT_COUNT srqlen); 49 /* 50 * dapli_get_dto_cq 51 * 52 * Obtain the cq_handle for a DTO EVD. If the EVD is NULL, use the 53 * null_ib_cq_handle. If it hasn't been created yet, create it now in 54 * the HCA structure. It will be cleaned up in dapls_ib_cqd_destroy(). 55 * 56 * This is strictly internal to IB. DAPL allows a NULL DTO EVD handle, 57 * but IB does not. So we create a CQ under the hood and make sure 58 * an error is generated if the user every tries to post, by 59 * setting the WQ length to 0 in ep_create and/or ep_modify. 60 * 61 * Returns 62 * A valid CQ handle 63 */ 64 static ib_cq_handle_t 65 dapli_get_dto_cq( 66 IN DAPL_IA *ia_ptr, 67 IN DAPL_EVD *evd_ptr) 68 { 69 dapl_evd_create_t create_msg; 70 ib_cq_handle_t cq_handle; 71 int ia_fd; 72 int retval; 73 mlnx_umap_cq_data_out_t *mcq; 74 75 if (evd_ptr != DAT_HANDLE_NULL) { 76 cq_handle = evd_ptr->ib_cq_handle; 77 } else if (ia_ptr->hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) { 78 cq_handle = ia_ptr->hca_ptr->null_ib_cq_handle; 79 } else { 80 cq_handle = (ib_cq_handle_t) 81 dapl_os_alloc(sizeof (struct dapls_ib_cq_handle)); 82 if (cq_handle == NULL) { 83 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 84 "dapli_get_dto_cq: cq malloc failed\n"); 85 ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE; 86 return (IB_INVALID_HANDLE); 87 } 88 89 /* 90 * create a fake a CQ, we don't bother to mmap this CQ 91 * since nobody know about it to reap events from it. 92 */ 93 (void) dapl_os_memzero(&create_msg, sizeof (create_msg)); 94 create_msg.evd_flags = DAT_EVD_DTO_FLAG; 95 mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out; 96 97 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd; 98 99 /* call into driver to allocate cq */ 100 retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg); 101 if (retval != 0) { 102 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 103 "dapli_get_dto_cq: DAPL_EVD_CREATE failed\n"); 104 dapl_os_free(cq_handle, 105 sizeof (struct dapls_ib_cq_handle)); 106 ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE; 107 return (IB_INVALID_HANDLE); 108 } 109 110 (void) dapl_os_memzero(cq_handle, 111 sizeof (struct dapls_ib_cq_handle)); 112 dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock); 113 cq_handle->evd_hkey = create_msg.evd_hkey; 114 cq_handle->cq_addr = NULL; 115 cq_handle->cq_map_offset = mcq->mcq_mapoffset; 116 cq_handle->cq_map_len = mcq->mcq_maplen; 117 cq_handle->cq_num = mcq->mcq_cqnum; 118 cq_handle->cq_size = create_msg.evd_cq_real_size; 119 cq_handle->cq_cqesz = mcq->mcq_cqesz; 120 cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar; 121 122 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 123 "dapli_get_dto_cq: cq 0x%p created, hkey 0x%016llx\n", 124 cq_handle, create_msg.evd_hkey); 125 126 /* save this dummy CQ handle into the hca */ 127 ia_ptr->hca_ptr->null_ib_cq_handle = cq_handle; 128 } 129 return (cq_handle); 130 } 131 132 133 /* 134 * dapl_ib_qp_alloc 135 * 136 * Alloc a QP 137 * 138 * Input: 139 * *ep_ptr pointer to EP INFO 140 * ib_hca_handle provider HCA handle 141 * ib_pd_handle provider protection domain handle 142 * cq_recv provider recv CQ handle 143 * cq_send provider send CQ handle 144 * 145 * Output: 146 * none 147 * 148 * Returns: 149 * DAT_SUCCESS 150 * DAT_INSUFFICIENT_RESOURCES 151 * 152 */ 153 DAT_RETURN 154 dapls_ib_qp_alloc( 155 IN DAPL_IA *ia_ptr, 156 IN DAPL_EP *ep_ptr, 157 IN DAPL_EP *ep_ctx_ptr) 158 { 159 dapl_ep_create_t ep_args; 160 dapl_ep_free_t epf_args; 161 ib_qp_handle_t qp_p; 162 DAPL_SRQ *srq_p; 163 ib_cq_handle_t cq_recv; 164 ib_cq_handle_t cq_send; 165 DAPL_PZ *pz_handle; 166 DAPL_EVD *evd_handle; 167 uint32_t mpt_mask; 168 size_t premev_size; 169 uint32_t i; 170 int ia_fd; 171 int hca_fd; 172 DAT_RETURN dat_status; 173 int retval; 174 mlnx_umap_qp_data_out_t *mqp; 175 176 /* check parameters */ 177 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) { 178 dapl_dbg_log(DAPL_DBG_TYPE_EP, 179 "qp_alloc: hca_handle == NULL\n"); 180 return (DAT_INVALID_PARAMETER); 181 } 182 183 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd; 184 hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd; 185 dapl_os_assert(ep_ptr->param.pz_handle != NULL); 186 dapl_os_assert(ep_ptr->param.connect_evd_handle != NULL); 187 188 /* fill in args for ep_create */ 189 (void) dapl_os_memzero(&ep_args, sizeof (ep_args)); 190 mqp = (mlnx_umap_qp_data_out_t *)ep_args.ep_qp_data_out; 191 pz_handle = (DAPL_PZ *)ep_ptr->param.pz_handle; 192 ep_args.ep_pd_hkey = pz_handle->pd_handle->pd_hkey; 193 194 cq_recv = dapli_get_dto_cq(ia_ptr, 195 (DAPL_EVD *)ep_ptr->param.recv_evd_handle); 196 ep_args.ep_rcv_evd_hkey = cq_recv->evd_hkey; 197 198 cq_send = dapli_get_dto_cq(ia_ptr, 199 (DAPL_EVD *)ep_ptr->param.request_evd_handle); 200 ep_args.ep_snd_evd_hkey = cq_send->evd_hkey; 201 202 evd_handle = (DAPL_EVD *)ep_ptr->param.connect_evd_handle; 203 ep_args.ep_conn_evd_hkey = evd_handle->ib_cq_handle->evd_hkey; 204 205 ep_args.ep_ch_sizes.dcs_sq = ep_ptr->param.ep_attr.max_request_dtos; 206 ep_args.ep_ch_sizes.dcs_sq_sgl = ep_ptr->param.ep_attr.max_request_iov; 207 208 qp_p = (ib_qp_handle_t)dapl_os_alloc( 209 sizeof (struct dapls_ib_qp_handle)); 210 if (qp_p == NULL) { 211 dapl_dbg_log(DAPL_DBG_TYPE_EP, 212 "qp_alloc: os_alloc failed\n"); 213 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 214 DAT_RESOURCE_MEMORY)); 215 } 216 217 (void) dapl_os_memzero(qp_p, sizeof (*qp_p)); 218 219 if (ep_ptr->param.srq_handle == NULL) { 220 premev_size = ep_ptr->param.ep_attr.max_recv_dtos * 221 sizeof (ib_work_completion_t); 222 if (premev_size != 0) { 223 qp_p->qp_premature_events = (ib_work_completion_t *) 224 dapl_os_alloc(premev_size); 225 if (qp_p->qp_premature_events == NULL) { 226 dapl_dbg_log(DAPL_DBG_TYPE_EP, 227 "qp_alloc:alloc premature_events failed\n"); 228 dapl_os_free(qp_p, sizeof (*qp_p)); 229 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 230 DAT_RESOURCE_MEMORY)); 231 } 232 } 233 qp_p->qp_num_premature_events = 0; 234 ep_args.ep_srq_hkey = 0; 235 ep_args.ep_srq_attached = 0; 236 ep_args.ep_ch_sizes.dcs_rq = 237 ep_ptr->param.ep_attr.max_recv_dtos; 238 ep_args.ep_ch_sizes.dcs_rq_sgl = 239 ep_ptr->param.ep_attr.max_recv_iov; 240 } else { 241 premev_size = 0; 242 srq_p = (DAPL_SRQ *)ep_ptr->param.srq_handle; 243 /* premature events for EPs with SRQ sit on the SRQ */ 244 qp_p->qp_premature_events = srq_p->srq_handle-> 245 srq_premature_events; 246 qp_p->qp_num_premature_events = 0; 247 ep_args.ep_srq_hkey = srq_p->srq_handle->srq_hkey; 248 ep_args.ep_srq_attached = 1; 249 ep_args.ep_ch_sizes.dcs_rq = 0; 250 ep_args.ep_ch_sizes.dcs_rq_sgl = 0; 251 } 252 253 /* 254 * there are cases when ep_ptr is a dummy container ep, and the orig 255 * ep pointer is passed in ep_ctx_ptr. eg - dapl_ep_modify does this. 256 * ep_cookie should be the actual ep pointer, not the dummy container 257 * ep since the kernel returns this via events and the CM callback 258 * routines 259 */ 260 ep_args.ep_cookie = (uintptr_t)ep_ctx_ptr; 261 262 dapl_dbg_log(DAPL_DBG_TYPE_EP, 263 "qp_alloc: ep_ptr 0x%p, pz 0x%p (0x%llx), rcv_evd 0x%p (0x%llx)\n" 264 " snd_evd 0x%p (0x%llx), conn_evd 0x%p (0x%llx)\n" 265 " srq_hdl 0x%p (0x%llx)\n" 266 " sq_sz %d, rq_sz %d, sq_sgl_sz %d, rq_sgl_sz %d\n", 267 ep_ptr, pz_handle, ep_args.ep_pd_hkey, 268 ep_ptr->param.recv_evd_handle, ep_args.ep_rcv_evd_hkey, 269 ep_ptr->param.request_evd_handle, ep_args.ep_snd_evd_hkey, 270 ep_ptr->param.connect_evd_handle, ep_args.ep_conn_evd_hkey, 271 ep_ptr->param.srq_handle, ep_args.ep_srq_hkey, 272 ep_args.ep_ch_sizes.dcs_sq, ep_args.ep_ch_sizes.dcs_rq, 273 ep_args.ep_ch_sizes.dcs_sq_sgl, ep_args.ep_ch_sizes.dcs_rq_sgl); 274 275 /* The next line is only needed for backward compatibility */ 276 mqp->mqp_rev = MLNX_UMAP_IF_VERSION; 277 retval = ioctl(ia_fd, DAPL_EP_CREATE, &ep_args); 278 if (retval != 0 || mqp->mqp_rev != MLNX_UMAP_IF_VERSION) { 279 dapl_dbg_log(DAPL_DBG_TYPE_EP, 280 "qp_alloc: ep_create failed errno %d, retval %d\n", 281 errno, retval); 282 if (premev_size != 0) { 283 dapl_os_free(qp_p->qp_premature_events, premev_size); 284 } 285 dapl_os_free(qp_p, sizeof (*qp_p)); 286 return (dapls_convert_error(errno, retval)); 287 } 288 289 /* In the case of Arbel or Hermon */ 290 if (mqp->mqp_sdbr_mapoffset != 0 || mqp->mqp_sdbr_maplen != 0) 291 qp_p->qp_sq_dbp = dapls_ib_get_dbp(mqp->mqp_sdbr_maplen, 292 hca_fd, mqp->mqp_sdbr_mapoffset, mqp->mqp_sdbr_offset); 293 if (mqp->mqp_rdbr_mapoffset != 0 || mqp->mqp_rdbr_maplen != 0) 294 qp_p->qp_rq_dbp = dapls_ib_get_dbp(mqp->mqp_rdbr_maplen, 295 hca_fd, mqp->mqp_rdbr_mapoffset, mqp->mqp_rdbr_offset); 296 297 qp_p->qp_addr = mmap64((void *)0, mqp->mqp_maplen, 298 (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd, 299 mqp->mqp_mapoffset); 300 301 if (qp_p->qp_addr == MAP_FAILED || 302 qp_p->qp_sq_dbp == MAP_FAILED || 303 qp_p->qp_rq_dbp == MAP_FAILED) { 304 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 305 "qp_alloc: mmap failed(%d)\n", errno); 306 epf_args.epf_hkey = ep_args.ep_hkey; 307 retval = ioctl(ia_fd, DAPL_EP_FREE, &epf_args); 308 if (retval != 0) { 309 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 310 "qp_alloc: EP_FREE err:%d\n", errno); 311 } 312 if (premev_size != 0) { 313 dapl_os_free(qp_p->qp_premature_events, premev_size); 314 } 315 dapl_os_free(qp_p, sizeof (*qp_p)); 316 return (dapls_convert_error(errno, 0)); 317 } 318 319 qp_p->qp_map_len = mqp->mqp_maplen; 320 qp_p->qp_num = mqp->mqp_qpnum; 321 qp_p->qp_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar; 322 qp_p->qp_ia_bf = ia_ptr->hca_ptr->ib_hca_handle->ia_bf; 323 qp_p->qp_ia_bf_toggle = ia_ptr->hca_ptr->ib_hca_handle->ia_bf_toggle; 324 325 evd_handle = (DAPL_EVD *)ep_ptr->param.request_evd_handle; 326 qp_p->qp_sq_cqhdl = evd_handle->ib_cq_handle; 327 qp_p->qp_sq_lastwqeaddr = NULL; 328 qp_p->qp_sq_wqhdr = NULL; 329 qp_p->qp_sq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_sq_off); 330 qp_p->qp_sq_desc_addr = mqp->mqp_sq_desc_addr; 331 qp_p->qp_sq_numwqe = mqp->mqp_sq_numwqe; 332 qp_p->qp_sq_wqesz = mqp->mqp_sq_wqesz; 333 qp_p->qp_sq_sgl = ep_ptr->param.ep_attr.max_request_iov; 334 qp_p->qp_sq_inline = ia_ptr->hca_ptr->max_inline_send; 335 qp_p->qp_sq_headroom = mqp->mqp_sq_headroomwqes; 336 337 evd_handle = (DAPL_EVD *)ep_ptr->param.recv_evd_handle; 338 qp_p->qp_rq_cqhdl = evd_handle->ib_cq_handle; 339 qp_p->qp_rq_lastwqeaddr = NULL; 340 qp_p->qp_rq_wqhdr = NULL; 341 qp_p->qp_rq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_rq_off); 342 qp_p->qp_rq_desc_addr = mqp->mqp_rq_desc_addr; 343 qp_p->qp_rq_numwqe = mqp->mqp_rq_numwqe; 344 qp_p->qp_rq_wqesz = mqp->mqp_rq_wqesz; 345 qp_p->qp_rq_sgl = ep_ptr->param.ep_attr.max_recv_iov; 346 347 dapl_dbg_log(DAPL_DBG_TYPE_EP, 348 "qp_alloc: created, qp_sq_buf %p, qp_rq_buf %p\n", 349 qp_p->qp_sq_buf, qp_p->qp_rq_buf); 350 dapl_dbg_log(DAPL_DBG_TYPE_EP, 351 "qp_alloc: created, sq numwqe %x wqesz %x, rq numwqe %x wqesz %x\n", 352 qp_p->qp_sq_numwqe, qp_p->qp_sq_wqesz, 353 qp_p->qp_rq_numwqe, qp_p->qp_rq_wqesz); 354 dapl_dbg_log(DAPL_DBG_TYPE_EP, 355 "qp_alloc: created, qp_sq_desc_addr %x, qp_rq_desc_addr %x\n", 356 mqp->mqp_sq_desc_addr, mqp->mqp_rq_desc_addr); 357 dapl_dbg_log(DAPL_DBG_TYPE_EP, 358 "qp_alloc: created, ep_ptr 0x%p, ep_hkey 0x%016llx\n\n", 359 ep_ptr, ep_args.ep_hkey); 360 361 qp_p->ep_hkey = ep_args.ep_hkey; 362 363 /* 364 * Calculate the number of bits in max_rmrs - this is indirectly 365 * the max number of entried in the MPT table (defaults to 512K 366 * but is configurable). This value is used while creating new 367 * rkeys in bind processing (see dapl_tavor_hw.c). 368 * Stash this value in the qp handle, don't want to do this math 369 * for every bind 370 */ 371 mpt_mask = (uint32_t)ia_ptr->hca_ptr->ia_attr.max_rmrs - 1; 372 for (i = 0; mpt_mask > 0; mpt_mask = (mpt_mask >> 1), i++) 373 ; 374 qp_p->qp_num_mpt_shift = (uint32_t)i; 375 376 ep_ptr->qpn = qp_p->qp_num; 377 /* update the qp handle in the ep ptr */ 378 ep_ptr->qp_handle = qp_p; 379 /* 380 * ibt_alloc_rc_channel transitions the qp state to INIT. 381 * hence we directly transition from UNATTACHED to INIT 382 */ 383 ep_ptr->qp_state = IBT_STATE_INIT; 384 385 if (ep_ptr->param.srq_handle) { 386 /* insert ep into the SRQ's ep_table */ 387 dat_status = dapli_ib_srq_add_ep(srq_p->srq_handle, 388 qp_p->qp_num, ep_ptr); 389 if (dat_status != DAT_SUCCESS) { 390 dapl_dbg_log(DAPL_DBG_TYPE_EP, 391 "qp_alloc: srq_add_ep failed ep_ptr 0x%p, 0x%x\n", 392 ep_ptr, dat_status); 393 (void) dapls_ib_qp_free(ia_ptr, ep_ptr); 394 return (DAT_INVALID_PARAMETER); 395 } 396 qp_p->qp_srq_enabled = 1; 397 qp_p->qp_srq = srq_p->srq_handle; 398 } else { 399 qp_p->qp_srq_enabled = 0; 400 qp_p->qp_srq = NULL; 401 } 402 DAPL_INIT_QP(ia_ptr)(qp_p); 403 404 if (dapls_tavor_wrid_init(qp_p) != DAT_SUCCESS) { 405 (void) dapls_ib_qp_free(ia_ptr, ep_ptr); 406 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 407 DAT_RESOURCE_MEMORY)); 408 } 409 410 return (DAT_SUCCESS); 411 } 412 413 414 /* 415 * dapls_ib_qp_free 416 * 417 * Free a QP 418 * 419 * Input: 420 * *ep_ptr pointer to EP INFO 421 * ib_hca_handle provider HCA handle 422 * 423 * Output: 424 * none 425 * 426 * Returns: 427 * none 428 * 429 */ 430 DAT_RETURN 431 dapls_ib_qp_free(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr) 432 { 433 ib_qp_handle_t qp_p = ep_ptr->qp_handle; 434 ib_hca_handle_t ib_hca_handle = ia_ptr->hca_ptr->ib_hca_handle; 435 dapl_ep_free_t args; 436 int retval; 437 438 if ((ep_ptr->qp_handle != IB_INVALID_HANDLE) && 439 (ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED)) { 440 if (munmap((void *)qp_p->qp_addr, qp_p->qp_map_len) < 0) { 441 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 442 "qp_free: munmap failed(%d)\n", errno); 443 } 444 args.epf_hkey = qp_p->ep_hkey; 445 retval = ioctl(ib_hca_handle->ia_fd, DAPL_EP_FREE, &args); 446 if (retval != 0) { 447 dapl_dbg_log(DAPL_DBG_TYPE_EP, 448 "qp_free: ioctl errno = %d, retval = %d\n", 449 errno, retval); 450 } 451 dapl_dbg_log(DAPL_DBG_TYPE_EP, 452 "qp_free: freed, ep_ptr 0x%p, ep_hkey 0x%016llx\n", 453 ep_ptr, qp_p->ep_hkey); 454 455 if (qp_p->qp_srq) { 456 dapli_ib_srq_remove_ep(qp_p->qp_srq, qp_p->qp_num); 457 } else { 458 if (qp_p->qp_premature_events) { 459 dapl_os_free(qp_p->qp_premature_events, 460 ep_ptr->param.ep_attr.max_recv_dtos * 461 sizeof (ib_work_completion_t)); 462 } 463 } 464 dapl_os_free(qp_p, sizeof (*qp_p)); 465 ep_ptr->qp_handle = NULL; 466 } 467 return (DAT_SUCCESS); 468 } 469 470 471 /* 472 * dapl_ib_qp_modify 473 * 474 * Set the QP to the parameters specified in an EP_PARAM 475 * 476 * We can't be sure what state the QP is in so we first obtain the state 477 * from the driver. The EP_PARAM structure that is provided has been 478 * sanitized such that only non-zero values are valid. 479 * 480 * Input: 481 * ib_hca_handle HCA handle 482 * qp_handle QP handle 483 * ep_attr Sanitized EP Params 484 * 485 * Output: 486 * none 487 * 488 * Returns: 489 * DAT_SUCCESS 490 * DAT_INSUFFICIENT_RESOURCES 491 * DAT_INVALID_PARAMETER 492 * 493 */ 494 DAT_RETURN 495 dapls_ib_qp_modify(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr, 496 IN DAT_EP_ATTR *ep_attr) 497 { 498 dapl_ep_modify_t epm_args; 499 boolean_t epm_needed; 500 int ia_fd; 501 int retval; 502 503 504 if (ep_ptr->qp_handle == NULL) { 505 dapl_dbg_log(DAPL_DBG_TYPE_EP, 506 "qp_modify: qp_handle == NULL\n"); 507 return (DAT_INVALID_PARAMETER); 508 } 509 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) { 510 dapl_dbg_log(DAPL_DBG_TYPE_EP, 511 "qp_modify: hca_handle == NULL\n"); 512 return (DAT_INVALID_PARAMETER); 513 } 514 515 epm_needed = B_FALSE; 516 517 /* 518 * NOTE: ep_attr->max_mtu_size indicates the maximum message 519 * size, which is always 2GB for IB. Nothing to do with the IB 520 * implementation, nothing to set up. 521 */ 522 523 if (ep_attr->max_rdma_size > 0) { 524 if (ep_attr->max_rdma_size > DAPL_IB_MAX_MESSAGE_SIZE) { 525 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 526 } 527 } 528 529 (void) memset((void *)&epm_args, 0, sizeof (epm_args)); 530 /* 531 * The following parameters are dealt by creating a new qp 532 * in dapl_ep_modify. 533 * - max_recv_dtos 534 * - max_request_dtos 535 * - max_recv_iov 536 * - max_request_iov 537 */ 538 539 if (ep_attr->max_rdma_read_in > 0) { 540 epm_args.epm_flags |= IBT_CEP_SET_RDMARA_IN; 541 epm_args.epm_rdma_ra_in = ep_attr->max_rdma_read_in; 542 epm_needed = B_TRUE; 543 } 544 if (ep_attr->max_rdma_read_out > 0) { 545 epm_args.epm_flags |= IBT_CEP_SET_RDMARA_OUT; 546 epm_args.epm_rdma_ra_out = ep_attr->max_rdma_read_out; 547 epm_needed = B_TRUE; 548 } 549 550 if (!epm_needed) { 551 dapl_dbg_log(DAPL_DBG_TYPE_EP, 552 "qp_modify: ep_hkey = %016llx nothing to do\n", 553 ep_ptr->qp_handle->ep_hkey); 554 return (DAT_SUCCESS); 555 } 556 557 epm_args.epm_hkey = ep_ptr->qp_handle->ep_hkey; 558 559 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd; 560 561 retval = ioctl(ia_fd, DAPL_EP_MODIFY, &epm_args); 562 if (retval != 0) { 563 dapl_dbg_log(DAPL_DBG_TYPE_EP, 564 "qp_modify: ioctl failed errno %d, retval %d\n", 565 errno, retval); 566 return (dapls_convert_error(errno, retval)); 567 } 568 569 dapl_dbg_log(DAPL_DBG_TYPE_EP, 570 "qp_modify: ep_hkey = %016llx\n", ep_ptr->qp_handle->ep_hkey); 571 return (DAT_SUCCESS); 572 } 573 574 /* 575 * Allocate the srq data structure as well as the kernel resource 576 * corresponding to it. 577 */ 578 DAT_RETURN 579 dapls_ib_srq_alloc(IN DAPL_IA *ia_ptr, IN DAPL_SRQ *srq_ptr) 580 { 581 dapl_srq_create_t srqc_args; 582 dapl_srq_free_t srqf_args; 583 ib_srq_handle_t ibsrq_p; 584 DAPL_PZ *pz_handle; 585 uint32_t i; 586 size_t premev_size; 587 size_t freeev_size; 588 int ia_fd; 589 int hca_fd; 590 int retval; 591 mlnx_umap_srq_data_out_t *msrq; 592 593 /* check parameters */ 594 if (ia_ptr->hca_ptr->ib_hca_handle == NULL) { 595 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 596 "srq_alloc: hca_handle == NULL\n"); 597 return (DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1)); 598 } 599 600 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd; 601 hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd; 602 dapl_os_assert(srq_ptr->param.pz_handle != NULL); 603 604 /* fill in args for srq_create */ 605 pz_handle = (DAPL_PZ *)srq_ptr->param.pz_handle; 606 607 ibsrq_p = (ib_srq_handle_t)dapl_os_alloc(sizeof (*ibsrq_p)); 608 if (ibsrq_p == NULL) { 609 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 610 "srq_alloc: os_alloc failed\n"); 611 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 612 DAT_RESOURCE_MEMORY)); 613 } 614 (void) dapl_os_memzero(ibsrq_p, sizeof (*ibsrq_p)); 615 616 (void) dapl_os_memzero(&srqc_args, sizeof (srqc_args)); 617 msrq = (mlnx_umap_srq_data_out_t *)srqc_args.srqc_data_out; 618 srqc_args.srqc_pd_hkey = pz_handle->pd_handle->pd_hkey; 619 srqc_args.srqc_sizes.srqs_sz = srq_ptr->param.max_recv_dtos; 620 srqc_args.srqc_sizes.srqs_sgl = srq_ptr->param.max_recv_iov; 621 622 dapl_dbg_log(DAPL_DBG_TYPE_EP, 623 "srq_alloc: srq_ptr 0x%p, pz 0x%p (0x%llx), srq_sz %d" 624 " srq_sgl %d\n", 625 srq_ptr, pz_handle, srqc_args.srqc_pd_hkey, 626 srqc_args.srqc_sizes.srqs_sz, srqc_args.srqc_sizes.srqs_sgl); 627 628 /* The next line is only needed for backward compatibility */ 629 msrq->msrq_rev = MLNX_UMAP_IF_VERSION; 630 retval = ioctl(ia_fd, DAPL_SRQ_CREATE, &srqc_args); 631 if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) { 632 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 633 "srq_alloc: srq_create failed errno %d, retval %d\n", 634 errno, retval); 635 dapl_os_free(ibsrq_p, sizeof (*ibsrq_p)); 636 return (dapls_convert_error(errno, retval)); 637 } 638 639 /* In the case of Arbel or Hermon */ 640 if (msrq->msrq_rdbr_mapoffset != 0 || msrq->msrq_rdbr_maplen != 0) 641 ibsrq_p->srq_dbp = dapls_ib_get_dbp( 642 msrq->msrq_rdbr_maplen, hca_fd, 643 msrq->msrq_rdbr_mapoffset, msrq->msrq_rdbr_offset); 644 645 ibsrq_p->srq_addr = mmap64((void *)0, 646 msrq->msrq_maplen, (PROT_READ | PROT_WRITE), 647 MAP_SHARED, hca_fd, msrq->msrq_mapoffset); 648 649 if (ibsrq_p->srq_addr == MAP_FAILED || 650 ibsrq_p->srq_dbp == MAP_FAILED) { 651 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 652 "srq_alloc: mmap failed(%d)\n", errno); 653 srqf_args.srqf_hkey = srqc_args.srqc_hkey; 654 retval = ioctl(ia_fd, DAPL_SRQ_FREE, &srqf_args); 655 if (retval != 0) { 656 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 657 "srq_alloc: SRQ_FREE err:%d\n", errno); 658 } 659 dapl_os_free(ibsrq_p, sizeof (*ibsrq_p)); 660 return (dapls_convert_error(errno, 0)); 661 } 662 663 ibsrq_p->srq_hkey = srqc_args.srqc_hkey; 664 ibsrq_p->srq_map_len = msrq->msrq_maplen; 665 ibsrq_p->srq_map_offset = msrq->msrq_mapoffset; 666 ibsrq_p->srq_num = msrq->msrq_srqnum; 667 ibsrq_p->srq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar; 668 /* since 0 is a valid index, -1 indicates invalid value */ 669 ibsrq_p->srq_wq_lastwqeindex = -1; 670 ibsrq_p->srq_wq_desc_addr = msrq->msrq_desc_addr; 671 ibsrq_p->srq_wq_numwqe = msrq->msrq_numwqe; 672 ibsrq_p->srq_wq_wqesz = msrq->msrq_wqesz; 673 ibsrq_p->srq_wq_sgl = srqc_args.srqc_real_sizes.srqs_sgl; 674 675 /* 676 * update the srq handle in the srq ptr, this is needed since from 677 * here on cleanup is done by calling dapls_ib_srq_free() 678 */ 679 srq_ptr->srq_handle = ibsrq_p; 680 681 premev_size = ibsrq_p->srq_wq_numwqe * sizeof (ib_work_completion_t); 682 ibsrq_p->srq_premature_events = (ib_work_completion_t *) 683 dapl_os_alloc(premev_size); 684 if (ibsrq_p->srq_premature_events == NULL) { 685 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 686 "srq_alloc: os_alloc premature_events failed\n"); 687 dapls_ib_srq_free(ia_ptr, srq_ptr); 688 srq_ptr->srq_handle = NULL; 689 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 690 DAT_RESOURCE_MEMORY)); 691 } 692 693 freeev_size = ibsrq_p->srq_wq_numwqe * sizeof (uint32_t); 694 ibsrq_p->srq_freepr_events = (uint32_t *)dapl_os_alloc(freeev_size); 695 if (ibsrq_p->srq_freepr_events == NULL) { 696 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 697 "srq_alloc: os_alloc freepr_events failed\n"); 698 dapls_ib_srq_free(ia_ptr, srq_ptr); 699 srq_ptr->srq_handle = NULL; 700 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 701 DAT_RESOURCE_MEMORY)); 702 } 703 ibsrq_p->srq_freepr_head = 0; 704 ibsrq_p->srq_freepr_tail = 0; 705 ibsrq_p->srq_freepr_num_events = ibsrq_p->srq_wq_numwqe; 706 707 /* initialize the free list of premature events */ 708 for (i = 0; i < ibsrq_p->srq_freepr_num_events; i++) { 709 ibsrq_p->srq_freepr_events[i] = i; 710 /* 711 * wc_res_hash field is used to mark entries in the premature 712 * events list 713 */ 714 DAPL_SET_CQE_INVALID(&(ibsrq_p->srq_premature_events[i])); 715 } 716 717 dapl_dbg_log(DAPL_DBG_TYPE_EP, 718 "srq_alloc: created, srq_ptr 0x%p, srq_hkey 0x%016llx\n", 719 srq_ptr, srqc_args.srqc_hkey); 720 721 DAPL_INIT_SRQ(ia_ptr)(ibsrq_p); 722 723 if (dapls_tavor_srq_wrid_init(ibsrq_p) != DAT_SUCCESS) { 724 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 725 "srq_alloc: wridlist alloc failed\n"); 726 dapls_ib_srq_free(ia_ptr, srq_ptr); 727 srq_ptr->srq_handle = NULL; 728 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 729 DAT_RESOURCE_MEMORY)); 730 } 731 ibsrq_p->srq_ep_table = NULL; 732 /* allocate a hash table to to store EPs */ 733 retval = dapls_hash_create(DAPL_HASH_TABLE_DEFAULT_CAPACITY, 734 DAT_FALSE, &ibsrq_p->srq_ep_table); 735 if (retval != DAT_SUCCESS) { 736 dapl_dbg_log(DAPL_DBG_TYPE_ERR, "dapls_ib_srq_alloc hash " 737 "create failed %d\n", retval); 738 dapls_ib_srq_free(ia_ptr, srq_ptr); 739 srq_ptr->srq_handle = NULL; 740 return (retval); 741 } 742 743 return (DAT_SUCCESS); 744 } 745 746 747 /* 748 * SRQ Free routine 749 */ 750 void 751 dapls_ib_srq_free(IN DAPL_IA *ia_handle, IN DAPL_SRQ *srq_ptr) 752 { 753 ib_srq_handle_t srq_handle = srq_ptr->srq_handle; 754 ib_hca_handle_t ib_hca_handle = ia_handle->hca_ptr->ib_hca_handle; 755 dapl_srq_free_t srqf_args; 756 int retval; 757 758 if (srq_handle == IB_INVALID_HANDLE) { 759 return; /* nothing to do */ 760 } 761 762 if (munmap((void *)srq_handle->srq_addr, srq_handle->srq_map_len) < 0) { 763 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 764 "srq_free: munmap failed(%d)\n", errno); 765 } 766 srqf_args.srqf_hkey = srq_handle->srq_hkey; 767 retval = ioctl(ib_hca_handle->ia_fd, DAPL_SRQ_FREE, &srqf_args); 768 if (retval != 0) { 769 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 770 "srq_free: ioctl errno = %d, retval = %d\n", errno, retval); 771 } 772 dapl_dbg_log(DAPL_DBG_TYPE_EP, 773 "srq_free: freed, srq_ptr 0x%p, srq_hkey 0x%016llx\n", 774 srq_ptr, srq_handle->srq_hkey); 775 if (srq_handle->srq_ep_table) { 776 (void) dapls_hash_free(srq_handle->srq_ep_table); 777 } 778 if (srq_handle->srq_wridlist) { 779 dapls_tavor_srq_wrid_free(srq_handle); 780 } 781 if (srq_handle->srq_freepr_events) { 782 dapl_os_free(srq_handle->srq_freepr_events, 783 srq_handle->srq_wq_numwqe * sizeof (ib_work_completion_t)); 784 } 785 if (srq_handle->srq_premature_events) { 786 dapl_os_free(srq_handle->srq_premature_events, 787 srq_handle->srq_wq_numwqe * sizeof (uint32_t)); 788 } 789 dapl_os_free(srq_handle, sizeof (*srq_handle)); 790 srq_ptr->srq_handle = NULL; 791 } 792 793 /* 794 * Adds EP to a hashtable in SRQ 795 */ 796 static DAT_RETURN 797 dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num, 798 IN DAPL_EP *ep_ptr) 799 { 800 DAPL_HASH_TABLE *htable; 801 DAPL_HASH_KEY key; 802 803 dapl_os_assert(srq_ptr); 804 805 htable = srq_ptr->srq_ep_table; 806 key = qp_num; 807 dapl_dbg_log(DAPL_DBG_TYPE_EP, 808 "srq_insert_ep:%p %p %llx\n", srq_ptr, htable, key); 809 return (dapls_hash_insert(htable, key, ep_ptr)); 810 } 811 812 /* 813 * Removes an EP from the hashtable in SRQ 814 */ 815 static void 816 dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num) 817 { 818 DAPL_HASH_TABLE *htable; 819 DAPL_HASH_KEY key; 820 DAPL_EP *epp; 821 DAT_RETURN retval; 822 823 dapl_os_assert(srq_ptr); 824 825 htable = srq_ptr->srq_ep_table; 826 key = qp_num; 827 828 retval = dapls_hash_remove(htable, key, (DAPL_HASH_DATA *)&epp); 829 if (retval != DAT_SUCCESS) { 830 dapl_dbg_log(DAPL_DBG_TYPE_EP, 831 "srq_remove_ep(%d): %p %llx\n", retval, htable, key); 832 } 833 } 834 835 /* 836 * Lookup an EP from the hashtable in SRQ 837 */ 838 DAPL_EP * 839 dapls_ib_srq_lookup_ep(IN DAPL_SRQ *srq_ptr, IN ib_work_completion_t *cqe_ptr) 840 { 841 DAPL_HASH_TABLE *htable; 842 DAPL_HASH_KEY key; 843 DAPL_EP *epp; 844 DAT_RETURN retval; 845 846 dapl_os_assert(srq_ptr && srq_ptr->srq_handle); 847 848 htable = srq_ptr->srq_handle->srq_ep_table; 849 key = DAPL_GET_CQE_QPN(cqe_ptr); 850 epp = NULL; 851 852 retval = dapls_hash_search(htable, key, (DAPL_HASH_DATA *)&epp); 853 if (retval != DAT_SUCCESS) { 854 dapl_dbg_log(DAPL_DBG_TYPE_EP, 855 "srq_lookup_ep(%x): %p %llx\n", retval, htable, key); 856 } 857 return (epp); 858 } 859 860 861 /* 862 * dapl_ib_srq_resize 863 * 864 * Resize an SRQ 865 * 866 * Input: 867 * srq_ptr pointer to SRQ struct 868 * srqlen new length of the SRQ 869 * Output: 870 * none 871 * 872 * Returns: 873 * DAT_SUCCESS 874 * DAT_INVALID_HANDLE 875 * DAT_INTERNAL_ERROR 876 * DAT_INSUFFICIENT_RESOURCES 877 * 878 */ 879 DAT_RETURN 880 dapls_ib_srq_resize( 881 IN DAPL_SRQ *srq_ptr, 882 IN DAT_COUNT srqlen) 883 { 884 ib_srq_handle_t srq_handle; 885 DAT_RETURN dat_status; 886 887 dat_status = dapli_ib_srq_resize_internal(srq_ptr, srqlen); 888 if (DAT_INSUFFICIENT_RESOURCES == DAT_GET_TYPE(dat_status)) { 889 srq_handle = srq_ptr->srq_handle; 890 /* attempt to resize back to the current size */ 891 dat_status = dapli_ib_srq_resize_internal(srq_ptr, 892 srq_handle->srq_wq_numwqe); 893 if (DAT_SUCCESS != dat_status) { 894 /* 895 * XXX this is catastrophic need to post an event 896 * to the async evd 897 */ 898 return (DAT_INTERNAL_ERROR); 899 } 900 } 901 902 return (dat_status); 903 } 904 905 /* 906 * dapli_ib_srq_resize_internal 907 * 908 * An internal routine to resize a SRQ. 909 * 910 * Input: 911 * srq_ptr pointer to SRQ struct 912 * srqlen new length of the srq 913 * Output: 914 * none 915 * 916 * Returns: 917 * DAT_SUCCESS 918 * DAT_INVALID_HANDLE 919 * DAT_INSUFFICIENT_RESOURCES 920 * 921 */ 922 static DAT_RETURN 923 dapli_ib_srq_resize_internal( 924 IN DAPL_SRQ *srq_ptr, 925 IN DAT_COUNT srqlen) 926 { 927 ib_srq_handle_t srq_handle; 928 dapl_srq_resize_t resize_msg; 929 int ia_fd; 930 int hca_fd; 931 ib_work_completion_t *new_premature_events; 932 ib_work_completion_t *old_premature_events; 933 uint32_t *new_freepr_events; 934 uint32_t *old_freepr_events; 935 size_t old_premature_size; 936 size_t old_freepr_size; 937 size_t new_premature_size; 938 size_t new_freepr_size; 939 int idx, i; 940 int retval; 941 mlnx_umap_srq_data_out_t *msrq; 942 943 dapl_dbg_log(DAPL_DBG_TYPE_EP, 944 "dapls_ib_srq_resize: srq 0x%p srq_hdl 0x%p " 945 "srq_hkey 0x%016llx srqlen %d\n", 946 srq_ptr, (void *)srq_ptr->srq_handle, 947 srq_ptr->srq_handle->srq_hkey, srqlen); 948 949 srq_handle = srq_ptr->srq_handle; 950 /* 951 * Since SRQs are created in powers of 2 its possible that the 952 * previously allocated SRQ has sufficient entries. If the current 953 * SRQ is big enough and it is mapped we are done. 954 */ 955 if ((srqlen < srq_handle->srq_wq_numwqe) && (srq_handle->srq_addr)) { 956 return (DAT_SUCCESS); 957 } 958 959 /* unmap the SRQ before resizing it */ 960 if ((srq_handle->srq_addr) && (munmap((char *)srq_handle->srq_addr, 961 srq_handle->srq_map_len) < 0)) { 962 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 963 "srq_resize: munmap(%p:0x%llx) failed(%d)\n", 964 srq_handle->srq_addr, srq_handle->srq_map_len, errno); 965 return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ)); 966 } 967 /* srq_addr is unmapped and no longer valid */ 968 srq_handle->srq_addr = NULL; 969 970 ia_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd; 971 hca_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->hca_fd; 972 973 (void) dapl_os_memzero(&resize_msg, sizeof (resize_msg)); 974 resize_msg.srqr_hkey = srq_handle->srq_hkey; 975 resize_msg.srqr_new_size = srqlen; 976 msrq = (mlnx_umap_srq_data_out_t *)resize_msg.srqr_data_out; 977 978 /* The next line is only needed for backward compatibility */ 979 msrq->msrq_rev = MLNX_UMAP_IF_VERSION; 980 retval = ioctl(ia_fd, DAPL_SRQ_RESIZE, &resize_msg); 981 if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) { 982 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 983 "dapls_ib_srq_resize: srq 0x%p, err: %s\n", 984 srq_ptr, strerror(errno)); 985 if (errno == EINVAL) { /* Couldn't find this srq */ 986 return (DAT_ERROR(DAT_INVALID_HANDLE, 987 DAT_INVALID_HANDLE_SRQ)); 988 } else { /* Need to retry resize with a smaller qlen */ 989 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 990 DAT_RESOURCE_SRQ)); 991 } 992 } 993 994 dapl_os_assert(srq_handle->srq_num == msrq->msrq_srqnum); 995 996 /* In the case of Arbel or Hermon */ 997 if (msrq->msrq_rdbr_mapoffset != 0 || 998 msrq->msrq_rdbr_maplen != 0) 999 srq_handle->srq_dbp = dapls_ib_get_dbp( 1000 msrq->msrq_rdbr_maplen, 1001 hca_fd, msrq->msrq_rdbr_mapoffset, 1002 msrq->msrq_rdbr_offset); 1003 1004 srq_handle->srq_addr = mmap64((void *)0, 1005 msrq->msrq_maplen, (PROT_READ | PROT_WRITE), 1006 MAP_SHARED, hca_fd, msrq->msrq_mapoffset); 1007 1008 if (srq_handle->srq_addr == MAP_FAILED || 1009 srq_handle->srq_dbp == MAP_FAILED) { 1010 srq_handle->srq_addr = NULL; 1011 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1012 "srq_resize: mmap failed(%d)\n", errno); 1013 /* Need to retry resize with a smaller qlen */ 1014 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 1015 DAT_RESOURCE_MEMORY)); 1016 } 1017 1018 old_freepr_size = srq_handle->srq_wq_numwqe * sizeof (uint32_t); 1019 old_premature_size = srq_handle->srq_wq_numwqe * 1020 sizeof (ib_work_completion_t); 1021 1022 old_freepr_events = srq_handle->srq_freepr_events; 1023 old_premature_events = srq_handle->srq_premature_events; 1024 1025 new_freepr_size = resize_msg.srqr_real_size * sizeof (uint32_t); 1026 new_premature_size = resize_msg.srqr_real_size * 1027 sizeof (ib_work_completion_t); 1028 1029 new_freepr_events = (uint32_t *)dapl_os_alloc(new_freepr_size); 1030 if (new_freepr_events == NULL) { 1031 goto bail; 1032 } 1033 new_premature_events = (ib_work_completion_t *)dapl_os_alloc( 1034 new_premature_size); 1035 if (new_premature_events == NULL) { 1036 goto bail; 1037 } 1038 if (!dapls_tavor_srq_wrid_resize(srq_handle, 1039 resize_msg.srqr_real_size)) { 1040 goto bail; 1041 } 1042 idx = 0; 1043 /* copy valid premature events */ 1044 for (i = 0; i < srq_handle->srq_wq_numwqe; i++) { 1045 if (!DAPL_CQE_IS_VALID(&old_premature_events[i])) { 1046 continue; 1047 } 1048 (void) dapl_os_memcpy(&new_premature_events[idx], 1049 &old_premature_events[i], sizeof (ib_work_completion_t)); 1050 idx++; 1051 } 1052 dapl_os_assert(srq_handle->srq_wq_numwqe - idx == 1053 srq_handle->srq_freepr_num_events); 1054 1055 /* Initialize free events lists */ 1056 for (i = 0; i < resize_msg.srqr_real_size - idx; i++) { 1057 new_freepr_events[i] = idx + i; 1058 } 1059 1060 srq_handle->srq_freepr_events = new_freepr_events; 1061 srq_handle->srq_premature_events = new_premature_events; 1062 srq_handle->srq_freepr_num_events = resize_msg.srqr_real_size - idx; 1063 srq_handle->srq_freepr_head = 0; 1064 /* a full freepr list has tail at 0 */ 1065 if (idx == 0) { 1066 srq_handle->srq_freepr_tail = 0; 1067 } else { 1068 srq_handle->srq_freepr_tail = srq_handle->srq_freepr_num_events; 1069 } 1070 1071 if (old_freepr_events) { 1072 old_freepr_size = old_freepr_size; /* pacify lint */ 1073 dapl_os_free(old_freepr_events, old_freepr_size); 1074 } 1075 if (old_premature_events) { 1076 old_premature_size = old_premature_size; /* pacify lint */ 1077 dapl_os_free(old_premature_events, old_premature_size); 1078 } 1079 1080 /* 1081 * update the srq fields, 1082 * note: the srq_wq_lastwqeindex doesn't change since the old 1083 * work queue is copied as a whole into the new work queue. 1084 */ 1085 srq_handle->srq_map_offset = msrq->msrq_mapoffset; 1086 srq_handle->srq_map_len = msrq->msrq_maplen; 1087 srq_handle->srq_wq_desc_addr = msrq->msrq_desc_addr; 1088 srq_handle->srq_wq_numwqe = msrq->msrq_numwqe; 1089 srq_handle->srq_wq_wqesz = msrq->msrq_wqesz; 1090 1091 return (DAT_SUCCESS); 1092 bail: 1093 if (new_freepr_events) { 1094 dapl_os_free(new_freepr_events, new_freepr_size); 1095 } 1096 if (new_premature_events) { 1097 dapl_os_free(new_premature_events, new_premature_size); 1098 } 1099 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY)); 1100 } 1101