1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include "dapl.h" 28 #include "dapl_adapter_util.h" 29 #include "dapl_evd_util.h" 30 #include "dapl_cr_util.h" 31 #include "dapl_lmr_util.h" 32 #include "dapl_rmr_util.h" 33 #include "dapl_cookie.h" 34 #include "dapl_ring_buffer_util.h" 35 #include "dapl_vendor.h" 36 #include "dapl_tavor_ibtf_impl.h" 37 38 /* Function prototypes */ 39 static DAT_RETURN dapli_ib_cq_resize_internal(DAPL_EVD *, DAT_COUNT); 40 41 /* 42 * The following declarations/fn are to used by the base library 43 * place holder for now 44 */ 45 46 int g_loopback_connection = 0; 47 48 /* 49 * dapl_ib_cq_alloc 50 * 51 * Alloc a CQ 52 * 53 * Input: 54 * ia_handle IA handle 55 * evd_ptr pointer to EVD struct 56 * cno_ptr pointer to CNO struct 57 * cqlen minimum QLen 58 * 59 * Output: 60 * none 61 * 62 * Returns: 63 * DAT_SUCCESS 64 * DAT_INSUFFICIENT_RESOURCES 65 * 66 */ 67 DAT_RETURN 68 dapls_ib_cq_alloc( 69 IN DAPL_IA *ia_ptr, 70 IN DAPL_EVD *evd_ptr, 71 IN DAPL_CNO *cno_ptr, 72 IN DAT_COUNT *cqlen) 73 { 74 dapl_evd_create_t create_msg; 75 dapl_evd_free_t free_msg; 76 ib_cq_handle_t cq_handle = IB_INVALID_HANDLE; 77 int ia_fd; 78 int hca_fd; 79 int retval; 80 mlnx_umap_cq_data_out_t *mcq; 81 82 /* cq handle is created even for non-cq type events */ 83 /* since cq handle is where the evd fd gets stored. */ 84 cq_handle = (ib_cq_handle_t) 85 dapl_os_alloc(sizeof (struct dapls_ib_cq_handle)); 86 if (cq_handle == NULL) { 87 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 88 "cq_alloc: evd_ptr 0x%p, cq_handle == NULL\n", 89 evd_ptr); 90 return (DAT_INSUFFICIENT_RESOURCES); 91 } 92 93 (void) dapl_os_memzero(cq_handle, sizeof (*cq_handle)); 94 95 /* get the hca information from ia_ptr */ 96 (void) dapl_os_memzero(&create_msg, sizeof (create_msg)); 97 create_msg.evd_flags = evd_ptr->evd_flags; 98 create_msg.evd_cookie = (uintptr_t)evd_ptr; 99 if (cno_ptr != NULL) { 100 create_msg.evd_cno_hkey = 101 (uint64_t)cno_ptr->ib_cno_handle; 102 } 103 if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) { 104 create_msg.evd_cq_size = (uint32_t)*cqlen; 105 } 106 107 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 108 "cq_alloc: evd 0x%p, flags 0x%x, cookie 0x%llx, hkey 0x%llx,\n" 109 " cno_hkey 0x%llx, cq_size %d\n", evd_ptr, 110 create_msg.evd_flags, create_msg.evd_cookie, create_msg.evd_hkey, 111 create_msg.evd_cno_hkey, create_msg.evd_cq_size); 112 113 ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd; 114 hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd; 115 mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out; 116 117 /* The next line is only needed for backward compatibility */ 118 mcq->mcq_rev = MLNX_UMAP_IF_VERSION; 119 120 /* call into driver to allocate cq */ 121 retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg); 122 if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) { 123 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 124 "cq_alloc: evd_create failed, %s\n", strerror(errno)); 125 dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle)); 126 return (dapls_convert_error(errno, retval)); 127 } 128 (void) dapl_os_memzero(cq_handle, sizeof (struct dapls_ib_cq_handle)); 129 dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock); 130 131 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 132 "cq_alloc: created, evd 0x%p, hkey 0x%016llx\n\n", evd_ptr, 133 create_msg.evd_hkey); 134 135 cq_handle->evd_hkey = create_msg.evd_hkey; 136 137 if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) { 138 139 /* 140 * allocate a hash table for wrid management, the key is 141 * a combination of QPnumber and SEND/RECV type. This is 142 * required only for evd which have a CQ mapped to 143 * it. 144 */ 145 if (DAT_SUCCESS != dapls_hash_create(DAPL_MED_HASHSIZE, 146 DAT_FALSE, &cq_handle->cq_wrid_wqhdr_list)) { 147 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 148 "cq_alloc: hash_create failed\n"); 149 dapl_os_free(cq_handle, 150 sizeof (struct dapls_ib_cq_handle)); 151 return (DAT_INSUFFICIENT_RESOURCES | 152 DAT_RESOURCE_MEMORY); 153 } 154 155 dapl_os_assert(create_msg.evd_cq_real_size > 0); 156 157 /* In the case of Arbel or Hermon */ 158 if (mcq->mcq_polldbr_mapoffset != 0 || 159 mcq->mcq_polldbr_maplen != 0) 160 cq_handle->cq_poll_dbp = dapls_ib_get_dbp( 161 mcq->mcq_polldbr_maplen, hca_fd, 162 mcq->mcq_polldbr_mapoffset, 163 mcq->mcq_polldbr_offset); 164 if (mcq->mcq_armdbr_mapoffset != 0 || 165 mcq->mcq_armdbr_maplen != 0) 166 cq_handle->cq_arm_dbp = dapls_ib_get_dbp( 167 mcq->mcq_armdbr_maplen, hca_fd, 168 mcq->mcq_armdbr_mapoffset, 169 mcq->mcq_armdbr_offset); 170 171 cq_handle->cq_addr = (tavor_hw_cqe_t *)(void *) mmap64( 172 (void *)0, mcq->mcq_maplen, 173 (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd, 174 mcq->mcq_mapoffset); 175 176 if (cq_handle->cq_addr == MAP_FAILED || 177 cq_handle->cq_poll_dbp == MAP_FAILED || 178 cq_handle->cq_arm_dbp == MAP_FAILED) { 179 free_msg.evf_hkey = cq_handle->evd_hkey; 180 retval = ioctl(ia_fd, DAPL_EVD_FREE, &free_msg); 181 if (retval != 0) { 182 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 183 "cq_alloc: EVD_FREE err:%s\n", 184 strerror(errno)); 185 } 186 187 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 188 "cq_alloc: DAPL_CQ_ALLOC failed\n"); 189 /* free the hash table we created */ 190 (void) dapls_hash_free(cq_handle->cq_wrid_wqhdr_list); 191 dapl_os_free(cq_handle, 192 sizeof (struct dapls_ib_cq_handle)); 193 return (DAT_INSUFFICIENT_RESOURCES); 194 } 195 196 cq_handle->cq_map_offset = mcq->mcq_mapoffset; 197 cq_handle->cq_map_len = mcq->mcq_maplen; 198 cq_handle->cq_num = mcq->mcq_cqnum; 199 /* 200 * cq_size is the actual depth of the CQ which is 1 more 201 * than what ibt_alloc_cq reports. However the application 202 * can only use (cq_size - 1) entries. 203 */ 204 cq_handle->cq_size = create_msg.evd_cq_real_size + 1; 205 cq_handle->cq_cqesz = mcq->mcq_cqesz; 206 cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar; 207 *cqlen = create_msg.evd_cq_real_size; 208 209 DAPL_INIT_CQ(ia_ptr)(cq_handle); 210 } 211 212 evd_ptr->ib_cq_handle = cq_handle; 213 return (DAT_SUCCESS); 214 } 215 216 217 /* 218 * dapl_ib_cq_resize 219 * 220 * Resize a CQ 221 * 222 * Input: 223 * evd_ptr pointer to EVD struct 224 * cqlen new length of the cq 225 * Output: 226 * none 227 * 228 * Returns: 229 * DAT_SUCCESS 230 * DAT_INVALID_HANDLE 231 * DAT_INTERNAL_ERROR 232 * DAT_INSUFFICIENT_RESOURCES 233 * 234 */ 235 DAT_RETURN 236 dapls_ib_cq_resize( 237 IN DAPL_EVD *evd_ptr, 238 IN DAT_COUNT cqlen) 239 { 240 ib_cq_handle_t cq_handle; 241 DAT_RETURN dat_status; 242 243 dat_status = dapli_ib_cq_resize_internal(evd_ptr, cqlen); 244 if (DAT_INSUFFICIENT_RESOURCES == dat_status) { 245 cq_handle = evd_ptr->ib_cq_handle; 246 /* attempt to resize back to the current size */ 247 dat_status = dapli_ib_cq_resize_internal(evd_ptr, 248 cq_handle->cq_size - 1); 249 if (DAT_SUCCESS != dat_status) { 250 /* 251 * XXX this is catastrophic need to post an event 252 * to the async evd 253 */ 254 return (DAT_INTERNAL_ERROR); 255 } 256 } 257 258 return (dat_status); 259 } 260 261 /* 262 * dapli_ib_cq_resize_internal 263 * 264 * An internal routine to resize a CQ. 265 * 266 * Input: 267 * evd_ptr pointer to EVD struct 268 * cqlen new length of the cq 269 * Output: 270 * none 271 * 272 * Returns: 273 * DAT_SUCCESS 274 * DAT_INVALID_HANDLE 275 * DAT_INSUFFICIENT_RESOURCES 276 * 277 */ 278 static DAT_RETURN 279 dapli_ib_cq_resize_internal( 280 IN DAPL_EVD *evd_ptr, 281 IN DAT_COUNT cqlen) 282 { 283 ib_cq_handle_t cq_handle; 284 dapl_cq_resize_t resize_msg; 285 int ia_fd; 286 int hca_fd; 287 int retval; 288 mlnx_umap_cq_data_out_t *mcq; 289 DAPL_HCA *hca_ptr; 290 dapls_hw_cqe_t cq_addr; 291 292 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 293 "dapls_ib_cq_resize: evd 0x%p cq 0x%p " 294 "evd_hkey 0x%016llx cqlen %d\n", 295 evd_ptr, (void *)evd_ptr->ib_cq_handle, 296 evd_ptr->ib_cq_handle->evd_hkey, cqlen); 297 298 cq_handle = evd_ptr->ib_cq_handle; 299 /* 300 * Since CQs are created in powers of 2 with one non-usable slot, 301 * its possible that the previously allocated CQ has sufficient 302 * entries. If the current cq is big enough and it is mapped in 303 * we are done. 304 */ 305 if ((cqlen < cq_handle->cq_size) && (cq_handle->cq_addr)) { 306 return (DAT_SUCCESS); 307 } 308 309 hca_ptr = evd_ptr->header.owner_ia->hca_ptr; 310 311 /* unmap the CQ before resizing it */ 312 if (hca_ptr->hermon_resize_cq == 0) { 313 if ((cq_handle->cq_addr) && 314 (munmap((char *)cq_handle->cq_addr, 315 cq_handle->cq_map_len) < 0)) { 316 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 317 "cq_resize: munmap(%p:0x%llx) failed(%d)\n", 318 cq_handle->cq_addr, cq_handle->cq_map_len, errno); 319 return (DAT_INVALID_HANDLE); 320 } 321 /* cq_addr is unmapped and no longer valid */ 322 cq_handle->cq_addr = NULL; 323 } 324 325 ia_fd = hca_ptr->ib_hca_handle->ia_fd; 326 hca_fd = hca_ptr->ib_hca_handle->hca_fd; 327 328 (void) dapl_os_memzero(&resize_msg, sizeof (resize_msg)); 329 mcq = (mlnx_umap_cq_data_out_t *)resize_msg.cqr_cq_data_out; 330 resize_msg.cqr_evd_hkey = cq_handle->evd_hkey; 331 resize_msg.cqr_cq_new_size = cqlen; 332 333 /* The next line is only needed for backward compatibility */ 334 mcq->mcq_rev = MLNX_UMAP_IF_VERSION; 335 retval = ioctl(ia_fd, DAPL_CQ_RESIZE, &resize_msg); 336 if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) { 337 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 338 "dapls_ib_cq_resize: evd 0x%p, err: %s\n", 339 evd_ptr, strerror(errno)); 340 if (errno == EINVAL) { /* Couldn't find evd for this cq */ 341 return (DAT_INVALID_HANDLE); 342 } else { /* Need to retry resize with a smaller qlen */ 343 return (DAT_INSUFFICIENT_RESOURCES); 344 } 345 } 346 347 dapl_os_assert(cq_handle->cq_num == mcq->mcq_cqnum); 348 349 /* In the case of Arbel or Hermon */ 350 if (mcq->mcq_polldbr_mapoffset != 0 || 351 mcq->mcq_polldbr_maplen != 0) 352 cq_handle->cq_poll_dbp = dapls_ib_get_dbp( 353 mcq->mcq_polldbr_maplen, hca_fd, 354 mcq->mcq_polldbr_mapoffset, 355 mcq->mcq_polldbr_offset); 356 if (mcq->mcq_armdbr_mapoffset != 0 || 357 mcq->mcq_armdbr_maplen != 0) 358 cq_handle->cq_arm_dbp = dapls_ib_get_dbp( 359 mcq->mcq_armdbr_maplen, hca_fd, 360 mcq->mcq_armdbr_mapoffset, 361 mcq->mcq_armdbr_offset); 362 363 cq_addr = (tavor_hw_cqe_t *)(void *)mmap64((void *)0, 364 mcq->mcq_maplen, (PROT_READ | PROT_WRITE), 365 MAP_SHARED, hca_fd, mcq->mcq_mapoffset); 366 367 if (cq_addr == MAP_FAILED || 368 cq_handle->cq_poll_dbp == MAP_FAILED || 369 cq_handle->cq_arm_dbp == MAP_FAILED) { 370 if (hca_ptr->hermon_resize_cq == 0) 371 cq_handle->cq_addr = NULL; 372 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 373 "cq_resize: mmap failed(%d)\n", errno); 374 /* Need to retry resize with a smaller qlen */ 375 return (DAT_INSUFFICIENT_RESOURCES); 376 } 377 378 if (hca_ptr->hermon_resize_cq == 0) { 379 cq_handle->cq_addr = cq_addr; 380 cq_handle->cq_map_offset = mcq->mcq_mapoffset; 381 cq_handle->cq_map_len = mcq->mcq_maplen; 382 cq_handle->cq_size = resize_msg.cqr_cq_real_size + 1; 383 cq_handle->cq_cqesz = mcq->mcq_cqesz; 384 /* 385 * upon resize the old events are moved to the start of the CQ 386 * hence we need to reset the consumer index too 387 */ 388 cq_handle->cq_consindx = 0; 389 } else { /* Hermon */ 390 cq_handle->cq_resize_addr = cq_addr; 391 cq_handle->cq_resize_map_offset = mcq->mcq_mapoffset; 392 cq_handle->cq_resize_map_len = mcq->mcq_maplen; 393 cq_handle->cq_resize_size = resize_msg.cqr_cq_real_size + 1; 394 cq_handle->cq_resize_cqesz = mcq->mcq_cqesz; 395 } 396 397 return (DAT_SUCCESS); 398 } 399 400 /* 401 * dapl_ib_cq_free 402 * 403 * Free a CQ 404 * 405 * Input: 406 * ia_handle IA handle 407 * evd_ptr pointer to EVD struct 408 * Output: 409 * none 410 * 411 * Returns: 412 * DAT_SUCCESS 413 * DAT_INVALID_HANDLE 414 * DAT_INSUFFICIENT_RESOURCES 415 * 416 */ 417 DAT_RETURN 418 dapls_ib_cq_free( 419 IN DAPL_IA *ia_ptr, 420 IN DAPL_EVD *evd_ptr) 421 { 422 dapl_evd_free_t args; 423 int retval; 424 ib_cq_handle_t cq_handle = evd_ptr->ib_cq_handle; 425 426 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 427 "dapls_ib_cq_free: evd 0x%p cq 0x%p hkey %016llx\n", evd_ptr, 428 (void *)evd_ptr->ib_cq_handle, evd_ptr->ib_cq_handle->evd_hkey); 429 430 /* If the cq was mmap'd unmap it before freeing it */ 431 if ((cq_handle->cq_addr) && 432 (munmap((char *)cq_handle->cq_addr, cq_handle->cq_map_len) < 0)) { 433 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 434 "cq_free: (%p:0x%llx)\n", cq_handle->cq_addr, 435 cq_handle->cq_map_len); 436 } 437 438 439 args.evf_hkey = cq_handle->evd_hkey; 440 441 retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd, 442 DAPL_EVD_FREE, &args); 443 if (retval != 0) { 444 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 445 "dapls_ib_cq_free: evd 0x%p, err: %s\n", 446 evd_ptr, strerror(errno)); 447 return (dapls_convert_error(errno, retval)); 448 } 449 450 dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle)); 451 evd_ptr->ib_cq_handle = NULL; 452 453 return (DAT_SUCCESS); 454 } 455 456 /* 457 * dapl_set_cq_notify 458 * 459 * Set up CQ completion notifications 460 * 461 * Input: 462 * ia_handle IA handle 463 * evd_ptr pointer to EVD struct 464 * 465 * Output: 466 * none 467 * 468 * Returns: 469 * DAT_SUCCESS 470 * DAT_INVALID_HANDLE 471 * DAT_INSUFFICIENT_RESOURCES 472 * 473 */ 474 /* ARGSUSED */ 475 DAT_RETURN 476 dapls_set_cq_notify( 477 IN DAPL_IA *ia_ptr, 478 IN DAPL_EVD *evd_ptr) 479 { 480 int retval; 481 ib_cq_handle_t cq_handle = evd_ptr->ib_cq_handle; 482 483 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 484 "dapls_ib_cq_notify: evd 0x%p cq 0x%p\n", evd_ptr, 485 (void *)cq_handle); 486 487 retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_COMP, 0); 488 489 return (retval); 490 491 } 492 493 /* ARGSUSED */ 494 DAT_RETURN 495 dapls_set_cqN_notify( 496 IN DAPL_IA *ia_ptr, 497 IN DAPL_EVD *evd_ptr, 498 IN uint32_t num_events) 499 { 500 int retval; 501 ib_cq_handle_t cq_handle = evd_ptr->ib_cq_handle; 502 503 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 504 "dapls_set_cqN_notify:evd %p cq %p num_events %d\n", evd_ptr, 505 (void *)cq_handle, num_events); 506 507 retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_NCOMP, 508 num_events); 509 510 return (retval); 511 512 } 513 514 /* 515 * dapls_ib_cqd_create 516 * 517 * Set up CQ notification event thread 518 * 519 * Input: 520 * ia_handle IA handle 521 * 522 * Output: 523 * none 524 * 525 * Returns: 526 * DAT_SUCCESS 527 * DAT_INVALID_HANDLE 528 * DAT_INSUFFICIENT_RESOURCES 529 * 530 */ 531 /* ARGSUSED */ 532 DAT_RETURN 533 dapls_ib_cqd_create( 534 IN DAPL_HCA *hca_ptr) 535 { 536 return (DAT_SUCCESS); 537 } 538 539 540 /* 541 * dapl_cqd_destroy 542 * 543 * Destroy CQ notification event thread 544 * 545 * Input: 546 * ia_handle IA handle 547 * 548 * Output: 549 * none 550 * 551 * Returns: 552 * DAT_SUCCESS 553 * DAT_INVALID_HANDLE 554 * DAT_INSUFFICIENT_RESOURCES 555 * 556 */ 557 DAT_RETURN 558 dapls_ib_cqd_destroy( 559 IN DAPL_HCA *hca_ptr) 560 { 561 dapl_evd_free_t args; 562 ib_cq_handle_t cq_handle; 563 int retval; 564 565 if (hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) { 566 /* free up the dummy cq */ 567 cq_handle = hca_ptr->null_ib_cq_handle; 568 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 569 "dapls_ib_cqd_destroy: cq %p\n", (void *)cq_handle); 570 571 args.evf_hkey = cq_handle->evd_hkey; 572 573 retval = ioctl(hca_ptr->ib_hca_handle->ia_fd, 574 DAPL_EVD_FREE, &args); 575 if (retval != 0) { 576 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 577 "dapls_ib_cqd_destroy: EVD_FREE err:%d errno:%d\n", 578 retval, errno); 579 } 580 581 dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle)); 582 hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE; 583 } 584 585 return (DAT_SUCCESS); 586 } 587 588 589 /* 590 * dapl_ib_pd_alloc 591 * 592 * Alloc a PD 593 * 594 * Input: 595 * ia_handle IA handle 596 * PZ_ptr pointer to PZEVD struct 597 * 598 * Output: 599 * none 600 * 601 * Returns: 602 * DAT_SUCCESS 603 * DAT_INSUFFICIENT_RESOURCES 604 * 605 */ 606 DAT_RETURN 607 dapls_ib_pd_alloc( 608 IN DAPL_IA *ia, 609 IN DAPL_PZ *pz) 610 { 611 struct dapls_ib_pd_handle *pd_p; 612 dapl_pd_alloc_t args; 613 int retval; 614 615 pd_p = (struct dapls_ib_pd_handle *)dapl_os_alloc(sizeof (*pd_p)); 616 if (pd_p == NULL) { 617 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 618 "pd_alloc: ia 0x%p, pz 0x%p, cannot allocate pd\n", 619 ia, pz); 620 return (DAT_INSUFFICIENT_RESOURCES); 621 } 622 retval = ioctl(ia->hca_ptr->ib_hca_handle->ia_fd, 623 DAPL_PD_ALLOC, &args); 624 if (retval != 0) { 625 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 626 "pd_alloc: ia 0x%p, pz 0x%p, cannot create pd, " 627 "err: %s\n", ia, pz, strerror(errno)); 628 dapl_os_free(pd_p, sizeof (*pd_p)); 629 return (dapls_convert_error(errno, retval)); 630 } 631 632 pd_p->pd_hkey = args.pda_hkey; 633 pz->pd_handle = pd_p; 634 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 635 "pd_alloc: successful, ia 0x%p, pz 0x%p, hkey %016llx\n", 636 ia, pz, args.pda_hkey); 637 638 return (DAT_SUCCESS); 639 } 640 641 642 /* 643 * dapl_ib_pd_free 644 * 645 * Free a PD 646 * 647 * Input: 648 * ia_handle IA handle 649 * PZ_ptr pointer to PZ struct 650 * 651 * Output: 652 * none 653 * 654 * Returns: 655 * DAT_SUCCESS 656 * DAT_INSUFFICIENT_RESOURCES 657 * 658 */ 659 DAT_RETURN 660 dapls_ib_pd_free( 661 IN DAPL_PZ *pz) 662 { 663 struct dapls_ib_pd_handle *pd_p; 664 dapl_pd_free_t args; 665 int retval; 666 667 pd_p = (struct dapls_ib_pd_handle *)pz->pd_handle; 668 args.pdf_hkey = pd_p->pd_hkey; 669 670 retval = ioctl(pz->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd, 671 DAPL_PD_FREE, &args); 672 if (retval != 0) { 673 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 674 "pd_free: pz 0x%p, cannot free pd\n", pz); 675 return (dapls_convert_error(errno, retval)); 676 } 677 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 678 "pd_free: pz 0x%p, hkey %016llx, freed\n", pz, pd_p->pd_hkey); 679 dapl_os_free((void *)pd_p, sizeof (*pd_p)); 680 pz->pd_handle = NULL; 681 return (DAT_SUCCESS); 682 } 683 684 685 /* 686 * dapl_ib_mr_register 687 * 688 * Register a virtual memory region 689 * 690 * Input: 691 * ia_handle IA handle 692 * lmr pointer to dapl_lmr struct 693 * virt_addr virtual address of beginning of mem region 694 * length length of memory region 695 * 696 * Output: 697 * none 698 * 699 * Returns: 700 * DAT_SUCCESS 701 * DAT_INSUFFICIENT_RESOURCES 702 * 703 */ 704 DAT_RETURN 705 dapls_ib_mr_register( 706 IN DAPL_IA *ia, 707 IN DAPL_LMR *lmr, 708 IN DAT_PVOID virt_addr, 709 IN DAT_VLEN length, 710 IN DAT_MEM_PRIV_FLAGS privileges) 711 { 712 dapl_mr_register_t reg_msg; 713 ib_mr_handle_t mr_handle; 714 DAPL_PZ * pz_handle; 715 int ia_fd; 716 int retval; 717 718 ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd; 719 mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle)); 720 if (mr_handle == NULL) { 721 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 722 "mr_register: lmr 0x%p, ia 0x%p, " 723 "cannot alloc mr_handle\n", lmr, ia); 724 return (DAT_INSUFFICIENT_RESOURCES); 725 } 726 pz_handle = ((DAPL_PZ *)lmr->param.pz_handle); 727 if (pz_handle == NULL) { 728 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 729 "mr_register: lmr 0x%p, ia 0x%p, " 730 "pz_handle == NULL!\n", lmr, ia); 731 dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle)); 732 return (DAT_INVALID_PARAMETER); 733 } 734 reg_msg.mr_pd_hkey = pz_handle->pd_handle->pd_hkey; 735 reg_msg.mr_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr; 736 reg_msg.mr_len = (ib_memlen_t)length; 737 reg_msg.mr_flags = (ibt_mr_flags_t) 738 dapl_lmr_convert_privileges(privileges); 739 reg_msg.mr_flags |= IBT_MR_ENABLE_WINDOW_BIND; 740 741 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 742 "mr_register: lmr 0x%p, pd_hkey 0x%016llx, vaddr 0x%016llx, " 743 "len %llu, flags 0x%x\n", lmr, reg_msg.mr_pd_hkey, 744 reg_msg.mr_vaddr, reg_msg.mr_len, reg_msg.mr_flags); 745 746 /* call into driver to allocate MR resource */ 747 retval = ioctl(ia_fd, DAPL_MR_REGISTER, ®_msg); 748 if (retval != 0) { 749 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 750 "mr_register: lmr 0x%p, failed (%s)\n", 751 lmr, strerror(errno)); 752 dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle)); 753 return (dapls_convert_error(errno, retval)); 754 } 755 mr_handle->mr_hkey = reg_msg.mr_hkey; 756 lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mr_lkey; 757 lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mr_rkey; 758 lmr->param.registered_address = reg_msg.mr_vaddr; 759 lmr->param.registered_size = reg_msg.mr_len; 760 lmr->mr_handle = mr_handle; 761 762 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 763 "mr_register: successful, lmr 0x%p, mr_hkey 0x%016llx, " 764 "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mr_hkey, 765 reg_msg.mr_lkey); 766 return (DAT_SUCCESS); 767 } 768 769 /* 770 * dapl_ib_mr_register_shared 771 * 772 * Register a shared virtual memory region 773 * 774 * Input: 775 * ia_handle IA handle 776 * lmr pointer to dapl_lmr struct 777 * virt_addr virtual address of beginning of mem region 778 * cookie shared memory identifer 779 * length length of memory region 780 * 781 * Output: 782 * none 783 * 784 * Returns: 785 * DAT_SUCCESS 786 * DAT_INSUFFICIENT_RESOURCES 787 * 788 */ 789 DAT_RETURN 790 dapls_ib_mr_register_shared( 791 IN DAPL_IA *ia, 792 IN DAPL_LMR *lmr, 793 IN DAT_PVOID virt_addr, 794 IN DAT_VLEN length, 795 IN DAT_LMR_COOKIE cookie, 796 IN DAT_MEM_PRIV_FLAGS privileges) 797 { 798 dapl_mr_register_shared_t reg_msg; 799 ib_mr_handle_t mr_handle; 800 DAPL_PZ *pz_handle; 801 int ia_fd, i; 802 int retval; 803 804 ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd; 805 mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle)); 806 if (mr_handle == NULL) { 807 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 808 "mr_register_shared: lmr 0x%p, ia 0x%p, " 809 "cannot alloc mr_handle\n", lmr, ia); 810 return (DAT_INSUFFICIENT_RESOURCES); 811 } 812 pz_handle = ((DAPL_PZ *)lmr->param.pz_handle); 813 if (pz_handle == NULL) { 814 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 815 "mr_register_shared: lmr 0x%p, ia 0x%p, " 816 "pz_handle == NULL!\n", lmr, ia); 817 dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle)); 818 return (DAT_INVALID_PARAMETER); 819 } 820 reg_msg.mrs_pd_hkey = pz_handle->pd_handle->pd_hkey; 821 reg_msg.mrs_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr; 822 reg_msg.mrs_len = (ib_memlen_t)length; 823 reg_msg.mrs_flags = (ibt_mr_flags_t) 824 dapl_lmr_convert_privileges(privileges); 825 reg_msg.mrs_flags |= IBT_MR_ENABLE_WINDOW_BIND; 826 /*CONSTCOND*/ 827 dapl_os_assert(DAT_LMR_COOKIE_SIZE == sizeof (reg_msg.mrs_shm_cookie)); 828 (void) dapl_os_memcpy((void *)®_msg.mrs_shm_cookie, (void *)cookie, 829 DAT_LMR_COOKIE_SIZE); 830 831 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 832 "mr_register_shared: lmr 0x%p, pd_hkey 0x%016llx, " 833 "vaddr 0x%016llx, len %llu, flags 0x%x\n", 834 lmr, reg_msg.mrs_pd_hkey, reg_msg.mrs_vaddr, reg_msg.mrs_len, 835 reg_msg.mrs_flags); 836 837 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 838 "mr_register_shared: cookie \n0x"); 839 for (i = 4; i >= 0; i--) { 840 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 841 "%016llx", reg_msg.mrs_shm_cookie.mc_uint_arr[i]); 842 } 843 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "\n"); 844 845 /* call into driver to allocate MR resource */ 846 retval = ioctl(ia_fd, DAPL_MR_REGISTER_SHARED, ®_msg); 847 if (retval != 0) { 848 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 849 "mr_register_shared: lmr 0x%p, failed (%s)\n", 850 lmr, strerror(errno)); 851 dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle)); 852 return (dapls_convert_error(errno, retval)); 853 } 854 mr_handle->mr_hkey = reg_msg.mrs_hkey; 855 lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mrs_lkey; 856 lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mrs_rkey; 857 lmr->param.registered_address = reg_msg.mrs_vaddr; 858 lmr->param.registered_size = reg_msg.mrs_len; 859 lmr->mr_handle = mr_handle; 860 861 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 862 "mr_register_shared: successful, lmr 0x%p, mr_hkey 0x%016llx, " 863 "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mrs_hkey, 864 reg_msg.mrs_lkey); 865 return (DAT_SUCCESS); 866 } 867 868 /* 869 * dapl_ib_mr_deregister 870 * 871 * Free a memory region 872 * 873 * Input: 874 * lmr pointer to dapl_lmr struct 875 * 876 * Output: 877 * none 878 * 879 * Returns: 880 * DAT_SUCCESS 881 * DAT_INSUFFICIENT_RESOURCES 882 * 883 */ 884 DAT_RETURN 885 dapls_ib_mr_deregister( 886 IN DAPL_LMR *lmr) 887 { 888 dapl_mr_deregister_t args; 889 int retval; 890 891 args.mrd_hkey = lmr->mr_handle->mr_hkey; 892 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 893 "mr_deregister: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n" 894 " vaddr 0x%016llx, len %llu, flags 0x%x\n", 895 lmr, args.mrd_hkey, lmr->param.lmr_context, 896 lmr->param.registered_address, lmr->param.registered_size, 897 dapl_lmr_convert_privileges(lmr->param.mem_priv) | 898 IBT_MR_ENABLE_WINDOW_BIND); 899 900 /* call into driver to do MR deregister */ 901 retval = ioctl(lmr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd, 902 DAPL_MR_DEREGISTER, &args); 903 904 if (retval != 0) { 905 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 906 "mr_deregister: lmr 0x%p, failed (%s)\n", 907 lmr, strerror(errno)); 908 return (dapls_convert_error(errno, retval)); 909 } 910 911 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 912 "mr_deregister: successful\n\n"); 913 dapl_os_free(lmr->mr_handle, sizeof (struct dapls_ib_mr_handle)); 914 lmr->mr_handle = NULL; 915 return (DAT_SUCCESS); 916 } 917 918 919 /* 920 * dapl_ib_mr_register_lmr 921 * 922 * Register a memory region based on attributes of an existing one 923 * 924 * Input: 925 * ia_handle IA handle 926 * lmr pointer to dapl_lmr struct 927 * virt_addr virtual address of beginning of mem region 928 * length length of memory region 929 * 930 * Output: 931 * none 932 * 933 * Returns: 934 * DAT_SUCCESS 935 * DAT_INSUFFICIENT_RESOURCES 936 * 937 */ 938 DAT_RETURN 939 dapls_ib_mr_register_lmr( 940 IN DAPL_IA *ia, 941 IN DAPL_LMR *lmr, 942 IN DAT_MEM_PRIV_FLAGS privileges) 943 { 944 dapl_mr_register_lmr_t regl_msg; 945 DAPL_LMR *orig_lmr; 946 struct dapls_ib_mr_handle *orig_mr_handle; 947 ib_mr_handle_t mr_handle; 948 int ia_fd; 949 int retval; 950 951 ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd; 952 mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle)); 953 if (mr_handle == NULL) { 954 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 955 "mr_register_lmr: lmr 0x%p, ia 0x%p, " 956 "cannot alloc mr_handle\n", lmr, ia); 957 return (DAT_INSUFFICIENT_RESOURCES); 958 } 959 960 orig_lmr = (DAPL_LMR *)lmr->param.region_desc.for_lmr_handle; 961 orig_mr_handle = (struct dapls_ib_mr_handle *)orig_lmr->mr_handle; 962 regl_msg.mrl_orig_hkey = orig_mr_handle->mr_hkey; 963 regl_msg.mrl_flags = (ibt_mr_flags_t) 964 dapl_lmr_convert_privileges(privileges); 965 regl_msg.mrl_flags |= IBT_MR_ENABLE_WINDOW_BIND; 966 regl_msg.mrl_lkey = regl_msg.mrl_rkey = 0; 967 968 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 969 "mr_register_lmr: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n" 970 " vaddr 0x%016llx, len %llu, flags 0x%x\n", 971 lmr, mr_handle->mr_hkey, lmr->param.lmr_context, 972 orig_lmr->param.registered_address, 973 orig_lmr->param.registered_size, 974 dapl_lmr_convert_privileges(orig_lmr->param.mem_priv) | 975 IBT_MR_ENABLE_WINDOW_BIND); 976 977 978 /* call into driver to allocate MR resource */ 979 retval = ioctl(ia_fd, DAPL_MR_REGISTER_LMR, ®l_msg); 980 if (retval != 0) { 981 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 982 "mr_register_lmr: failed (%s), orig_hkey (%016llx)\n", 983 strerror(errno), orig_mr_handle->mr_hkey); 984 dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle)); 985 return (dapls_convert_error(errno, retval)); 986 } 987 988 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 989 "mr_registered_lmr: successful, lmr 0x%p, hkey 0x%016llx\n", 990 lmr, regl_msg.mrl_hkey); 991 992 mr_handle->mr_hkey = regl_msg.mrl_hkey; 993 lmr->param.lmr_context = (DAT_LMR_CONTEXT)regl_msg.mrl_lkey; 994 lmr->param.rmr_context = (DAT_RMR_CONTEXT)regl_msg.mrl_rkey; 995 lmr->param.registered_address = orig_lmr->param.registered_address; 996 lmr->param.registered_size = orig_lmr->param.registered_size; 997 lmr->mr_handle = mr_handle; 998 999 return (DAT_SUCCESS); 1000 } 1001 1002 1003 /* 1004 * dapls_ib_mw_alloc 1005 * 1006 * Bind a protection domain to a memory window 1007 * 1008 * Input: 1009 * rmr Initialized rmr to hold binding handles 1010 * 1011 * Output: 1012 * none 1013 * 1014 * Returns: 1015 * DAT_SUCCESS 1016 * DAT_INSUFFICIENT_RESOURCES 1017 * 1018 */ 1019 DAT_RETURN 1020 dapls_ib_mw_alloc( 1021 IN DAPL_RMR *rmr) 1022 { 1023 DAPL_IA *ia_hdl = (DAPL_IA *)rmr->param.ia_handle; 1024 DAPL_PZ *pz_hdl = rmr->param.pz_handle; 1025 dapl_mw_alloc_t args; 1026 ib_mw_handle_t mw_handle; 1027 int ia_fd; 1028 int retval; 1029 1030 ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr-> 1031 ib_hca_handle))->ia_fd; 1032 1033 mw_handle = dapl_os_alloc(sizeof (struct dapls_ib_mw_handle)); 1034 if (mw_handle == NULL) { 1035 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1036 "mw_alloc: rmr 0x%p, cannot alloc mw_handle\n", rmr); 1037 return (DAT_INSUFFICIENT_RESOURCES); 1038 } 1039 args.mw_pd_hkey = ((struct dapls_ib_pd_handle *) 1040 (pz_hdl->pd_handle))->pd_hkey; 1041 1042 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1043 "mw_alloc: rmr 0x%p, pd_hkey 0x%016llx\n", 1044 rmr, args.mw_pd_hkey); 1045 1046 retval = ioctl(ia_fd, DAPL_MW_ALLOC, &args); 1047 if (retval != 0) { 1048 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1049 "mw_alloc: rmr 0x%p, failed (%s)\n", rmr, strerror(errno)); 1050 dapl_os_free(mw_handle, sizeof (struct dapls_ib_mr_handle)); 1051 return (dapls_convert_error(errno, retval)); 1052 } 1053 1054 mw_handle->mw_hkey = args.mw_hkey; 1055 rmr->mw_handle = mw_handle; 1056 rmr->param.rmr_context = (DAT_RMR_CONTEXT) args.mw_rkey; 1057 1058 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1059 "mw_alloc: successful, rmr 0x%p, mw_hkey 0x%llx, " 1060 "rmr_ctx 0x%x\n\n", rmr, (uint64_t)args.mw_hkey, 1061 rmr->param.rmr_context); 1062 1063 return (DAT_SUCCESS); 1064 } 1065 1066 1067 /* 1068 * dapls_ib_mw_free 1069 * 1070 * Release bindings of a protection domain to a memory window 1071 * 1072 * Input: 1073 * rmr Initialized rmr to hold binding handles 1074 * 1075 * Output: 1076 * none 1077 * 1078 * Returns: 1079 * DAT_SUCCESS 1080 * DAT_INSUFFICIENT_RESOURCES 1081 * 1082 */ 1083 DAT_RETURN 1084 dapls_ib_mw_free( 1085 IN DAPL_RMR *rmr) 1086 { 1087 DAPL_IA *ia_hdl = rmr->param.ia_handle; 1088 dapl_mw_free_t args; 1089 int ia_fd; 1090 int retval; 1091 1092 ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr-> 1093 ib_hca_handle))->ia_fd; 1094 1095 args.mw_hkey = rmr->mw_handle->mw_hkey; 1096 1097 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1098 "mw_free: rmr 0x%p, mw_hkey 0x%016llx\n", rmr, args.mw_hkey); 1099 1100 retval = ioctl(ia_fd, DAPL_MW_FREE, &args); 1101 if (retval != 0) { 1102 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1103 "mw_free: rmr 0x%p, failed (%s)\n", rmr, strerror(errno)); 1104 return (dapls_convert_error(errno, retval)); 1105 } 1106 1107 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "mw_free: successful\n\n"); 1108 dapl_os_free(rmr->mw_handle, sizeof (struct dapls_ib_mr_handle)); 1109 rmr->mw_handle = NULL; 1110 1111 return (DAT_SUCCESS); 1112 } 1113 1114 /* 1115 * dapls_ib_mw_bind 1116 * 1117 * Bind a protection domain to a memory window 1118 * 1119 * Input: 1120 * rmr Initialized rmr to hold binding handles 1121 * 1122 * Output: 1123 * none 1124 * 1125 * Returns: 1126 * DAT_SUCCESS 1127 * DAT_INSUFFICIENT_RESOURCES 1128 * 1129 */ 1130 DAT_RETURN 1131 dapls_ib_mw_bind( 1132 IN DAPL_RMR *rmr, 1133 IN DAT_LMR_CONTEXT lmr_context, 1134 IN DAPL_EP *ep, 1135 IN DAPL_COOKIE *cookie, 1136 IN DAT_VADDR virtual_address, 1137 IN DAT_VLEN length, 1138 IN DAT_MEM_PRIV_FLAGS mem_priv, 1139 IN DAT_COMPLETION_FLAGS completion_flags) 1140 { 1141 ibt_send_wr_t wre; 1142 ibt_wr_bind_t wrbind; 1143 boolean_t suppress_notification; 1144 int retval; 1145 1146 if (length > 0) { 1147 wrbind.bind_flags = (ibt_bind_flags_t) 1148 (dapl_rmr_convert_privileges(mem_priv) | 1149 IBT_WR_BIND_ATOMIC); 1150 } else { 1151 wrbind.bind_flags = (ibt_bind_flags_t)NULL; 1152 } 1153 wrbind.bind_rkey = rmr->param.rmr_context; 1154 wrbind.bind_va = virtual_address; 1155 wrbind.bind_len = length; 1156 wrbind.bind_lkey = lmr_context; 1157 1158 wre.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 1159 /* 1160 * wre.wr_flags = (is_signaled) ? IBT_WR_SEND_SIGNAL : 1161 * IBT_WR_NO_FLAGS; 1162 * Till we fix the chan alloc flags do the following - 1163 */ 1164 /* Translate dapl flags */ 1165 wre.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG & 1166 completion_flags) ? IBT_WR_SEND_FENCE : 0; 1167 /* suppress completions */ 1168 wre.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & 1169 completion_flags) ? 0 : IBT_WR_SEND_SIGNAL; 1170 1171 wre.wr_trans = IBT_RC_SRV; 1172 wre.wr_opcode = IBT_WRC_BIND; 1173 wre.wr_nds = 0; 1174 wre.wr_sgl = NULL; 1175 wre.wr.rc.rcwr.bind = &wrbind; 1176 1177 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1178 "mw_bind: rmr 0x%p, wr_flags 0x%x, rkey 0x%x, bind_flags 0x%x\n" 1179 " bind_va 0x%llx, bind_len 0x%llx, mem_priv 0x%x\n", 1180 rmr, wre.wr_flags, wrbind.bind_rkey, wrbind.bind_flags, 1181 wrbind.bind_va, wrbind.bind_len, mem_priv); 1182 1183 if (ep->param.ep_attr.recv_completion_flags & 1184 DAT_COMPLETION_UNSIGNALLED_FLAG) { 1185 /* This flag is used to control notification of completions */ 1186 suppress_notification = (completion_flags & 1187 DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE; 1188 } else { 1189 /* 1190 * The evd waiter will use threshold to control wakeups 1191 * Hence the event notification will be done via arming the 1192 * CQ so we do not need special notification generation 1193 * hence set suppression to true 1194 */ 1195 suppress_notification = B_TRUE; 1196 } 1197 1198 retval = DAPL_SEND(ep)(ep, &wre, suppress_notification); 1199 1200 if (retval != 0) { 1201 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1202 "mw_bind: rmr 0x%p, failed (%s)\n", rmr, strerror(errno)); 1203 return (dapls_convert_error(errno, retval)); 1204 } 1205 1206 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1207 "mw_bind: new_rkey = 0x%x\n", wrbind.bind_rkey_out); 1208 rmr->param.rmr_context = (DAT_RMR_CONTEXT) wrbind.bind_rkey_out; 1209 1210 return (DAT_SUCCESS); 1211 } 1212 1213 /* 1214 * dapls_ib_mw_unbind 1215 * 1216 * Unbind a protection domain from a memory window 1217 * 1218 * Input: 1219 * rmr Initialized rmr to hold binding handles 1220 * 1221 * Output: 1222 * none 1223 * 1224 * Returns: 1225 * DAT_SUCCESS 1226 * DAT_INSUFFICIENT_RESOURCES 1227 * 1228 */ 1229 DAT_RETURN 1230 dapls_ib_mw_unbind( 1231 IN DAPL_RMR *rmr, 1232 IN DAT_LMR_CONTEXT lmr_context, 1233 IN DAPL_EP *ep, 1234 IN DAPL_COOKIE *cookie, 1235 IN DAT_COMPLETION_FLAGS completion_flags) 1236 { 1237 DAT_RETURN retval; 1238 1239 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1240 "mw_unbind: rmr 0x%p, enter\n", rmr); 1241 1242 retval = dapls_ib_mw_bind(rmr, lmr_context, ep, cookie, 1243 (DAT_VADDR)NULL, (DAT_VLEN)0, (DAT_MEM_PRIV_FLAGS)NULL, 1244 completion_flags); 1245 1246 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1247 "mw_unbind: rmr 0x%p, exit\n\n", rmr); 1248 1249 return (retval); 1250 } 1251 1252 /* 1253 * Processes async events and calls appropriate callbacks so that events 1254 * can be posted to the async evd. 1255 */ 1256 void 1257 dapls_ib_async_callback( 1258 IN DAPL_EVD *async_evd, 1259 IN ib_hca_handle_t hca_handle, 1260 IN ib_error_record_t *event_ptr, 1261 IN void *context) 1262 { 1263 DAPL_IA *ia_ptr; 1264 DAPL_EP *ep_ptr; 1265 DAPL_EVD *evd_ptr; 1266 dapl_ib_async_event_t *async_evp; 1267 1268 ia_ptr = (DAPL_IA *)context; 1269 1270 dapl_os_assert(event_ptr != NULL); 1271 async_evp = (dapl_ib_async_event_t *)event_ptr; 1272 1273 switch (async_evp->ibae_type) { 1274 case IBT_ERROR_INVALID_REQUEST_CHAN: 1275 case IBT_ERROR_CATASTROPHIC_CHAN: 1276 /* 1277 * Walk the EPs to match this EP, then invoke the 1278 * routine when we have the EP we need 1279 */ 1280 dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->ep_list_head)); 1281 dapl_os_lock(&ia_ptr->header.lock); 1282 1283 ep_ptr = (DAPL_EP *)dapl_llist_next_entry(&ia_ptr->ep_list_head, 1284 NULL); 1285 while (ep_ptr != NULL) { 1286 if (ep_ptr == 1287 (DAPL_EP *)(uintptr_t)async_evp->ibae_cookie) { 1288 break; 1289 } 1290 1291 ep_ptr = (DAPL_EP *) dapl_llist_next_entry( 1292 &ia_ptr->ep_list_head, 1293 &ep_ptr->header.ia_list_entry); 1294 } 1295 1296 dapl_os_unlock(&ia_ptr->header.lock); 1297 dapl_os_assert(ep_ptr != NULL); 1298 dapl_evd_qp_async_error_callback(hca_handle, NULL, event_ptr, 1299 (void *)ep_ptr); 1300 break; 1301 case IBT_ERROR_CQ: 1302 /* 1303 * Walk the EVDs to match this EVD, then invoke the 1304 * routine when we have the EVD we need 1305 */ 1306 dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->evd_list_head)); 1307 dapl_os_lock(&ia_ptr->header.lock); 1308 1309 evd_ptr = (DAPL_EVD *) dapl_llist_next_entry( 1310 &ia_ptr->evd_list_head, NULL); 1311 while (evd_ptr != NULL) { 1312 if (evd_ptr == 1313 (DAPL_EVD *)(uintptr_t)async_evp->ibae_cookie) { 1314 break; 1315 } 1316 evd_ptr = (DAPL_EVD *) 1317 dapl_llist_next_entry(&ia_ptr->evd_list_head, 1318 &evd_ptr->header.ia_list_entry); 1319 } 1320 dapl_os_unlock(&ia_ptr->header.lock); 1321 dapl_os_assert(evd_ptr != NULL); 1322 dapl_evd_cq_async_error_callback(hca_handle, NULL, event_ptr, 1323 (void *)evd_ptr); 1324 break; 1325 case IBT_ERROR_PORT_DOWN: 1326 case IBT_ERROR_LOCAL_CATASTROPHIC: 1327 dapl_evd_un_async_error_callback(hca_handle, event_ptr, 1328 (void *)async_evd); 1329 break; 1330 default: 1331 /* 1332 * We are not interested in the following events 1333 * case IBT_EVENT_PATH_MIGRATED: 1334 * case IBT_EVENT_COM_EST: 1335 * case IBT_EVENT_SQD: 1336 * case IBT_ERROR_PATH_MIGRATE_REQ: 1337 * case IBT_EVENT_PORT_UP: 1338 */ 1339 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1340 "dapls_ib_async_callback: unhandled async code:%x\n", 1341 async_evp->ibae_type); 1342 break; 1343 } 1344 } 1345 1346 /* 1347 * dapls_ib_setup_async_callback 1348 * The reference implementation calls this to register callbacks, 1349 * but since our model of polling for events is based on retrieving 1350 * events by the waiting thread itself this is a NOOP for us. 1351 */ 1352 /* ARGSUSED */ 1353 DAT_RETURN 1354 dapls_ib_setup_async_callback( 1355 IN DAPL_IA *ia_ptr, 1356 IN DAPL_ASYNC_HANDLER_TYPE handler_type, 1357 IN unsigned int *callback_handle, 1358 IN ib_async_handler_t callback, 1359 IN void *context) 1360 { 1361 return (DAT_SUCCESS); 1362 } 1363 1364 /* 1365 * dapls_ib_query_hca 1366 * 1367 * Set up an asynchronous callbacks of various kinds 1368 * 1369 * Input: 1370 * hca_handl hca handle 1371 * ep_attr attribute of the ep 1372 * 1373 * Output: 1374 * none 1375 * 1376 * Returns: 1377 * DAT_SUCCESS 1378 * DAT_INVALID_PARAMETER 1379 */ 1380 1381 /* these are just arbitrary values for now */ 1382 1383 static DAT_RETURN 1384 dapls_ib_query_ia( 1385 IN dapl_hca_attr_t *hca_attr, 1386 IN DAPL_HCA *hca_ptr, 1387 OUT DAT_IA_ATTR *ia_attr) 1388 { 1389 (void) dapl_os_memzero(ia_attr, sizeof (*ia_attr)); 1390 1391 (void) dapl_os_strcpy(ia_attr->adapter_name, VN_ADAPTER_NAME); 1392 1393 (void) sprintf(ia_attr->vendor_name, "0x%08x:0x%08x", 1394 hca_attr->dhca_vendor_id, hca_attr->dhca_device_id); 1395 1396 ia_attr->hardware_version_major = (DAT_UINT32)hca_attr->dhca_version_id; 1397 ia_attr->ia_address_ptr = (DAT_IA_ADDRESS_PTR)&hca_ptr->hca_address; 1398 ia_attr->max_eps = (DAT_COUNT)hca_attr->dhca_max_chans; 1399 ia_attr->max_dto_per_ep = (DAT_COUNT)hca_attr->dhca_max_chan_sz; 1400 ia_attr->max_rdma_read_per_ep_in = hca_attr->dhca_max_rdma_in_chan; 1401 ia_attr->max_rdma_read_per_ep_out = hca_attr->dhca_max_rdma_out_chan; 1402 ia_attr->max_evds = (DAT_COUNT)hca_attr->dhca_max_cq; 1403 ia_attr->max_evd_qlen = (DAT_COUNT)hca_attr->dhca_max_cq_sz; 1404 /* max_iov_segments_per_dto is for non-RDMA */ 1405 ia_attr->max_iov_segments_per_dto = (DAT_COUNT)hca_attr->dhca_max_sgl; 1406 ia_attr->max_lmrs = (DAT_COUNT)hca_attr->dhca_max_memr; 1407 ia_attr->max_lmr_block_size = (DAT_VLEN)hca_attr->dhca_max_memr_len; 1408 ia_attr->max_lmr_virtual_address = (DAT_VADDR)DAPL_MAX_ADDRESS; 1409 ia_attr->max_pzs = (DAT_COUNT)hca_attr->dhca_max_pd; 1410 ia_attr->max_mtu_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE; 1411 ia_attr->max_rdma_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE; 1412 ia_attr->max_rmrs = (DAT_COUNT)hca_attr->dhca_max_mem_win; 1413 ia_attr->max_rmr_target_address = (DAT_VADDR)DAPL_MAX_ADDRESS; 1414 ia_attr->max_iov_segments_per_rdma_read = 1415 (DAT_COUNT)hca_attr->dhca_max_sgl; 1416 ia_attr->max_iov_segments_per_rdma_write = 1417 (DAT_COUNT)hca_attr->dhca_max_sgl; 1418 /* all instances of IA */ 1419 ia_attr->max_rdma_read_in = hca_attr->dhca_max_rdma_in_chan * 1420 hca_attr->dhca_max_chans; 1421 ia_attr->max_rdma_read_out = hca_attr->dhca_max_rdma_out_chan * 1422 hca_attr->dhca_max_chans; 1423 ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE; 1424 ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE; 1425 ia_attr->max_srqs = (DAT_COUNT)hca_attr->dhca_max_srqs; 1426 ia_attr->max_ep_per_srq = ia_attr->max_eps; 1427 ia_attr->max_recv_per_srq = (DAT_COUNT)hca_attr->dhca_max_srqs_sz; 1428 1429 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "IA Attributes:\n" 1430 "\tadapter_name %s\n " 1431 "\tvendor_name %s\n " 1432 "\thardware_version_major 0x%08x\n" 1433 "\tmax_eps %d\n" 1434 "\tmax_dto_per_ep %d\n" 1435 "\tmax_rdma_read_per_ep_in %d\n" 1436 "\tmax_rdma_read_per_ep_out %d\n" 1437 "\tmax_evds %d\n" 1438 "\tmax_evd_qlen %d\n" 1439 "\tmax_iov_segments_per_dto %d\n" 1440 "\tmax_lmrs %d\n" 1441 "\tmax_lmr_block_size 0x%016llx\n" 1442 "\tmax_lmr_virtual_address 0x%016llx\n" 1443 "\tmax_pzs %d\n" 1444 "\tmax_mtu_size 0x%016llx\n" 1445 "\tmax_rdma_size 0x%016llx\n" 1446 "\tmax_rmrs %d\n" 1447 "\tmax_rmr_target_address 0x%016llx\n" 1448 "\tmax_iov_segments_per_rdma_read %d\n" 1449 "\tmax_iov_segments_per_rdma_write %d\n" 1450 "\tmax_rdma_read_in %d\n" 1451 "\tmax_rdma_read_out %d\n" 1452 "\tmax_srqs %d\n" 1453 "\tmax_ep_per_srq %d\n" 1454 "\tmax_recv_per_srq %d\n" 1455 "\n", 1456 ia_attr->adapter_name, 1457 ia_attr->vendor_name, 1458 ia_attr->hardware_version_major, 1459 ia_attr->max_eps, 1460 ia_attr->max_dto_per_ep, 1461 ia_attr->max_rdma_read_per_ep_in, 1462 ia_attr->max_rdma_read_per_ep_out, 1463 ia_attr->max_evds, 1464 ia_attr->max_evd_qlen, 1465 ia_attr->max_iov_segments_per_dto, 1466 ia_attr->max_lmrs, 1467 ia_attr->max_lmr_block_size, 1468 ia_attr->max_lmr_virtual_address, 1469 ia_attr->max_pzs, 1470 ia_attr->max_mtu_size, 1471 ia_attr->max_rdma_size, 1472 ia_attr->max_rmrs, 1473 ia_attr->max_rmr_target_address, 1474 ia_attr->max_iov_segments_per_rdma_read, 1475 ia_attr->max_iov_segments_per_rdma_write, 1476 ia_attr->max_rdma_read_in, 1477 ia_attr->max_rdma_read_out, 1478 ia_attr->max_srqs, 1479 ia_attr->max_ep_per_srq, 1480 ia_attr->max_recv_per_srq); 1481 1482 return (DAT_SUCCESS); 1483 } 1484 1485 /* ARGSUSED */ 1486 static DAT_RETURN 1487 dapls_ib_query_ep( 1488 IN dapl_hca_attr_t *hca_attr, 1489 IN DAPL_HCA *hca_ptr, 1490 OUT DAT_EP_ATTR *ep_attr) 1491 { 1492 (void) dapl_os_memzero(ep_attr, sizeof (*ep_attr)); 1493 ep_attr->service_type = DAT_SERVICE_TYPE_RC; 1494 ep_attr->max_mtu_size = DAPL_IB_MAX_MESSAGE_SIZE; 1495 ep_attr->max_rdma_size = DAPL_IB_MAX_MESSAGE_SIZE; 1496 ep_attr->qos = DAT_QOS_BEST_EFFORT; 1497 ep_attr->max_recv_dtos = hca_attr->dhca_max_chan_sz; 1498 ep_attr->max_request_dtos = hca_attr->dhca_max_chan_sz; 1499 ep_attr->max_recv_iov = hca_attr->dhca_max_sgl; 1500 ep_attr->max_request_iov = hca_attr->dhca_max_sgl; 1501 ep_attr->request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG; 1502 ep_attr->recv_completion_flags = DAT_COMPLETION_DEFAULT_FLAG; 1503 ep_attr->srq_soft_hw = DAT_HW_DEFAULT; 1504 return (DAT_SUCCESS); 1505 } 1506 1507 static void 1508 dapls_ib_query_srq( 1509 IN dapl_hca_attr_t *hca_attr, 1510 OUT DAT_SRQ_ATTR *srq_attr) 1511 { 1512 (void) dapl_os_memzero(srq_attr, sizeof (*srq_attr)); 1513 srq_attr->max_recv_dtos = hca_attr->dhca_max_srqs_sz; 1514 srq_attr->max_recv_iov = hca_attr->dhca_max_srq_sgl; 1515 srq_attr->low_watermark = DAT_SRQ_LW_DEFAULT; 1516 } 1517 1518 /* ARGSUSED */ 1519 DAT_RETURN 1520 dapls_ib_query_hca( 1521 IN DAPL_HCA *hca_ptr, 1522 OUT DAT_IA_ATTR *ia_attr, 1523 OUT DAT_EP_ATTR *ep_attr, 1524 OUT DAT_SOCK_ADDR6 *ip_addr, 1525 OUT DAT_SRQ_ATTR *srq_attr) 1526 { 1527 dapl_ia_query_t args; 1528 int ia_fd, retval; 1529 1530 if (hca_ptr == NULL) { 1531 return (DAT_INVALID_PARAMETER); 1532 } 1533 1534 ia_fd = hca_ptr->ib_hca_handle->ia_fd; 1535 retval = ioctl(ia_fd, DAPL_IA_QUERY, &args); 1536 if (retval != 0) { 1537 return (dapls_convert_error(errno, retval)); 1538 } 1539 1540 if (ia_attr != NULL) { 1541 (void) dapls_ib_query_ia(&args.hca_attr, hca_ptr, ia_attr); 1542 } 1543 if (ep_attr != NULL) { 1544 (void) dapls_ib_query_ep(&args.hca_attr, hca_ptr, ep_attr); 1545 } 1546 if (srq_attr != NULL) { 1547 (void) dapls_ib_query_srq(&args.hca_attr, srq_attr); 1548 } 1549 if (ia_attr == NULL && ep_attr == NULL && srq_attr == NULL) { 1550 return (DAT_INVALID_PARAMETER); 1551 } 1552 return (DAT_SUCCESS); 1553 } 1554 1555 void 1556 dapls_ib_store_premature_events( 1557 IN ib_qp_handle_t qp_ptr, 1558 IN ib_work_completion_t *cqe_ptr) 1559 { 1560 ib_srq_handle_t srqp; 1561 int head; 1562 1563 if (qp_ptr->qp_srq_enabled) { 1564 /* 1565 * For QPs with SRQ attached store the premature event in the 1566 * SRQ's premature event list 1567 */ 1568 srqp = qp_ptr->qp_srq; 1569 dapl_os_assert(srqp->srq_freepr_num_events > 0); 1570 head = srqp->srq_freepr_events[srqp->srq_freepr_head]; 1571 /* 1572 * mark cqe as valid before storing it in the 1573 * premature events list 1574 */ 1575 DAPL_SET_CQE_VALID(cqe_ptr); 1576 (void) dapl_os_memcpy(&(srqp->srq_premature_events[head]), 1577 cqe_ptr, sizeof (*cqe_ptr)); 1578 srqp->srq_freepr_head = (srqp->srq_freepr_head + 1) % 1579 srqp->srq_wq_numwqe; 1580 srqp->srq_freepr_num_events--; 1581 } else { 1582 (void) dapl_os_memcpy(&(qp_ptr->qp_premature_events[ 1583 qp_ptr->qp_num_premature_events]), 1584 cqe_ptr, sizeof (*cqe_ptr)); 1585 } 1586 qp_ptr->qp_num_premature_events++; 1587 } 1588 1589 void 1590 dapls_ib_poll_premature_events( 1591 IN DAPL_EP *ep_ptr, 1592 OUT ib_work_completion_t **cqe_ptr, 1593 OUT int *nevents) 1594 { 1595 ib_qp_handle_t qp = ep_ptr->qp_handle; 1596 1597 if (qp->qp_srq_enabled) { 1598 *cqe_ptr = qp->qp_srq->srq_premature_events; 1599 } else { 1600 *cqe_ptr = qp->qp_premature_events; 1601 } 1602 1603 *nevents = qp->qp_num_premature_events; 1604 qp->qp_num_premature_events = 0; 1605 } 1606 1607 /* 1608 * Return the premature events to the free list after processing it 1609 * This function is called only for premature events on the SRQ 1610 */ 1611 void 1612 dapls_ib_free_premature_events( 1613 IN DAPL_EP *ep_ptr, 1614 IN int free_index) 1615 { 1616 ib_qp_handle_t qp_ptr; 1617 ib_srq_handle_t srq_ptr; 1618 int tail; 1619 1620 qp_ptr = ep_ptr->qp_handle; 1621 srq_ptr = qp_ptr->qp_srq; 1622 1623 dapl_os_assert(qp_ptr->qp_srq_enabled); 1624 1625 tail = srq_ptr->srq_freepr_tail; 1626 srq_ptr->srq_freepr_events[tail] = free_index; 1627 srq_ptr->srq_freepr_tail = (tail + 1) % srq_ptr->srq_wq_numwqe; 1628 srq_ptr->srq_freepr_num_events++; 1629 DAPL_SET_CQE_INVALID(&srq_ptr->srq_premature_events[free_index]); 1630 } 1631 1632 /* 1633 * dapls_ib_get_async_event 1634 * 1635 * Translate an asynchronous event type to the DAT event. 1636 * Note that different providers have different sets of errors. 1637 * 1638 * Input: 1639 * cause_ptr provider event cause 1640 * 1641 * Output: 1642 * async_event DAT mapping of error 1643 * 1644 * Returns: 1645 * DAT_SUCCESS 1646 * DAT_NOT_IMPLEMENTED Caller is not interested this event 1647 */ 1648 1649 DAT_RETURN dapls_ib_get_async_event( 1650 IN ib_error_record_t *cause_ptr, 1651 OUT DAT_EVENT_NUMBER *async_event) 1652 { 1653 ibt_async_code_t code; 1654 DAT_RETURN dat_status; 1655 1656 dat_status = DAT_SUCCESS; 1657 code = (ibt_async_code_t)((dapl_ib_async_event_t *)cause_ptr-> 1658 ibae_type); 1659 1660 switch (code) { 1661 case IBT_ERROR_CQ: 1662 case IBT_ERROR_ACCESS_VIOLATION_CHAN: 1663 case IBT_ERROR_INVALID_REQUEST_CHAN: 1664 *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR; 1665 break; 1666 /* CATASTROPHIC errors */ 1667 case IBT_ERROR_CATASTROPHIC_CHAN: 1668 case IBT_ERROR_LOCAL_CATASTROPHIC: 1669 case IBT_ERROR_PORT_DOWN: 1670 *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC; 1671 break; 1672 default: 1673 /* 1674 * Errors we are not interested in reporting: 1675 * IBT_EVENT_PATH_MIGRATED 1676 * IBT_ERROR_PATH_MIGRATE_REQ 1677 * IBT_EVENT_COM_EST 1678 * IBT_EVENT_SQD 1679 * IBT_EVENT_PORT_UP 1680 */ 1681 dat_status = DAT_NOT_IMPLEMENTED; 1682 } 1683 return (dat_status); 1684 } 1685 1686 DAT_RETURN 1687 dapls_ib_event_poll( 1688 IN DAPL_EVD *evd_ptr, 1689 IN uint64_t timeout, 1690 IN uint_t threshold, 1691 OUT dapl_ib_event_t *evp_ptr, 1692 OUT int *num_events) 1693 { 1694 dapl_event_poll_t evp_msg; 1695 int ia_fd; 1696 int retval; 1697 1698 *num_events = 0; 1699 ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd; 1700 1701 evp_msg.evp_evd_hkey = evd_ptr->ib_cq_handle->evd_hkey; 1702 evp_msg.evp_threshold = threshold; 1703 evp_msg.evp_timeout = timeout; 1704 evp_msg.evp_ep = evp_ptr; 1705 if (evp_ptr) { 1706 evp_msg.evp_num_ev = 1707 DAPL_MAX(evd_ptr->threshold, NUM_EVENTS_PER_POLL); 1708 } else { 1709 evp_msg.evp_num_ev = 0; 1710 } 1711 evp_msg.evp_num_polled = 0; 1712 1713 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1714 "event_poll: evd 0x%p, hkey 0x%llx, threshold %d,\n" 1715 " timeout 0x%llx, evp_ptr 0x%p, num_ev %d\n", 1716 evd_ptr, evp_msg.evp_evd_hkey, evp_msg.evp_threshold, 1717 timeout, evp_ptr, evp_msg.evp_num_ev); 1718 1719 /* 1720 * Poll the EVD and if there are no events then we wait in 1721 * the kernel. 1722 */ 1723 retval = ioctl(ia_fd, DAPL_EVENT_POLL, &evp_msg); 1724 if (retval != 0) { 1725 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 1726 "event_poll: evd 0x%p, retval %d err: %s\n", 1727 evd_ptr, retval, strerror(errno)); 1728 *num_events = evp_msg.evp_num_polled; 1729 return (dapls_convert_error(errno, retval)); 1730 } 1731 1732 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 1733 "dapls_ib_event_poll: evd %p nevents %d\n", evd_ptr, 1734 evp_msg.evp_num_polled); 1735 1736 *num_events = evp_msg.evp_num_polled; 1737 1738 return (DAT_SUCCESS); 1739 } 1740 1741 DAT_RETURN 1742 dapls_ib_event_wakeup( 1743 IN DAPL_EVD *evd_ptr) 1744 { 1745 dapl_event_wakeup_t evw_msg; 1746 int ia_fd; 1747 int retval; 1748 1749 ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd; 1750 1751 evw_msg.evw_hkey = evd_ptr->ib_cq_handle->evd_hkey; 1752 1753 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1754 "event_wakeup: evd 0x%p, hkey 0x%llx\n", 1755 evd_ptr, evw_msg.evw_hkey); 1756 1757 /* 1758 * Wakeup any thread waiting in the kernel on this EVD 1759 */ 1760 retval = ioctl(ia_fd, DAPL_EVENT_WAKEUP, &evw_msg); 1761 if (retval != 0) { 1762 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 1763 "event_wakeup: evd 0x%p, retval %d err: %s\n", 1764 evd_ptr, retval, strerror(errno)); 1765 return (dapls_convert_error(errno, retval)); 1766 } 1767 1768 return (DAT_SUCCESS); 1769 } 1770 1771 /* 1772 * dapls_ib_cq_peek is used by dapl_cno_wait(). After the CQ has been 1773 * inspected we arm the CQ if it was empty. 1774 * 1775 */ 1776 void dapls_ib_cq_peek( 1777 IN DAPL_EVD *evd_ptr, 1778 OUT int *num_cqe) 1779 { 1780 DAPL_IA *ia_ptr; 1781 1782 *num_cqe = 0; 1783 if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) { 1784 DAPL_PEEK(evd_ptr)(evd_ptr->ib_cq_handle, num_cqe); 1785 /* No events found in CQ arm it now */ 1786 if (*num_cqe == 0) { 1787 ia_ptr = evd_ptr->header.owner_ia; 1788 (void) dapls_set_cq_notify(ia_ptr, evd_ptr); 1789 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 1790 "dapls_ib_cq_peek: set_cq_notify\n"); 1791 } 1792 } 1793 } 1794 1795 /* 1796 * Modifies the CNO associated to an EVD 1797 */ 1798 DAT_RETURN dapls_ib_modify_cno( 1799 IN DAPL_EVD *evd_ptr, 1800 IN DAPL_CNO *cno_ptr) 1801 { 1802 dapl_evd_modify_cno_t evmc_msg; 1803 int ia_fd; 1804 int retval; 1805 1806 ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd; 1807 1808 evmc_msg.evmc_hkey = evd_ptr->ib_cq_handle->evd_hkey; 1809 1810 if (cno_ptr) { 1811 evmc_msg.evmc_cno_hkey = (uint64_t)cno_ptr->ib_cno_handle; 1812 } else { 1813 evmc_msg.evmc_cno_hkey = 0; 1814 } 1815 1816 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1817 "modify_cno: evd 0x%p, hkey 0x%llx, cno 0x%p, cno_hkey 0x%llx\n", 1818 evd_ptr, evmc_msg.evmc_hkey, cno_ptr, evmc_msg.evmc_cno_hkey); 1819 1820 /* 1821 * modify CNO associated with the EVD 1822 */ 1823 retval = ioctl(ia_fd, DAPL_EVD_MODIFY_CNO, &evmc_msg); 1824 if (retval != 0) { 1825 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 1826 "modify_cno: evd 0x%p, cno %p retval %d err: %s\n", 1827 evd_ptr, cno_ptr, retval, strerror(errno)); 1828 return (dapls_convert_error(errno, retval)); 1829 } 1830 1831 return (DAT_SUCCESS); 1832 } 1833 1834 DAT_RETURN 1835 dapls_ib_cno_wait( 1836 IN DAPL_CNO *cno_ptr, 1837 IN DAT_TIMEOUT timeout, 1838 IN DAPL_EVD **evd_ptr_p) 1839 { 1840 dapl_cno_wait_t args; 1841 int retval; 1842 1843 args.cnw_hkey = (uint64_t)cno_ptr->ib_cno_handle; 1844 if (timeout == DAT_TIMEOUT_INFINITE) { 1845 args.cnw_timeout = UINT64_MAX; 1846 } else { 1847 args.cnw_timeout = (uint64_t)timeout & 0x00000000ffffffff; 1848 } 1849 1850 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1851 "cno_wait: cno 0x%p, hkey 0x%016llx, timeout 0x%016llx\n", 1852 cno_ptr, args.cnw_hkey, args.cnw_timeout); 1853 1854 retval = ioctl(cno_ptr->header.owner_ia->hca_ptr-> 1855 ib_hca_handle->ia_fd, DAPL_CNO_WAIT, &args); 1856 1857 if (retval != 0) { 1858 *evd_ptr_p = (DAPL_EVD *)NULL; 1859 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1860 "cno_wait: cno 0x%p ioctl err: %s\n", 1861 cno_ptr, strerror(errno)); 1862 return (dapls_convert_error(errno, retval)); 1863 } 1864 1865 *evd_ptr_p = (DAPL_EVD *)(uintptr_t)args.cnw_evd_cookie; 1866 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1867 "cno_wait: woken up, cno 0x%p, evd 0x%p\n\n", 1868 cno_ptr, *evd_ptr_p); 1869 1870 return (DAT_SUCCESS); 1871 } 1872 1873 DAT_RETURN 1874 dapls_ib_cno_alloc( 1875 IN DAPL_IA *ia_ptr, 1876 IN DAPL_CNO *cno_ptr) 1877 { 1878 dapl_cno_alloc_t args; 1879 int retval; 1880 1881 if (cno_ptr->cno_wait_agent.instance_data != NULL || 1882 cno_ptr->cno_wait_agent.proxy_agent_func != NULL) { 1883 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1884 "cno_alloc: cno 0x%p, wait_agent != NULL\n", cno_ptr); 1885 return (DAT_NOT_IMPLEMENTED); 1886 } 1887 1888 retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd, 1889 DAPL_CNO_ALLOC, &args); 1890 if (retval != 0) { 1891 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1892 "cno_alloc: cno 0x%p ioctl err: %s\n", 1893 cno_ptr, strerror(errno)); 1894 return (dapls_convert_error(errno, retval)); 1895 } 1896 1897 cno_ptr->ib_cno_handle = (ib_cno_handle_t)args.cno_hkey; 1898 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1899 "cno_alloc: cno 0x%p allocated, ia_ptr 0x%p, hkey 0x%016llx\n", 1900 cno_ptr, ia_ptr, args.cno_hkey); 1901 1902 return (DAT_SUCCESS); 1903 } 1904 1905 DAT_RETURN 1906 dapls_ib_cno_free( 1907 IN DAPL_CNO *cno_ptr) 1908 { 1909 dapl_cno_free_t args; 1910 int retval; 1911 1912 args.cnf_hkey = (uint64_t)cno_ptr->ib_cno_handle; 1913 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1914 "cno_free: cno 0x%p, hkey 0x%016llx\n", 1915 cno_ptr, args.cnf_hkey); 1916 1917 retval = ioctl(cno_ptr->header.owner_ia->hca_ptr-> 1918 ib_hca_handle->ia_fd, DAPL_CNO_FREE, &args); 1919 1920 if (retval != 0) { 1921 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1922 "cno_free: cno 0x%p ioctl err: %s\n", 1923 cno_ptr, strerror(errno)); 1924 return (dapls_convert_error(errno, retval)); 1925 } 1926 1927 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 1928 "cno_free: cno 0x%p freed\n", cno_ptr); 1929 1930 return (DAT_SUCCESS); 1931 } 1932 1933 DAT_RETURN 1934 dapls_convert_error(int errnum, int retval) 1935 { 1936 if (retval < 0) { 1937 switch (errnum) { 1938 case EINVAL: 1939 return (DAT_INVALID_PARAMETER); 1940 case ENOMEM: 1941 return (DAT_INSUFFICIENT_RESOURCES); 1942 case ETIME: 1943 return (DAT_TIMEOUT_EXPIRED); 1944 case EINTR: 1945 return (DAT_INTERRUPTED_CALL); 1946 case EFAULT: 1947 return (DAT_INTERNAL_ERROR); 1948 default: 1949 return (DAT_INTERNAL_ERROR); 1950 } 1951 } else { 1952 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 1953 "ERROR: got IBTF error %d\n", retval); 1954 switch (retval) { 1955 case IBT_SERVICE_RECORDS_NOT_FOUND: 1956 /* 1957 * Connecting to a non-existant conn qual gets 1958 * us here 1959 */ 1960 return (DAT_ERROR(DAT_INVALID_PARAMETER, 1961 DAT_INVALID_ADDRESS_UNREACHABLE)); 1962 case IBT_INSUFF_RESOURCE: 1963 case IBT_INSUFF_KERNEL_RESOURCE: 1964 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 1965 case IBT_AR_NOT_REGISTERED: 1966 /* 1967 * forward ipaddr lookup failed 1968 */ 1969 return (DAT_ERROR(DAT_INVALID_ADDRESS, 0)); 1970 default: 1971 return (DAT_INTERNAL_ERROR); 1972 } 1973 } 1974 } 1975 1976 typedef struct dapls_ib_dbp_page_s { 1977 uint32_t *dbp_page_addr; 1978 uint64_t dbp_mapoffset; 1979 struct dapls_ib_dbp_page_s *next; 1980 int fd; 1981 } dapls_ib_dbp_page_t; 1982 1983 dapls_ib_dbp_page_t *dapls_ib_pagelist = NULL; 1984 1985 /* Function that returns a pointer to the specified doorbell entry */ 1986 uint32_t *dapls_ib_get_dbp(uint64_t maplen, int fd, uint64_t mapoffset, 1987 uint32_t offset) 1988 { 1989 dapls_ib_dbp_page_t *new_page; 1990 dapls_ib_dbp_page_t *cur_page; 1991 1992 dapl_os_lock(&dapls_ib_dbp_lock); 1993 /* Check to see if page already mapped for entry */ 1994 for (cur_page = dapls_ib_pagelist; cur_page != NULL; 1995 cur_page = cur_page->next) 1996 if (cur_page->dbp_mapoffset == mapoffset && 1997 cur_page->fd == fd) { 1998 dapl_os_unlock(&dapls_ib_dbp_lock); 1999 return ((uint32_t *) 2000 (offset + (uintptr_t)cur_page->dbp_page_addr)); 2001 } 2002 2003 /* If not, map a new page and prepend to pagelist */ 2004 new_page = malloc(sizeof (dapls_ib_dbp_page_t)); 2005 if (new_page == NULL) { 2006 dapl_os_unlock(&dapls_ib_dbp_lock); 2007 return (MAP_FAILED); 2008 } 2009 new_page->dbp_page_addr = (uint32_t *)(void *)mmap64((void *)0, 2010 maplen, (PROT_READ | PROT_WRITE), MAP_SHARED, fd, mapoffset); 2011 if (new_page->dbp_page_addr == MAP_FAILED) { 2012 free(new_page); 2013 dapl_os_unlock(&dapls_ib_dbp_lock); 2014 return (MAP_FAILED); 2015 } 2016 new_page->next = dapls_ib_pagelist; 2017 new_page->dbp_mapoffset = mapoffset; 2018 new_page->fd = fd; 2019 dapls_ib_pagelist = new_page; 2020 dapl_os_unlock(&dapls_ib_dbp_lock); 2021 return ((uint32_t *)(offset + (uintptr_t)new_page->dbp_page_addr)); 2022 } 2023