1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include "dapl.h" 28 #include "dapl_adapter_util.h" 29 #include "dapl_lmr_util.h" 30 #include "dapl_rmr_util.h" 31 #include "dapl_cookie.h" 32 33 #include "dapl_tavor_ibtf_impl.h" 34 35 /* 36 * 37 * 38 * MODULE: dapl_tavor_ibtf_dto.c 39 * 40 * PURPOSE: Utility routines for data transfer operations 41 * 42 */ 43 44 45 /* 46 * dapls_ib_post_recv 47 * 48 * Provider specific Post RECV function 49 */ 50 DAT_RETURN 51 dapls_ib_post_recv( 52 IN DAPL_EP *ep_ptr, 53 IN DAPL_COOKIE *cookie, 54 IN DAT_COUNT num_segments, 55 IN DAT_LMR_TRIPLET *local_iov, 56 IN DAT_COMPLETION_FLAGS completion_flags) 57 { 58 ibt_recv_wr_t pr_wr; 59 ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV]; 60 ibt_wr_ds_t *pr_sgl; 61 boolean_t suppress_notification; 62 DAT_COUNT total_len; 63 int retval; 64 int i; 65 66 total_len = 0; 67 68 if (ep_ptr->qp_handle == NULL) { 69 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: " 70 "qp_handle == NULL\n"); 71 return (DAT_INVALID_PARAMETER); 72 } 73 74 /* allocate scatter-gather list on the heap if its large */ 75 if (num_segments > DAPL_MAX_IOV) { 76 pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t)); 77 if (NULL == pr_sgl) { 78 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 79 "dapls_ib_post_recv: pr_sgl alloc failed"); 80 return (DAT_INSUFFICIENT_RESOURCES); 81 } 82 } else { 83 pr_sgl = pr_sgl_arr; 84 } 85 86 for (i = 0; i < num_segments; i++) { 87 pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address; 88 pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context; 89 pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length; 90 91 total_len += pr_sgl[i].ds_len; 92 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: " 93 "i(%d) va(%p), lmrctxt(0x%x), len(%llu)\n", i, 94 pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len); 95 } 96 97 if (cookie != NULL) { 98 cookie->val.dto.size = total_len; 99 dapl_dbg_log(DAPL_DBG_TYPE_EP, 100 "dapls_ib_post_recv: dto_cookie(%p), num_seg(%d), " 101 "size(%d) hkey(%016llx)\n", cookie, num_segments, 102 cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey); 103 } 104 105 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 106 pr_wr.wr_nds = (uint32_t)num_segments; 107 if (num_segments > 0) { 108 pr_wr.wr_sgl = &pr_sgl[0]; 109 } else { 110 pr_wr.wr_sgl = NULL; 111 } 112 113 if (ep_ptr->param.ep_attr.recv_completion_flags & 114 DAT_COMPLETION_UNSIGNALLED_FLAG) { 115 /* This flag is used to control notification of completions */ 116 suppress_notification = (completion_flags & 117 DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE; 118 } else { 119 /* 120 * The evd waiter will use threshold to control wakeups 121 * Hence the event notification will be done via arming the 122 * CQ so we do not need special notification generation 123 * hence set suppression to true 124 */ 125 suppress_notification = B_TRUE; 126 } 127 128 retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification); 129 130 if (retval != 0) { 131 dapl_dbg_log(DAPL_DBG_TYPE_EP, 132 "dapls_ib_post_recv: post_recv failed %s\n", 133 strerror(errno)); 134 } 135 136 /* free the pr_sgl if we had allocated it */ 137 if (num_segments > DAPL_MAX_IOV) { 138 dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t)); 139 } 140 141 return (retval); 142 } 143 144 /* 145 * dapls_ib_post_recv_one 146 * 147 * Provider specific Post RECV function 148 */ 149 DAT_RETURN 150 dapls_ib_post_recv_one( 151 IN DAPL_EP *ep_ptr, 152 IN DAPL_COOKIE *cookie, 153 IN DAT_LMR_TRIPLET *local_iov) 154 { 155 ibt_recv_wr_t pr_wr; 156 ibt_wr_ds_t pr_sgl; 157 boolean_t suppress_notification; 158 DAT_COUNT total_len; 159 int retval; 160 161 if (ep_ptr->qp_handle == NULL) { 162 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: " 163 "qp_handle == NULL\n"); 164 return (DAT_INVALID_PARAMETER); 165 } 166 167 pr_sgl.ds_va = (ib_vaddr_t)local_iov->virtual_address; 168 pr_sgl.ds_key = (ibt_lkey_t)local_iov->lmr_context; 169 pr_sgl.ds_len = (ib_msglen_t)local_iov->segment_length; 170 171 total_len = pr_sgl.ds_len; 172 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: " 173 "va(%p), lmrctxt(0x%x), len(%llu)\n", 174 pr_sgl.ds_va, pr_sgl.ds_key, pr_sgl.ds_len); 175 176 if (cookie != NULL) { 177 cookie->val.dto.size = total_len; 178 dapl_dbg_log(DAPL_DBG_TYPE_EP, 179 "dapls_ib_post_recv_one: dto_cookie(%p), num_seg(1), " 180 "size(%d) hkey(%016llx)\n", cookie, 181 cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey); 182 } 183 184 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 185 pr_wr.wr_nds = 1; 186 pr_wr.wr_sgl = &pr_sgl; 187 188 if (ep_ptr->param.ep_attr.recv_completion_flags & 189 DAT_COMPLETION_UNSIGNALLED_FLAG) { 190 /* This flag is used to control notification of completions */ 191 suppress_notification = B_FALSE; 192 } else { 193 /* 194 * The evd waiter will use threshold to control wakeups 195 * Hence the event notification will be done via arming the 196 * CQ so we do not need special notification generation 197 * hence set suppression to true 198 */ 199 suppress_notification = B_TRUE; 200 } 201 202 retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification); 203 204 if (retval != 0) { 205 dapl_dbg_log(DAPL_DBG_TYPE_EP, 206 "dapls_ib_post_recv_one: post_recv failed %s\n", 207 strerror(errno)); 208 } 209 210 return (retval); 211 } 212 213 /* 214 * dapls_ib_srq_post_recv 215 * 216 * Provider specific SRQ Post RECV function 217 */ 218 DAT_RETURN 219 dapls_ib_post_srq( 220 IN DAPL_SRQ *srq_ptr, 221 IN DAPL_COOKIE *cookie, 222 IN DAT_COUNT num_segments, 223 IN DAT_LMR_TRIPLET *local_iov) 224 { 225 ibt_recv_wr_t pr_wr; 226 ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV]; 227 ibt_wr_ds_t *pr_sgl; 228 DAT_COUNT total_len; 229 int retval; 230 int i; 231 232 total_len = 0; 233 234 if (srq_ptr->srq_handle == NULL) { 235 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: " 236 "srq_handle == NULL\n"); 237 return (DAT_INVALID_PARAMETER); 238 } 239 240 /* allocate scatter-gather list on the heap if its large */ 241 if (num_segments > DAPL_MAX_IOV) { 242 pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t)); 243 if (NULL == pr_sgl) { 244 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 245 "dapls_ib_post_srq: pr_sgl alloc failed"); 246 return (DAT_INSUFFICIENT_RESOURCES); 247 } 248 } else { 249 pr_sgl = pr_sgl_arr; 250 } 251 252 for (i = 0; i < num_segments; i++) { 253 pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address; 254 pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context; 255 pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length; 256 257 total_len += pr_sgl[i].ds_len; 258 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: " 259 "i(%d) va(%p), lmrctxt(0x%x), len(%u)\n", i, 260 pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len); 261 } 262 263 if (cookie != NULL) { 264 cookie->val.dto.size = total_len; 265 dapl_dbg_log(DAPL_DBG_TYPE_EP, 266 "dapls_ib_post_srq: dto_cookie(%p), num_seg(%d), " 267 "size(%d) hkey(%016llx)\n", cookie, num_segments, 268 cookie->val.dto.size, srq_ptr->srq_handle->srq_hkey); 269 } 270 271 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 272 pr_wr.wr_nds = (uint32_t)num_segments; 273 if (num_segments > 0) { 274 pr_wr.wr_sgl = &pr_sgl[0]; 275 } else { 276 pr_wr.wr_sgl = NULL; 277 } 278 279 retval = DAPL_SRECV(srq_ptr)(srq_ptr, &pr_wr, B_TRUE); 280 281 if (retval != 0) { 282 dapl_dbg_log(DAPL_DBG_TYPE_EP, 283 "dapls_ib_post_srq: post_recv failed %s\n", 284 strerror(errno)); 285 } 286 287 /* free the pr_sgl if we had allocated it */ 288 if (num_segments > DAPL_MAX_IOV) { 289 dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t)); 290 } 291 292 return (retval); 293 } 294 295 /* 296 * dapls_ib_post_send 297 * 298 * Provider specific Post SEND function 299 */ 300 DAT_RETURN 301 dapls_ib_post_send(IN DAPL_EP *ep_ptr, 302 IN ib_send_op_type_t op_type, 303 IN DAPL_COOKIE *cookie, 304 IN DAT_COUNT num_segments, 305 IN DAT_LMR_TRIPLET *local_iov, 306 IN const DAT_RMR_TRIPLET *remote_iov, 307 IN DAT_COMPLETION_FLAGS completion_flags) 308 { 309 ibt_send_wr_t ps_wr; 310 ibt_wr_ds_t ps_sgl_arr[DAPL_MAX_IOV]; 311 ibt_wr_ds_t *ps_sgl; 312 DAT_COUNT total_len; 313 boolean_t suppress_notification; 314 int retval; 315 int i; 316 317 total_len = 0; 318 retval = DAT_SUCCESS; 319 320 if (ep_ptr->qp_handle == NULL) { 321 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: " 322 "qp_handle == NULL\n"); 323 return (DAT_INVALID_PARAMETER); 324 } 325 326 /* allocate scatter-gather list on the heap if its large */ 327 if (num_segments > DAPL_MAX_IOV) { 328 ps_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t)); 329 if (NULL == ps_sgl) { 330 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 331 "dapls_ib_post_send: pr_sgl alloc failed"); 332 return (DAT_INSUFFICIENT_RESOURCES); 333 } 334 } else { 335 ps_sgl = ps_sgl_arr; 336 } 337 338 for (i = 0; i < num_segments; i++) { 339 ps_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address; 340 ps_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context; 341 ps_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length; 342 total_len += ps_sgl[i].ds_len; 343 344 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: " 345 "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n", 346 i, ps_sgl[i].ds_va, ps_sgl[i].ds_key, ps_sgl[i].ds_len); 347 } 348 349 if (cookie != NULL) { 350 cookie->val.dto.size = total_len; 351 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 352 "dapls_ib_post_send: op_type(%d), cookie(%p) " 353 "num_seg(%d) size(%d) hkey(%016llx)\n", op_type, 354 cookie, num_segments, cookie->val.dto.size, 355 ep_ptr->qp_handle->ep_hkey); 356 } 357 358 ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 359 /* Translate dapl flags */ 360 ps_wr.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG & 361 completion_flags) ? IBT_WR_SEND_FENCE : 0; 362 /* suppress completions */ 363 ps_wr.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & 364 completion_flags) ? 0 : IBT_WR_SEND_SIGNAL; 365 366 /* Solicited wait flag is valid only for post_send */ 367 if (op_type == OP_SEND) { 368 ps_wr.wr_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & 369 completion_flags) ? IBT_WR_SEND_SOLICIT : 0; 370 } 371 372 ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type; 373 ps_wr.wr_nds = (uint32_t)num_segments; 374 if (num_segments > 0) { 375 ps_wr.wr_sgl = &ps_sgl[0]; 376 if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) { 377 if (remote_iov == NULL) { 378 /* free the ps_sgl if we had allocated it */ 379 if (num_segments > DAPL_MAX_IOV) { 380 dapl_os_free(ps_sgl, 381 num_segments*sizeof (ibt_wr_ds_t)); 382 } 383 dapl_dbg_log(DAPL_DBG_TYPE_EP, 384 "dapls_ib_post_send: " 385 "remote_iov == NULL\n"); 386 return (DAT_INVALID_PARAMETER); 387 } 388 389 if (remote_iov->segment_length != (DAT_VLEN)total_len) { 390 /* free the ps_sgl if we had allocated it */ 391 if (num_segments > DAPL_MAX_IOV) { 392 dapl_os_free(ps_sgl, 393 num_segments*sizeof (ibt_wr_ds_t)); 394 } 395 dapl_dbg_log(DAPL_DBG_TYPE_EP, 396 "dapls_ib_post_send: " 397 "remote_iov length(%llu != %llu)\n", 398 (DAT_VLEN)total_len, 399 remote_iov->segment_length); 400 return (DAT_LENGTH_ERROR); 401 } 402 403 ps_wr.wr.rc.rcwr.rdma.rdma_raddr = 404 (ib_vaddr_t)remote_iov->target_address; 405 ps_wr.wr.rc.rcwr.rdma.rdma_rkey = 406 (ibt_rkey_t)remote_iov->rmr_context; 407 408 dapl_dbg_log(DAPL_DBG_TYPE_EP, 409 "dapls_ib_post_send: remote_iov taddr(0x%llx), " 410 "rmr(0x%x)\n", remote_iov->target_address, 411 remote_iov->rmr_context); 412 } 413 } else { 414 ps_wr.wr_sgl = NULL; 415 } 416 417 if (ep_ptr->param.ep_attr.recv_completion_flags & 418 DAT_COMPLETION_UNSIGNALLED_FLAG) { 419 /* This flag is used to control notification of completions */ 420 suppress_notification = (completion_flags & 421 DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE; 422 } else { 423 /* 424 * The evd waiter will use threshold to control wakeups 425 * Hence the event notification will be done via arming the 426 * CQ so we do not need special notification generation 427 * hence set suppression to true 428 */ 429 suppress_notification = B_TRUE; 430 } 431 432 retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification); 433 434 if (retval != 0) { 435 dapl_dbg_log(DAPL_DBG_TYPE_EP, 436 "dapls_ib_post_send: post_send failed %d\n", retval); 437 } 438 439 /* free the pr_sgl if we had allocated it */ 440 if (num_segments > DAPL_MAX_IOV) { 441 dapl_os_free(ps_sgl, num_segments*sizeof (ibt_wr_ds_t)); 442 } 443 444 return (retval); 445 } 446 447 /* 448 * dapls_ib_post_send_one 449 * 450 * Provider specific Post SEND function - special case for the common case of 451 * sgl num_segments == 1 and completion_flags == DAT_COMPLETION_DEFAULT_FLAG. 452 */ 453 DAT_RETURN 454 dapls_ib_post_send_one(IN DAPL_EP *ep_ptr, 455 IN ib_send_op_type_t op_type, 456 IN DAPL_COOKIE *cookie, 457 IN DAT_LMR_TRIPLET *local_iov, 458 IN const DAT_RMR_TRIPLET *remote_iov) 459 { 460 ibt_send_wr_t ps_wr; 461 ibt_wr_ds_t ps_sgl; 462 boolean_t suppress_notification; 463 int retval; 464 465 if (ep_ptr->qp_handle == NULL) { 466 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: " 467 "qp_handle == NULL\n"); 468 return (DAT_INVALID_PARAMETER); 469 } 470 471 ps_sgl.ds_va = (ib_vaddr_t)local_iov[0].virtual_address; 472 ps_sgl.ds_key = (ibt_lkey_t)local_iov[0].lmr_context; 473 ps_sgl.ds_len = (ib_msglen_t)local_iov[0].segment_length; 474 475 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: " 476 "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n", 477 0, ps_sgl.ds_va, ps_sgl.ds_key, ps_sgl.ds_len); 478 479 cookie->val.dto.size = ps_sgl.ds_len; 480 dapl_dbg_log(DAPL_DBG_TYPE_EVD, 481 "dapls_ib_post_send_one: op_type(%d), cookie(%p) " 482 "num_seg(%d) size(%d) hkey(%016llx)\n", op_type, 483 cookie, 1, cookie->val.dto.size, 484 ep_ptr->qp_handle->ep_hkey); 485 486 ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie; 487 /* suppress completions */ 488 ps_wr.wr_flags = IBT_WR_SEND_SIGNAL; 489 490 ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type; 491 ps_wr.wr_nds = 1; 492 493 ps_wr.wr_sgl = &ps_sgl; 494 if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) { 495 if (remote_iov == NULL) { 496 /* free the ps_sgl if we had allocated it */ 497 dapl_dbg_log(DAPL_DBG_TYPE_EP, 498 "dapls_ib_post_send_one: " 499 "remote_iov == NULL\n"); 500 return (DAT_INVALID_PARAMETER); 501 } 502 503 if (remote_iov->segment_length != (DAT_VLEN)ps_sgl.ds_len) { 504 dapl_dbg_log(DAPL_DBG_TYPE_EP, 505 "dapls_ib_post_send_one: " 506 "remote_iov length(%llu != %llu)\n", 507 (DAT_VLEN)ps_sgl.ds_len, 508 remote_iov->segment_length); 509 return (DAT_LENGTH_ERROR); 510 } 511 512 ps_wr.wr.rc.rcwr.rdma.rdma_raddr = 513 (ib_vaddr_t)remote_iov->target_address; 514 ps_wr.wr.rc.rcwr.rdma.rdma_rkey = 515 (ibt_rkey_t)remote_iov->rmr_context; 516 517 dapl_dbg_log(DAPL_DBG_TYPE_EP, 518 "dapls_ib_post_send_one: remote_iov taddr(0x%llx), " 519 "rmr(0x%x)\n", remote_iov->target_address, 520 remote_iov->rmr_context); 521 } 522 523 if (ep_ptr->param.ep_attr.recv_completion_flags & 524 DAT_COMPLETION_UNSIGNALLED_FLAG) { 525 /* This flag is used to control notification of completions */ 526 suppress_notification = B_FALSE; 527 } else { 528 /* 529 * The evd waiter will use threshold to control wakeups 530 * Hence the event notification will be done via arming the 531 * CQ so we do not need special notification generation 532 * hence set suppression to true 533 */ 534 suppress_notification = B_TRUE; 535 } 536 537 retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification); 538 539 if (retval != 0) { 540 dapl_dbg_log(DAPL_DBG_TYPE_EP, 541 "dapls_ib_post_send_one: post_send failed %d\n", retval); 542 } 543 544 return (retval); 545 } 546