1 /* 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 3 */ 4 5 /* 6 * This file contains code imported from the OFED rds source file rdma.c 7 * Oracle elects to have and use the contents of rdma.c under and governed 8 * by the OpenIB.org BSD license (see below for full license text). However, 9 * the following notice accompanied the original version of this file: 10 */ 11 12 /* 13 * Copyright (c) 2007 Oracle. All rights reserved. 14 * 15 * This software is available to you under a choice of one of two 16 * licenses. You may choose to be licensed under the terms of the GNU 17 * General Public License (GPL) Version 2, available from the file 18 * COPYING in the main directory of this source tree, or the 19 * OpenIB.org BSD license below: 20 * 21 * Redistribution and use in source and binary forms, with or 22 * without modification, are permitted provided that the following 23 * conditions are met: 24 * 25 * - Redistributions of source code must retain the above 26 * copyright notice, this list of conditions and the following 27 * disclaimer. 28 * 29 * - Redistributions in binary form must reproduce the above 30 * copyright notice, this list of conditions and the following 31 * disclaimer in the documentation and/or other materials 32 * provided with the distribution. 33 * 34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 41 * SOFTWARE. 42 * 43 */ 44 #include <sys/ib/clients/of/rdma/ib_verbs.h> 45 #include <sys/ib/clients/of/rdma/ib_addr.h> 46 #include <sys/ib/clients/of/rdma/rdma_cm.h> 47 48 #include <sys/ib/clients/rdsv3/ib.h> 49 #include <sys/ib/clients/rdsv3/rdma.h> 50 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 51 52 #define DMA_TO_DEVICE 0 53 #define DMA_FROM_DEVICE 1 54 #define RB_CLEAR_NODE(nodep) AVL_SETPARENT(nodep, nodep); 55 56 /* 57 * XXX 58 * - build with sparse 59 * - should we limit the size of a mr region? let transport return failure? 60 * - should we detect duplicate keys on a socket? hmm. 61 * - an rdma is an mlock, apply rlimit? 62 */ 63 64 /* 65 * get the number of pages by looking at the page indices that the start and 66 * end addresses fall in. 67 * 68 * Returns 0 if the vec is invalid. It is invalid if the number of bytes 69 * causes the address to wrap or overflows an unsigned int. This comes 70 * from being stored in the 'length' member of 'struct rdsv3_scatterlist'. 71 */ 72 static unsigned int 73 rdsv3_pages_in_vec(struct rdsv3_iovec *vec) 74 { 75 if ((vec->addr + vec->bytes <= vec->addr) || 76 (vec->bytes > (uint64_t)UINT_MAX)) { 77 return (0); 78 } 79 80 return (((vec->addr + vec->bytes + PAGESIZE - 1) >> 81 PAGESHIFT) - (vec->addr >> PAGESHIFT)); 82 } 83 84 static struct rdsv3_mr * 85 rdsv3_mr_tree_walk(struct avl_tree *root, uint32_t key, 86 struct rdsv3_mr *insert) 87 { 88 struct rdsv3_mr *mr; 89 avl_index_t where; 90 91 mr = avl_find(root, &key, &where); 92 if ((mr == NULL) && (insert != NULL)) { 93 avl_insert(root, (void *)insert, where); 94 atomic_add_32(&insert->r_refcount, 1); 95 return (NULL); 96 } 97 98 return (mr); 99 } 100 101 /* 102 * Destroy the transport-specific part of a MR. 103 */ 104 static void 105 rdsv3_destroy_mr(struct rdsv3_mr *mr) 106 { 107 struct rdsv3_sock *rs = mr->r_sock; 108 void *trans_private = NULL; 109 avl_node_t *np; 110 111 RDSV3_DPRINTF5("rdsv3_destroy_mr", 112 "RDS: destroy mr key is %x refcnt %u", 113 mr->r_key, atomic_get(&mr->r_refcount)); 114 115 if (test_and_set_bit(RDSV3_MR_DEAD, &mr->r_state)) 116 return; 117 118 mutex_enter(&rs->rs_rdma_lock); 119 np = &mr->r_rb_node; 120 if (AVL_XPARENT(np) != np) 121 avl_remove(&rs->rs_rdma_keys, mr); 122 trans_private = mr->r_trans_private; 123 mr->r_trans_private = NULL; 124 mutex_exit(&rs->rs_rdma_lock); 125 126 if (trans_private) 127 mr->r_trans->free_mr(trans_private, mr->r_invalidate); 128 } 129 130 void 131 __rdsv3_put_mr_final(struct rdsv3_mr *mr) 132 { 133 rdsv3_destroy_mr(mr); 134 kmem_free(mr, sizeof (*mr)); 135 } 136 137 /* 138 * By the time this is called we can't have any more ioctls called on 139 * the socket so we don't need to worry about racing with others. 140 */ 141 void 142 rdsv3_rdma_drop_keys(struct rdsv3_sock *rs) 143 { 144 struct rdsv3_mr *mr; 145 struct avl_node *node; 146 147 /* Release any MRs associated with this socket */ 148 mutex_enter(&rs->rs_rdma_lock); 149 while ((node = avl_first(&rs->rs_rdma_keys))) { 150 mr = container_of(node, struct rdsv3_mr, r_rb_node); 151 if (mr->r_trans == rs->rs_transport) 152 mr->r_invalidate = 0; 153 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 154 RB_CLEAR_NODE(&mr->r_rb_node) 155 mutex_exit(&rs->rs_rdma_lock); 156 rdsv3_destroy_mr(mr); 157 rdsv3_mr_put(mr); 158 mutex_enter(&rs->rs_rdma_lock); 159 } 160 mutex_exit(&rs->rs_rdma_lock); 161 162 if (rs->rs_transport && rs->rs_transport->flush_mrs) 163 rs->rs_transport->flush_mrs(); 164 } 165 166 static int 167 __rdsv3_rdma_map(struct rdsv3_sock *rs, struct rdsv3_get_mr_args *args, 168 uint64_t *cookie_ret, struct rdsv3_mr **mr_ret) 169 { 170 struct rdsv3_mr *mr = NULL, *found; 171 void *trans_private; 172 rdsv3_rdma_cookie_t cookie; 173 unsigned int nents = 0; 174 int ret; 175 176 if (rs->rs_bound_addr == 0) { 177 ret = -ENOTCONN; /* XXX not a great errno */ 178 goto out; 179 } 180 181 if (!rs->rs_transport->get_mr) { 182 ret = -EOPNOTSUPP; 183 goto out; 184 } 185 186 mr = kmem_zalloc(sizeof (struct rdsv3_mr), KM_NOSLEEP); 187 if (!mr) { 188 ret = -ENOMEM; 189 goto out; 190 } 191 192 mr->r_refcount = 1; 193 RB_CLEAR_NODE(&mr->r_rb_node); 194 mr->r_trans = rs->rs_transport; 195 mr->r_sock = rs; 196 197 if (args->flags & RDSV3_RDMA_USE_ONCE) 198 mr->r_use_once = 1; 199 if (args->flags & RDSV3_RDMA_INVALIDATE) 200 mr->r_invalidate = 1; 201 if (args->flags & RDSV3_RDMA_READWRITE) 202 mr->r_write = 1; 203 204 /* 205 * Obtain a transport specific MR. If this succeeds, the 206 * s/g list is now owned by the MR. 207 * Note that dma_map() implies that pending writes are 208 * flushed to RAM, so no dma_sync is needed here. 209 */ 210 trans_private = rs->rs_transport->get_mr(&args->vec, nents, rs, 211 &mr->r_key); 212 213 if (IS_ERR(trans_private)) { 214 ret = PTR_ERR(trans_private); 215 goto out; 216 } 217 218 mr->r_trans_private = trans_private; 219 220 /* 221 * The user may pass us an unaligned address, but we can only 222 * map page aligned regions. So we keep the offset, and build 223 * a 64bit cookie containing <R_Key, offset> and pass that 224 * around. 225 */ 226 cookie = rdsv3_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGEMASK); 227 if (cookie_ret) 228 *cookie_ret = cookie; 229 230 /* 231 * copy value of cookie to user address at args->cookie_addr 232 */ 233 if (args->cookie_addr) { 234 ret = ddi_copyout((void *)&cookie, 235 (void *)((intptr_t)args->cookie_addr), 236 sizeof (rdsv3_rdma_cookie_t), 0); 237 if (ret != 0) { 238 ret = -EFAULT; 239 goto out; 240 } 241 } 242 243 RDSV3_DPRINTF5("__rdsv3_rdma_map", 244 "RDS: get_mr mr 0x%p addr 0x%llx key 0x%x", 245 mr, args->vec.addr, mr->r_key); 246 /* 247 * Inserting the new MR into the rbtree bumps its 248 * reference count. 249 */ 250 mutex_enter(&rs->rs_rdma_lock); 251 found = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr); 252 mutex_exit(&rs->rs_rdma_lock); 253 254 ASSERT(!(found && found != mr)); 255 256 if (mr_ret) { 257 atomic_add_32(&mr->r_refcount, 1); 258 *mr_ret = mr; 259 } 260 261 ret = 0; 262 out: 263 if (mr) 264 rdsv3_mr_put(mr); 265 return (ret); 266 } 267 268 int 269 rdsv3_get_mr(struct rdsv3_sock *rs, const void *optval, int optlen) 270 { 271 struct rdsv3_get_mr_args args; 272 273 if (optlen != sizeof (struct rdsv3_get_mr_args)) 274 return (-EINVAL); 275 276 #if 1 277 bcopy((struct rdsv3_get_mr_args *)optval, &args, 278 sizeof (struct rdsv3_get_mr_args)); 279 #else 280 if (ddi_copyin(optval, &args, optlen, 0)) 281 return (-EFAULT); 282 #endif 283 284 return (__rdsv3_rdma_map(rs, &args, NULL, NULL)); 285 } 286 287 int 288 rdsv3_get_mr_for_dest(struct rdsv3_sock *rs, const void *optval, 289 int optlen) 290 { 291 struct rdsv3_get_mr_for_dest_args args; 292 struct rdsv3_get_mr_args new_args; 293 294 if (optlen != sizeof (struct rdsv3_get_mr_for_dest_args)) 295 return (-EINVAL); 296 297 #if 1 298 bcopy((struct rdsv3_get_mr_for_dest_args *)optval, &args, 299 sizeof (struct rdsv3_get_mr_for_dest_args)); 300 #else 301 if (ddi_copyin(optval, &args, optlen, 0)) 302 return (-EFAULT); 303 #endif 304 305 /* 306 * Initially, just behave like get_mr(). 307 * TODO: Implement get_mr as wrapper around this 308 * and deprecate it. 309 */ 310 new_args.vec = args.vec; 311 new_args.cookie_addr = args.cookie_addr; 312 new_args.flags = args.flags; 313 314 return (__rdsv3_rdma_map(rs, &new_args, NULL, NULL)); 315 } 316 317 /* 318 * Free the MR indicated by the given R_Key 319 */ 320 int 321 rdsv3_free_mr(struct rdsv3_sock *rs, const void *optval, int optlen) 322 { 323 struct rdsv3_free_mr_args args; 324 struct rdsv3_mr *mr; 325 326 if (optlen != sizeof (struct rdsv3_free_mr_args)) 327 return (-EINVAL); 328 329 #if 1 330 bcopy((struct rdsv3_free_mr_args *)optval, &args, 331 sizeof (struct rdsv3_free_mr_args)); 332 #else 333 if (ddi_copyin((struct rdsv3_free_mr_args *)optval, &args, 334 sizeof (struct rdsv3_free_mr_args), 0)) 335 return (-EFAULT); 336 #endif 337 338 /* Special case - a null cookie means flush all unused MRs */ 339 if (args.cookie == 0) { 340 if (!rs->rs_transport || !rs->rs_transport->flush_mrs) 341 return (-EINVAL); 342 rs->rs_transport->flush_mrs(); 343 return (0); 344 } 345 346 /* 347 * Look up the MR given its R_key and remove it from the rbtree 348 * so nobody else finds it. 349 * This should also prevent races with rdsv3_rdma_unuse. 350 */ 351 mutex_enter(&rs->rs_rdma_lock); 352 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, 353 rdsv3_rdma_cookie_key(args.cookie), NULL); 354 if (mr) { 355 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 356 RB_CLEAR_NODE(&mr->r_rb_node); 357 if (args.flags & RDSV3_RDMA_INVALIDATE) 358 mr->r_invalidate = 1; 359 } 360 mutex_exit(&rs->rs_rdma_lock); 361 362 if (!mr) 363 return (-EINVAL); 364 365 /* 366 * call rdsv3_destroy_mr() ourselves so that we're sure it's done 367 * by time we return. If we let rdsv3_mr_put() do it it might not 368 * happen until someone else drops their ref. 369 */ 370 rdsv3_destroy_mr(mr); 371 rdsv3_mr_put(mr); 372 return (0); 373 } 374 375 /* 376 * This is called when we receive an extension header that 377 * tells us this MR was used. It allows us to implement 378 * use_once semantics 379 */ 380 void 381 rdsv3_rdma_unuse(struct rdsv3_sock *rs, uint32_t r_key, int force) 382 { 383 struct rdsv3_mr *mr; 384 int zot_me = 0; 385 386 RDSV3_DPRINTF4("rdsv3_rdma_unuse", "Enter rkey: 0x%x", r_key); 387 388 mutex_enter(&rs->rs_rdma_lock); 389 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); 390 if (!mr) { 391 RDSV3_DPRINTF4("rdsv3_rdma_unuse", 392 "rdsv3: trying to unuse MR with unknown r_key %u!", r_key); 393 mutex_exit(&rs->rs_rdma_lock); 394 return; 395 } 396 397 if (mr->r_use_once || force) { 398 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 399 RB_CLEAR_NODE(&mr->r_rb_node); 400 zot_me = 1; 401 } 402 mutex_exit(&rs->rs_rdma_lock); 403 404 /* 405 * May have to issue a dma_sync on this memory region. 406 * Note we could avoid this if the operation was a RDMA READ, 407 * but at this point we can't tell. 408 */ 409 if (mr->r_trans->sync_mr) 410 mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE); 411 412 /* 413 * If the MR was marked as invalidate, this will 414 * trigger an async flush. 415 */ 416 if (zot_me) 417 rdsv3_destroy_mr(mr); 418 rdsv3_mr_put(mr); 419 RDSV3_DPRINTF4("rdsv3_rdma_unuse", "Return"); 420 } 421 422 void 423 rdsv3_rdma_free_op(struct rdsv3_rdma_op *ro) 424 { 425 unsigned int i; 426 427 /* deallocate RDMA resources on rdsv3_message */ 428 for (i = 0; i < ro->r_nents; i++) { 429 ddi_umem_unlock(ro->r_rdma_sg[i].umem_cookie); 430 } 431 432 if (ro->r_notifier) 433 kmem_free(ro->r_notifier, sizeof (*ro->r_notifier)); 434 kmem_free(ro, sizeof (*ro)); 435 } 436 437 /* 438 * args is a pointer to an in-kernel copy in the sendmsg cmsg. 439 */ 440 static struct rdsv3_rdma_op * 441 rdsv3_rdma_prepare(struct rdsv3_sock *rs, struct rdsv3_rdma_args *args) 442 { 443 struct rdsv3_iovec vec; 444 struct rdsv3_rdma_op *op = NULL; 445 unsigned int nr_bytes; 446 struct rdsv3_iovec *local_vec; 447 unsigned int nr; 448 unsigned int i; 449 ddi_umem_cookie_t umem_cookie; 450 size_t umem_len; 451 caddr_t umem_addr; 452 int ret; 453 454 if (rs->rs_bound_addr == 0) { 455 ret = -ENOTCONN; /* XXX not a great errno */ 456 goto out; 457 } 458 459 if (args->nr_local > (uint64_t)UINT_MAX) { 460 ret = -EMSGSIZE; 461 goto out; 462 } 463 464 op = kmem_zalloc(offsetof(struct rdsv3_rdma_op, 465 r_rdma_sg[args->nr_local]), KM_NOSLEEP); 466 if (op == NULL) { 467 ret = -ENOMEM; 468 goto out; 469 } 470 471 op->r_write = !!(args->flags & RDSV3_RDMA_READWRITE); 472 op->r_fence = !!(args->flags & RDSV3_RDMA_FENCE); 473 op->r_notify = !!(args->flags & RDSV3_RDMA_NOTIFY_ME); 474 op->r_recverr = rs->rs_recverr; 475 476 if (op->r_notify || op->r_recverr) { 477 /* 478 * We allocate an uninitialized notifier here, because 479 * we don't want to do that in the completion handler. We 480 * would have to use GFP_ATOMIC there, and don't want to deal 481 * with failed allocations. 482 */ 483 op->r_notifier = kmem_alloc(sizeof (struct rdsv3_notifier), 484 KM_NOSLEEP); 485 if (!op->r_notifier) { 486 ret = -ENOMEM; 487 goto out; 488 } 489 op->r_notifier->n_user_token = args->user_token; 490 op->r_notifier->n_status = RDSV3_RDMA_SUCCESS; 491 } 492 493 /* 494 * The cookie contains the R_Key of the remote memory region, and 495 * optionally an offset into it. This is how we implement RDMA into 496 * unaligned memory. 497 * When setting up the RDMA, we need to add that offset to the 498 * destination address (which is really an offset into the MR) 499 * FIXME: We may want to move this into ib_rdma.c 500 */ 501 op->r_key = rdsv3_rdma_cookie_key(args->cookie); 502 op->r_remote_addr = args->remote_vec.addr + 503 rdsv3_rdma_cookie_offset(args->cookie); 504 505 nr_bytes = 0; 506 507 RDSV3_DPRINTF5("rdsv3_rdma_prepare", 508 "RDS: rdma prepare nr_local %llu rva %llx rkey %x", 509 (unsigned long long)args->nr_local, 510 (unsigned long long)args->remote_vec.addr, 511 op->r_key); 512 513 local_vec = (struct rdsv3_iovec *)(unsigned long) args->local_vec_addr; 514 515 /* pin the scatter list of user buffers */ 516 for (i = 0; i < args->nr_local; i++) { 517 if (ddi_copyin(&local_vec[i], &vec, 518 sizeof (struct rdsv3_iovec), 0)) { 519 ret = -EFAULT; 520 goto out; 521 } 522 523 nr = rdsv3_pages_in_vec(&vec); 524 if (nr == 0) { 525 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 526 "rdsv3_pages_in_vec returned 0"); 527 ret = -EINVAL; 528 goto out; 529 } 530 531 rs->rs_user_addr = vec.addr; 532 rs->rs_user_bytes = vec.bytes; 533 534 /* pin user memory pages */ 535 umem_len = ptob(btopr(vec.bytes + 536 ((uintptr_t)vec.addr & PAGEOFFSET))); 537 umem_addr = (caddr_t)((uintptr_t)vec.addr & ~PAGEOFFSET); 538 ret = umem_lockmemory(umem_addr, umem_len, 539 DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ, 540 &umem_cookie, NULL, NULL); 541 if (ret != 0) { 542 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 543 "umem_lockmemory() returned %d", ret); 544 ret = -EFAULT; 545 goto out; 546 } 547 op->r_rdma_sg[i].umem_cookie = umem_cookie; 548 op->r_rdma_sg[i].iovec = vec; 549 nr_bytes += vec.bytes; 550 551 RDSV3_DPRINTF5("rdsv3_rdma_prepare", 552 "RDS: nr_bytes %u nr %u vec.bytes %llu vec.addr %llx", 553 nr_bytes, nr, vec.bytes, vec.addr); 554 } 555 op->r_nents = i; 556 557 if (nr_bytes > args->remote_vec.bytes) { 558 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 559 "RDS nr_bytes %u remote_bytes %u do not match", 560 nr_bytes, (unsigned int) args->remote_vec.bytes); 561 ret = -EINVAL; 562 goto out; 563 } 564 op->r_bytes = nr_bytes; 565 566 ret = 0; 567 out: 568 if (ret) { 569 if (op) 570 rdsv3_rdma_free_op(op); 571 op = ERR_PTR(ret); 572 } 573 return (op); 574 } 575 576 /* 577 * The application asks for a RDMA transfer. 578 * Extract all arguments and set up the rdma_op 579 */ 580 int 581 rdsv3_cmsg_rdma_args(struct rdsv3_sock *rs, struct rdsv3_message *rm, 582 struct cmsghdr *cmsg) 583 { 584 struct rdsv3_rdma_op *op; 585 struct rdsv3_rdma_args *ap; 586 587 if (cmsg->cmsg_len < CMSG_LEN(sizeof (struct rdsv3_rdma_args)) || 588 rm->m_rdma_op != NULL) 589 return (-EINVAL); 590 591 /* uint64_t alignment on struct rdsv3_get_mr_args */ 592 ap = (struct rdsv3_rdma_args *)kmem_alloc(cmsg->cmsg_len, KM_SLEEP); 593 bcopy(CMSG_DATA(cmsg), ap, cmsg->cmsg_len); 594 op = rdsv3_rdma_prepare(rs, ap); 595 kmem_free(ap, cmsg->cmsg_len); 596 if (IS_ERR(op)) 597 return (PTR_ERR(op)); 598 rdsv3_stats_inc(s_send_rdma); 599 rm->m_rdma_op = op; 600 return (0); 601 } 602 603 /* 604 * The application wants us to pass an RDMA destination (aka MR) 605 * to the remote 606 */ 607 int 608 rdsv3_cmsg_rdma_dest(struct rdsv3_sock *rs, struct rdsv3_message *rm, 609 struct cmsghdr *cmsg) 610 { 611 struct rdsv3_mr *mr; 612 uint32_t r_key; 613 int err = 0; 614 615 if (cmsg->cmsg_len < CMSG_LEN(sizeof (rdsv3_rdma_cookie_t)) || 616 rm->m_rdma_cookie != 0) 617 return (-EINVAL); 618 619 (void) memcpy(&rm->m_rdma_cookie, CMSG_DATA(cmsg), 620 sizeof (rm->m_rdma_cookie)); 621 622 /* 623 * We are reusing a previously mapped MR here. Most likely, the 624 * application has written to the buffer, so we need to explicitly 625 * flush those writes to RAM. Otherwise the HCA may not see them 626 * when doing a DMA from that buffer. 627 */ 628 r_key = rdsv3_rdma_cookie_key(rm->m_rdma_cookie); 629 630 mutex_enter(&rs->rs_rdma_lock); 631 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); 632 if (!mr) 633 err = -EINVAL; /* invalid r_key */ 634 else 635 atomic_add_32(&mr->r_refcount, 1); 636 mutex_exit(&rs->rs_rdma_lock); 637 638 if (mr) { 639 mr->r_trans->sync_mr(mr->r_trans_private, DMA_TO_DEVICE); 640 rm->m_rdma_mr = mr; 641 } 642 return (err); 643 } 644 645 /* 646 * The application passes us an address range it wants to enable RDMA 647 * to/from. We map the area, and save the <R_Key,offset> pair 648 * in rm->m_rdma_cookie. This causes it to be sent along to the peer 649 * in an extension header. 650 */ 651 int 652 rdsv3_cmsg_rdma_map(struct rdsv3_sock *rs, struct rdsv3_message *rm, 653 struct cmsghdr *cmsg) 654 { 655 struct rdsv3_get_mr_args *mrp; 656 int status; 657 658 if (cmsg->cmsg_len < CMSG_LEN(sizeof (struct rdsv3_get_mr_args)) || 659 rm->m_rdma_cookie != 0) 660 return (-EINVAL); 661 662 /* uint64_t alignment on struct rdsv3_get_mr_args */ 663 mrp = (struct rdsv3_get_mr_args *)kmem_alloc(cmsg->cmsg_len, KM_SLEEP); 664 bcopy(CMSG_DATA(cmsg), mrp, cmsg->cmsg_len); 665 status = __rdsv3_rdma_map(rs, mrp, &rm->m_rdma_cookie, &rm->m_rdma_mr); 666 kmem_free(mrp, cmsg->cmsg_len); 667 return (status); 668 } 669