1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 27 /* 28 * NAME: sol_uverbs_comp.c 29 * 30 * OFED User Verbs Kernel completion queue/processing implementation 31 * 32 */ 33 #include <sys/file.h> 34 #include <sys/vfs.h> 35 #include <sys/vnode.h> 36 #include <sys/errno.h> 37 #include <sys/cred.h> 38 #include <sys/uio.h> 39 #include <sys/semaphore.h> 40 #include <sys/ddi.h> 41 42 #include <sys/ib/clients/of/ofa_solaris.h> 43 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 44 #include <sys/ib/ibtl/ibvti.h> 45 #include <sys/ib/clients/of/ofed_kernel.h> 46 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h> 47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_comp.h> 48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h> 49 50 extern char *sol_uverbs_dbg_str; 51 52 /* 53 * Function: 54 * uverbs_convert_wc 55 * Input: 56 * uctxt - Pointer to the callers user context. 57 * ibt_wc - Pointer to IBT work completion. 58 * Output: 59 * ofed_wc - Pointer to hold converted OFED work completion. 60 * Returns: 61 * None 62 * Description: 63 * Convert and IBT work completion to an OFED work completion. 64 */ 65 /* ARGSUSED */ 66 static void 67 uverbs_convert_wc(uverbs_uctxt_uobj_t *uctxt, ibt_wc_t *ibt_wc, 68 struct ib_uverbs_wc *ofed_wc) 69 { 70 ASSERT(uctxt != NULL); 71 ASSERT(ibt_wc != NULL); 72 ASSERT(ofed_wc != NULL); 73 74 ofed_wc->wr_id = ibt_wc->wc_id; 75 76 switch (ibt_wc->wc_status) { 77 78 case IBT_WC_SUCCESS: 79 ofed_wc->status = IB_WC_SUCCESS; 80 break; 81 case IBT_WC_LOCAL_LEN_ERR: 82 ofed_wc->status = IB_WC_LOC_LEN_ERR; 83 break; 84 case IBT_WC_LOCAL_CHAN_OP_ERR: 85 ofed_wc->status = IB_WC_LOC_QP_OP_ERR; 86 break; 87 case IBT_WC_LOCAL_PROTECT_ERR: 88 ofed_wc->status = IB_WC_LOC_PROT_ERR; 89 break; 90 case IBT_WC_WR_FLUSHED_ERR: 91 ofed_wc->status = IB_WC_WR_FLUSH_ERR; 92 break; 93 case IBT_WC_MEM_WIN_BIND_ERR: 94 ofed_wc->status = IB_WC_MW_BIND_ERR; 95 break; 96 case IBT_WC_BAD_RESPONSE_ERR: 97 ofed_wc->status = IB_WC_BAD_RESP_ERR; 98 break; 99 case IBT_WC_LOCAL_ACCESS_ERR: 100 ofed_wc->status = IB_WC_LOC_ACCESS_ERR; 101 break; 102 case IBT_WC_REMOTE_INVALID_REQ_ERR: 103 ofed_wc->status = IB_WC_REM_INV_REQ_ERR; 104 break; 105 case IBT_WC_REMOTE_ACCESS_ERR: 106 ofed_wc->status = IB_WC_REM_ACCESS_ERR; 107 break; 108 case IBT_WC_REMOTE_OP_ERR: 109 ofed_wc->status = IB_WC_REM_OP_ERR; 110 break; 111 case IBT_WC_TRANS_TIMEOUT_ERR: 112 case IBT_WC_RNR_NAK_TIMEOUT_ERR: 113 ofed_wc->status = IB_WC_RESP_TIMEOUT_ERR; 114 break; 115 default: 116 ofed_wc->status = IB_WC_FATAL_ERR; 117 break; 118 } 119 120 switch (ibt_wc->wc_type) { 121 122 case IBT_WRC_SEND: 123 ofed_wc->opcode = IB_WC_SEND; 124 break; 125 case IBT_WRC_RDMAR: 126 ofed_wc->opcode = IB_WC_RDMA_READ; 127 break; 128 case IBT_WRC_RDMAW: 129 ofed_wc->opcode = IB_WC_RDMA_WRITE; 130 break; 131 case IBT_WRC_CSWAP: 132 ofed_wc->opcode = IB_WC_COMP_SWAP; 133 break; 134 case IBT_WRC_FADD: 135 ofed_wc->opcode = IB_WC_FETCH_ADD; 136 break; 137 case IBT_WRC_BIND: 138 ofed_wc->opcode = IB_WC_BIND_MW; 139 break; 140 case IBT_WRC_RECV: 141 ofed_wc->opcode = IB_WC_RECV; 142 break; 143 case IBT_WRC_RECV_RDMAWI: 144 ofed_wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 145 break; 146 147 case IBT_WRC_FAST_REG_PMR: 148 case IBT_WRC_LOCAL_INVALIDATE: 149 default: 150 ofed_wc->opcode = -1; /* (?) What to do here */ 151 break; 152 } 153 154 ofed_wc->vendor_err = 0; 155 ofed_wc->byte_len = ibt_wc->wc_bytes_xfer; 156 ofed_wc->imm_data = ibt_wc->wc_immed_data; 157 ofed_wc->qp_num = ibt_wc->wc_local_qpn; 158 ofed_wc->src_qp = ibt_wc->wc_qpn; 159 ofed_wc->wc_flags = 0; 160 161 if (ibt_wc->wc_flags & IBT_WC_GRH_PRESENT) { 162 ofed_wc->wc_flags |= IB_WC_GRH; 163 } 164 165 if (ibt_wc->wc_flags & IBT_WC_IMMED_DATA_PRESENT) { 166 ofed_wc->wc_flags |= IB_WC_WITH_IMM; 167 } 168 169 ofed_wc->pkey_index = ibt_wc->wc_pkey_ix; 170 ofed_wc->slid = ibt_wc->wc_slid; 171 ofed_wc->sl = ibt_wc->wc_sl; 172 ofed_wc->dlid_path_bits = ibt_wc->wc_path_bits; 173 ofed_wc->port_num = 0; 174 ofed_wc->reserved = 0; 175 } 176 177 /* 178 * Function: 179 * sol_uverbs_create_cq 180 * Input: 181 * uctxt - Pointer to the callers user context. 182 * buf - Pointer to kernel buffer containing create CQ command. 183 * in_len - Length in bytes of input command buffer. 184 * out_len - Length in bytes of output response buffer. 185 * Output: 186 * The command output buffer is updated with command results. 187 * Returns: 188 * DDI_SUCCESS on success, else error code. 189 * Description: 190 * User verbs entry point to create a device CQ. 191 */ 192 /* ARGSUSED */ 193 int 194 sol_uverbs_create_cq(uverbs_uctxt_uobj_t *uctxt, char *buf, 195 int in_len, int out_len) 196 { 197 struct ib_uverbs_create_cq cmd; 198 struct ib_uverbs_create_cq_resp resp; 199 uverbs_ucq_uobj_t *ucq; 200 ibt_cq_attr_t cq_attr; 201 uint_t real_size; 202 int rc; 203 204 (void) memcpy(&cmd, buf, sizeof (cmd)); 205 (void) memset(&resp, 0, sizeof (resp)); 206 (void) memset(&cq_attr, 0, sizeof (cq_attr)); 207 208 cq_attr.cq_size = cmd.cqe; 209 cq_attr.cq_sched = 0; 210 cq_attr.cq_flags = IBT_CQ_USER_MAP; 211 212 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: " 213 "num_cqe=%d, sched=%x, flags=%x", 214 cq_attr.cq_size, cq_attr.cq_sched, cq_attr.cq_flags); 215 216 /* 217 * To be consistent with OFED semantics, we fail a CQ that is being 218 * created with 0 CQ entries. 219 */ 220 if (!cmd.cqe) { 221 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "create_cq: 0 cqe"); 222 rc = EINVAL; 223 goto err_out; 224 } 225 226 ucq = kmem_zalloc(sizeof (*ucq), KM_NOSLEEP); 227 if (!ucq) { 228 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 229 "create_cq: mem alloc failure"); 230 rc = ENOMEM; 231 goto err_out; 232 } 233 sol_ofs_uobj_init(&ucq->uobj, cmd.user_handle, 234 SOL_UVERBS_UCQ_UOBJ_TYPE); 235 rw_enter(&ucq->uobj.uo_lock, RW_WRITER); 236 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 237 "create_cq: ucq %p, comp_chan %d", ucq, cmd.comp_channel); 238 239 /* 240 * If a event completion channel was specified look it up and 241 * assign the channel to the user CQ object. Note that this 242 * places a reference on the file itself. 243 */ 244 if ((int)cmd.comp_channel > SOL_UVERBS_DRIVER_MAX_MINOR) { 245 uverbs_uctxt_uobj_t *compl_uctxt = NULL; 246 uverbs_ufile_uobj_t *ufile; 247 248 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: " 249 "cmd.comp_chan %d", cmd.comp_channel); 250 compl_uctxt = uverbs_uobj_get_uctxt_write( 251 cmd.comp_channel - SOL_UVERBS_DRIVER_MAX_MINOR); 252 if (!compl_uctxt) { 253 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 254 "create_cq: Invalid comp channel %d", 255 cmd.comp_channel); 256 rc = EINVAL; 257 goto chan_err; 258 } 259 if (compl_uctxt->uctxt_verbs_id != uctxt->uobj.uo_id + 260 SOL_UVERBS_DRIVER_MAX_MINOR) { 261 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 262 "create_cq: Invalid comp channel %d, " 263 "verbs id %d mismatch", 264 cmd.comp_channel, 265 compl_uctxt->uctxt_verbs_id); 266 rc = EINVAL; 267 sol_ofs_uobj_put(&compl_uctxt->uobj); 268 goto chan_err; 269 } 270 ufile = compl_uctxt->comp_evfile; 271 ucq->comp_chan = ufile; 272 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 273 ufile->ufile_cq_cnt++; 274 rw_exit(&ufile->uobj.uo_lock); 275 sol_ofs_uobj_put(&compl_uctxt->uobj); 276 277 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: " 278 "ucq %p, comp_chan %p", ucq, ucq->comp_chan); 279 } else { 280 ucq->comp_chan = NULL; 281 } 282 283 llist_head_init(&ucq->async_list, NULL); 284 llist_head_init(&ucq->comp_list, NULL); 285 286 rc = ibt_alloc_cq(uctxt->hca->hdl, &cq_attr, &ucq->cq, &real_size); 287 288 if (rc != IBT_SUCCESS) { 289 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 290 "create_cq: ibt_alloc_cq() (rc=%d)", rc); 291 rc = sol_uverbs_ibt_to_kernel_status(rc); 292 ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t); 293 goto alloc_err; 294 } 295 296 ibt_set_cq_private(ucq->cq, ucq); 297 298 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 299 "create_cq: ib_alloc_cq() (rc=%d), real_size=%d", 300 rc, real_size); 301 /* 302 * Query underlying hardware for data used in mapping CQ back to user 303 * space, we will return this information in the user verbs command 304 * response. 305 */ 306 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ, 307 (void *) ucq->cq, &resp.drv_out, sizeof (resp.drv_out)); 308 if (rc != IBT_SUCCESS) { 309 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 310 "create_cq: ibt_ci_data_out() rc=%d", rc); 311 rc = EFAULT; 312 ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t); 313 goto err_cq_destroy; 314 } 315 316 if (sol_ofs_uobj_add(&uverbs_ucq_uo_tbl, &ucq->uobj) != 0) { 317 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 318 "create_cq: User object add failed"); 319 rc = ENOMEM; 320 goto err_cq_destroy; 321 } 322 323 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 324 "create_cq: ibt_ci_data_out: 0x%16llx 0x%16llx " 325 "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1], 326 resp.drv_out[2], resp.drv_out[3]); 327 328 resp.cqe = real_size; 329 resp.cq_handle = ucq->uobj.uo_id; 330 331 #ifdef _LP64 332 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp)); 333 #else 334 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp)); 335 #endif 336 if (rc != 0) { 337 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 338 "create_cq: copyout failed %x", rc); 339 rc = EFAULT; 340 goto err_uo_delete; 341 } 342 343 ucq->uctxt = uctxt; 344 345 mutex_enter(&uctxt->lock); 346 ucq->list_entry = add_genlist(&uctxt->cq_list, (uintptr_t)ucq, uctxt); 347 mutex_exit(&uctxt->lock); 348 349 if (!ucq->list_entry) { 350 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 351 "create_cq: Error adding ucq to cq_list"); 352 rc = ENOMEM; 353 goto err_uo_delete; 354 } 355 356 if (ucq->comp_chan) { 357 ibt_set_cq_handler(ucq->cq, sol_uverbs_comp_event_handler, ucq); 358 } 359 360 ucq->uobj.uo_live = 1; 361 rw_exit(&ucq->uobj.uo_lock); 362 363 return (DDI_SUCCESS); 364 365 err_uo_delete: 366 /* 367 * Need to set uo_live, so sol_ofs_uobj_remove() will 368 * remove the object from the object table. 369 */ 370 ucq->uobj.uo_live = 1; 371 (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj); 372 373 err_cq_destroy: 374 (void) ibt_free_cq(ucq->cq); 375 376 alloc_err: 377 if (ucq->comp_chan) { 378 uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq); 379 } 380 381 chan_err: 382 rw_exit(&ucq->uobj.uo_lock); 383 sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free); 384 385 err_out: 386 return (rc); 387 } 388 389 int 390 uverbs_ucq_free(uverbs_ucq_uobj_t *ucq, uverbs_uctxt_uobj_t *uctxt) 391 { 392 int rc; 393 394 rc = ibt_free_cq(ucq->cq); 395 if (rc != IBT_SUCCESS) { 396 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 397 "destroy_id: ibt_free_cq() rc=%d", 398 rc); 399 rc = sol_uverbs_ibt_to_kernel_status(rc); 400 sol_ofs_uobj_put(&ucq->uobj); 401 return (rc); 402 } 403 (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj); 404 sol_ofs_uobj_put(&ucq->uobj); 405 406 if (ucq->list_entry) { 407 mutex_enter(&uctxt->lock); 408 delete_genlist(&uctxt->cq_list, ucq->list_entry); 409 mutex_exit(&uctxt->lock); 410 } 411 sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free); 412 return (0); 413 } 414 415 /* 416 * Function: 417 * sol_uverbs_destroy_cq 418 * Input: 419 * uctxt - Pointer to the callers user context. 420 * buf - Pointer to kernel buffer containing a destroy CQ command. 421 * in_len - Length in bytes of input command buffer. 422 * out_len - Length in bytes of output response buffer. 423 * Output: 424 * The command output buffer is updated with command results. 425 * Returns: 426 * DDI_SUCCESS on success, else error code. 427 * Description: 428 * User verbs entry point to destroy a device CQ. 429 */ 430 /* ARGSUSED */ 431 int 432 sol_uverbs_destroy_cq(uverbs_uctxt_uobj_t *uctxt, char *buf, 433 int in_len, int out_len) 434 { 435 struct ib_uverbs_destroy_cq cmd; 436 struct ib_uverbs_destroy_cq_resp resp; 437 uverbs_ucq_uobj_t *ucq; 438 int rc; 439 440 (void) memcpy(&cmd, buf, sizeof (cmd)); 441 (void) memset(&resp, 0, sizeof (resp)); 442 443 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 444 "destroy_cq(cq_handle=%d)", cmd.cq_handle); 445 446 ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle); 447 if (ucq == NULL) { 448 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 449 "destroy_cq: Invalid handle: %d", 450 cmd.cq_handle); 451 rc = EINVAL; 452 goto err_out; 453 } 454 455 uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq); 456 cmd.cq_handle = 0; 457 resp.comp_events_reported = ucq->comp_events_reported; 458 resp.async_events_reported = ucq->async_events_reported; 459 460 if (ucq->active_qp_cnt) { 461 sol_ofs_uobj_put(&ucq->uobj); 462 return (EBUSY); 463 } else { 464 rc = uverbs_ucq_free(ucq, uctxt); 465 if (rc) 466 goto err_out; 467 } 468 469 #ifdef _LP64 470 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp)); 471 #else 472 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp)); 473 #endif 474 if (rc != 0) { 475 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 476 "destroy_cq: copuout failed %x", rc); 477 rc = EFAULT; 478 goto err_out; 479 } 480 481 return (DDI_SUCCESS); 482 483 err_out: 484 return (rc); 485 } 486 487 /* 488 * Function: 489 * sol_uverbs_resize_cq 490 * Input: 491 * uctxt - Pointer to the callers user context. 492 * buf - Pointer to kernel buffer containing a resize CQ command. 493 * in_len - Length in bytes of input command buffer. 494 * out_len - Length in bytes of output response buffer. 495 * Output: 496 * The command output buffer is updated with command results. 497 * Returns: 498 * DDI_SUCCESS on success, else error code. 499 * Description: 500 * User verbs entry point to resize a device CQ. 501 */ 502 /* ARGSUSED */ 503 int 504 sol_uverbs_resize_cq(uverbs_uctxt_uobj_t *uctxt, char *buf, 505 int in_len, int out_len) 506 { 507 struct ib_uverbs_resize_cq cmd; 508 struct ib_uverbs_resize_cq_resp resp; 509 uverbs_ucq_uobj_t *ucq; 510 int rc; 511 int resize_status; 512 513 (void) memcpy(&cmd, buf, sizeof (cmd)); 514 (void) memset(&resp, 0, sizeof (resp)); 515 516 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 517 "resize_cq(cq_handle=%d)", cmd.cq_handle); 518 519 ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle); 520 if (ucq == NULL) { 521 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 522 "resize_cq: Invalid handle: %d", 523 cmd.cq_handle); 524 rc = EINVAL; 525 goto err_out; 526 } 527 528 /* 529 * If CQ resize fails, note the error but keep going. In this case we 530 * expect the old CQ size to be returned, and when we extract the 531 * mapping data, we expect the offset and length are approrpriate for 532 * the old CQ. 533 */ 534 resize_status = ibt_resize_cq(ucq->cq, cmd.cqe, &resp.cqe); 535 if (resize_status != IBT_SUCCESS) { 536 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 537 "resize_cq: ibt_resize_cq() (resize_status=%d), using " 538 "original CQ", resize_status); 539 rc = sol_uverbs_ibt_to_kernel_status(resize_status); 540 goto err_out; 541 } 542 543 sol_ofs_uobj_put(&ucq->uobj); 544 545 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ, 546 (void *) ucq->cq, &resp.drv_out, sizeof (resp.drv_out)); 547 if (rc != IBT_SUCCESS) { 548 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 549 "resize_cq: Error in ibt_ci_data_out() " 550 "(rc=%d)", rc); 551 rc = EFAULT; 552 goto err_out; 553 } 554 555 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 556 "resize_cq: ibt_ci_data_out: 0x%16llx 0x%16llx " 557 "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1], 558 resp.drv_out[2], resp.drv_out[3]); 559 560 #ifdef _LP64 561 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp)); 562 #else 563 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp)); 564 #endif 565 if (rc != 0) { 566 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 567 "resize_cq: copyout %d", rc); 568 rc = EFAULT; 569 goto err_out; 570 } 571 572 return (resize_status); 573 574 err_out: 575 return (rc); 576 } 577 578 /* 579 * Function: 580 * sol_uverbs_req_notify_cq 581 * Input: 582 * uctxt - Pointer to the callers user context. 583 * buf - Pointer to kernel buffer containing request notify 584 * command. 585 * in_len - Length in bytes of input command buffer. 586 * out_len - Length in bytes of output response buffer. 587 * Output: 588 * The command output buffer is updated with command results. 589 * Returns: 590 * DDI_SUCCESS on success, else error code. 591 * Description: 592 * User verbs entry point to request notification of CQ events. 593 */ 594 /* ARGSUSED */ 595 int 596 sol_uverbs_req_notify_cq(uverbs_uctxt_uobj_t *uctxt, char *buf, 597 int in_len, int out_len) 598 { 599 struct ib_uverbs_req_notify_cq cmd; 600 ibt_cq_notify_flags_t flag; 601 uverbs_ucq_uobj_t *ucq; 602 int rc; 603 604 (void) memcpy(&cmd, buf, sizeof (cmd)); 605 606 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 607 "req_notify_cq(cq_handle=%d)", cmd.cq_handle); 608 609 ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle); 610 if (ucq == NULL) { 611 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 612 "req_notify_cq: List lookup failure"); 613 rc = EINVAL; 614 goto err_out; 615 } 616 617 flag = IBT_NEXT_COMPLETION; 618 619 if (cmd.solicited_only != 0) { 620 flag = IBT_NEXT_SOLICITED; 621 } 622 623 rc = ibt_enable_cq_notify(ucq->cq, flag); 624 if (rc != IBT_SUCCESS) { 625 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 626 "req_notify_cq: ibt_enable_cq_notify() " 627 "(rc=%d)", rc); 628 rc = sol_uverbs_ibt_to_kernel_status(rc); 629 goto err_put; 630 } 631 632 sol_ofs_uobj_put(&ucq->uobj); 633 return (DDI_SUCCESS); 634 635 err_put: 636 sol_ofs_uobj_put(&ucq->uobj); 637 638 err_out: 639 return (rc); 640 } 641 642 /* 643 * Function: 644 * sol_uverbs_poll_cq 645 * Input: 646 * uctxt - Pointer to the callers user context. 647 * buf - Pointer to kernel buffer containing poll CQ command. 648 * in_len - Length in bytes of input command buffer. 649 * out_len - Length in bytes of output response buffer. 650 * Output: 651 * The command output buffer is updated with command results. 652 * Returns: 653 * DDI_SUCCESS on success, else error code. 654 * Description: 655 * User verbs entry point to poll a CQ for completion events. Note that 656 * this is a non OS Bypass version, the CQ normally would be polled 657 * directly from the user space driver. 658 */ 659 /* ARGSUSED */ 660 int 661 sol_uverbs_poll_cq(uverbs_uctxt_uobj_t *uctxt, char *buf, 662 int in_len, int out_len) 663 { 664 struct ib_uverbs_poll_cq cmd; 665 struct ib_uverbs_poll_cq_resp resp; 666 uverbs_ucq_uobj_t *ucq; 667 ibt_wc_t *completions; 668 struct ib_uverbs_wc ofed_wc; 669 int rc; 670 int i; 671 672 (void) memcpy(&cmd, buf, sizeof (cmd)); 673 (void) memset(&resp, 0, sizeof (resp)); 674 675 #ifdef DEBUG 676 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 677 "poll_cq(cq_handle=%d)", cmd.cq_handle); 678 #endif 679 680 ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle); 681 if (ucq == NULL) { 682 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 683 "poll_cq: List lookup failure"); 684 rc = EINVAL; 685 goto err_find; 686 } 687 688 completions = (ibt_wc_t *)kmem_zalloc(sizeof (ibt_wc_t) * cmd.ne, 689 KM_NOSLEEP); 690 if (completions == NULL) { 691 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 692 "poll_cq: Allocation Error"); 693 rc = ENOMEM; 694 goto err_alloc; 695 } 696 697 rc = ibt_poll_cq(ucq->cq, completions, cmd.ne, &resp.count); 698 if (rc != IBT_SUCCESS) { 699 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 700 "poll_cq: ibt_poll_cq() (rc=%d)", rc); 701 rc = sol_uverbs_ibt_to_kernel_status(rc); 702 goto err_poll; 703 } 704 705 #ifdef _LP64 706 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp)); 707 #else 708 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp)); 709 #endif 710 if (rc != 0) { 711 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 712 "poll_cq: copyout %x", rc); 713 rc = EFAULT; 714 goto err_poll; 715 } 716 717 for (i = 0; i < resp.count; i++) { 718 (void) memset(&ofed_wc, 0, sizeof (ofed_wc)); 719 uverbs_convert_wc(uctxt, &completions[i], &ofed_wc); 720 721 #ifdef _LP64 722 rc = copyout((void*)&ofed_wc, 723 (void *)cmd.response.r_laddr, sizeof (ofed_wc)); 724 #else 725 rc = copyout((void*)&ofed_wc, 726 (void *)cmd.response.r_addr, sizeof (ofed_wc)); 727 #endif 728 if (rc != 0) { 729 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 730 "poll_cq: copyout wc data %x", rc); 731 rc = EFAULT; 732 goto err_poll; 733 } 734 } 735 736 kmem_free((void*)completions, sizeof (ibt_wc_t) * cmd.ne); 737 sol_ofs_uobj_put(&ucq->uobj); 738 739 return (DDI_SUCCESS); 740 741 err_poll: 742 kmem_free((void *)completions, sizeof (ibt_wc_t) * cmd.ne); 743 744 err_alloc: 745 sol_ofs_uobj_put(&ucq->uobj); 746 747 err_find: 748 return (rc); 749 } 750 751 /* 752 * Function: 753 * sol_uverbs_comp_event_handler 754 * Input: 755 * ibt_cq - Handle to the IBT CQ. 756 * arg - Address of the associated Solaris User Verbs CQ 757 * object. 758 * Output: 759 * None 760 * Returns: 761 * None 762 * Description: 763 * Solaris User Verbs completion channel IBT CQ callback. Queue 764 * the completion event and wakeup/notify consumers that may be 765 * blocked waiting for the completion. 766 */ 767 /* ARGSUSED */ 768 void 769 sol_uverbs_comp_event_handler(ibt_cq_hdl_t ibt_cq, void *arg) 770 { 771 uverbs_ucq_uobj_t *ucq = arg; 772 uverbs_ufile_uobj_t *ufile; 773 uverbs_event_t *entry; 774 775 if (!ucq || !ucq->comp_chan) { 776 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr " 777 "ucq %p ucq->comp_chan %p", ucq, (ucq) ? ucq->comp_chan : 778 NULL); 779 return; 780 } 781 782 #ifdef DEBUG 783 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr(%p, %p) - ", 784 "ucq = %p, ucq->cq = %p, ucq->uctxt = %p, ucq->comp_chan =%p", 785 ibt_cq, arg, ucq, ucq->cq, ucq->uctxt, ucq->comp_chan); 786 #endif 787 788 ufile = ucq->comp_chan; 789 790 mutex_enter(&ufile->lock); 791 if (!ufile->uctxt) { 792 mutex_exit(&ufile->lock); 793 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr " 794 "ufile->uctxt %p", ufile->uctxt); 795 return; 796 } 797 798 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP); 799 if (!entry) { 800 mutex_exit(&ufile->lock); 801 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr() " 802 "memory allocation error"); 803 return; 804 } 805 806 entry->ev_desc.comp.cq_handle = ucq->uobj.uo_user_handle; 807 entry->ev_counter = &ucq->comp_events_reported; 808 809 /* 810 * Add to list of entries associated with completion channel 811 * and the list associated with the specific CQ. 812 */ 813 llist_head_init(&entry->ev_list, entry); 814 llist_head_init(&entry->ev_obj_list, entry); 815 816 #ifdef DEBUG 817 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr() " 818 "Add COMP entry->ev_list=%p, &entry->ev_obj_list, entry=%p", 819 &entry->ev_list, &entry->ev_obj_list, entry); 820 #endif 821 822 llist_add_tail(&entry->ev_list, &ufile->event_list); 823 llist_add_tail(&entry->ev_obj_list, &ucq->comp_list); 824 825 /* Do not notify users if sol_ucma has disabled CQ notify */ 826 if (ufile->ufile_notify_enabled == 827 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE) { 828 mutex_exit(&ufile->lock); 829 return; 830 } 831 832 cv_signal(&ufile->poll_wait); 833 mutex_exit(&ufile->lock); 834 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM); 835 } 836