1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * This file implements the Work Queue Entry (WQE) management in IBMF. 31 */ 32 33 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 34 35 extern int ibmf_trace_level; 36 extern int ibmf_send_wqes_per_port, ibmf_recv_wqes_per_port; 37 38 #define IBMF_INIT_SG_ELEMENT(sg, mem, lkey, size) { \ 39 (sg).ds_va = (ib_vaddr_t)(uintptr_t)(mem); \ 40 (sg).ds_key = (lkey); \ 41 (sg).ds_len = (size); \ 42 } 43 44 #define IBMF_ADDR_TO_SEND_WR_ID(ptr, id) \ 45 (id) = (ibt_wrid_t)(uintptr_t)(ptr) 46 47 #define IBMF_ADDR_TO_RECV_WR_ID(ptr, id) \ 48 (id) = ((ibt_wrid_t)(uintptr_t)(ptr) | IBMF_RCV_CQE) 49 50 #define IBMF_INIT_RMPP_HDR(hdrp, ver, type, respt, flg, status, seg, lennwl) { \ 51 (hdrp)->rmpp_version = (ver); \ 52 (hdrp)->rmpp_type = (type); \ 53 (hdrp)->rmpp_resp_time = (respt); \ 54 (hdrp)->rmpp_flags = (flg); \ 55 (hdrp)->rmpp_status = (status); \ 56 (hdrp)->rmpp_segnum = (h2b32(seg)); \ 57 (hdrp)->rmpp_pyldlen_nwl = (h2b32(lennwl)); \ 58 } 59 60 static int ibmf_send_wqe_cache_constructor(void *buf, void *cdrarg, 61 int kmflags); 62 static void ibmf_send_wqe_cache_destructor(void *buf, void *cdrarg); 63 static int ibmf_recv_wqe_cache_constructor(void *buf, void *cdrarg, 64 int kmflags); 65 static void ibmf_recv_wqe_cache_destructor(void *buf, void *cdrarg); 66 static int ibmf_i_extend_wqe_mem(ibmf_ci_t *cip, 67 ibmf_qp_handle_t ibmf_qp_handle, ibmf_wqe_mgt_t *wqe_mgt, 68 boolean_t block); 69 70 /* 71 * ibmf_send_wqe_cache_constructor(): 72 * Constructor for the kmem cache used for send WQEs for special QPs 73 */ 74 /* ARGSUSED */ 75 static int 76 ibmf_send_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags) 77 { 78 ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf; 79 ibmf_ci_t *cip = (ibmf_ci_t *)cdrarg; 80 ibmf_wqe_mgt_t *wqe_mgt; 81 82 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 83 ibmf_send_wqe_cache_constructor_start, IBMF_TNF_TRACE, "", 84 "ibmf_send_wqe_cache_constructor() enter, buf = %p, cdarg = %p\n", 85 tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 86 87 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe)) 88 89 /* initialize send WQE context */ 90 send_wqe->send_sg_mem = 91 (ib_vaddr_t)(uintptr_t)vmem_alloc(cip->ci_wqe_ib_vmem, 92 IBMF_MEM_PER_WQE, kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 93 if (send_wqe->send_sg_mem == NULL) { 94 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 95 ibmf_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, "", 96 "ibmf_send_wqe_cache_constructor(): %s\n", tnf_string, msg, 97 "Failed vmem allocation in send WQE cache constructor"); 98 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 99 ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 100 "ibmf_send_wqe_cache_constructor() exit\n"); 101 return (-1); 102 } 103 104 mutex_enter(&cip->ci_wqe_mutex); 105 wqe_mgt = cip->ci_wqe_mgt_list; 106 107 /* Look for the WQE management struct that includes this address */ 108 while (wqe_mgt != NULL) { 109 mutex_enter(&wqe_mgt->wqes_mutex); 110 if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) && 111 (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem + 112 wqe_mgt->wqes_kmem_sz))) { 113 mutex_exit(&wqe_mgt->wqes_mutex); 114 break; 115 } 116 mutex_exit(&wqe_mgt->wqes_mutex); 117 wqe_mgt = wqe_mgt->wqe_mgt_next; 118 } 119 120 if (wqe_mgt == NULL) { 121 mutex_exit(&cip->ci_wqe_mutex); 122 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 123 ibmf_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, "", 124 "ibmf_send_wqe_cache_constructor(): %s\n", tnf_string, msg, 125 "Address not found in WQE mgt list"); 126 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 127 ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 128 "ibmf_send_wqe_cache_constructor() exit\n"); 129 return (-1); 130 } 131 132 mutex_enter(&wqe_mgt->wqes_mutex); 133 134 send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem + 135 (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem)); 136 bzero(send_wqe->send_mem, IBMF_MEM_PER_WQE); 137 send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey; 138 send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 139 send_wqe->send_wqe_flags = 0; 140 send_wqe->send_wqe_next = NULL; 141 142 mutex_exit(&wqe_mgt->wqes_mutex); 143 mutex_exit(&cip->ci_wqe_mutex); 144 145 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 146 ibmf_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 147 "ibmf_send_wqe_cache_constructor() exit\n"); 148 149 return (0); 150 } 151 152 /* 153 * ibmf_send_wqe_cache_destructor(): 154 * Destructor for send WQE kmem cache for special QPs 155 */ 156 /* ARGSUSED */ 157 static void 158 ibmf_send_wqe_cache_destructor(void *buf, void *cdrarg) 159 { 160 ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf; 161 ibmf_ci_t *cip = (ibmf_ci_t *)cdrarg; 162 163 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 164 ibmf_i_send_wqe_cache_destructor_start, IBMF_TNF_TRACE, "", 165 "ibmf_send_wqe_cache_destructor() enter, buf = %p, cdarg = %p\n", 166 tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 167 168 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe)) 169 170 /* Free the vmem allocated for the WQE */ 171 vmem_free(cip->ci_wqe_ib_vmem, 172 (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE); 173 send_wqe->send_mem = NULL; 174 175 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 176 ibmf_i_send_wqe_cache_destructor_end, IBMF_TNF_TRACE, "", 177 "ibmf_send_wqe_cache_destructor() exit\n"); 178 } 179 180 /* 181 * ibmf_recv_wqe_cache_constructor(): 182 * Constructor for receive WQE kmem cache for special QPs 183 */ 184 /* ARGSUSED */ 185 static int 186 ibmf_recv_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags) 187 { 188 ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf; 189 ibmf_ci_t *cip = (ibmf_ci_t *)cdrarg; 190 ibmf_wqe_mgt_t *wqe_mgt; 191 192 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 193 ibmf_i_recv_wqe_cache_constructor_start, IBMF_TNF_TRACE, "", 194 "ibmf_recv_wqe_cache_constructor() enter, buf = %p, cdarg = %p\n", 195 tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 196 197 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe)) 198 199 /* initialize recv WQE context */ 200 recv_wqe->recv_sg_mem = 201 (ib_vaddr_t)(uintptr_t)vmem_alloc(cip->ci_wqe_ib_vmem, 202 IBMF_MEM_PER_WQE, kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 203 if (recv_wqe->recv_sg_mem == NULL) { 204 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 205 ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "", 206 "ibmf_recv_wqe_cache_constructor(): %s\n", tnf_string, msg, 207 "Failed vmem allocation in receive WQE cache constructor"); 208 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 209 ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 210 "ibmf_recv_wqe_cache_constructor() exit\n"); 211 return (-1); 212 } 213 214 mutex_enter(&cip->ci_wqe_mutex); 215 wqe_mgt = cip->ci_wqe_mgt_list; 216 217 /* Look for the WQE management struct that includes this address */ 218 while (wqe_mgt != NULL) { 219 mutex_enter(&wqe_mgt->wqes_mutex); 220 if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) && 221 (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem + 222 wqe_mgt->wqes_kmem_sz))) { 223 mutex_exit(&wqe_mgt->wqes_mutex); 224 break; 225 } 226 mutex_exit(&wqe_mgt->wqes_mutex); 227 wqe_mgt = wqe_mgt->wqe_mgt_next; 228 } 229 230 if (wqe_mgt == NULL) { 231 mutex_exit(&cip->ci_wqe_mutex); 232 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 233 ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "", 234 "ibmf_recv_wqe_cache_constructor(): %s\n", tnf_string, msg, 235 "Address not found in WQE mgt list"); 236 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 237 ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 238 "ibmf_recv_wqe_cache_constructor() exit\n"); 239 return (-1); 240 } 241 242 mutex_enter(&wqe_mgt->wqes_mutex); 243 244 recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem + 245 (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem)); 246 bzero(recv_wqe->recv_mem, IBMF_MEM_PER_WQE); 247 recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey; 248 recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 249 recv_wqe->recv_wqe_next = NULL; 250 recv_wqe->recv_msg = NULL; 251 recv_wqe->recv_wqe_flags = 0; 252 253 mutex_exit(&wqe_mgt->wqes_mutex); 254 mutex_exit(&cip->ci_wqe_mutex); 255 256 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 257 ibmf_i_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 258 "ibmf_recv_wqe_cache_constructor() exit\n"); 259 260 return (0); 261 } 262 263 /* 264 * ibmf_recv_wqe_cache_destructor(): 265 * Destructor for receive WQE kmem cache for special QPs 266 */ 267 /* ARGSUSED */ 268 static void 269 ibmf_recv_wqe_cache_destructor(void *buf, void *cdrarg) 270 { 271 ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf; 272 ibmf_ci_t *cip = (ibmf_ci_t *)cdrarg; 273 274 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 275 ibmf_i_recv_wqe_cache_destructor_start, IBMF_TNF_TRACE, "", 276 "ibmf_recv_wqe_cache_destructor() enter, buf = %p, cdarg = %p\n", 277 tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 278 279 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe)) 280 281 /* Free the vmem allocated for the WQE */ 282 vmem_free(cip->ci_wqe_ib_vmem, 283 (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE); 284 recv_wqe->recv_mem = NULL; 285 286 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 287 ibmf_i_recv_wqe_cache_destructor_end, IBMF_TNF_TRACE, "", 288 "ibmf_recv_wqe_cache_destructor() exit\n"); 289 } 290 291 /* 292 * ibmf_altqp_send_wqe_cache_constructor(): 293 * Constructor for the kmem cache used for send WQEs for alternate QPs 294 */ 295 /* ARGSUSED */ 296 int 297 ibmf_altqp_send_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags) 298 { 299 ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf; 300 ibmf_alt_qp_t *qp_ctx = (ibmf_alt_qp_t *)cdrarg; 301 ibmf_wqe_mgt_t *wqe_mgt; 302 303 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 304 ibmf_altqp_send_wqe_cache_constructor_start, IBMF_TNF_TRACE, "", 305 "ibmf_altqp_send_wqe_cache_constructor() enter, buf = %p, " 306 "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 307 308 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe)) 309 310 /* initialize send WQE context */ 311 send_wqe->send_sg_mem = (ib_vaddr_t)(uintptr_t)vmem_alloc( 312 qp_ctx->isq_wqe_ib_vmem, IBMF_MEM_PER_WQE, 313 kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 314 if (send_wqe->send_sg_mem == NULL) { 315 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 316 ibmf_altqp_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, 317 "", "ibmf_altqp_send_wqe_cache_constructor(): %s\n", 318 tnf_string, msg, "Failed vmem allocation in " 319 "alternate QP send WQE cache constructor"); 320 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 321 ibmf_altqp_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, 322 "", "ibmf_altqp_send_wqe_cache_constructor() exit\n"); 323 return (-1); 324 } 325 326 mutex_enter(&qp_ctx->isq_wqe_mutex); 327 wqe_mgt = qp_ctx->isq_wqe_mgt_list; 328 329 /* Look for the WQE management struct that includes this address */ 330 while (wqe_mgt != NULL) { 331 mutex_enter(&wqe_mgt->wqes_mutex); 332 if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) && 333 (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem + 334 wqe_mgt->wqes_kmem_sz))) { 335 mutex_exit(&wqe_mgt->wqes_mutex); 336 break; 337 } 338 mutex_exit(&wqe_mgt->wqes_mutex); 339 wqe_mgt = wqe_mgt->wqe_mgt_next; 340 } 341 342 if (wqe_mgt == NULL) { 343 mutex_exit(&qp_ctx->isq_wqe_mutex); 344 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 345 ibmf_altqp_send_wqe_cache_constructor_err, IBMF_TNF_ERROR, 346 "", "ibmf_altqp_send_wqe_cache_constructor(): %s\n", 347 tnf_string, msg, "Address not found in WQE mgt list"); 348 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 349 ibmf_altqp_send_wqe_cache_constructor_end, 350 IBMF_TNF_TRACE, "", 351 "ibmf_altqp_send_wqe_cache_constructor() exit\n"); 352 return (-1); 353 } 354 355 mutex_enter(&wqe_mgt->wqes_mutex); 356 357 send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem + 358 (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem)); 359 bzero(send_wqe->send_mem, IBMF_MEM_PER_WQE); 360 send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey; 361 send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 362 send_wqe->send_wqe_flags = 0; 363 364 mutex_exit(&wqe_mgt->wqes_mutex); 365 mutex_exit(&qp_ctx->isq_wqe_mutex); 366 367 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 368 ibmf_i_altqp_send_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 369 "ibmf_altqp_send_wqe_cache_constructor() exit\n"); 370 371 return (0); 372 } 373 374 /* 375 * ibmf_altqp_send_wqe_cache_destructor(): 376 * Destructor for send WQE kmem cache for alternate QPs 377 */ 378 /* ARGSUSED */ 379 void 380 ibmf_altqp_send_wqe_cache_destructor(void *buf, void *cdrarg) 381 { 382 ibmf_send_wqe_t *send_wqe = (ibmf_send_wqe_t *)buf; 383 ibmf_alt_qp_t *qp_ctx = (ibmf_alt_qp_t *)cdrarg; 384 385 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 386 ibmf_i_altqp_send_wqe_cache_destructor_start, IBMF_TNF_TRACE, "", 387 "ibmf_altqp_send_wqe_cache_destructor() enter, buf = %p, " 388 "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 389 390 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqe)) 391 392 /* Free the vmem allocated for the WQE */ 393 vmem_free(qp_ctx->isq_wqe_ib_vmem, 394 (void *)(uintptr_t)send_wqe->send_sg_mem, IBMF_MEM_PER_WQE); 395 send_wqe->send_mem = NULL; 396 397 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 398 ibmf_i_altqp_send_wqe_cache_destructor_end, IBMF_TNF_TRACE, "", 399 "ibmf_altqp_send_wqe_cache_destructor() exit\n"); 400 } 401 402 /* 403 * ibmf_altqp_recv_wqe_cache_constructor(): 404 * Constructor for receive WQE kmem cache for alternate QPs 405 */ 406 /* ARGSUSED */ 407 int 408 ibmf_altqp_recv_wqe_cache_constructor(void *buf, void *cdrarg, int kmflags) 409 { 410 ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf; 411 ibmf_alt_qp_t *qp_ctx = (ibmf_alt_qp_t *)cdrarg; 412 ibmf_wqe_mgt_t *wqe_mgt; 413 414 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 415 ibmf_i_altqp_recv_wqe_cache_constructor_start, IBMF_TNF_TRACE, "", 416 "ibmf_altqp_recv_wqe_cache_constructor() enter, buf = %p, " 417 "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 418 419 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe)) 420 421 /* initialize recv WQE context */ 422 recv_wqe->recv_sg_mem = (ib_vaddr_t)(uintptr_t)vmem_alloc( 423 qp_ctx->isq_wqe_ib_vmem, IBMF_MEM_PER_WQE, 424 kmflags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 425 if (recv_wqe->recv_sg_mem == NULL) { 426 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 427 ibmf_altqp_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, 428 "", "ibmf_altqp_recv_wqe_cache_constructor(): %s\n", 429 tnf_string, msg, 430 "Failed vmem allocation in recv WQE cache constructor"); 431 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 432 ibmf_altqp_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, 433 "", "ibmf_altqp_recv_wqe_cache_constructor() exit\n"); 434 return (-1); 435 } 436 437 mutex_enter(&qp_ctx->isq_wqe_mutex); 438 wqe_mgt = qp_ctx->isq_wqe_mgt_list; 439 440 /* Look for the WQE management struct that includes this address */ 441 while (wqe_mgt != NULL) { 442 mutex_enter(&wqe_mgt->wqes_mutex); 443 if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) && 444 (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem + 445 wqe_mgt->wqes_kmem_sz))) { 446 mutex_exit(&wqe_mgt->wqes_mutex); 447 break; 448 } 449 mutex_exit(&wqe_mgt->wqes_mutex); 450 wqe_mgt = wqe_mgt->wqe_mgt_next; 451 } 452 453 if (wqe_mgt == NULL) { 454 mutex_exit(&qp_ctx->isq_wqe_mutex); 455 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 456 ibmf_recv_wqe_cache_constructor_err, IBMF_TNF_ERROR, "", 457 "ibmf_altqp_recv_wqe_cache_constructor(): %s\n", 458 tnf_string, msg, "Address not found in WQE mgt list"); 459 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 460 ibmf_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 461 "ibmf_recv_wqe_cache_constructor() exit\n"); 462 return (-1); 463 } 464 465 mutex_enter(&wqe_mgt->wqes_mutex); 466 467 recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem + 468 (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem)); 469 bzero(recv_wqe->recv_mem, IBMF_MEM_PER_WQE); 470 recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey; 471 recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 472 recv_wqe->recv_wqe_flags = 0; 473 474 mutex_exit(&wqe_mgt->wqes_mutex); 475 mutex_exit(&qp_ctx->isq_wqe_mutex); 476 477 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 478 ibmf_i_altqp_recv_wqe_cache_constructor_end, IBMF_TNF_TRACE, "", 479 "ibmf_altqp_recv_wqe_cache_constructor() exit\n"); 480 481 return (0); 482 } 483 484 /* 485 * ibmf_altqp_recv_wqe_cache_destructor(): 486 * Destructor for receive WQE kmem cache for alternate QPs 487 */ 488 /* ARGSUSED */ 489 void 490 ibmf_altqp_recv_wqe_cache_destructor(void *buf, void *cdrarg) 491 { 492 ibmf_recv_wqe_t *recv_wqe = (ibmf_recv_wqe_t *)buf; 493 ibmf_alt_qp_t *qp_ctx = (ibmf_alt_qp_t *)cdrarg; 494 495 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 496 ibmf_i_altqp_recv_wqe_cache_destructor_start, IBMF_TNF_TRACE, "", 497 "ibmf_altqp_recv_wqe_cache_destructor() enter, buf = %p, " 498 "cdarg = %p\n", tnf_opaque, buf, buf, tnf_opaque, cdrarg, cdrarg); 499 500 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*recv_wqe)) 501 502 /* Free the vmem allocated for the WQE */ 503 vmem_free(qp_ctx->isq_wqe_ib_vmem, 504 (void *)(uintptr_t)recv_wqe->recv_sg_mem, IBMF_MEM_PER_WQE); 505 recv_wqe->recv_mem = NULL; 506 507 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 508 ibmf_i_altqp_recv_wqe_cache_destructor_end, IBMF_TNF_TRACE, "", 509 "ibmf_altqp_recv_wqe_cache_destructor() exit\n"); 510 } 511 512 /* 513 * ibmf_i_init_wqes(): 514 * Create the kmem cache for send and receive WQEs 515 */ 516 int 517 ibmf_i_init_wqes(ibmf_ci_t *cip) 518 { 519 ibt_status_t status; 520 ibt_mr_hdl_t mem_hdl; 521 ibt_mr_desc_t mem_desc; 522 ibt_mr_attr_t mem_attr; 523 ibmf_wqe_mgt_t *wqe_mgtp; 524 char string[128]; 525 526 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_wqes_start, 527 IBMF_TNF_TRACE, "", "ibmf_i_init_wqes() enter, cip = %p\n", 528 tnf_opaque, cip, cip); 529 530 bzero(string, 128); 531 (void) sprintf(string, "ibmf_%016" PRIx64 "_swqe", cip->ci_node_guid); 532 /* create a kmem cache for the send WQEs */ 533 cip->ci_send_wqes_cache = kmem_cache_create(string, 534 sizeof (ibmf_send_wqe_t), 0, ibmf_send_wqe_cache_constructor, 535 ibmf_send_wqe_cache_destructor, NULL, (void *)cip, NULL, 0); 536 537 bzero(string, 128); 538 (void) sprintf(string, "ibmf_%016" PRIx64 "_rwqe", cip->ci_node_guid); 539 /* create a kmem cache for the receive WQEs */ 540 cip->ci_recv_wqes_cache = kmem_cache_create(string, 541 sizeof (ibmf_recv_wqe_t), 0, ibmf_recv_wqe_cache_constructor, 542 ibmf_recv_wqe_cache_destructor, NULL, (void *)cip, NULL, 0); 543 544 /* 545 * Allocate memory for the WQE management structure 546 */ 547 wqe_mgtp = kmem_zalloc(sizeof (ibmf_wqe_mgt_t), KM_SLEEP); 548 mutex_init(&wqe_mgtp->wqes_mutex, NULL, MUTEX_DRIVER, NULL); 549 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgtp)) 550 551 /* 552 * Allocate memory for the WQEs to be used by the special QPs on this CI 553 * There are two special QPs per CI port 554 */ 555 wqe_mgtp->wqes_kmem_sz = cip->ci_nports * 2 * 556 ((IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) + 557 (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port)); 558 wqe_mgtp->wqes_kmem = 559 kmem_zalloc(wqe_mgtp->wqes_kmem_sz, KM_SLEEP); 560 561 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgtp->wqes_kmem; 562 mem_attr.mr_len = wqe_mgtp->wqes_kmem_sz; 563 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE; 564 mem_attr.mr_as = NULL; 565 566 /* Register the allocated memory */ 567 status = ibt_register_mr(cip->ci_ci_handle, cip->ci_pd, &mem_attr, 568 &mem_hdl, &mem_desc); 569 if (status != IBT_SUCCESS) { 570 kmem_free(wqe_mgtp->wqes_kmem, 571 wqe_mgtp->wqes_kmem_sz); 572 kmem_cache_destroy(cip->ci_recv_wqes_cache); 573 kmem_cache_destroy(cip->ci_send_wqes_cache); 574 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 575 ibmf_i_init_wqes_err, IBMF_TNF_ERROR, "", 576 "ibmf_i_init_wqes(): %s, status = %d\n", tnf_string, msg, 577 "register of WQE mem failed", tnf_uint, status, status); 578 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 579 ibmf_i_init_wqes_end, IBMF_TNF_TRACE, "", 580 "ibmf_i_init_wqes() exit\n"); 581 return (IBMF_NO_RESOURCES); 582 } 583 584 /* Store the memory registration information */ 585 wqe_mgtp->wqes_ib_mem = mem_desc.md_vaddr; 586 wqe_mgtp->wqes_ib_lkey = mem_desc.md_lkey; 587 wqe_mgtp->wqes_ib_mem_hdl = mem_hdl; 588 589 /* Create a vmem arena for the IB virtual address space */ 590 bzero(string, 128); 591 (void) sprintf(string, "ibmf_%016" PRIx64 "_wqes", cip->ci_node_guid); 592 cip->ci_wqe_ib_vmem = vmem_create(string, 593 (void *)(uintptr_t)wqe_mgtp->wqes_ib_mem, wqe_mgtp->wqes_kmem_sz, 594 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 595 596 mutex_enter(&cip->ci_wqe_mutex); 597 cip->ci_wqe_mgt_list = wqe_mgtp; 598 mutex_exit(&cip->ci_wqe_mutex); 599 600 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_wqes_end, 601 IBMF_TNF_TRACE, "", "ibmf_i_init_wqes() exit\n"); 602 603 return (IBMF_SUCCESS); 604 } 605 606 /* 607 * ibmf_i_fini_wqes(): 608 * Destroy the kmem cache for send and receive WQEs 609 */ 610 void 611 ibmf_i_fini_wqes(ibmf_ci_t *cip) 612 { 613 ibmf_wqe_mgt_t *wqe_mgt; 614 ibt_mr_hdl_t wqe_ib_mem_hdl; 615 void *wqe_kmem; 616 uint64_t wqe_kmem_sz; 617 618 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_start, 619 IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() enter, cip = %p\n", 620 tnf_opaque, cip, cip); 621 622 mutex_enter(&cip->ci_wqe_mutex); 623 624 wqe_mgt = cip->ci_wqe_mgt_list; 625 while (wqe_mgt != NULL) { 626 /* Remove the WQE mgt struct from the list */ 627 cip->ci_wqe_mgt_list = wqe_mgt->wqe_mgt_next; 628 mutex_exit(&cip->ci_wqe_mutex); 629 630 mutex_enter(&wqe_mgt->wqes_mutex); 631 wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 632 wqe_kmem = wqe_mgt->wqes_kmem; 633 wqe_kmem_sz = wqe_mgt->wqes_kmem_sz; 634 mutex_exit(&wqe_mgt->wqes_mutex); 635 636 /* Deregister the memory allocated for the WQEs */ 637 (void) ibt_deregister_mr(cip->ci_ci_handle, wqe_ib_mem_hdl); 638 639 /* Free the kmem allocated for the WQEs */ 640 kmem_free(wqe_kmem, wqe_kmem_sz); 641 642 /* Destroy the mutex */ 643 mutex_destroy(&wqe_mgt->wqes_mutex); 644 645 /* Free the WQE management structure */ 646 kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t)); 647 648 mutex_enter(&cip->ci_wqe_mutex); 649 wqe_mgt = cip->ci_wqe_mgt_list; 650 } 651 652 mutex_exit(&cip->ci_wqe_mutex); 653 654 /* Destroy the kmem_cache for the send WQE */ 655 kmem_cache_destroy(cip->ci_send_wqes_cache); 656 /* Destroy the kmem_cache for the receive WQE */ 657 kmem_cache_destroy(cip->ci_recv_wqes_cache); 658 659 /* 660 * Destroy the vmem arena for the WQEs 661 * This must be done after the kmem_cache_destroy() calls since 662 * the cache destructors call vmem_free() 663 */ 664 vmem_destroy((void *)cip->ci_wqe_ib_vmem); 665 666 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_end, 667 IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() exit\n"); 668 } 669 670 /* 671 * ibmf_i_init_altqp_wqes(): 672 * Create the kmem cache for send and receive WQEs used by alternate QPs 673 */ 674 int 675 ibmf_i_init_altqp_wqes(ibmf_alt_qp_t *qp_ctx) 676 { 677 ibt_status_t status; 678 ibt_mr_hdl_t mem_hdl; 679 ibt_mr_desc_t mem_desc; 680 ibt_mr_attr_t mem_attr; 681 ibmf_wqe_mgt_t *wqe_mgtp; 682 char string[128]; 683 684 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_altqp_wqes_start, 685 IBMF_TNF_TRACE, "", "ibmf_i_init_altqp_wqes() enter, qp_ctx = %p\n", 686 tnf_opaque, qp, qp_ctx); 687 688 bzero(string, 128); 689 /* 690 * CAUTION: Do not exceed 32 characters for the kmem cache name, else, 691 * mdb does not exit (bug 4878751). There is some connection between 692 * mdb walkers and kmem_caches with the limitation likely to be in the 693 * mdb code. 694 */ 695 (void) sprintf(string, "ibmf%016" PRIx64 "_%xs", 696 qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn); 697 /* create a kmem cache for the send WQEs */ 698 qp_ctx->isq_send_wqes_cache = kmem_cache_create(string, 699 sizeof (ibmf_send_wqe_t), 0, ibmf_altqp_send_wqe_cache_constructor, 700 ibmf_altqp_send_wqe_cache_destructor, NULL, (void *)qp_ctx, 701 NULL, 0); 702 703 bzero(string, 128); 704 (void) sprintf(string, "ibmf%016" PRIx64 "_%xr", 705 qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn); 706 /* create a kmem cache for the receive WQEs */ 707 qp_ctx->isq_recv_wqes_cache = kmem_cache_create(string, 708 sizeof (ibmf_recv_wqe_t), 0, ibmf_altqp_recv_wqe_cache_constructor, 709 ibmf_altqp_recv_wqe_cache_destructor, NULL, (void *)qp_ctx, 710 NULL, 0); 711 712 /* 713 * Allocate memory for the WQE management structure 714 */ 715 wqe_mgtp = kmem_zalloc(sizeof (ibmf_wqe_mgt_t), KM_SLEEP); 716 mutex_init(&wqe_mgtp->wqes_mutex, NULL, MUTEX_DRIVER, NULL); 717 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgtp)) 718 719 /* 720 * Allocate memory for all the WQEs to be used by this alternate QP 721 */ 722 wqe_mgtp->wqes_kmem_sz = (IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) + 723 (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port); 724 wqe_mgtp->wqes_kmem = kmem_zalloc(wqe_mgtp->wqes_kmem_sz, KM_SLEEP); 725 726 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgtp->wqes_kmem; 727 mem_attr.mr_len = wqe_mgtp->wqes_kmem_sz; 728 mem_attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE; 729 mem_attr.mr_as = NULL; 730 731 /* Register the allocated memory */ 732 status = ibt_register_mr(qp_ctx->isq_client_hdl->ic_myci->ci_ci_handle, 733 qp_ctx->isq_client_hdl->ic_myci->ci_pd, &mem_attr, &mem_hdl, 734 &mem_desc); 735 if (status != IBT_SUCCESS) { 736 kmem_free(wqe_mgtp->wqes_kmem, wqe_mgtp->wqes_kmem_sz); 737 kmem_cache_destroy(qp_ctx->isq_recv_wqes_cache); 738 kmem_cache_destroy(qp_ctx->isq_send_wqes_cache); 739 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 740 ibmf_i_init_altqp_wqes_err, IBMF_TNF_ERROR, "", 741 "ibmf_i_init_altqp_wqes(): %s, status = %d\n", 742 tnf_string, msg, 743 "register of WQE mem failed", tnf_uint, status, status); 744 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 745 ibmf_i_init_altqp_wqes_end, IBMF_TNF_TRACE, "", 746 "ibmf_i_init_altqp_wqes() exit\n"); 747 return (IBMF_NO_RESOURCES); 748 } 749 750 /* Store the memory registration information */ 751 wqe_mgtp->wqes_ib_mem = mem_desc.md_vaddr; 752 wqe_mgtp->wqes_ib_lkey = mem_desc.md_lkey; 753 wqe_mgtp->wqes_ib_mem_hdl = mem_hdl; 754 755 /* Create a vmem arena for the IB virtual address space */ 756 bzero(string, 128); 757 (void) sprintf(string, "ibmf_%016" PRIx64 "_%x_wqes", 758 qp_ctx->isq_client_hdl->ic_client_info.ci_guid, qp_ctx->isq_qpn); 759 qp_ctx->isq_wqe_ib_vmem = vmem_create(string, 760 (void *)(uintptr_t)wqe_mgtp->wqes_ib_mem, wqe_mgtp->wqes_kmem_sz, 761 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 762 763 mutex_enter(&qp_ctx->isq_wqe_mutex); 764 qp_ctx->isq_wqe_mgt_list = wqe_mgtp; 765 mutex_exit(&qp_ctx->isq_wqe_mutex); 766 767 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_altqp_wqes_end, 768 IBMF_TNF_TRACE, "", "ibmf_i_init_altqp_wqes() exit\n"); 769 770 return (IBMF_SUCCESS); 771 } 772 773 /* 774 * ibmf_i_fini_altqp_wqes(): 775 * Destroy the kmem cache for send and receive WQEs for alternate QPs 776 */ 777 void 778 ibmf_i_fini_altqp_wqes(ibmf_alt_qp_t *qp_ctx) 779 { 780 ibmf_wqe_mgt_t *wqe_mgt; 781 ibt_mr_hdl_t wqe_ib_mem_hdl; 782 void *wqe_kmem; 783 uint64_t wqe_kmem_sz; 784 785 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_start, 786 IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() enter, qp_ctx = %p\n", 787 tnf_opaque, qp, qp_ctx); 788 789 mutex_enter(&qp_ctx->isq_wqe_mutex); 790 wqe_mgt = qp_ctx->isq_wqe_mgt_list; 791 while (wqe_mgt != NULL) { 792 /* Remove the WQE mgt struct from the list */ 793 qp_ctx->isq_wqe_mgt_list = wqe_mgt->wqe_mgt_next; 794 mutex_exit(&qp_ctx->isq_wqe_mutex); 795 796 mutex_enter(&wqe_mgt->wqes_mutex); 797 wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl; 798 wqe_kmem = wqe_mgt->wqes_kmem; 799 wqe_kmem_sz = wqe_mgt->wqes_kmem_sz; 800 mutex_exit(&wqe_mgt->wqes_mutex); 801 802 /* Deregister the memory allocated for the WQEs */ 803 (void) ibt_deregister_mr( 804 qp_ctx->isq_client_hdl->ic_myci->ci_ci_handle, 805 wqe_ib_mem_hdl); 806 807 /* Free the kmem allocated for the WQEs */ 808 kmem_free(wqe_kmem, wqe_kmem_sz); 809 810 /* Destroy the WQE mgt struct mutex */ 811 mutex_destroy(&wqe_mgt->wqes_mutex); 812 813 /* Free the WQE management structure */ 814 kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t)); 815 816 mutex_enter(&qp_ctx->isq_wqe_mutex); 817 wqe_mgt = qp_ctx->isq_wqe_mgt_list; 818 } 819 820 mutex_exit(&qp_ctx->isq_wqe_mutex); 821 822 /* Destroy the kmem_cache for the send WQE */ 823 kmem_cache_destroy(qp_ctx->isq_send_wqes_cache); 824 /* Destroy the kmem_cache for the receive WQE */ 825 kmem_cache_destroy(qp_ctx->isq_recv_wqes_cache); 826 827 /* 828 * Destroy the vmem arena for the WQEs 829 * This must be done after the kmem_cache_destroy() calls since 830 * the cache destructors call vmem_free() 831 */ 832 vmem_destroy((void *)qp_ctx->isq_wqe_ib_vmem); 833 834 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_fini_wqes_end, 835 IBMF_TNF_TRACE, "", "ibmf_i_fini_wqes() exit\n"); 836 } 837 838 /* 839 * ibmf_i_init_send_wqe(): 840 * Initialize a send WQE 841 */ 842 /* ARGSUSED */ 843 void 844 ibmf_i_init_send_wqe(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp, 845 ibt_wr_ds_t *sglp, ibmf_send_wqe_t *wqep, ibt_ud_dest_hdl_t ud_dest, 846 ibt_qp_hdl_t ibt_qp_handle, ibmf_qp_handle_t ibmf_qp_handle) 847 { 848 ibmf_msg_bufs_t *ipbufs = &msgimplp->im_msgbufs_send; 849 ibmf_msg_bufs_t *hdr_ipbufs; 850 ib_mad_hdr_t *ibmadhdrp; 851 ibmf_rmpp_ctx_t *rmpp_ctx = &msgimplp->im_rmpp_ctx; 852 ibmf_rmpp_hdr_t *rmpp_hdr; 853 ibt_send_wr_t *swrp; 854 uchar_t *buf; 855 size_t data_sz, offset; 856 uint32_t cl_hdr_sz, cl_hdr_off; 857 858 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_send_wqe_start, 859 IBMF_TNF_TRACE, "", "ibmf_i_init_send_wqe() enter, " 860 "clientp = %p, msg = %p, sglp = %p , wqep = %p, qp_hdl = %p\n", 861 tnf_opaque, clientp, clientp, tnf_opaque, msg, msgimplp, 862 tnf_opaque, sglp, sglp, tnf_opaque, wqep, wqep, 863 tnf_opaque, qp_hdl, ibmf_qp_handle); 864 865 _NOTE(ASSUMING_PROTECTED(*wqep)) 866 _NOTE(ASSUMING_PROTECTED(*sglp)) 867 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrp)) 868 869 swrp = &wqep->send_wr; 870 /* use send wqe pointer as the WR ID */ 871 IBMF_ADDR_TO_SEND_WR_ID(wqep, swrp->wr_id); 872 ASSERT(swrp->wr_id != NULL); 873 swrp->wr_flags = IBT_WR_NO_FLAGS; 874 swrp->wr_opcode = IBT_WRC_SEND; 875 swrp->wr_trans = IBT_UD_SRV; 876 wqep->send_client = clientp; 877 wqep->send_msg = msgimplp; 878 879 IBMF_INIT_SG_ELEMENT(sglp[0], wqep->send_mem, wqep->send_sg_lkey, 880 IBMF_MAD_SIZE); 881 882 bzero(wqep->send_mem, IBMF_MAD_SIZE); 883 if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) { 884 buf = (uchar_t *)ipbufs->im_bufs_cl_data + 885 (rmpp_ctx->rmpp_ns - 1) * rmpp_ctx->rmpp_pkt_data_sz; 886 data_sz = (rmpp_ctx->rmpp_ns == rmpp_ctx->rmpp_num_pkts) ? 887 rmpp_ctx->rmpp_last_pkt_sz : rmpp_ctx->rmpp_pkt_data_sz; 888 } else { 889 buf = ipbufs->im_bufs_cl_data; 890 data_sz = ipbufs->im_bufs_cl_data_len; 891 } 892 893 /* 894 * We pick the correct msgbuf based on the nature of the transaction. 895 * Where the send msgbuf is available, we pick it to provide the 896 * context of the outgoing MAD. Note that if this is a termination 897 * context, then the send buffer is invalid even if the sequenced 898 * flags is set because the termination message only has a receive 899 * buffer set up. 900 */ 901 if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) && 902 ((msgimplp->im_flags & IBMF_MSG_FLAGS_TERMINATION) == 0)) { 903 hdr_ipbufs = &msgimplp->im_msgbufs_send; 904 } else if (msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) { 905 hdr_ipbufs = &msgimplp->im_msgbufs_recv; 906 } else if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) { 907 hdr_ipbufs = &msgimplp->im_msgbufs_send; 908 } else { 909 if (msgimplp->im_unsolicited == B_TRUE) { 910 hdr_ipbufs = &msgimplp->im_msgbufs_recv; 911 } else { 912 hdr_ipbufs = &msgimplp->im_msgbufs_send; 913 } 914 } 915 916 bcopy((void *)hdr_ipbufs->im_bufs_mad_hdr, 917 (void *)wqep->send_mem, sizeof (ib_mad_hdr_t)); 918 919 /* 920 * For unsolicited messages, we only have the sender's MAD at hand. 921 * So, we must flip the response bit in the method for the outgoing MAD. 922 */ 923 ibmadhdrp = (ib_mad_hdr_t *)wqep->send_mem; 924 if (msgimplp->im_unsolicited == B_TRUE) { 925 ibmadhdrp->R_Method = IBMF_FLIP_RESP_BIT(ibmadhdrp->R_Method); 926 } 927 928 offset = sizeof (ib_mad_hdr_t); 929 930 if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) || 931 (msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP)) { 932 933 rmpp_hdr = (ibmf_rmpp_hdr_t *) 934 ((uintptr_t)wqep->send_mem + offset); 935 936 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rmpp_hdr)); 937 938 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_init_send_wqe, 939 IBMF_TNF_TRACE, "", 940 "ibmf_init_send_wqe: msgimplp = %p, rmpp_type = %d," 941 " next_seg = %d, num_pkts = %d\n", 942 tnf_opaque, msgimplp, msgimplp, 943 tnf_opaque, rmpp_type, rmpp_ctx->rmpp_type, 944 tnf_opaque, next_seg, rmpp_ctx->rmpp_ns, 945 tnf_opaque, num_pkts, rmpp_ctx->rmpp_num_pkts); 946 947 /* 948 * Initialize the RMPP header 949 */ 950 rmpp_ctx->rmpp_flags = IBMF_RMPP_FLAGS_ACTIVE; 951 952 /* first, last packet flags set only for type DATA */ 953 if (rmpp_ctx->rmpp_type == IBMF_RMPP_TYPE_DATA) { 954 955 if (rmpp_ctx->rmpp_ns == 1) 956 rmpp_ctx->rmpp_flags |= 957 IBMF_RMPP_FLAGS_FIRST_PKT; 958 else 959 rmpp_ctx->rmpp_respt = IBMF_RMPP_DEFAULT_RRESPT; 960 961 if (rmpp_ctx->rmpp_ns == rmpp_ctx->rmpp_num_pkts) 962 rmpp_ctx->rmpp_flags |= 963 IBMF_RMPP_FLAGS_LAST_PKT; 964 } else { 965 data_sz = 0; 966 rmpp_ctx->rmpp_respt = IBMF_RMPP_TERM_RRESPT; 967 } 968 969 IBMF_INIT_RMPP_HDR(rmpp_hdr, 970 IBMF_RMPP_VERSION, rmpp_ctx->rmpp_type, 971 rmpp_ctx->rmpp_respt, rmpp_ctx->rmpp_flags, 972 rmpp_ctx->rmpp_status, rmpp_ctx->rmpp_word3, 973 rmpp_ctx->rmpp_word4) 974 975 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_i_init_send_wqe, 976 IBMF_TNF_TRACE, "", 977 "ibmf_init_send_wqe: msgimplp = %p, rmpp_type = %d," 978 " rmpp_flags = 0x%x, rmpp_segnum = %d, pyld_nwl = %d\n", 979 tnf_opaque, msgimplp, msgimplp, 980 tnf_opaque, rmpp_type, rmpp_hdr->rmpp_type, 981 tnf_opaque, rmpp_flags, rmpp_hdr->rmpp_flags, 982 tnf_opaque, rmpp_segnum, b2h32(rmpp_hdr->rmpp_segnum), 983 tnf_opaque, pyld_nwl, b2h32(rmpp_hdr->rmpp_pyldlen_nwl)); 984 985 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(rmpp_hdr)); 986 } 987 988 /* determine offset to start class header */ 989 ibmf_i_mgt_class_to_hdr_sz_off( 990 hdr_ipbufs->im_bufs_mad_hdr->MgmtClass, 991 &cl_hdr_sz, &cl_hdr_off); 992 offset += cl_hdr_off; 993 if (hdr_ipbufs->im_bufs_cl_hdr != NULL) { 994 bcopy((void *)hdr_ipbufs->im_bufs_cl_hdr, 995 (void *)((uintptr_t)wqep->send_mem + offset), 996 hdr_ipbufs->im_bufs_cl_hdr_len); 997 offset += hdr_ipbufs->im_bufs_cl_hdr_len; 998 } 999 bcopy((void *)buf, (void *)((uintptr_t)wqep->send_mem + offset), 1000 data_sz); 1001 swrp->wr_sgl = sglp; 1002 swrp->wr_nds = 1; 1003 swrp->wr.ud.udwr_dest = ud_dest; 1004 wqep->send_port_num = clientp->ic_client_info.port_num; 1005 wqep->send_qp_handle = ibt_qp_handle; 1006 wqep->send_ibmf_qp_handle = ibmf_qp_handle; 1007 1008 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*swrp)) 1009 1010 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_send_wqe_end, 1011 IBMF_TNF_TRACE, "", "ibmf_i_init_send_wqe() exit\n"); 1012 } 1013 1014 /* 1015 * ibmf_i_init_recv_wqe(): 1016 * Initialize a receive WQE 1017 */ 1018 void 1019 ibmf_i_init_recv_wqe(ibmf_qp_t *qpp, ibt_wr_ds_t *sglp, 1020 ibmf_recv_wqe_t *wqep, ibt_qp_hdl_t ibt_qp_handle, 1021 ibmf_qp_handle_t ibmf_qp_handle) 1022 { 1023 ibt_recv_wr_t *rwrp; 1024 1025 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_recv_wqe_start, 1026 IBMF_TNF_TRACE, "", "ibmf_i_init_recv_wqe() enter, " 1027 "qpp = %p, sglp = %p , wqep = %p, ud_dest = %p, qp_hdl = %p\n", 1028 tnf_opaque, qpp, qpp, tnf_opaque, sglp, sglp, tnf_opaque, 1029 wqep, wqep, tnf_opaque, qp_hdl, ibmf_qp_handle); 1030 1031 _NOTE(ASSUMING_PROTECTED(*wqep)) 1032 _NOTE(ASSUMING_PROTECTED(*sglp)) 1033 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rwrp)) 1034 1035 rwrp = &wqep->recv_wr; 1036 1037 /* 1038 * we set a bit in the WR ID to be able to easily distinguish 1039 * between send completions and recv completions 1040 */ 1041 IBMF_ADDR_TO_RECV_WR_ID(wqep, rwrp->wr_id); 1042 1043 IBMF_INIT_SG_ELEMENT(sglp[0], wqep->recv_mem, wqep->recv_sg_lkey, 1044 sizeof (ib_grh_t) + IBMF_MAD_SIZE); 1045 1046 rwrp->wr_sgl = sglp; 1047 rwrp->wr_nds = IBMF_MAX_RQ_WR_SGL_ELEMENTS; 1048 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 1049 wqep->recv_port_num = qpp->iq_port_num; 1050 } else { 1051 ibmf_alt_qp_t *altqp = (ibmf_alt_qp_t *)ibmf_qp_handle; 1052 wqep->recv_port_num = altqp->isq_port_num; 1053 } 1054 wqep->recv_qpp = qpp; 1055 wqep->recv_qp_handle = ibt_qp_handle; 1056 wqep->recv_ibmf_qp_handle = ibmf_qp_handle; 1057 1058 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*rwrp)) 1059 1060 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_init_recv_wqe_end, 1061 IBMF_TNF_TRACE, "", "ibmf_i_init_recv_wqe() exit\n"); 1062 } 1063 1064 /* 1065 * ibmf_i_extend_wqe_cache(): 1066 * Extend the kmem WQE cache 1067 */ 1068 int 1069 ibmf_i_extend_wqe_cache(ibmf_ci_t *cip, ibmf_qp_handle_t ibmf_qp_handle, 1070 boolean_t block) 1071 { 1072 ibmf_wqe_mgt_t *wqe_mgt; 1073 1074 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4, 1075 ibmf_i_extend_wqe_cache_start, IBMF_TNF_TRACE, "", 1076 "ibmf_i_extend_wqe_cache() enter, cip = %p, qp_hdl = %p, " 1077 " block = %d\n", tnf_opaque, cip, cip, tnf_opaque, qp_hdl, 1078 ibmf_qp_handle, tnf_uint, block, block); 1079 1080 /* 1081 * Allocate memory for the WQE management structure 1082 */ 1083 wqe_mgt = kmem_zalloc(sizeof (ibmf_wqe_mgt_t), 1084 (block == B_TRUE ? KM_SLEEP : KM_NOSLEEP)); 1085 if (wqe_mgt == NULL) { 1086 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1087 ibmf_i_extend_wqe_cache_err, IBMF_TNF_ERROR, "", 1088 "ibmf_i_extend_wqe_cache(): %s\n", 1089 tnf_string, msg, "wqe mgt alloc failed"); 1090 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1091 ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "", 1092 "ibmf_i_extend_wqe_cache() exit\n"); 1093 return (IBMF_NO_RESOURCES); 1094 } 1095 mutex_init(&wqe_mgt->wqes_mutex, NULL, MUTEX_DRIVER, NULL); 1096 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt)) 1097 1098 /* Allocate and register more WQE memory */ 1099 if (ibmf_i_extend_wqe_mem(cip, ibmf_qp_handle, wqe_mgt, 1100 block) != IBMF_SUCCESS) { 1101 mutex_destroy(&wqe_mgt->wqes_mutex); 1102 kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t)); 1103 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1104 ibmf_i_extend_wqe_cache_err, IBMF_TNF_ERROR, "", 1105 "ibmf_i_extend_wqe_cache(): %s\n", 1106 tnf_string, msg, "extension of WQE pool failed"); 1107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1108 ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "", 1109 "ibmf_i_extend_wqe_cache() exit\n"); 1110 return (IBMF_NO_RESOURCES); 1111 } 1112 1113 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1114 ibmf_i_extend_wqe_cache_end, IBMF_TNF_TRACE, "", 1115 "ibmf_i_extend_wqe_cache() exit\n"); 1116 1117 return (IBMF_SUCCESS); 1118 } 1119 1120 /* 1121 * ibmf_i_extend_wqe_mem(): 1122 * Allocate and register more WQE memory, and expand the VMEM arena 1123 */ 1124 static int 1125 ibmf_i_extend_wqe_mem(ibmf_ci_t *cip, ibmf_qp_handle_t ibmf_qp_handle, 1126 ibmf_wqe_mgt_t *wqe_mgt, boolean_t block) 1127 { 1128 ibt_status_t status; 1129 ibt_mr_hdl_t mem_hdl; 1130 ibt_mr_desc_t mem_desc; 1131 ibt_mr_attr_t mem_attr; 1132 ibmf_alt_qp_t *qp_ctx; 1133 ibmf_wqe_mgt_t *pwqe_mgt; 1134 vmem_t *wqe_vmem_arena; 1135 1136 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt)) 1137 1138 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, 1139 ibmf_i_extend_wqe_cache_start, IBMF_TNF_TRACE, "", 1140 "ibmf_i_extend_wqe_cache() enter, cip = %p, qp_hdl = %p" 1141 "wqe_mgt = %p, block = %d\n", 1142 tnf_opaque, cip, cip, tnf_opaque, qp_hdl, ibmf_qp_handle, 1143 tnf_opaque, wqe_mgt, wqe_mgt, tnf_uint, block, block); 1144 1145 /* 1146 * Allocate more memory for the WQEs to be used by the 1147 * specified QP 1148 */ 1149 wqe_mgt->wqes_kmem_sz = cip->ci_nports * 2 * 1150 ((IBMF_MEM_PER_WQE * ibmf_send_wqes_per_port) + 1151 (IBMF_MEM_PER_WQE * ibmf_recv_wqes_per_port)); 1152 wqe_mgt->wqes_kmem = kmem_zalloc(wqe_mgt->wqes_kmem_sz, 1153 (block == B_TRUE ? KM_SLEEP : KM_NOSLEEP)); 1154 if (wqe_mgt->wqes_kmem == NULL) { 1155 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1156 ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "", 1157 "ibmf_i_extend_wqe_mem(): %s\n", 1158 tnf_string, msg, "extension of WQE pool failed"); 1159 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1160 ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "", 1161 "ibmf_i_extend_wqe_mem() exit\n"); 1162 return (IBMF_NO_RESOURCES); 1163 } 1164 1165 mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgt->wqes_kmem; 1166 mem_attr.mr_len = wqe_mgt->wqes_kmem_sz; 1167 mem_attr.mr_flags = (block == B_TRUE ? IBT_MR_SLEEP : IBT_MR_NOSLEEP) 1168 | IBT_MR_ENABLE_LOCAL_WRITE; 1169 mem_attr.mr_as = NULL; 1170 1171 /* Register the allocated memory */ 1172 status = ibt_register_mr(cip->ci_ci_handle, cip->ci_pd, 1173 &mem_attr, &mem_hdl, &mem_desc); 1174 if (status != IBT_SUCCESS) { 1175 kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz); 1176 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1177 ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "", 1178 "ibmf_i_extend_wqe_mem(): %s\n", 1179 tnf_string, msg, "wqe extension MR failed"); 1180 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1181 ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "", 1182 "ibmf_i_extend_wqe_mem() exit\n"); 1183 return (IBMF_NO_RESOURCES); 1184 } 1185 1186 /* Store the memory registration information */ 1187 wqe_mgt->wqes_ib_mem = mem_desc.md_vaddr; 1188 wqe_mgt->wqes_ib_lkey = mem_desc.md_lkey; 1189 wqe_mgt->wqes_ib_mem_hdl = mem_hdl; 1190 1191 /* Get the VMEM arena based on the QP type */ 1192 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 1193 wqe_vmem_arena = cip->ci_wqe_ib_vmem; 1194 } else { 1195 qp_ctx = (ibmf_alt_qp_t *)ibmf_qp_handle; 1196 wqe_vmem_arena = qp_ctx->isq_wqe_ib_vmem; 1197 } 1198 1199 /* Add these addresses to the vmem arena */ 1200 if (vmem_add(wqe_vmem_arena, (void *)(uintptr_t)wqe_mgt->wqes_ib_mem, 1201 wqe_mgt->wqes_kmem_sz, 1202 (block == B_TRUE ? VM_SLEEP : VM_NOSLEEP)) == NULL) { 1203 (void) ibt_deregister_mr(cip->ci_ci_handle, 1204 wqe_mgt->wqes_ib_mem_hdl); 1205 kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz); 1206 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1207 ibmf_i_extend_wqe_mem_err, IBMF_TNF_ERROR, "", 1208 "ibmf_i_extend_wqe_mem(): %s\n", 1209 tnf_string, msg, "wqe extension vmem_add failed"); 1210 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1211 ibmf_i_extend_wqe_mem_end, IBMF_TNF_TRACE, "", 1212 "ibmf_i_extend_wqe_mem() exit\n"); 1213 return (IBMF_NO_RESOURCES); 1214 } 1215 1216 /* Get the WQE management pointers based on the QP type */ 1217 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 1218 mutex_enter(&cip->ci_wqe_mutex); 1219 pwqe_mgt = cip->ci_wqe_mgt_list; 1220 1221 /* Add the new wqe management struct to the end of the list */ 1222 while (pwqe_mgt->wqe_mgt_next != NULL) 1223 pwqe_mgt = pwqe_mgt->wqe_mgt_next; 1224 pwqe_mgt->wqe_mgt_next = wqe_mgt; 1225 1226 mutex_exit(&cip->ci_wqe_mutex); 1227 } else { 1228 mutex_enter(&qp_ctx->isq_wqe_mutex); 1229 pwqe_mgt = qp_ctx->isq_wqe_mgt_list; 1230 1231 /* Add the new wqe management struct to the end of the list */ 1232 while (pwqe_mgt->wqe_mgt_next != NULL) 1233 pwqe_mgt = pwqe_mgt->wqe_mgt_next; 1234 pwqe_mgt->wqe_mgt_next = wqe_mgt; 1235 1236 mutex_exit(&qp_ctx->isq_wqe_mutex); 1237 } 1238 1239 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_extend_wqe_mem_end, 1240 IBMF_TNF_TRACE, "", "ibmf_i_extend_wqe_mem() exit\n"); 1241 1242 return (IBMF_SUCCESS); 1243 } 1244 1245 /* 1246 * ibmf_i_alloc_send_resources(): 1247 * Allocate send resources (the send WQE) 1248 */ 1249 int 1250 ibmf_i_alloc_send_resources(ibmf_ci_t *cip, ibmf_msg_impl_t *msgimplp, 1251 boolean_t block, ibmf_send_wqe_t **swqepp) 1252 { 1253 ibmf_send_wqe_t *send_wqep; 1254 struct kmem_cache *kmem_cachep; 1255 ibmf_qp_handle_t ibmf_qp_handle = msgimplp->im_qp_hdl; 1256 ibmf_alt_qp_t *altqp; 1257 1258 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4, 1259 ibmf_i_alloc_send_resources_start, IBMF_TNF_TRACE, "", 1260 "ibmf_i_alloc_send_resources() enter, cip = %p, msg = %p, " 1261 " block = %d\n", tnf_opaque, cip, cip, tnf_opaque, msg, 1262 msgimplp, tnf_uint, block, block); 1263 1264 /* Get the WQE kmem cache pointer based on the QP type */ 1265 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) 1266 kmem_cachep = cip->ci_send_wqes_cache; 1267 else { 1268 altqp = (ibmf_alt_qp_t *)ibmf_qp_handle; 1269 kmem_cachep = altqp->isq_send_wqes_cache; 1270 } 1271 1272 /* 1273 * Allocate a send WQE from the send WQE kmem cache 1274 * Do not block here as we are holding the msgimpl mutex. 1275 */ 1276 send_wqep = kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 1277 if (send_wqep == NULL) { 1278 /* 1279 * Attempt to extend the cache and then retry the 1280 * kmem_cache_alloc() 1281 * The block argument (third) is set to B_FALSE. 1282 */ 1283 if (ibmf_i_extend_wqe_cache(cip, ibmf_qp_handle, B_FALSE) == 1284 IBMF_NO_RESOURCES) { 1285 mutex_enter(&cip->ci_mutex); 1286 IBMF_ADD32_PORT_KSTATS(cip, swqe_allocs_failed, 1); 1287 mutex_exit(&cip->ci_mutex); 1288 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1289 ibmf_i_alloc_send_resources_err, IBMF_TNF_ERROR, "", 1290 "ibmf_i_alloc_send_resources(): %s\n", 1291 tnf_string, msg, "alloc send_wqe failed"); 1292 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1293 ibmf_i_alloc_send_resources_end, IBMF_TNF_TRACE, "", 1294 "ibmf_i_alloc_send_resources() exit\n"); 1295 return (IBMF_NO_RESOURCES); 1296 } else { 1297 send_wqep = kmem_cache_alloc(kmem_cachep, KM_NOSLEEP); 1298 if (send_wqep == NULL) { 1299 /* Allocation failed again. Give up here. */ 1300 mutex_enter(&cip->ci_mutex); 1301 IBMF_ADD32_PORT_KSTATS(cip, swqe_allocs_failed, 1302 1); 1303 mutex_exit(&cip->ci_mutex); 1304 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1305 ibmf_i_alloc_send_resources_err, 1306 IBMF_TNF_ERROR, "", 1307 "ibmf_i_alloc_send_resources(): %s\n", 1308 tnf_string, msg, "alloc send_wqe failed"); 1309 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1310 ibmf_i_alloc_send_resources_end, 1311 IBMF_TNF_TRACE, "", 1312 "ibmf_i_alloc_send_resources() exit\n"); 1313 return (IBMF_NO_RESOURCES); 1314 } 1315 } 1316 } 1317 1318 mutex_enter(&cip->ci_mutex); 1319 IBMF_ADD32_PORT_KSTATS(cip, send_wqes_alloced, 1); 1320 mutex_exit(&cip->ci_mutex); 1321 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 1322 mutex_enter(&cip->ci_mutex); 1323 cip->ci_wqes_alloced++; 1324 mutex_exit(&cip->ci_mutex); 1325 } else { 1326 mutex_enter(&altqp->isq_mutex); 1327 altqp->isq_wqes_alloced++; 1328 mutex_exit(&altqp->isq_mutex); 1329 } 1330 1331 send_wqep->send_msg = msgimplp; 1332 *swqepp = send_wqep; 1333 1334 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1335 ibmf_i_alloc_send_resources_end, IBMF_TNF_TRACE, "", 1336 "ibmf_i_alloc_send_resources() exit\n"); 1337 1338 return (IBMF_SUCCESS); 1339 } 1340 1341 /* 1342 * ibmf_i_free_send_resources(): 1343 * Free send resources (just the send WQE) 1344 */ 1345 /* ARGSUSED */ 1346 void 1347 ibmf_i_free_send_resources(ibmf_ci_t *cip, ibmf_msg_impl_t *msgimplp, 1348 ibmf_send_wqe_t *swqep) 1349 { 1350 struct kmem_cache *kmem_cachep; 1351 ibmf_qp_handle_t ibmf_qp_handle = msgimplp->im_qp_hdl; 1352 ibmf_alt_qp_t *altqp; 1353 1354 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L4, 1355 ibmf_i_free_send_resources_start, IBMF_TNF_TRACE, "", 1356 "ibmf_i_free_send_resources() enter, cip = %p, msg = %p, " 1357 " swqep = %p\n", tnf_opaque, cip, cip, tnf_opaque, msg, 1358 msgimplp, tnf_opaque, swqep, swqep); 1359 1360 /* Get the WQE kmem cache pointer based on the QP type */ 1361 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) 1362 kmem_cachep = cip->ci_send_wqes_cache; 1363 else { 1364 altqp = (ibmf_alt_qp_t *)ibmf_qp_handle; 1365 kmem_cachep = altqp->isq_send_wqes_cache; 1366 } 1367 1368 /* return the send WQE to the kmem cache */ 1369 kmem_cache_free(kmem_cachep, swqep); 1370 1371 mutex_enter(&cip->ci_mutex); 1372 IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); 1373 mutex_exit(&cip->ci_mutex); 1374 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 1375 mutex_enter(&cip->ci_mutex); 1376 cip->ci_wqes_alloced--; 1377 if (cip->ci_wqes_alloced == 0) 1378 cv_signal(&cip->ci_wqes_cv); 1379 mutex_exit(&cip->ci_mutex); 1380 } else { 1381 mutex_enter(&altqp->isq_mutex); 1382 altqp->isq_wqes_alloced--; 1383 if (altqp->isq_wqes_alloced == 0) 1384 cv_signal(&altqp->isq_wqes_cv); 1385 mutex_exit(&altqp->isq_mutex); 1386 } 1387 1388 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1389 ibmf_i_free_send_resources_end, IBMF_TNF_TRACE, "", 1390 "ibmf_i_free_send_resources() exit\n"); 1391 } 1392