1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor_rsrc.c 29 * Tavor Resource Management Routines 30 * 31 * Implements all the routines necessary for setup, teardown, and 32 * alloc/free of all Tavor resources, including those that are managed 33 * by Tavor hardware or which live in Tavor's direct attached DDR memory. 34 */ 35 36 #include <sys/sysmacros.h> 37 #include <sys/types.h> 38 #include <sys/conf.h> 39 #include <sys/ddi.h> 40 #include <sys/sunddi.h> 41 #include <sys/modctl.h> 42 #include <sys/vmem.h> 43 #include <sys/bitmap.h> 44 45 #include <sys/ib/adapters/tavor/tavor.h> 46 47 /* 48 * The following routines are used for initializing and destroying 49 * the resource pools used by the Tavor resource allocation routines. 50 * They consist of four classes of object: 51 * 52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor 53 * command interface routines. Mailboxes are used to pass information 54 * back and forth to the Tavor firmware. Either type of mailbox may 55 * be allocated from Tavor's direct attached DDR memory or from system 56 * memory (although currently all "In" mailboxes are in DDR and all "out" 57 * mailboxes come from system memory. 58 * 59 * HW entry objects: These objects represent resources required by the Tavor 60 * hardware. These objects include things like Queue Pair contexts (QPC), 61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for 62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory 63 * Protection Table entries (MPT), Memory Translation Table entries (MTT). 64 * 65 * What these objects all have in common is that they are each required 66 * to come from DDR memory, they are always allocated from tables, and 67 * they are not to be directly accessed (read or written) by driver 68 * software. 69 * One notable exceptions to this rule are the Extended QP contexts (EQPC), 70 * and the UAR scratch area (UAR_SCR), both of which are not directly 71 * accessible through the Tavor resource allocation routines, but both 72 * of which are also required to reside in DDR memory and are not to be 73 * manipulated by driver software (they are privately managed by Tavor 74 * hardware). 75 * The other notable exceptions are the UAR pages (UAR_PG) which are 76 * allocated from the UAR address space rather than DDR, and the UD 77 * address vectors (UDAV) which are similar to the common object types 78 * with the major difference being that UDAVs _are_ directly read and 79 * written by driver software. 80 * 81 * SW handle objects: These objects represent resources required by Tavor 82 * driver software. They are primarily software tracking structures, 83 * which are allocated from system memory (using kmem_cache). Several of 84 * the objects have both a "constructor" and "destructor" method 85 * associated with them (see below). 86 * 87 * Protection Domain (PD) handle objects: These objects are very much like 88 * a SW handle object with the notable difference that all PD handle 89 * objects have an actual Protection Domain number (PD) associated with 90 * them (and the PD number is allocated/managed through a separate 91 * vmem_arena specifically set aside for this purpose. 92 */ 93 94 static int tavor_rsrc_mbox_init(tavor_state_t *state, 95 tavor_rsrc_mbox_info_t *info); 96 static void tavor_rsrc_mbox_fini(tavor_state_t *state, 97 tavor_rsrc_mbox_info_t *info); 98 99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state, 100 tavor_rsrc_hw_entry_info_t *info); 101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state, 102 tavor_rsrc_hw_entry_info_t *info); 103 104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state, 105 tavor_rsrc_sw_hdl_info_t *info); 106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state, 107 tavor_rsrc_sw_hdl_info_t *info); 108 109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state, 110 tavor_rsrc_sw_hdl_info_t *info); 111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state, 112 tavor_rsrc_sw_hdl_info_t *info); 113 114 /* 115 * The following routines are used for allocating and freeing the specific 116 * types of objects described above from their associated resource pools. 117 */ 118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, 119 uint_t num, tavor_rsrc_t *hdl); 120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, 121 tavor_rsrc_t *hdl); 122 123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, 124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle, 125 uint_t sleepflag, tavor_rsrc_t *hdl); 126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, 127 tavor_rsrc_t *hdl); 128 129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 130 uint_t sleepflag, tavor_rsrc_t *hdl); 131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, 132 tavor_rsrc_t *hdl); 133 134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 135 uint_t sleepflag, tavor_rsrc_t *hdl); 136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, 137 tavor_rsrc_t *hdl); 138 139 /* 140 * The following routines are the constructors and destructors for several 141 * of the SW handle type objects. For certain types of SW handles objects 142 * (all of which are implemented using kmem_cache), we need to do some 143 * special field initialization (specifically, mutex_init/destroy). These 144 * routines enable that init and teardown. 145 */ 146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags); 147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state); 148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags); 149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state); 150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags); 151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state); 152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags); 153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state); 154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags); 155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state); 156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags); 157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state); 158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags); 159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state); 160 161 /* 162 * Special routine to calculate and return the size of a MCG object based 163 * on current driver configuration (specifically, the number of QP per MCG 164 * that has been configured. 165 */ 166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, 167 uint_t *mcg_size_shift); 168 169 170 /* 171 * tavor_rsrc_alloc() 172 * 173 * Context: Can be called from interrupt or base context. 174 * The "sleepflag" parameter is used by all object allocators to 175 * determine whether to SLEEP for resources or not. 176 */ 177 int 178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num, 179 uint_t sleepflag, tavor_rsrc_t **hdl) 180 { 181 tavor_rsrc_pool_info_t *rsrc_pool; 182 tavor_rsrc_t *tmp_rsrc_hdl; 183 int flag, status = DDI_FAILURE; 184 185 ASSERT(state != NULL); 186 ASSERT(hdl != NULL); 187 188 rsrc_pool = &state->ts_rsrc_hdl[rsrc]; 189 ASSERT(rsrc_pool != NULL); 190 191 /* 192 * Allocate space for the object used to track the resource handle 193 */ 194 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 195 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache, 196 flag); 197 if (tmp_rsrc_hdl == NULL) { 198 return (DDI_FAILURE); 199 } 200 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl)) 201 202 /* 203 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call 204 * to know what type of resource is being freed. 205 */ 206 tmp_rsrc_hdl->rsrc_type = rsrc; 207 208 /* 209 * Depending on resource type, call the appropriate alloc routine 210 */ 211 switch (rsrc_pool->rsrc_type) { 212 case TAVOR_IN_MBOX: 213 case TAVOR_OUT_MBOX: 214 case TAVOR_INTR_IN_MBOX: 215 case TAVOR_INTR_OUT_MBOX: 216 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl); 217 break; 218 219 case TAVOR_QPC: 220 case TAVOR_CQC: 221 case TAVOR_SRQC: 222 case TAVOR_EQC: 223 case TAVOR_RDB: 224 /* 225 * Because these objects are NOT accessed by Tavor driver 226 * software, we set the acc_handle parameter to zero. But 227 * if they are allocated in multiples, we specify here that 228 * they must be aligned on a more restrictive boundary. 229 */ 230 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0, 231 sleepflag, tmp_rsrc_hdl); 232 break; 233 234 case TAVOR_MPT: 235 /* 236 * Because these MPT objects are sometimes accessed by Tavor 237 * driver software (FMR), we set the acc_handle parameter. But 238 * if they are allocated in multiples, we specify here that 239 * they must be aligned on a more restrictive boundary. 240 */ 241 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 242 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 243 break; 244 245 case TAVOR_MCG: 246 /* 247 * Tavor MCG entries are also NOT accessed by Tavor driver 248 * software, but because MCG entries do not have the same 249 * alignnment restrictions we loosen the constraint here. 250 */ 251 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0, 252 sleepflag, tmp_rsrc_hdl); 253 break; 254 255 case TAVOR_MTT: 256 case TAVOR_UDAV: 257 /* 258 * Because MTT segments are among the few HW resources that 259 * may be allocated in odd numbers, we specify a less 260 * restrictive alignment than for the above resources. 261 * 262 * Also because both UDAV and MTT segment objects are read 263 * and/or written by Tavor driver software, we set the 264 * acc_handle parameter to point to the ddi_acc_handle_t for 265 * the Tavor DDR memory. 266 */ 267 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 268 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 269 break; 270 271 case TAVOR_UARPG: 272 /* 273 * Because UAR pages are written by Tavor driver software (for 274 * doorbells), we set the acc_handle parameter to point to 275 * the ddi_acc_handle_t for the Tavor UAR memory. 276 */ 277 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 278 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl); 279 break; 280 281 case TAVOR_MRHDL: 282 case TAVOR_EQHDL: 283 case TAVOR_CQHDL: 284 case TAVOR_SRQHDL: 285 case TAVOR_AHHDL: 286 case TAVOR_QPHDL: 287 case TAVOR_REFCNT: 288 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag, 289 tmp_rsrc_hdl); 290 break; 291 292 case TAVOR_PDHDL: 293 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag, 294 tmp_rsrc_hdl); 295 break; 296 297 default: 298 TAVOR_WARNING(state, "unexpected resource type in alloc"); 299 break; 300 } 301 302 /* 303 * If the resource allocation failed, then free the special resource 304 * tracking structure and return failure. Otherwise return the 305 * handle for the resource tracking structure. 306 */ 307 if (status != DDI_SUCCESS) { 308 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl); 309 tmp_rsrc_hdl = NULL; 310 return (DDI_FAILURE); 311 } else { 312 *hdl = tmp_rsrc_hdl; 313 return (DDI_SUCCESS); 314 } 315 } 316 317 318 /* 319 * tavor_rsrc_free() 320 * Context: Can be called from interrupt or base context. 321 */ 322 void 323 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl) 324 { 325 tavor_rsrc_pool_info_t *rsrc_pool; 326 327 ASSERT(state != NULL); 328 ASSERT(hdl != NULL); 329 330 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type]; 331 ASSERT(rsrc_pool != NULL); 332 333 /* 334 * Depending on resource type, call the appropriate free routine 335 */ 336 switch (rsrc_pool->rsrc_type) { 337 case TAVOR_IN_MBOX: 338 case TAVOR_OUT_MBOX: 339 case TAVOR_INTR_IN_MBOX: 340 case TAVOR_INTR_OUT_MBOX: 341 tavor_rsrc_mbox_free(rsrc_pool, *hdl); 342 break; 343 344 case TAVOR_QPC: 345 case TAVOR_CQC: 346 case TAVOR_SRQC: 347 case TAVOR_EQC: 348 case TAVOR_RDB: 349 case TAVOR_MCG: 350 case TAVOR_MPT: 351 case TAVOR_MTT: 352 case TAVOR_UDAV: 353 case TAVOR_UARPG: 354 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl); 355 break; 356 357 case TAVOR_MRHDL: 358 case TAVOR_EQHDL: 359 case TAVOR_CQHDL: 360 case TAVOR_SRQHDL: 361 case TAVOR_AHHDL: 362 case TAVOR_QPHDL: 363 case TAVOR_REFCNT: 364 tavor_rsrc_swhdl_free(rsrc_pool, *hdl); 365 break; 366 367 case TAVOR_PDHDL: 368 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl); 369 break; 370 371 default: 372 TAVOR_WARNING(state, "unexpected resource type in free"); 373 break; 374 } 375 376 /* 377 * Free the special resource tracking structure, set the handle to 378 * NULL, and return. 379 */ 380 kmem_cache_free(state->ts_rsrc_cache, *hdl); 381 *hdl = NULL; 382 } 383 384 385 /* 386 * tavor_rsrc_init_phase1() 387 * 388 * Completes the first phase of Tavor resource/configuration init. 389 * This involves creating the kmem_cache for the "tavor_rsrc_t" 390 * structs, allocating the space for the resource pool handles, 391 * and setting up the "Out" mailboxes. 392 * 393 * When this function completes, the Tavor driver is ready to 394 * post the following commands which return information only in the 395 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER 396 * If any of these commands are to be posted at this time, they must be 397 * done so only when "spinning" (as the outstanding command list and 398 * EQ setup code has not yet run) 399 * 400 * Context: Only called from attach() path context 401 */ 402 int 403 tavor_rsrc_init_phase1(tavor_state_t *state) 404 { 405 tavor_rsrc_pool_info_t *rsrc_pool; 406 tavor_rsrc_mbox_info_t mbox_info; 407 tavor_rsrc_cleanup_level_t cleanup; 408 tavor_cfg_profile_t *cfgprof; 409 uint64_t num, size; 410 int status; 411 char *rsrc_name; 412 413 ASSERT(state != NULL); 414 415 /* This is where Phase 1 of resource initialization begins */ 416 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0; 417 418 /* Build kmem cache name from Tavor instance */ 419 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 420 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE); 421 422 /* 423 * Create the kmem_cache for "tavor_rsrc_t" structures 424 * (kmem_cache_create will SLEEP until successful) 425 */ 426 state->ts_rsrc_cache = kmem_cache_create(rsrc_name, 427 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 428 429 /* 430 * Allocate an array of tavor_rsrc_pool_info_t's (used in all 431 * subsequent resource allocations) 432 */ 433 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES * 434 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP); 435 436 cfgprof = state->ts_cfg_profile; 437 438 /* 439 * Initialize the resource pool for "Out" mailboxes. Notice that 440 * the number of "Out" mailboxes, their size, and their location 441 * (DDR or system memory) is configurable. By default, however, 442 * all "Out" mailboxes are located in system memory only (because 443 * they are primarily read from and never written to) 444 */ 445 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox); 446 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 447 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 448 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX; 449 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 450 rsrc_pool->rsrc_pool_size = (size * num); 451 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 452 rsrc_pool->rsrc_quantum = size; 453 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 454 rsrc_pool->rsrc_state = state; 455 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM); 456 mbox_info.mbi_num = num; 457 mbox_info.mbi_size = size; 458 mbox_info.mbi_rsrcpool = rsrc_pool; 459 mbox_info.mbi_rsrcname = rsrc_name; 460 status = tavor_rsrc_mbox_init(state, &mbox_info); 461 if (status != DDI_SUCCESS) { 462 tavor_rsrc_fini(state, cleanup); 463 goto rsrcinitp1_fail; 464 } 465 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1; 466 467 /* 468 * Initialize the Tavor "Out" mailbox list. This step actually uses 469 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 470 * "Out" mailboxes, bind them for DMA access, and arrange them into 471 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 472 * for more details) 473 */ 474 status = tavor_outmbox_list_init(state); 475 if (status != DDI_SUCCESS) { 476 tavor_rsrc_fini(state, cleanup); 477 goto rsrcinitp1_fail; 478 } 479 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2; 480 481 /* 482 * Initialize the resource pool for interrupt "Out" mailboxes. Notice 483 * that the number of interrupt "Out" mailboxes, their size, and their 484 * location (DDR or system memory) is configurable. By default, 485 * however, all interrupt "Out" mailboxes are located in system memory 486 * only (because they are primarily read from and never written to) 487 */ 488 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox); 489 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 490 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX]; 491 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX; 492 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 493 rsrc_pool->rsrc_pool_size = (size * num); 494 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 495 rsrc_pool->rsrc_quantum = size; 496 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 497 rsrc_pool->rsrc_state = state; 498 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM); 499 mbox_info.mbi_num = num; 500 mbox_info.mbi_size = size; 501 mbox_info.mbi_rsrcpool = rsrc_pool; 502 mbox_info.mbi_rsrcname = rsrc_name; 503 status = tavor_rsrc_mbox_init(state, &mbox_info); 504 if (status != DDI_SUCCESS) { 505 tavor_rsrc_fini(state, cleanup); 506 goto rsrcinitp1_fail; 507 } 508 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3; 509 510 /* 511 * Initialize the Tavor "Out" mailbox list. This step actually uses 512 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 513 * "Out" mailboxes, bind them for DMA access, and arrange them into 514 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 515 * for more details) 516 */ 517 status = tavor_intr_outmbox_list_init(state); 518 if (status != DDI_SUCCESS) { 519 tavor_rsrc_fini(state, cleanup); 520 goto rsrcinitp1_fail; 521 } 522 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 523 524 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 525 return (DDI_SUCCESS); 526 527 rsrcinitp1_fail: 528 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 529 return (status); 530 } 531 532 533 /* 534 * tavor_rsrc_init_phase2() 535 * Context: Only called from attach() path context 536 */ 537 int 538 tavor_rsrc_init_phase2(tavor_state_t *state) 539 { 540 tavor_rsrc_sw_hdl_info_t hdl_info; 541 tavor_rsrc_hw_entry_info_t entry_info; 542 tavor_rsrc_mbox_info_t mbox_info; 543 tavor_rsrc_pool_info_t *rsrc_pool; 544 tavor_rsrc_cleanup_level_t cleanup; 545 tavor_cfg_profile_t *cfgprof; 546 uint64_t num, max, size, num_prealloc; 547 uint64_t ddr_size, fw_size; 548 uint_t mcg_size, mcg_size_shift; 549 uint_t uarscr_size, mttsegment_sz; 550 int status; 551 char *rsrc_name; 552 553 ASSERT(state != NULL); 554 555 /* Phase 2 initialization begins where Phase 1 left off */ 556 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 557 558 /* 559 * Calculate the extent of the DDR size and portion of which that 560 * is already reserved for Tavor firmware. (Note: this information 561 * is available because the QUERY_DDR and QUERY_FW commands have 562 * been posted to Tavor firmware prior to calling this routine) 563 */ 564 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1; 565 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1; 566 567 /* Build the DDR vmem arena name from Tavor instance */ 568 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 569 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM); 570 571 /* 572 * Do a vmem_create for the entire DDR range (not including the 573 * portion consumed by Tavor firmware). This creates the vmem arena 574 * from which all other DDR objects (specifically, tables of HW 575 * entries) will be allocated. 576 */ 577 state->ts_ddrvmem = vmem_create(rsrc_name, 578 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size), 579 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 580 if (state->ts_ddrvmem == NULL) { 581 tavor_rsrc_fini(state, cleanup); 582 goto rsrcinitp2_fail; 583 } 584 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5; 585 586 /* 587 * Initialize the resource pools for all objects that exist in 588 * Tavor DDR memory. This includes ("In") mailboxes, context tables 589 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects. 590 */ 591 cfgprof = state->ts_cfg_profile; 592 593 /* 594 * Initialize the resource pool for the MPT table entries. Notice 595 * that the number of MPTs is configurable. The configured value must 596 * be less that the maximum value (obtained from the QUERY_DEV_LIM 597 * command) or the initialization will fail. Note also that a certain 598 * number of MPTs must be set aside for Tavor firmware use. 599 */ 600 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 601 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 602 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); 603 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 604 rsrc_pool->rsrc_type = TAVOR_MPT; 605 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 606 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num); 607 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT; 608 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE; 609 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num); 610 rsrc_pool->rsrc_state = state; 611 rsrc_pool->rsrc_start = NULL; 612 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM); 613 entry_info.hwi_num = num; 614 entry_info.hwi_max = max; 615 entry_info.hwi_prealloc = num_prealloc; 616 entry_info.hwi_rsrcpool = rsrc_pool; 617 entry_info.hwi_rsrcname = rsrc_name; 618 status = tavor_rsrc_hw_entries_init(state, &entry_info); 619 if (status != DDI_SUCCESS) { 620 tavor_rsrc_fini(state, cleanup); 621 goto rsrcinitp2_fail; 622 } 623 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6; 624 625 /* 626 * Initialize the resource pool for the MTT table entries. Notice 627 * that the number of MTTs is configurable. The configured value must 628 * be less that the maximum value (obtained from the QUERY_DEV_LIM 629 * command) or the initialization will fail. Note also that a certain 630 * number of MTT segments must be set aside for Tavor firmware use. 631 */ 632 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT); 633 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg); 634 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg); 635 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg); 636 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 637 rsrc_pool->rsrc_type = TAVOR_MTT; 638 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 639 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num); 640 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT; 641 rsrc_pool->rsrc_quantum = mttsegment_sz; 642 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num); 643 rsrc_pool->rsrc_state = state; 644 rsrc_pool->rsrc_start = NULL; 645 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM); 646 entry_info.hwi_num = num; 647 entry_info.hwi_max = max; 648 entry_info.hwi_prealloc = num_prealloc; 649 entry_info.hwi_rsrcpool = rsrc_pool; 650 entry_info.hwi_rsrcname = rsrc_name; 651 status = tavor_rsrc_hw_entries_init(state, &entry_info); 652 if (status != DDI_SUCCESS) { 653 tavor_rsrc_fini(state, cleanup); 654 goto rsrcinitp2_fail; 655 } 656 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7; 657 658 /* 659 * Initialize the resource pool for the QPC table entries. Notice 660 * that the number of QPs is configurable. The configured value must 661 * be less that the maximum value (obtained from the QUERY_DEV_LIM 662 * command) or the initialization will fail. Note also that a certain 663 * number of QP contexts must be set aside for Tavor firmware use. 664 */ 665 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 666 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 667 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp); 668 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 669 rsrc_pool->rsrc_type = TAVOR_QPC; 670 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 671 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num); 672 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT; 673 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE; 674 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num); 675 rsrc_pool->rsrc_state = state; 676 rsrc_pool->rsrc_start = NULL; 677 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM); 678 entry_info.hwi_num = num; 679 entry_info.hwi_max = max; 680 entry_info.hwi_prealloc = num_prealloc; 681 entry_info.hwi_rsrcpool = rsrc_pool; 682 entry_info.hwi_rsrcname = rsrc_name; 683 status = tavor_rsrc_hw_entries_init(state, &entry_info); 684 if (status != DDI_SUCCESS) { 685 tavor_rsrc_fini(state, cleanup); 686 goto rsrcinitp2_fail; 687 } 688 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8; 689 690 /* 691 * Initialize the resource pool for the RDB table entries. Notice 692 * that the number of RDBs is configurable. The configured value must 693 * be less that the maximum value (obtained from the QUERY_DEV_LIM 694 * command) or the initialization will fail. 695 */ 696 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb); 697 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob); 698 num_prealloc = 0; 699 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 700 rsrc_pool->rsrc_type = TAVOR_RDB; 701 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 702 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num); 703 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT; 704 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE; 705 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num); 706 rsrc_pool->rsrc_state = state; 707 rsrc_pool->rsrc_start = NULL; 708 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM); 709 entry_info.hwi_num = num; 710 entry_info.hwi_max = max; 711 entry_info.hwi_prealloc = num_prealloc; 712 entry_info.hwi_rsrcpool = rsrc_pool; 713 entry_info.hwi_rsrcname = rsrc_name; 714 status = tavor_rsrc_hw_entries_init(state, &entry_info); 715 if (status != DDI_SUCCESS) { 716 tavor_rsrc_fini(state, cleanup); 717 goto rsrcinitp2_fail; 718 } 719 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9; 720 721 /* 722 * Initialize the resource pool for the CQC table entries. Notice 723 * that the number of CQs is configurable. The configured value must 724 * be less that the maximum value (obtained from the QUERY_DEV_LIM 725 * command) or the initialization will fail. Note also that a certain 726 * number of CQ contexts must be set aside for Tavor firmware use. 727 */ 728 num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 729 max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 730 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq); 731 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 732 rsrc_pool->rsrc_type = TAVOR_CQC; 733 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 734 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num); 735 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT; 736 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE; 737 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num); 738 rsrc_pool->rsrc_state = state; 739 rsrc_pool->rsrc_start = NULL; 740 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM); 741 entry_info.hwi_num = num; 742 entry_info.hwi_max = max; 743 entry_info.hwi_prealloc = num_prealloc; 744 entry_info.hwi_rsrcpool = rsrc_pool; 745 entry_info.hwi_rsrcname = rsrc_name; 746 status = tavor_rsrc_hw_entries_init(state, &entry_info); 747 if (status != DDI_SUCCESS) { 748 tavor_rsrc_fini(state, cleanup); 749 goto rsrcinitp2_fail; 750 } 751 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10; 752 753 /* 754 * Initialize the resource pool for the Extended QPC table entries. 755 * Notice that the number of EQPCs must be the same as the number 756 * of QP contexts. So the initialization is constructed in a 757 * similar way as above (for TAVOR_QPC). One notable difference 758 * here, however, is that by setting the rsrc_quantum field to 759 * zero (indicating a zero-sized object) we indicate that the 760 * object is not allocatable. The EQPC table is, in fact, managed 761 * internally by the hardware and it is, therefore, unnecessary to 762 * initialize an additional vmem_arena for this type of object. 763 */ 764 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 765 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 766 num_prealloc = 0; 767 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 768 rsrc_pool->rsrc_type = TAVOR_EQPC; 769 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 770 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num); 771 rsrc_pool->rsrc_shift = 0; 772 rsrc_pool->rsrc_quantum = 0; 773 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE; 774 rsrc_pool->rsrc_state = state; 775 rsrc_pool->rsrc_start = NULL; 776 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM); 777 entry_info.hwi_num = num; 778 entry_info.hwi_max = max; 779 entry_info.hwi_prealloc = num_prealloc; 780 entry_info.hwi_rsrcpool = rsrc_pool; 781 entry_info.hwi_rsrcname = rsrc_name; 782 status = tavor_rsrc_hw_entries_init(state, &entry_info); 783 if (status != DDI_SUCCESS) { 784 tavor_rsrc_fini(state, cleanup); 785 goto rsrcinitp2_fail; 786 } 787 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11; 788 789 /* 790 * Initialize the resource pool for the UD address vector table 791 * entries. Notice that the number of UDAVs is configurable. The 792 * configured value must be less that the maximum value (obtained 793 * from the QUERY_DEV_LIM command) or the initialization will fail. 794 */ 795 num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 796 max = ((uint64_t)1 << state->ts_devlim.log_max_av); 797 num_prealloc = 0; 798 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 799 rsrc_pool->rsrc_type = TAVOR_UDAV; 800 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 801 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num); 802 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT; 803 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE; 804 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE; 805 rsrc_pool->rsrc_state = state; 806 rsrc_pool->rsrc_start = NULL; 807 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM); 808 entry_info.hwi_num = num; 809 entry_info.hwi_max = max; 810 entry_info.hwi_prealloc = num_prealloc; 811 entry_info.hwi_rsrcpool = rsrc_pool; 812 entry_info.hwi_rsrcname = rsrc_name; 813 status = tavor_rsrc_hw_entries_init(state, &entry_info); 814 if (status != DDI_SUCCESS) { 815 tavor_rsrc_fini(state, cleanup); 816 goto rsrcinitp2_fail; 817 } 818 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12; 819 820 /* 821 * Initialize the resource pool for the UAR scratch table entries. 822 * Notice that the number of UARSCRs is configurable. The configured 823 * value must be less that the maximum value (obtained from the 824 * QUERY_DEV_LIM command) or the initialization will fail. 825 * Like the EQPCs above, UARSCR objects are not allocatable. The 826 * UARSCR table is also managed internally by the hardware and it 827 * is, therefore, unnecessary to initialize an additional vmem_arena 828 * for this type of object. We indicate this by setting the 829 * rsrc_quantum field to zero (indicating a zero-sized object). 830 */ 831 uarscr_size = state->ts_devlim.uarscr_entry_sz; 832 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 833 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 834 PAGESHIFT)); 835 num_prealloc = 0; 836 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 837 rsrc_pool->rsrc_type = TAVOR_UAR_SCR; 838 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 839 rsrc_pool->rsrc_pool_size = (uarscr_size * num); 840 rsrc_pool->rsrc_shift = 0; 841 rsrc_pool->rsrc_quantum = 0; 842 rsrc_pool->rsrc_align = uarscr_size; 843 rsrc_pool->rsrc_state = state; 844 rsrc_pool->rsrc_start = NULL; 845 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM); 846 entry_info.hwi_num = num; 847 entry_info.hwi_max = max; 848 entry_info.hwi_prealloc = num_prealloc; 849 entry_info.hwi_rsrcpool = rsrc_pool; 850 entry_info.hwi_rsrcname = rsrc_name; 851 status = tavor_rsrc_hw_entries_init(state, &entry_info); 852 if (status != DDI_SUCCESS) { 853 tavor_rsrc_fini(state, cleanup); 854 goto rsrcinitp2_fail; 855 } 856 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13; 857 858 /* 859 * Initialize the resource pool for the SRQC table entries. Notice 860 * that the number of SRQs is configurable. The configured value must 861 * be less that the maximum value (obtained from the QUERY_DEV_LIM 862 * command) or the initialization will fail. Note also that a certain 863 * number of SRQ contexts must be set aside for Tavor firmware use. 864 * 865 * Note: We only allocate these resources if SRQ is enabled in the 866 * config profile; see below. 867 */ 868 num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 869 max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 870 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq); 871 872 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 873 rsrc_pool->rsrc_type = TAVOR_SRQC; 874 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 875 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num); 876 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT; 877 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE; 878 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num); 879 rsrc_pool->rsrc_state = state; 880 rsrc_pool->rsrc_start = NULL; 881 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM); 882 entry_info.hwi_num = num; 883 entry_info.hwi_max = max; 884 entry_info.hwi_prealloc = num_prealloc; 885 entry_info.hwi_rsrcpool = rsrc_pool; 886 entry_info.hwi_rsrcname = rsrc_name; 887 888 /* 889 * SRQ support is configurable. Only if SRQ is enabled (the default) 890 * do we actually try to configure these resources. Otherwise, we 891 * simply set the cleanup level and continue on to the next resource 892 */ 893 if (state->ts_cfg_profile->cp_srq_enable != 0) { 894 status = tavor_rsrc_hw_entries_init(state, &entry_info); 895 if (status != DDI_SUCCESS) { 896 tavor_rsrc_fini(state, cleanup); 897 goto rsrcinitp2_fail; 898 } 899 } 900 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14; 901 902 /* 903 * Initialize the resource pool for "In" mailboxes. Notice that 904 * the number of "In" mailboxes, their size, and their location 905 * (DDR or system memory) is configurable. By default, however, 906 * all "In" mailboxes are located in system memory only (because 907 * they are primarily written to and rarely read from) 908 */ 909 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox); 910 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 911 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 912 rsrc_pool->rsrc_type = TAVOR_IN_MBOX; 913 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 914 rsrc_pool->rsrc_pool_size = (size * num); 915 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 916 rsrc_pool->rsrc_quantum = size; 917 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 918 rsrc_pool->rsrc_state = state; 919 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM); 920 mbox_info.mbi_num = num; 921 mbox_info.mbi_size = size; 922 mbox_info.mbi_rsrcpool = rsrc_pool; 923 mbox_info.mbi_rsrcname = rsrc_name; 924 status = tavor_rsrc_mbox_init(state, &mbox_info); 925 if (status != DDI_SUCCESS) { 926 tavor_rsrc_fini(state, cleanup); 927 goto rsrcinitp2_fail; 928 } 929 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15; 930 931 /* 932 * Initialize the Tavor "In" mailbox list. This step actually uses 933 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the 934 * "In" mailboxes, bind them for DMA access, and arrange them into 935 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 936 * for more details) 937 */ 938 status = tavor_inmbox_list_init(state); 939 if (status != DDI_SUCCESS) { 940 tavor_rsrc_fini(state, cleanup); 941 goto rsrcinitp2_fail; 942 } 943 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16; 944 945 /* 946 * Initialize the resource pool for interrupt "In" mailboxes. Notice 947 * that the number of interrupt "In" mailboxes, their size, and their 948 * location (DDR or system memory) is configurable. By default, 949 * however, all interrupt "In" mailboxes are located in system memory 950 * only (because they are primarily written to and rarely read from) 951 */ 952 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox); 953 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 954 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX]; 955 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX; 956 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 957 rsrc_pool->rsrc_pool_size = (size * num); 958 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 959 rsrc_pool->rsrc_quantum = size; 960 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 961 rsrc_pool->rsrc_state = state; 962 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM); 963 mbox_info.mbi_num = num; 964 mbox_info.mbi_size = size; 965 mbox_info.mbi_rsrcpool = rsrc_pool; 966 mbox_info.mbi_rsrcname = rsrc_name; 967 status = tavor_rsrc_mbox_init(state, &mbox_info); 968 if (status != DDI_SUCCESS) { 969 tavor_rsrc_fini(state, cleanup); 970 goto rsrcinitp2_fail; 971 } 972 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17; 973 974 /* 975 * Initialize the Tavor interrupt "In" mailbox list. This step 976 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to 977 * preallocate the interrupt "In" mailboxes, bind them for DMA access, 978 * and arrange them into an easily accessed fast-allocation mechanism 979 * (see tavor_cmd.c for more details) 980 */ 981 status = tavor_intr_inmbox_list_init(state); 982 if (status != DDI_SUCCESS) { 983 tavor_rsrc_fini(state, cleanup); 984 goto rsrcinitp2_fail; 985 } 986 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18; 987 988 /* 989 * Initialize the Tavor command handling interfaces. This step 990 * sets up the outstanding command tracking mechanism for easy access 991 * and fast allocation (see tavor_cmd.c for more details). 992 */ 993 status = tavor_outstanding_cmdlist_init(state); 994 if (status != DDI_SUCCESS) { 995 tavor_rsrc_fini(state, cleanup); 996 goto rsrcinitp2_fail; 997 } 998 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19; 999 1000 /* 1001 * Calculate (and validate) the size of Multicast Group (MCG) entries 1002 */ 1003 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift); 1004 if (status != DDI_SUCCESS) { 1005 tavor_rsrc_fini(state, cleanup); 1006 goto rsrcinitp2_fail; 1007 } 1008 mcg_size = TAVOR_MCGMEM_SZ(state); 1009 1010 /* 1011 * Initialize the resource pool for the MCG table entries. Notice 1012 * that the number of MCGs is configurable. The configured value must 1013 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1014 * command) or the initialization will fail. Note also that a certain 1015 * number of MCGs must be set aside for Tavor firmware use (they 1016 * correspond to the number of MCGs used by the internal hash 1017 * function. 1018 */ 1019 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1020 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1021 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash); 1022 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1023 rsrc_pool->rsrc_type = TAVOR_MCG; 1024 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1025 rsrc_pool->rsrc_pool_size = (mcg_size * num); 1026 rsrc_pool->rsrc_shift = mcg_size_shift; 1027 rsrc_pool->rsrc_quantum = mcg_size; 1028 rsrc_pool->rsrc_align = mcg_size; 1029 rsrc_pool->rsrc_state = state; 1030 rsrc_pool->rsrc_start = NULL; 1031 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM); 1032 entry_info.hwi_num = num; 1033 entry_info.hwi_max = max; 1034 entry_info.hwi_prealloc = num_prealloc; 1035 entry_info.hwi_rsrcpool = rsrc_pool; 1036 entry_info.hwi_rsrcname = rsrc_name; 1037 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1038 if (status != DDI_SUCCESS) { 1039 tavor_rsrc_fini(state, cleanup); 1040 goto rsrcinitp2_fail; 1041 } 1042 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20; 1043 1044 /* 1045 * Initialize the resource pool for the EQC table entries. Notice 1046 * that the number of EQs is hardcoded. The hardcoded value should 1047 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1048 * command) or the initialization will fail. 1049 */ 1050 num = TAVOR_NUM_EQ; 1051 max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1052 num_prealloc = 0; 1053 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1054 rsrc_pool->rsrc_type = TAVOR_EQC; 1055 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1056 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num); 1057 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT; 1058 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE; 1059 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num); 1060 rsrc_pool->rsrc_state = state; 1061 rsrc_pool->rsrc_start = NULL; 1062 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM); 1063 entry_info.hwi_num = num; 1064 entry_info.hwi_max = max; 1065 entry_info.hwi_prealloc = num_prealloc; 1066 entry_info.hwi_rsrcpool = rsrc_pool; 1067 entry_info.hwi_rsrcname = rsrc_name; 1068 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1069 if (status != DDI_SUCCESS) { 1070 tavor_rsrc_fini(state, cleanup); 1071 goto rsrcinitp2_fail; 1072 } 1073 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21; 1074 1075 /* 1076 * Initialize the resource pools for all objects that exist in 1077 * system memory. This includes PD handles, MR handle, EQ handles, 1078 * QP handles, etc. These objects are almost entirely managed using 1079 * kmem_cache routines. (See comment above for more detail) 1080 */ 1081 1082 /* 1083 * Initialize the resource pool for the PD handles. Notice 1084 * that the number of PDHDLs is configurable. The configured value 1085 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1086 * command) or the initialization will fail. Note also that the PD 1087 * handle has constructor and destructor methods associated with it. 1088 */ 1089 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1090 rsrc_pool->rsrc_type = TAVOR_PDHDL; 1091 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1092 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s); 1093 rsrc_pool->rsrc_state = state; 1094 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE); 1095 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd); 1096 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd); 1097 hdl_info.swi_rsrcpool = rsrc_pool; 1098 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor; 1099 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor; 1100 hdl_info.swi_rsrcname = rsrc_name; 1101 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1102 status = tavor_rsrc_pd_handles_init(state, &hdl_info); 1103 if (status != DDI_SUCCESS) { 1104 tavor_rsrc_fini(state, cleanup); 1105 goto rsrcinitp2_fail; 1106 } 1107 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22; 1108 1109 /* 1110 * Initialize the resource pool for the MR handles. Notice 1111 * that the number of MRHDLs is configurable. The configured value 1112 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1113 * command) or the initialization will fail. 1114 */ 1115 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1116 rsrc_pool->rsrc_type = TAVOR_MRHDL; 1117 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1118 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s); 1119 rsrc_pool->rsrc_state = state; 1120 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE); 1121 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1122 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1123 hdl_info.swi_rsrcpool = rsrc_pool; 1124 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor; 1125 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor; 1126 hdl_info.swi_rsrcname = rsrc_name; 1127 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1128 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1129 if (status != DDI_SUCCESS) { 1130 tavor_rsrc_fini(state, cleanup); 1131 goto rsrcinitp2_fail; 1132 } 1133 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23; 1134 1135 /* 1136 * Initialize the resource pool for the EQ handles. Notice 1137 * that the number of EQHDLs is hardcoded. The hardcoded value 1138 * should be less that the maximum value (obtained from the 1139 * QUERY_DEV_LIM command) or the initialization will fail. 1140 */ 1141 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1142 rsrc_pool->rsrc_type = TAVOR_EQHDL; 1143 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1144 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s); 1145 rsrc_pool->rsrc_state = state; 1146 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE); 1147 hdl_info.swi_num = TAVOR_NUM_EQ; 1148 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1149 hdl_info.swi_rsrcpool = rsrc_pool; 1150 hdl_info.swi_constructor = NULL; 1151 hdl_info.swi_destructor = NULL; 1152 hdl_info.swi_rsrcname = rsrc_name; 1153 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1154 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1155 if (status != DDI_SUCCESS) { 1156 tavor_rsrc_fini(state, cleanup); 1157 goto rsrcinitp2_fail; 1158 } 1159 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24; 1160 1161 /* 1162 * Initialize the resource pool for the CQ handles. Notice 1163 * that the number of CQHDLs is configurable. The configured value 1164 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1165 * command) or the initialization will fail. Note also that the CQ 1166 * handle has constructor and destructor methods associated with it. 1167 */ 1168 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1169 rsrc_pool->rsrc_type = TAVOR_CQHDL; 1170 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1171 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s); 1172 rsrc_pool->rsrc_state = state; 1173 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE); 1174 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 1175 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 1176 hdl_info.swi_rsrcpool = rsrc_pool; 1177 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor; 1178 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor; 1179 hdl_info.swi_rsrcname = rsrc_name; 1180 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1181 TAVOR_SWHDL_TABLE_INIT); 1182 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1183 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1184 if (status != DDI_SUCCESS) { 1185 tavor_rsrc_fini(state, cleanup); 1186 goto rsrcinitp2_fail; 1187 } 1188 1189 /* 1190 * Save away the pointer to the central list of CQ handle pointers 1191 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle 1192 * lookup during EQ event processing. The table is a list of 1193 * tavor_cqhdl_t allocated by the above routine because of the 1194 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t 1195 * as the number of CQs. 1196 */ 1197 state->ts_cqhdl = hdl_info.swi_table_ptr; 1198 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25; 1199 1200 /* 1201 * Initialize the resource pool for the SRQ handles. Notice 1202 * that the number of SRQHDLs is configurable. The configured value 1203 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1204 * command) or the initialization will fail. Note also that the SRQ 1205 * handle has constructor and destructor methods associated with it. 1206 * 1207 * Note: We only allocate these resources if SRQ is enabled in the 1208 * config profile; see below. 1209 */ 1210 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1211 rsrc_pool->rsrc_type = TAVOR_SRQHDL; 1212 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1213 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s); 1214 rsrc_pool->rsrc_state = state; 1215 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE); 1216 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 1217 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 1218 hdl_info.swi_rsrcpool = rsrc_pool; 1219 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor; 1220 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor; 1221 hdl_info.swi_rsrcname = rsrc_name; 1222 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1223 TAVOR_SWHDL_TABLE_INIT); 1224 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1225 1226 /* 1227 * SRQ support is configurable. Only if SRQ is enabled (the default) 1228 * do we actually try to configure these resources. Otherwise, we 1229 * simply set the cleanup level and continue on to the next resource 1230 */ 1231 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1232 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1233 if (status != DDI_SUCCESS) { 1234 tavor_rsrc_fini(state, cleanup); 1235 goto rsrcinitp2_fail; 1236 } 1237 1238 /* 1239 * Save away the pointer to the central list of SRQ handle 1240 * pointers This this is used as a mechanism to enable fast 1241 * SRQnumber-to-SRQhandle lookup. The table is a list of 1242 * tavor_srqhdl_t allocated by the above routine because of the 1243 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many 1244 * tavor_srqhdl_t as the number of SRQs. 1245 */ 1246 state->ts_srqhdl = hdl_info.swi_table_ptr; 1247 } 1248 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26; 1249 1250 /* 1251 * Initialize the resource pool for the address handles. Notice 1252 * that the number of AHHDLs is configurable. The configured value 1253 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1254 * command) or the initialization will fail. 1255 */ 1256 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1257 rsrc_pool->rsrc_type = TAVOR_AHHDL; 1258 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1259 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s); 1260 rsrc_pool->rsrc_state = state; 1261 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE); 1262 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 1263 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av); 1264 hdl_info.swi_rsrcpool = rsrc_pool; 1265 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor; 1266 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor; 1267 hdl_info.swi_rsrcname = rsrc_name; 1268 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1269 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1270 if (status != DDI_SUCCESS) { 1271 tavor_rsrc_fini(state, cleanup); 1272 goto rsrcinitp2_fail; 1273 } 1274 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27; 1275 1276 /* 1277 * Initialize the resource pool for the QP handles. Notice 1278 * that the number of QPHDLs is configurable. The configured value 1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1280 * command) or the initialization will fail. Note also that the QP 1281 * handle has constructor and destructor methods associated with it. 1282 */ 1283 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1284 rsrc_pool->rsrc_type = TAVOR_QPHDL; 1285 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1286 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s); 1287 rsrc_pool->rsrc_state = state; 1288 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE); 1289 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 1290 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 1291 hdl_info.swi_rsrcpool = rsrc_pool; 1292 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor; 1293 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor; 1294 hdl_info.swi_rsrcname = rsrc_name; 1295 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1296 TAVOR_SWHDL_TABLE_INIT); 1297 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1298 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1299 if (status != DDI_SUCCESS) { 1300 tavor_rsrc_fini(state, cleanup); 1301 goto rsrcinitp2_fail; 1302 } 1303 1304 /* 1305 * Save away the pointer to the central list of QP handle pointers 1306 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle 1307 * lookup during CQ event processing. The table is a list of 1308 * tavor_qphdl_t allocated by the above routine because of the 1309 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t 1310 * as the number of QPs. 1311 */ 1312 state->ts_qphdl = hdl_info.swi_table_ptr; 1313 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28; 1314 1315 /* 1316 * Initialize the resource pool for the reference count handles. 1317 * Notice that the number of REFCNTs is configurable, but it's value 1318 * is set to the number of MPTs. Since REFCNTs are used to support 1319 * shared memory regions, it is possible that we might require as 1320 * one REFCNT for every MPT. 1321 */ 1322 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1323 rsrc_pool->rsrc_type = TAVOR_REFCNT; 1324 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1325 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t); 1326 rsrc_pool->rsrc_state = state; 1327 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE); 1328 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1329 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1330 hdl_info.swi_rsrcpool = rsrc_pool; 1331 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor; 1332 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor; 1333 hdl_info.swi_rsrcname = rsrc_name; 1334 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1335 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1336 if (status != DDI_SUCCESS) { 1337 tavor_rsrc_fini(state, cleanup); 1338 goto rsrcinitp2_fail; 1339 } 1340 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29; 1341 1342 /* 1343 * Initialize the resource pool for the MCG handles. Notice that for 1344 * these MCG handles, we are allocating a table of structures (used to 1345 * keep track of the MCG entries that are being written to hardware 1346 * and to speed up multicast attach/detach operations). 1347 */ 1348 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1349 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1350 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT; 1351 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1352 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1353 if (status != DDI_SUCCESS) { 1354 tavor_rsrc_fini(state, cleanup); 1355 goto rsrcinitp2_fail; 1356 } 1357 state->ts_mcghdl = hdl_info.swi_table_ptr; 1358 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30; 1359 1360 /* 1361 * Initialize the resource pools for all objects that exist in 1362 * UAR memory. The only objects that are allocated from UAR memory 1363 * are the UAR pages which are used for holding Tavor hardware's 1364 * doorbell registers. 1365 */ 1366 1367 /* 1368 * Initialize the resource pool for the UAR pages. Notice 1369 * that the number of UARPGs is configurable. The configured value 1370 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1371 * command) or the initialization will fail. Note also that by 1372 * specifying the rsrc_start parameter in advance, we direct the 1373 * initialization routine not to attempt to allocated space from the 1374 * Tavor DDR vmem_arena. 1375 */ 1376 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 1377 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 1378 PAGESHIFT)); 1379 num_prealloc = 0; 1380 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1381 rsrc_pool->rsrc_type = TAVOR_UARPG; 1382 rsrc_pool->rsrc_loc = TAVOR_IN_UAR; 1383 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT); 1384 rsrc_pool->rsrc_shift = PAGESHIFT; 1385 rsrc_pool->rsrc_quantum = PAGESIZE; 1386 rsrc_pool->rsrc_align = PAGESIZE; 1387 rsrc_pool->rsrc_state = state; 1388 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr; 1389 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM); 1390 entry_info.hwi_num = num; 1391 entry_info.hwi_max = max; 1392 entry_info.hwi_prealloc = num_prealloc; 1393 entry_info.hwi_rsrcpool = rsrc_pool; 1394 entry_info.hwi_rsrcname = rsrc_name; 1395 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1396 if (status != DDI_SUCCESS) { 1397 tavor_rsrc_fini(state, cleanup); 1398 goto rsrcinitp2_fail; 1399 } 1400 cleanup = TAVOR_RSRC_CLEANUP_ALL; 1401 1402 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1403 return (DDI_SUCCESS); 1404 1405 rsrcinitp2_fail: 1406 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1407 return (status); 1408 } 1409 1410 1411 /* 1412 * tavor_rsrc_fini() 1413 * Context: Only called from attach() and/or detach() path contexts 1414 */ 1415 void 1416 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean) 1417 { 1418 tavor_rsrc_sw_hdl_info_t hdl_info; 1419 tavor_rsrc_hw_entry_info_t entry_info; 1420 tavor_rsrc_mbox_info_t mbox_info; 1421 tavor_cfg_profile_t *cfgprof; 1422 1423 ASSERT(state != NULL); 1424 1425 cfgprof = state->ts_cfg_profile; 1426 1427 switch (clean) { 1428 /* 1429 * If we add more resources that need to be cleaned up here, we should 1430 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e. 1431 * corresponds to the last resource allocated). 1432 */ 1433 case TAVOR_RSRC_CLEANUP_ALL: 1434 /* Cleanup the UAR page resource pool */ 1435 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1436 tavor_rsrc_hw_entries_fini(state, &entry_info); 1437 /* FALLTHROUGH */ 1438 1439 case TAVOR_RSRC_CLEANUP_LEVEL30: 1440 /* Cleanup the central MCG handle pointers list */ 1441 hdl_info.swi_rsrcpool = NULL; 1442 hdl_info.swi_table_ptr = state->ts_mcghdl; 1443 hdl_info.swi_num = 1444 ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1445 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1446 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1447 /* FALLTHROUGH */ 1448 1449 case TAVOR_RSRC_CLEANUP_LEVEL29: 1450 /* Cleanup the reference count resource pool */ 1451 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1452 hdl_info.swi_table_ptr = NULL; 1453 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1454 /* FALLTHROUGH */ 1455 1456 case TAVOR_RSRC_CLEANUP_LEVEL28: 1457 /* Cleanup the QP handle resource pool */ 1458 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1459 hdl_info.swi_table_ptr = state->ts_qphdl; 1460 hdl_info.swi_num = 1461 ((uint64_t)1 << cfgprof->cp_log_num_qp); 1462 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1463 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1464 /* FALLTHROUGH */ 1465 1466 case TAVOR_RSRC_CLEANUP_LEVEL27: 1467 /* Cleanup the address handle resource pool */ 1468 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1469 hdl_info.swi_table_ptr = NULL; 1470 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1471 /* FALLTHROUGH */ 1472 1473 case TAVOR_RSRC_CLEANUP_LEVEL26: 1474 /* 1475 * Cleanup the SRQ handle resource pool. 1476 * 1477 * Note: We only clean up if SRQ is enabled. Otherwise we 1478 * simply fallthrough to the next resource cleanup. 1479 */ 1480 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1481 hdl_info.swi_rsrcpool = 1482 &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1483 hdl_info.swi_table_ptr = state->ts_srqhdl; 1484 hdl_info.swi_num = 1485 ((uint64_t)1 << cfgprof->cp_log_num_srq); 1486 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1487 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1488 } 1489 /* FALLTHROUGH */ 1490 1491 case TAVOR_RSRC_CLEANUP_LEVEL25: 1492 /* Cleanup the CQ handle resource pool */ 1493 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1494 hdl_info.swi_table_ptr = state->ts_cqhdl; 1495 hdl_info.swi_num = 1496 ((uint64_t)1 << cfgprof->cp_log_num_cq); 1497 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1498 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1499 /* FALLTHROUGH */ 1500 1501 case TAVOR_RSRC_CLEANUP_LEVEL24: 1502 /* Cleanup the EQ handle resource pool */ 1503 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1504 hdl_info.swi_table_ptr = NULL; 1505 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1506 /* FALLTHROUGH */ 1507 1508 case TAVOR_RSRC_CLEANUP_LEVEL23: 1509 /* Cleanup the MR handle resource pool */ 1510 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1511 hdl_info.swi_table_ptr = NULL; 1512 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1513 /* FALLTHROUGH */ 1514 1515 case TAVOR_RSRC_CLEANUP_LEVEL22: 1516 /* Cleanup the PD handle resource pool */ 1517 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1518 hdl_info.swi_table_ptr = NULL; 1519 tavor_rsrc_pd_handles_fini(state, &hdl_info); 1520 /* FALLTHROUGH */ 1521 1522 case TAVOR_RSRC_CLEANUP_LEVEL21: 1523 /* Cleanup the EQC table resource pool */ 1524 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1525 tavor_rsrc_hw_entries_fini(state, &entry_info); 1526 /* FALLTHROUGH */ 1527 1528 case TAVOR_RSRC_CLEANUP_LEVEL20: 1529 /* Cleanup the MCG table resource pool */ 1530 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1531 tavor_rsrc_hw_entries_fini(state, &entry_info); 1532 /* FALLTHROUGH */ 1533 1534 case TAVOR_RSRC_CLEANUP_LEVEL19: 1535 /* Cleanup the outstanding command list */ 1536 tavor_outstanding_cmdlist_fini(state); 1537 /* FALLTHROUGH */ 1538 1539 case TAVOR_RSRC_CLEANUP_LEVEL18: 1540 /* Cleanup the "In" mailbox list */ 1541 tavor_intr_inmbox_list_fini(state); 1542 /* FALLTHROUGH */ 1543 1544 case TAVOR_RSRC_CLEANUP_LEVEL17: 1545 /* Cleanup the interrupt "In" mailbox resource pool */ 1546 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1547 TAVOR_INTR_IN_MBOX]; 1548 tavor_rsrc_mbox_fini(state, &mbox_info); 1549 /* FALLTHROUGH */ 1550 1551 case TAVOR_RSRC_CLEANUP_LEVEL16: 1552 /* Cleanup the "In" mailbox list */ 1553 tavor_inmbox_list_fini(state); 1554 /* FALLTHROUGH */ 1555 1556 case TAVOR_RSRC_CLEANUP_LEVEL15: 1557 /* Cleanup the "In" mailbox resource pool */ 1558 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 1559 tavor_rsrc_mbox_fini(state, &mbox_info); 1560 /* FALLTHROUGH */ 1561 1562 case TAVOR_RSRC_CLEANUP_LEVEL14: 1563 /* 1564 * Cleanup the SRQC table resource pool. 1565 * 1566 * Note: We only clean up if SRQ is enabled. Otherwise we 1567 * simply fallthrough to the next resource cleanup. 1568 */ 1569 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1570 entry_info.hwi_rsrcpool = 1571 &state->ts_rsrc_hdl[TAVOR_SRQC]; 1572 tavor_rsrc_hw_entries_fini(state, &entry_info); 1573 } 1574 /* FALLTHROUGH */ 1575 1576 case TAVOR_RSRC_CLEANUP_LEVEL13: 1577 /* Cleanup the UAR scratch table resource pool */ 1578 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 1579 tavor_rsrc_hw_entries_fini(state, &entry_info); 1580 /* FALLTHROUGH */ 1581 1582 case TAVOR_RSRC_CLEANUP_LEVEL12: 1583 /* Cleanup the UDAV table resource pool */ 1584 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 1585 tavor_rsrc_hw_entries_fini(state, &entry_info); 1586 /* FALLTHROUGH */ 1587 1588 case TAVOR_RSRC_CLEANUP_LEVEL11: 1589 /* Cleanup the EQPC table resource pool */ 1590 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 1591 tavor_rsrc_hw_entries_fini(state, &entry_info); 1592 /* FALLTHROUGH */ 1593 1594 case TAVOR_RSRC_CLEANUP_LEVEL10: 1595 /* Cleanup the CQC table resource pool */ 1596 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC]; 1597 tavor_rsrc_hw_entries_fini(state, &entry_info); 1598 /* FALLTHROUGH */ 1599 1600 case TAVOR_RSRC_CLEANUP_LEVEL9: 1601 /* Cleanup the RDB table resource pool */ 1602 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB]; 1603 tavor_rsrc_hw_entries_fini(state, &entry_info); 1604 /* FALLTHROUGH */ 1605 1606 case TAVOR_RSRC_CLEANUP_LEVEL8: 1607 /* Cleanup the QPC table resource pool */ 1608 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC]; 1609 tavor_rsrc_hw_entries_fini(state, &entry_info); 1610 /* FALLTHROUGH */ 1611 1612 case TAVOR_RSRC_CLEANUP_LEVEL7: 1613 /* Cleanup the MTT table resource pool */ 1614 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT]; 1615 tavor_rsrc_hw_entries_fini(state, &entry_info); 1616 /* FALLTHROUGH */ 1617 1618 case TAVOR_RSRC_CLEANUP_LEVEL6: 1619 /* Cleanup the MPT table resource pool */ 1620 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT]; 1621 tavor_rsrc_hw_entries_fini(state, &entry_info); 1622 /* FALLTHROUGH */ 1623 1624 case TAVOR_RSRC_CLEANUP_LEVEL5: 1625 /* Destroy the vmem arena for DDR memory */ 1626 vmem_destroy(state->ts_ddrvmem); 1627 break; 1628 1629 /* 1630 * The cleanup below comes from the "Phase 1" initialization step. 1631 * (see tavor_rsrc_init_phase1() above) 1632 */ 1633 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE: 1634 /* Cleanup the interrupt "Out" mailbox list */ 1635 tavor_intr_outmbox_list_fini(state); 1636 /* FALLTHROUGH */ 1637 1638 case TAVOR_RSRC_CLEANUP_LEVEL3: 1639 /* Cleanup the "Out" mailbox resource pool */ 1640 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1641 TAVOR_INTR_OUT_MBOX]; 1642 tavor_rsrc_mbox_fini(state, &mbox_info); 1643 /* FALLTHROUGH */ 1644 1645 case TAVOR_RSRC_CLEANUP_LEVEL2: 1646 /* Cleanup the "Out" mailbox list */ 1647 tavor_outmbox_list_fini(state); 1648 /* FALLTHROUGH */ 1649 1650 case TAVOR_RSRC_CLEANUP_LEVEL1: 1651 /* Cleanup the "Out" mailbox resource pool */ 1652 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 1653 tavor_rsrc_mbox_fini(state, &mbox_info); 1654 /* FALLTHROUGH */ 1655 1656 case TAVOR_RSRC_CLEANUP_LEVEL0: 1657 /* Free the array of tavor_rsrc_pool_info_t's */ 1658 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES * 1659 sizeof (tavor_rsrc_pool_info_t)); 1660 kmem_cache_destroy(state->ts_rsrc_cache); 1661 break; 1662 1663 default: 1664 TAVOR_WARNING(state, "unexpected resource cleanup level"); 1665 } 1666 } 1667 1668 1669 /* 1670 * tavor_rsrc_mbox_init() 1671 * Context: Only called from attach() path context 1672 */ 1673 static int 1674 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1675 { 1676 tavor_rsrc_pool_info_t *rsrc_pool; 1677 tavor_rsrc_priv_mbox_t *priv; 1678 vmem_t *vmp; 1679 uint64_t offset; 1680 uint_t dma_xfer_mode; 1681 1682 ASSERT(state != NULL); 1683 ASSERT(info != NULL); 1684 1685 rsrc_pool = info->mbi_rsrcpool; 1686 ASSERT(rsrc_pool != NULL); 1687 1688 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent; 1689 1690 /* Allocate and initialize mailbox private structure */ 1691 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP); 1692 priv->pmb_dip = state->ts_dip; 1693 priv->pmb_acchdl = state->ts_reg_ddrhdl; 1694 priv->pmb_devaccattr = state->ts_reg_accattr; 1695 priv->pmb_xfer_mode = dma_xfer_mode; 1696 1697 /* 1698 * Initialize many of the default DMA attributes. Then set alignment 1699 * and scatter-gather restrictions specific for mailbox memory. 1700 */ 1701 tavor_dma_attr_init(&priv->pmb_dmaattr); 1702 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN; 1703 priv->pmb_dmaattr.dma_attr_sgllen = 1; 1704 1705 rsrc_pool->rsrc_private = priv; 1706 1707 /* Is object in DDR memory or system memory? */ 1708 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1709 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1710 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1711 0, 0, NULL, NULL, VM_SLEEP); 1712 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1713 /* Unable to alloc space for mailboxes */ 1714 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1715 return (DDI_FAILURE); 1716 } 1717 1718 /* Calculate offset and starting point (in DDR) */ 1719 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1720 state->ts_ddr.ddr_baseaddr); 1721 rsrc_pool->rsrc_start = 1722 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1723 offset); 1724 1725 /* Create new vmem arena for the mailboxes */ 1726 vmp = vmem_create(info->mbi_rsrcname, 1727 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1728 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1729 if (vmp == NULL) { 1730 /* Unable to create vmem arena */ 1731 vmem_xfree(state->ts_ddrvmem, 1732 rsrc_pool->rsrc_ddr_offset, 1733 rsrc_pool->rsrc_pool_size); 1734 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1735 return (DDI_FAILURE); 1736 } 1737 rsrc_pool->rsrc_vmp = vmp; 1738 } else { 1739 rsrc_pool->rsrc_ddr_offset = NULL; 1740 rsrc_pool->rsrc_start = NULL; 1741 rsrc_pool->rsrc_vmp = NULL; 1742 } 1743 1744 return (DDI_SUCCESS); 1745 } 1746 1747 1748 /* 1749 * tavor_rsrc_mbox_fini() 1750 * Context: Only called from attach() and/or detach() path contexts 1751 */ 1752 static void 1753 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1754 { 1755 tavor_rsrc_pool_info_t *rsrc_pool; 1756 1757 ASSERT(state != NULL); 1758 ASSERT(info != NULL); 1759 1760 rsrc_pool = info->mbi_rsrcpool; 1761 ASSERT(rsrc_pool != NULL); 1762 1763 /* If mailboxes are DDR memory, then destroy and free up vmem */ 1764 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1765 1766 /* Destroy the specially created mbox vmem arena */ 1767 vmem_destroy(rsrc_pool->rsrc_vmp); 1768 1769 /* Free up the region from the ddr_vmem arena */ 1770 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 1771 rsrc_pool->rsrc_pool_size); 1772 } 1773 1774 /* Free up the private struct */ 1775 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t)); 1776 } 1777 1778 1779 /* 1780 * tavor_rsrc_hw_entries_init() 1781 * Context: Only called from attach() path context 1782 */ 1783 static int 1784 tavor_rsrc_hw_entries_init(tavor_state_t *state, 1785 tavor_rsrc_hw_entry_info_t *info) 1786 { 1787 tavor_rsrc_pool_info_t *rsrc_pool; 1788 tavor_rsrc_t *rsvd_rsrc = NULL; 1789 vmem_t *vmp; 1790 uint64_t num_hwentry, max_hwentry, num_prealloc; 1791 uint64_t offset; 1792 int status; 1793 1794 ASSERT(state != NULL); 1795 ASSERT(info != NULL); 1796 1797 rsrc_pool = info->hwi_rsrcpool; 1798 ASSERT(rsrc_pool != NULL); 1799 num_hwentry = info->hwi_num; 1800 max_hwentry = info->hwi_max; 1801 num_prealloc = info->hwi_prealloc; 1802 1803 /* Make sure number of HW entries makes sense */ 1804 if (num_hwentry > max_hwentry) { 1805 return (DDI_FAILURE); 1806 } 1807 1808 /* 1809 * Determine if we need to allocate DDR space to set up the 1810 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already 1811 * been initialized (as is the case for the UAR page init). 1812 */ 1813 if (rsrc_pool->rsrc_start == NULL) { 1814 /* Make sure HW entries table is aligned as specified */ 1815 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1816 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1817 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT); 1818 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1819 /* Unable to alloc space for aligned HW table */ 1820 return (DDI_FAILURE); 1821 } 1822 1823 /* Calculate offset and starting point (in DDR) */ 1824 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1825 state->ts_ddr.ddr_baseaddr); 1826 rsrc_pool->rsrc_start = 1827 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1828 offset); 1829 } else { 1830 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start; 1831 } 1832 1833 /* 1834 * Create new vmem arena for the HW entries table (if rsrc_quantum 1835 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW 1836 * entries are not going to be dynamically allocatable (i.e. they 1837 * won't be allocated/freed through tavor_rsrc_alloc/free). This 1838 * latter option is used for EQPC and UARSCR resource which are, in 1839 * fact, managed by the Tavor hardware. 1840 */ 1841 if (rsrc_pool->rsrc_quantum != 0) { 1842 vmp = vmem_create(info->hwi_rsrcname, 1843 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1844 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1845 if (vmp == NULL) { 1846 /* Unable to create vmem arena */ 1847 if (rsrc_pool->rsrc_ddr_offset != 1848 rsrc_pool->rsrc_start) { 1849 vmem_xfree(state->ts_ddrvmem, 1850 rsrc_pool->rsrc_ddr_offset, 1851 rsrc_pool->rsrc_pool_size); 1852 } 1853 return (DDI_FAILURE); 1854 } 1855 rsrc_pool->rsrc_vmp = vmp; 1856 } else { 1857 rsrc_pool->rsrc_vmp = NULL; 1858 } 1859 1860 /* The first HW entries may be reserved by Tavor firmware */ 1861 if (num_prealloc != 0) { 1862 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type, 1863 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc); 1864 if (status != DDI_SUCCESS) { 1865 /* Unable to preallocate the reserved HW entries */ 1866 if (rsrc_pool->rsrc_vmp != NULL) { 1867 vmem_destroy(rsrc_pool->rsrc_vmp); 1868 } 1869 if (rsrc_pool->rsrc_ddr_offset != 1870 rsrc_pool->rsrc_start) { 1871 vmem_xfree(state->ts_ddrvmem, 1872 rsrc_pool->rsrc_ddr_offset, 1873 rsrc_pool->rsrc_pool_size); 1874 } 1875 return (DDI_FAILURE); 1876 } 1877 } 1878 rsrc_pool->rsrc_private = rsvd_rsrc; 1879 1880 return (DDI_SUCCESS); 1881 } 1882 1883 1884 /* 1885 * tavor_rsrc_hw_entries_fini() 1886 * Context: Only called from attach() and/or detach() path contexts 1887 */ 1888 static void 1889 tavor_rsrc_hw_entries_fini(tavor_state_t *state, 1890 tavor_rsrc_hw_entry_info_t *info) 1891 { 1892 tavor_rsrc_pool_info_t *rsrc_pool; 1893 tavor_rsrc_t *rsvd_rsrc; 1894 1895 ASSERT(state != NULL); 1896 ASSERT(info != NULL); 1897 1898 rsrc_pool = info->hwi_rsrcpool; 1899 ASSERT(rsrc_pool != NULL); 1900 1901 /* Free up any "reserved" (i.e. preallocated) HW entries */ 1902 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private; 1903 if (rsvd_rsrc != NULL) { 1904 tavor_rsrc_free(state, &rsvd_rsrc); 1905 } 1906 1907 /* 1908 * If we've actually setup a vmem arena for the HW entries, then 1909 * destroy it now 1910 */ 1911 if (rsrc_pool->rsrc_vmp != NULL) { 1912 vmem_destroy(rsrc_pool->rsrc_vmp); 1913 } 1914 1915 /* 1916 * Determine if a region was allocated from the tavor_ddr_vmem 1917 * arena (and free it up if necessary) 1918 */ 1919 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) { 1920 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 1921 rsrc_pool->rsrc_pool_size); 1922 } 1923 } 1924 1925 1926 /* 1927 * tavor_rsrc_sw_handles_init() 1928 * Context: Only called from attach() path context 1929 */ 1930 /* ARGSUSED */ 1931 static int 1932 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 1933 { 1934 tavor_rsrc_pool_info_t *rsrc_pool; 1935 uint64_t num_swhdl, max_swhdl, prealloc_sz; 1936 1937 ASSERT(state != NULL); 1938 ASSERT(info != NULL); 1939 1940 rsrc_pool = info->swi_rsrcpool; 1941 ASSERT(rsrc_pool != NULL); 1942 num_swhdl = info->swi_num; 1943 max_swhdl = info->swi_max; 1944 prealloc_sz = info->swi_prealloc_sz; 1945 1946 /* Make sure number of SW handles makes sense */ 1947 if (num_swhdl > max_swhdl) { 1948 return (DDI_FAILURE); 1949 } 1950 1951 /* 1952 * Depending on the flags parameter, create a kmem_cache for some 1953 * number of software handle structures. Note: kmem_cache_create() 1954 * will SLEEP until successful. 1955 */ 1956 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) { 1957 rsrc_pool->rsrc_private = kmem_cache_create( 1958 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0, 1959 info->swi_constructor, info->swi_destructor, NULL, 1960 rsrc_pool->rsrc_state, NULL, 0); 1961 } 1962 1963 /* Allocate the central list of SW handle pointers */ 1964 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) { 1965 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz, 1966 KM_SLEEP); 1967 } 1968 1969 return (DDI_SUCCESS); 1970 } 1971 1972 1973 /* 1974 * tavor_rsrc_sw_handles_fini() 1975 * Context: Only called from attach() and/or detach() path contexts 1976 */ 1977 /* ARGSUSED */ 1978 static void 1979 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 1980 { 1981 tavor_rsrc_pool_info_t *rsrc_pool; 1982 uint64_t num_swhdl, prealloc_sz; 1983 1984 ASSERT(state != NULL); 1985 ASSERT(info != NULL); 1986 1987 rsrc_pool = info->swi_rsrcpool; 1988 num_swhdl = info->swi_num; 1989 prealloc_sz = info->swi_prealloc_sz; 1990 1991 /* 1992 * If a "software handle" kmem_cache exists for this resource, then 1993 * destroy it now 1994 */ 1995 if (rsrc_pool != NULL) { 1996 kmem_cache_destroy(rsrc_pool->rsrc_private); 1997 } 1998 1999 /* Free up this central list of SW handle pointers */ 2000 if (info->swi_table_ptr != NULL) { 2001 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz); 2002 } 2003 } 2004 2005 2006 /* 2007 * tavor_rsrc_pd_handles_init() 2008 * Context: Only called from attach() path context 2009 */ 2010 static int 2011 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2012 { 2013 tavor_rsrc_pool_info_t *rsrc_pool; 2014 vmem_t *vmp; 2015 char vmem_name[TAVOR_RSRC_NAME_MAXLEN]; 2016 int status; 2017 2018 ASSERT(state != NULL); 2019 ASSERT(info != NULL); 2020 2021 rsrc_pool = info->swi_rsrcpool; 2022 ASSERT(rsrc_pool != NULL); 2023 2024 /* Initialize the resource pool for software handle table */ 2025 status = tavor_rsrc_sw_handles_init(state, info); 2026 if (status != DDI_SUCCESS) { 2027 return (DDI_FAILURE); 2028 } 2029 2030 /* Build vmem arena name from Tavor instance */ 2031 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM); 2032 2033 /* Create new vmem arena for PD numbers */ 2034 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL, 2035 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 2036 if (vmp == NULL) { 2037 /* Unable to create vmem arena */ 2038 info->swi_table_ptr = NULL; 2039 tavor_rsrc_sw_handles_fini(state, info); 2040 return (DDI_FAILURE); 2041 } 2042 rsrc_pool->rsrc_vmp = vmp; 2043 2044 return (DDI_SUCCESS); 2045 } 2046 2047 2048 /* 2049 * tavor_rsrc_pd_handles_fini() 2050 * Context: Only called from attach() and/or detach() path contexts 2051 */ 2052 static void 2053 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2054 { 2055 tavor_rsrc_pool_info_t *rsrc_pool; 2056 2057 ASSERT(state != NULL); 2058 ASSERT(info != NULL); 2059 2060 rsrc_pool = info->swi_rsrcpool; 2061 2062 /* Destroy the specially created UAR scratch table vmem arena */ 2063 vmem_destroy(rsrc_pool->rsrc_vmp); 2064 2065 /* Destroy the "tavor_sw_pd_t" kmem_cache */ 2066 tavor_rsrc_sw_handles_fini(state, info); 2067 } 2068 2069 2070 /* 2071 * tavor_rsrc_mbox_alloc() 2072 * Context: Only called from attach() path context 2073 */ 2074 static int 2075 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2076 tavor_rsrc_t *hdl) 2077 { 2078 tavor_rsrc_priv_mbox_t *priv; 2079 void *addr; 2080 caddr_t kaddr; 2081 uint64_t offset; 2082 size_t real_len, temp_len; 2083 int status; 2084 2085 ASSERT(pool_info != NULL); 2086 ASSERT(hdl != NULL); 2087 2088 /* Get the private pointer for the mailboxes */ 2089 priv = pool_info->rsrc_private; 2090 ASSERT(priv != NULL); 2091 2092 /* 2093 * Allocate a DMA handle for the mailbox. This will be used for 2094 * two purposes (potentially). First, it could be used below in 2095 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from 2096 * system memory. Second, it is definitely used later to bind 2097 * the mailbox for DMA access from/by the hardware. 2098 */ 2099 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr, 2100 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl); 2101 if (status != DDI_SUCCESS) { 2102 return (DDI_FAILURE); 2103 } 2104 2105 /* Is mailbox in DDR memory or system memory? */ 2106 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2107 /* Use vmem_alloc() to get DDR address of mbox */ 2108 hdl->tr_len = (num * pool_info->rsrc_quantum); 2109 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len, 2110 VM_SLEEP); 2111 if (addr == NULL) { 2112 /* No more DDR available for mailbox entries */ 2113 ddi_dma_free_handle(&hdl->tr_dmahdl); 2114 return (DDI_FAILURE); 2115 } 2116 hdl->tr_acchdl = priv->pmb_acchdl; 2117 2118 /* Calculate kernel virtual address (from the DDR offset) */ 2119 offset = ((uintptr_t)addr - 2120 (uintptr_t)pool_info->rsrc_ddr_offset); 2121 hdl->tr_addr = (void *)(uintptr_t)(offset + 2122 (uintptr_t)pool_info->rsrc_start); 2123 2124 } else { /* TAVOR_IN_SYSMEM */ 2125 2126 /* Use ddi_dma_mem_alloc() to get memory for mailbox */ 2127 temp_len = (num * pool_info->rsrc_quantum); 2128 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len, 2129 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP, 2130 NULL, &kaddr, &real_len, &hdl->tr_acchdl); 2131 if (status != DDI_SUCCESS) { 2132 /* No more sys memory available for mailbox entries */ 2133 ddi_dma_free_handle(&hdl->tr_dmahdl); 2134 return (DDI_FAILURE); 2135 } 2136 hdl->tr_addr = (void *)kaddr; 2137 hdl->tr_len = real_len; 2138 } 2139 2140 return (DDI_SUCCESS); 2141 } 2142 2143 2144 /* 2145 * tavor_rsrc_mbox_free() 2146 * Context: Can be called from interrupt or base context. 2147 */ 2148 static void 2149 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2150 { 2151 void *addr; 2152 uint64_t offset; 2153 2154 ASSERT(pool_info != NULL); 2155 ASSERT(hdl != NULL); 2156 2157 /* Is mailbox in DDR memory or system memory? */ 2158 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2159 2160 /* Calculate the allocated address (the mbox's DDR offset) */ 2161 offset = ((uintptr_t)hdl->tr_addr - 2162 (uintptr_t)pool_info->rsrc_start); 2163 addr = (void *)(uintptr_t)(offset + 2164 (uintptr_t)pool_info->rsrc_ddr_offset); 2165 2166 /* Use vmem_free() to free up DDR memory for mailbox */ 2167 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len); 2168 2169 } else { /* TAVOR_IN_SYSMEM */ 2170 2171 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */ 2172 ddi_dma_mem_free(&hdl->tr_acchdl); 2173 } 2174 2175 /* Free the DMA handle for the mailbox */ 2176 ddi_dma_free_handle(&hdl->tr_dmahdl); 2177 } 2178 2179 2180 /* 2181 * tavor_rsrc_hw_entry_alloc() 2182 * Context: Can be called from interrupt or base context. 2183 */ 2184 static int 2185 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2186 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag, 2187 tavor_rsrc_t *hdl) 2188 { 2189 void *addr; 2190 uint64_t offset; 2191 uint32_t align; 2192 int flag; 2193 2194 ASSERT(pool_info != NULL); 2195 ASSERT(hdl != NULL); 2196 2197 /* 2198 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not 2199 * use dma_handle (because they are in Tavor locally attached DDR 2200 * memory) and, generally, don't use the acc_handle (because the 2201 * entries are not directly accessed by software). The exceptions 2202 * to this rule are the UARPG and UDAV entries. 2203 */ 2204 2205 /* 2206 * Use vmem_xalloc() to get a properly aligned pointer (based on 2207 * the number requested) to the HW entry(ies). This handles the 2208 * cases (for special QPCs and for RDB entries) where we need more 2209 * than one and need to ensure that they are properly aligned. 2210 */ 2211 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2212 hdl->tr_len = (num * pool_info->rsrc_quantum); 2213 align = (num_align * pool_info->rsrc_quantum); 2214 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len, 2215 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT); 2216 if (addr == NULL) { 2217 /* No more HW entries available */ 2218 return (DDI_FAILURE); 2219 } 2220 2221 /* If an access handle was provided, fill it in */ 2222 if (acc_handle != 0) { 2223 hdl->tr_acchdl = acc_handle; 2224 } 2225 2226 /* Calculate vaddr and HW table index (from the DDR offset) */ 2227 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset); 2228 hdl->tr_addr = (void *)(uintptr_t)(offset + 2229 (uintptr_t)pool_info->rsrc_start); 2230 hdl->tr_indx = (offset >> pool_info->rsrc_shift); 2231 2232 return (DDI_SUCCESS); 2233 } 2234 2235 2236 /* 2237 * tavor_rsrc_hw_entry_free() 2238 * Context: Can be called from interrupt or base context. 2239 */ 2240 static void 2241 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2242 { 2243 void *addr; 2244 uint64_t offset; 2245 2246 ASSERT(pool_info != NULL); 2247 ASSERT(hdl != NULL); 2248 2249 /* Calculate the allocated address (the entry's DDR offset) */ 2250 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start); 2251 addr = (void *)(uintptr_t)(offset + 2252 (uintptr_t)pool_info->rsrc_ddr_offset); 2253 2254 /* Use vmem_xfree() to free up the HW table entry */ 2255 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len); 2256 } 2257 2258 2259 /* 2260 * tavor_rsrc_swhdl_alloc() 2261 * Context: Can be called from interrupt or base context. 2262 */ 2263 static int 2264 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2265 tavor_rsrc_t *hdl) 2266 { 2267 void *addr; 2268 int flag; 2269 2270 ASSERT(pool_info != NULL); 2271 ASSERT(hdl != NULL); 2272 2273 /* Allocate the software handle structure */ 2274 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 2275 addr = kmem_cache_alloc(pool_info->rsrc_private, flag); 2276 if (addr == NULL) { 2277 return (DDI_FAILURE); 2278 } 2279 hdl->tr_len = pool_info->rsrc_quantum; 2280 hdl->tr_addr = addr; 2281 2282 return (DDI_SUCCESS); 2283 } 2284 2285 2286 /* 2287 * tavor_rsrc_swhdl_free() 2288 * Context: Can be called from interrupt or base context. 2289 */ 2290 static void 2291 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2292 { 2293 ASSERT(pool_info != NULL); 2294 ASSERT(hdl != NULL); 2295 2296 /* Free the software handle structure */ 2297 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr); 2298 } 2299 2300 2301 /* 2302 * tavor_rsrc_pdhdl_alloc() 2303 * Context: Can be called from interrupt or base context. 2304 */ 2305 static int 2306 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2307 tavor_rsrc_t *hdl) 2308 { 2309 tavor_pdhdl_t addr; 2310 void *tmpaddr; 2311 int flag, status; 2312 2313 ASSERT(pool_info != NULL); 2314 ASSERT(hdl != NULL); 2315 2316 /* Allocate the software handle */ 2317 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl); 2318 if (status != DDI_SUCCESS) { 2319 return (DDI_FAILURE); 2320 } 2321 addr = (tavor_pdhdl_t)hdl->tr_addr; 2322 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr)) 2323 2324 /* Allocate a PD number for the handle */ 2325 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2326 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag); 2327 if (tmpaddr == NULL) { 2328 /* No more PD number entries available */ 2329 tavor_rsrc_swhdl_free(pool_info, hdl); 2330 return (DDI_FAILURE); 2331 } 2332 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr; 2333 addr->pd_rsrcp = hdl; 2334 hdl->tr_indx = addr->pd_pdnum; 2335 2336 return (DDI_SUCCESS); 2337 } 2338 2339 2340 /* 2341 * tavor_rsrc_pdhdl_free() 2342 * Context: Can be called from interrupt or base context. 2343 */ 2344 static void 2345 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2346 { 2347 ASSERT(pool_info != NULL); 2348 ASSERT(hdl != NULL); 2349 2350 /* Use vmem_free() to free up the PD number */ 2351 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1); 2352 2353 /* Free the software handle structure */ 2354 tavor_rsrc_swhdl_free(pool_info, hdl); 2355 } 2356 2357 2358 /* 2359 * tavor_rsrc_pdhdl_constructor() 2360 * Context: Can be called from interrupt or base context. 2361 */ 2362 /* ARGSUSED */ 2363 static int 2364 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags) 2365 { 2366 tavor_pdhdl_t pdhdl; 2367 tavor_state_t *state; 2368 2369 pdhdl = (tavor_pdhdl_t)pd; 2370 state = (tavor_state_t *)priv; 2371 2372 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER, 2373 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2374 2375 return (DDI_SUCCESS); 2376 } 2377 2378 2379 /* 2380 * tavor_rsrc_pdhdl_destructor() 2381 * Context: Can be called from interrupt or base context. 2382 */ 2383 /* ARGSUSED */ 2384 static void 2385 tavor_rsrc_pdhdl_destructor(void *pd, void *priv) 2386 { 2387 tavor_pdhdl_t pdhdl; 2388 2389 pdhdl = (tavor_pdhdl_t)pd; 2390 2391 mutex_destroy(&pdhdl->pd_lock); 2392 } 2393 2394 2395 /* 2396 * tavor_rsrc_cqhdl_constructor() 2397 * Context: Can be called from interrupt or base context. 2398 */ 2399 /* ARGSUSED */ 2400 static int 2401 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags) 2402 { 2403 tavor_cqhdl_t cqhdl; 2404 tavor_state_t *state; 2405 2406 cqhdl = (tavor_cqhdl_t)cq; 2407 state = (tavor_state_t *)priv; 2408 2409 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER, 2410 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2411 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER, 2412 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2413 2414 return (DDI_SUCCESS); 2415 } 2416 2417 2418 /* 2419 * tavor_rsrc_cqhdl_destructor() 2420 * Context: Can be called from interrupt or base context. 2421 */ 2422 /* ARGSUSED */ 2423 static void 2424 tavor_rsrc_cqhdl_destructor(void *cq, void *priv) 2425 { 2426 tavor_cqhdl_t cqhdl; 2427 2428 cqhdl = (tavor_cqhdl_t)cq; 2429 2430 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock); 2431 mutex_destroy(&cqhdl->cq_lock); 2432 } 2433 2434 2435 /* 2436 * tavor_rsrc_qphdl_constructor() 2437 * Context: Can be called from interrupt or base context. 2438 */ 2439 /* ARGSUSED */ 2440 static int 2441 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags) 2442 { 2443 tavor_qphdl_t qphdl; 2444 tavor_state_t *state; 2445 2446 qphdl = (tavor_qphdl_t)qp; 2447 state = (tavor_state_t *)priv; 2448 2449 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER, 2450 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2451 2452 return (DDI_SUCCESS); 2453 } 2454 2455 2456 /* 2457 * tavor_rsrc_qphdl_destructor() 2458 * Context: Can be called from interrupt or base context. 2459 */ 2460 /* ARGSUSED */ 2461 static void 2462 tavor_rsrc_qphdl_destructor(void *qp, void *priv) 2463 { 2464 tavor_qphdl_t qphdl; 2465 2466 qphdl = (tavor_qphdl_t)qp; 2467 2468 mutex_destroy(&qphdl->qp_lock); 2469 } 2470 2471 2472 /* 2473 * tavor_rsrc_srqhdl_constructor() 2474 * Context: Can be called from interrupt or base context. 2475 */ 2476 /* ARGSUSED */ 2477 static int 2478 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags) 2479 { 2480 tavor_srqhdl_t srqhdl; 2481 tavor_state_t *state; 2482 2483 srqhdl = (tavor_srqhdl_t)srq; 2484 state = (tavor_state_t *)priv; 2485 2486 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER, 2487 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2488 2489 return (DDI_SUCCESS); 2490 } 2491 2492 2493 /* 2494 * tavor_rsrc_srqhdl_destructor() 2495 * Context: Can be called from interrupt or base context. 2496 */ 2497 /* ARGSUSED */ 2498 static void 2499 tavor_rsrc_srqhdl_destructor(void *srq, void *priv) 2500 { 2501 tavor_srqhdl_t srqhdl; 2502 2503 srqhdl = (tavor_srqhdl_t)srq; 2504 2505 mutex_destroy(&srqhdl->srq_lock); 2506 } 2507 2508 2509 /* 2510 * tavor_rsrc_refcnt_constructor() 2511 * Context: Can be called from interrupt or base context. 2512 */ 2513 /* ARGSUSED */ 2514 static int 2515 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags) 2516 { 2517 tavor_sw_refcnt_t *refcnt; 2518 tavor_state_t *state; 2519 2520 refcnt = (tavor_sw_refcnt_t *)rc; 2521 state = (tavor_state_t *)priv; 2522 2523 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER, 2524 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2525 2526 return (DDI_SUCCESS); 2527 } 2528 2529 2530 /* 2531 * tavor_rsrc_refcnt_destructor() 2532 * Context: Can be called from interrupt or base context. 2533 */ 2534 /* ARGSUSED */ 2535 static void 2536 tavor_rsrc_refcnt_destructor(void *rc, void *priv) 2537 { 2538 tavor_sw_refcnt_t *refcnt; 2539 2540 refcnt = (tavor_sw_refcnt_t *)rc; 2541 2542 mutex_destroy(&refcnt->swrc_lock); 2543 } 2544 2545 2546 /* 2547 * tavor_rsrc_ahhdl_constructor() 2548 * Context: Can be called from interrupt or base context. 2549 */ 2550 /* ARGSUSED */ 2551 static int 2552 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags) 2553 { 2554 tavor_ahhdl_t ahhdl; 2555 tavor_state_t *state; 2556 2557 ahhdl = (tavor_ahhdl_t)ah; 2558 state = (tavor_state_t *)priv; 2559 2560 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER, 2561 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2562 2563 return (DDI_SUCCESS); 2564 } 2565 2566 2567 /* 2568 * tavor_rsrc_ahhdl_destructor() 2569 * Context: Can be called from interrupt or base context. 2570 */ 2571 /* ARGSUSED */ 2572 static void 2573 tavor_rsrc_ahhdl_destructor(void *ah, void *priv) 2574 { 2575 tavor_ahhdl_t ahhdl; 2576 2577 ahhdl = (tavor_ahhdl_t)ah; 2578 2579 mutex_destroy(&ahhdl->ah_lock); 2580 } 2581 2582 2583 /* 2584 * tavor_rsrc_mrhdl_constructor() 2585 * Context: Can be called from interrupt or base context. 2586 */ 2587 /* ARGSUSED */ 2588 static int 2589 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags) 2590 { 2591 tavor_mrhdl_t mrhdl; 2592 tavor_state_t *state; 2593 2594 mrhdl = (tavor_mrhdl_t)mr; 2595 state = (tavor_state_t *)priv; 2596 2597 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER, 2598 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2599 2600 return (DDI_SUCCESS); 2601 } 2602 2603 2604 /* 2605 * tavor_rsrc_mrhdl_destructor() 2606 * Context: Can be called from interrupt or base context. 2607 */ 2608 /* ARGSUSED */ 2609 static void 2610 tavor_rsrc_mrhdl_destructor(void *mr, void *priv) 2611 { 2612 tavor_mrhdl_t mrhdl; 2613 2614 mrhdl = (tavor_mrhdl_t)mr; 2615 2616 mutex_destroy(&mrhdl->mr_lock); 2617 } 2618 2619 2620 /* 2621 * tavor_rsrc_mcg_entry_get_size() 2622 */ 2623 static int 2624 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift) 2625 { 2626 uint_t num_qp_per_mcg, max_qp_per_mcg, log2; 2627 2628 /* 2629 * Round the configured number of QP per MCG to next larger 2630 * power-of-2 size and update. 2631 */ 2632 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8; 2633 log2 = highbit(num_qp_per_mcg); 2634 if (ISP2(num_qp_per_mcg)) { 2635 log2 = log2 - 1; 2636 } 2637 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8; 2638 2639 /* Now make sure number of QP per MCG makes sense */ 2640 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg; 2641 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg); 2642 if (num_qp_per_mcg > max_qp_per_mcg) { 2643 return (DDI_FAILURE); 2644 } 2645 2646 /* Return the (shift) size of an individual MCG HW entry */ 2647 *mcg_size_shift = log2 + 2; 2648 2649 return (DDI_SUCCESS); 2650 } 2651