1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_rsrc.c
29 * Tavor Resource Management Routines
30 *
31 * Implements all the routines necessary for setup, teardown, and
32 * alloc/free of all Tavor resources, including those that are managed
33 * by Tavor hardware or which live in Tavor's direct attached DDR memory.
34 */
35
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/vmem.h>
43 #include <sys/bitmap.h>
44
45 #include <sys/ib/adapters/tavor/tavor.h>
46
47 /*
48 * The following routines are used for initializing and destroying
49 * the resource pools used by the Tavor resource allocation routines.
50 * They consist of four classes of object:
51 *
52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor
53 * command interface routines. Mailboxes are used to pass information
54 * back and forth to the Tavor firmware. Either type of mailbox may
55 * be allocated from Tavor's direct attached DDR memory or from system
56 * memory (although currently all "In" mailboxes are in DDR and all "out"
57 * mailboxes come from system memory.
58 *
59 * HW entry objects: These objects represent resources required by the Tavor
60 * hardware. These objects include things like Queue Pair contexts (QPC),
61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
63 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
64 *
65 * What these objects all have in common is that they are each required
66 * to come from DDR memory, they are always allocated from tables, and
67 * they are not to be directly accessed (read or written) by driver
68 * software.
69 * One notable exceptions to this rule are the Extended QP contexts (EQPC),
70 * and the UAR scratch area (UAR_SCR), both of which are not directly
71 * accessible through the Tavor resource allocation routines, but both
72 * of which are also required to reside in DDR memory and are not to be
73 * manipulated by driver software (they are privately managed by Tavor
74 * hardware).
75 * The other notable exceptions are the UAR pages (UAR_PG) which are
76 * allocated from the UAR address space rather than DDR, and the UD
77 * address vectors (UDAV) which are similar to the common object types
78 * with the major difference being that UDAVs _are_ directly read and
79 * written by driver software.
80 *
81 * SW handle objects: These objects represent resources required by Tavor
82 * driver software. They are primarily software tracking structures,
83 * which are allocated from system memory (using kmem_cache). Several of
84 * the objects have both a "constructor" and "destructor" method
85 * associated with them (see below).
86 *
87 * Protection Domain (PD) handle objects: These objects are very much like
88 * a SW handle object with the notable difference that all PD handle
89 * objects have an actual Protection Domain number (PD) associated with
90 * them (and the PD number is allocated/managed through a separate
91 * vmem_arena specifically set aside for this purpose.
92 */
93
94 static int tavor_rsrc_mbox_init(tavor_state_t *state,
95 tavor_rsrc_mbox_info_t *info);
96 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
97 tavor_rsrc_mbox_info_t *info);
98
99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
100 tavor_rsrc_hw_entry_info_t *info);
101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
102 tavor_rsrc_hw_entry_info_t *info);
103
104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
105 tavor_rsrc_sw_hdl_info_t *info);
106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
107 tavor_rsrc_sw_hdl_info_t *info);
108
109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
110 tavor_rsrc_sw_hdl_info_t *info);
111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
112 tavor_rsrc_sw_hdl_info_t *info);
113
114 /*
115 * The following routines are used for allocating and freeing the specific
116 * types of objects described above from their associated resource pools.
117 */
118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
119 uint_t num, tavor_rsrc_t *hdl);
120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
121 tavor_rsrc_t *hdl);
122
123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
125 uint_t sleepflag, tavor_rsrc_t *hdl);
126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
127 tavor_rsrc_t *hdl);
128
129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
130 uint_t sleepflag, tavor_rsrc_t *hdl);
131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
132 tavor_rsrc_t *hdl);
133
134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
135 uint_t sleepflag, tavor_rsrc_t *hdl);
136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
137 tavor_rsrc_t *hdl);
138
139 /*
140 * The following routines are the constructors and destructors for several
141 * of the SW handle type objects. For certain types of SW handles objects
142 * (all of which are implemented using kmem_cache), we need to do some
143 * special field initialization (specifically, mutex_init/destroy). These
144 * routines enable that init and teardown.
145 */
146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
160
161 /*
162 * Special routine to calculate and return the size of a MCG object based
163 * on current driver configuration (specifically, the number of QP per MCG
164 * that has been configured.
165 */
166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
167 uint_t *mcg_size_shift);
168
169
170 /*
171 * tavor_rsrc_alloc()
172 *
173 * Context: Can be called from interrupt or base context.
174 * The "sleepflag" parameter is used by all object allocators to
175 * determine whether to SLEEP for resources or not.
176 */
177 int
tavor_rsrc_alloc(tavor_state_t * state,tavor_rsrc_type_t rsrc,uint_t num,uint_t sleepflag,tavor_rsrc_t ** hdl)178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
179 uint_t sleepflag, tavor_rsrc_t **hdl)
180 {
181 tavor_rsrc_pool_info_t *rsrc_pool;
182 tavor_rsrc_t *tmp_rsrc_hdl;
183 int flag, status = DDI_FAILURE;
184
185 ASSERT(state != NULL);
186 ASSERT(hdl != NULL);
187
188 rsrc_pool = &state->ts_rsrc_hdl[rsrc];
189 ASSERT(rsrc_pool != NULL);
190
191 /*
192 * Allocate space for the object used to track the resource handle
193 */
194 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
195 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
196 flag);
197 if (tmp_rsrc_hdl == NULL) {
198 return (DDI_FAILURE);
199 }
200 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
201
202 /*
203 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call
204 * to know what type of resource is being freed.
205 */
206 tmp_rsrc_hdl->rsrc_type = rsrc;
207
208 /*
209 * Depending on resource type, call the appropriate alloc routine
210 */
211 switch (rsrc_pool->rsrc_type) {
212 case TAVOR_IN_MBOX:
213 case TAVOR_OUT_MBOX:
214 case TAVOR_INTR_IN_MBOX:
215 case TAVOR_INTR_OUT_MBOX:
216 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
217 break;
218
219 case TAVOR_QPC:
220 case TAVOR_CQC:
221 case TAVOR_SRQC:
222 case TAVOR_EQC:
223 case TAVOR_RDB:
224 /*
225 * Because these objects are NOT accessed by Tavor driver
226 * software, we set the acc_handle parameter to zero. But
227 * if they are allocated in multiples, we specify here that
228 * they must be aligned on a more restrictive boundary.
229 */
230 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
231 sleepflag, tmp_rsrc_hdl);
232 break;
233
234 case TAVOR_MPT:
235 /*
236 * Because these MPT objects are sometimes accessed by Tavor
237 * driver software (FMR), we set the acc_handle parameter. But
238 * if they are allocated in multiples, we specify here that
239 * they must be aligned on a more restrictive boundary.
240 */
241 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
242 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
243 break;
244
245 case TAVOR_MCG:
246 /*
247 * Tavor MCG entries are also NOT accessed by Tavor driver
248 * software, but because MCG entries do not have the same
249 * alignnment restrictions we loosen the constraint here.
250 */
251 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
252 sleepflag, tmp_rsrc_hdl);
253 break;
254
255 case TAVOR_MTT:
256 case TAVOR_UDAV:
257 /*
258 * Because MTT segments are among the few HW resources that
259 * may be allocated in odd numbers, we specify a less
260 * restrictive alignment than for the above resources.
261 *
262 * Also because both UDAV and MTT segment objects are read
263 * and/or written by Tavor driver software, we set the
264 * acc_handle parameter to point to the ddi_acc_handle_t for
265 * the Tavor DDR memory.
266 */
267 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
268 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
269 break;
270
271 case TAVOR_UARPG:
272 /*
273 * Because UAR pages are written by Tavor driver software (for
274 * doorbells), we set the acc_handle parameter to point to
275 * the ddi_acc_handle_t for the Tavor UAR memory.
276 */
277 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
278 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
279 break;
280
281 case TAVOR_MRHDL:
282 case TAVOR_EQHDL:
283 case TAVOR_CQHDL:
284 case TAVOR_SRQHDL:
285 case TAVOR_AHHDL:
286 case TAVOR_QPHDL:
287 case TAVOR_REFCNT:
288 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
289 tmp_rsrc_hdl);
290 break;
291
292 case TAVOR_PDHDL:
293 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
294 tmp_rsrc_hdl);
295 break;
296
297 default:
298 TAVOR_WARNING(state, "unexpected resource type in alloc");
299 break;
300 }
301
302 /*
303 * If the resource allocation failed, then free the special resource
304 * tracking structure and return failure. Otherwise return the
305 * handle for the resource tracking structure.
306 */
307 if (status != DDI_SUCCESS) {
308 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
309 tmp_rsrc_hdl = NULL;
310 return (DDI_FAILURE);
311 } else {
312 *hdl = tmp_rsrc_hdl;
313 return (DDI_SUCCESS);
314 }
315 }
316
317
318 /*
319 * tavor_rsrc_free()
320 * Context: Can be called from interrupt or base context.
321 */
322 void
tavor_rsrc_free(tavor_state_t * state,tavor_rsrc_t ** hdl)323 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
324 {
325 tavor_rsrc_pool_info_t *rsrc_pool;
326
327 ASSERT(state != NULL);
328 ASSERT(hdl != NULL);
329
330 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
331 ASSERT(rsrc_pool != NULL);
332
333 /*
334 * Depending on resource type, call the appropriate free routine
335 */
336 switch (rsrc_pool->rsrc_type) {
337 case TAVOR_IN_MBOX:
338 case TAVOR_OUT_MBOX:
339 case TAVOR_INTR_IN_MBOX:
340 case TAVOR_INTR_OUT_MBOX:
341 tavor_rsrc_mbox_free(rsrc_pool, *hdl);
342 break;
343
344 case TAVOR_QPC:
345 case TAVOR_CQC:
346 case TAVOR_SRQC:
347 case TAVOR_EQC:
348 case TAVOR_RDB:
349 case TAVOR_MCG:
350 case TAVOR_MPT:
351 case TAVOR_MTT:
352 case TAVOR_UDAV:
353 case TAVOR_UARPG:
354 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
355 break;
356
357 case TAVOR_MRHDL:
358 case TAVOR_EQHDL:
359 case TAVOR_CQHDL:
360 case TAVOR_SRQHDL:
361 case TAVOR_AHHDL:
362 case TAVOR_QPHDL:
363 case TAVOR_REFCNT:
364 tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
365 break;
366
367 case TAVOR_PDHDL:
368 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
369 break;
370
371 default:
372 TAVOR_WARNING(state, "unexpected resource type in free");
373 break;
374 }
375
376 /*
377 * Free the special resource tracking structure, set the handle to
378 * NULL, and return.
379 */
380 kmem_cache_free(state->ts_rsrc_cache, *hdl);
381 *hdl = NULL;
382 }
383
384
385 /*
386 * tavor_rsrc_init_phase1()
387 *
388 * Completes the first phase of Tavor resource/configuration init.
389 * This involves creating the kmem_cache for the "tavor_rsrc_t"
390 * structs, allocating the space for the resource pool handles,
391 * and setting up the "Out" mailboxes.
392 *
393 * When this function completes, the Tavor driver is ready to
394 * post the following commands which return information only in the
395 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
396 * If any of these commands are to be posted at this time, they must be
397 * done so only when "spinning" (as the outstanding command list and
398 * EQ setup code has not yet run)
399 *
400 * Context: Only called from attach() path context
401 */
402 int
tavor_rsrc_init_phase1(tavor_state_t * state)403 tavor_rsrc_init_phase1(tavor_state_t *state)
404 {
405 tavor_rsrc_pool_info_t *rsrc_pool;
406 tavor_rsrc_mbox_info_t mbox_info;
407 tavor_rsrc_cleanup_level_t cleanup;
408 tavor_cfg_profile_t *cfgprof;
409 uint64_t num, size;
410 int status;
411 char *rsrc_name;
412
413 ASSERT(state != NULL);
414
415 /* This is where Phase 1 of resource initialization begins */
416 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
417
418 /* Build kmem cache name from Tavor instance */
419 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
420 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
421
422 /*
423 * Create the kmem_cache for "tavor_rsrc_t" structures
424 * (kmem_cache_create will SLEEP until successful)
425 */
426 state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
427 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
428
429 /*
430 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
431 * subsequent resource allocations)
432 */
433 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
434 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
435
436 cfgprof = state->ts_cfg_profile;
437
438 /*
439 * Initialize the resource pool for "Out" mailboxes. Notice that
440 * the number of "Out" mailboxes, their size, and their location
441 * (DDR or system memory) is configurable. By default, however,
442 * all "Out" mailboxes are located in system memory only (because
443 * they are primarily read from and never written to)
444 */
445 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
446 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
447 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
448 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX;
449 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
450 rsrc_pool->rsrc_pool_size = (size * num);
451 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
452 rsrc_pool->rsrc_quantum = size;
453 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
454 rsrc_pool->rsrc_state = state;
455 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
456 mbox_info.mbi_num = num;
457 mbox_info.mbi_size = size;
458 mbox_info.mbi_rsrcpool = rsrc_pool;
459 mbox_info.mbi_rsrcname = rsrc_name;
460 status = tavor_rsrc_mbox_init(state, &mbox_info);
461 if (status != DDI_SUCCESS) {
462 tavor_rsrc_fini(state, cleanup);
463 goto rsrcinitp1_fail;
464 }
465 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
466
467 /*
468 * Initialize the Tavor "Out" mailbox list. This step actually uses
469 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
470 * "Out" mailboxes, bind them for DMA access, and arrange them into
471 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
472 * for more details)
473 */
474 status = tavor_outmbox_list_init(state);
475 if (status != DDI_SUCCESS) {
476 tavor_rsrc_fini(state, cleanup);
477 goto rsrcinitp1_fail;
478 }
479 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
480
481 /*
482 * Initialize the resource pool for interrupt "Out" mailboxes. Notice
483 * that the number of interrupt "Out" mailboxes, their size, and their
484 * location (DDR or system memory) is configurable. By default,
485 * however, all interrupt "Out" mailboxes are located in system memory
486 * only (because they are primarily read from and never written to)
487 */
488 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
489 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
490 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
491 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX;
492 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
493 rsrc_pool->rsrc_pool_size = (size * num);
494 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
495 rsrc_pool->rsrc_quantum = size;
496 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
497 rsrc_pool->rsrc_state = state;
498 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
499 mbox_info.mbi_num = num;
500 mbox_info.mbi_size = size;
501 mbox_info.mbi_rsrcpool = rsrc_pool;
502 mbox_info.mbi_rsrcname = rsrc_name;
503 status = tavor_rsrc_mbox_init(state, &mbox_info);
504 if (status != DDI_SUCCESS) {
505 tavor_rsrc_fini(state, cleanup);
506 goto rsrcinitp1_fail;
507 }
508 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
509
510 /*
511 * Initialize the Tavor "Out" mailbox list. This step actually uses
512 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
513 * "Out" mailboxes, bind them for DMA access, and arrange them into
514 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
515 * for more details)
516 */
517 status = tavor_intr_outmbox_list_init(state);
518 if (status != DDI_SUCCESS) {
519 tavor_rsrc_fini(state, cleanup);
520 goto rsrcinitp1_fail;
521 }
522 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
523
524 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
525 return (DDI_SUCCESS);
526
527 rsrcinitp1_fail:
528 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
529 return (status);
530 }
531
532
533 /*
534 * tavor_rsrc_init_phase2()
535 * Context: Only called from attach() path context
536 */
537 int
tavor_rsrc_init_phase2(tavor_state_t * state)538 tavor_rsrc_init_phase2(tavor_state_t *state)
539 {
540 tavor_rsrc_sw_hdl_info_t hdl_info;
541 tavor_rsrc_hw_entry_info_t entry_info;
542 tavor_rsrc_mbox_info_t mbox_info;
543 tavor_rsrc_pool_info_t *rsrc_pool;
544 tavor_rsrc_cleanup_level_t cleanup;
545 tavor_cfg_profile_t *cfgprof;
546 uint64_t num, max, size, num_prealloc;
547 uint64_t ddr_size, fw_size;
548 uint_t mcg_size, mcg_size_shift;
549 uint_t uarscr_size, mttsegment_sz;
550 int status;
551 char *rsrc_name;
552
553 ASSERT(state != NULL);
554
555 /* Phase 2 initialization begins where Phase 1 left off */
556 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
557
558 /*
559 * Calculate the extent of the DDR size and portion of which that
560 * is already reserved for Tavor firmware. (Note: this information
561 * is available because the QUERY_DDR and QUERY_FW commands have
562 * been posted to Tavor firmware prior to calling this routine)
563 */
564 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
565 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
566
567 /* Build the DDR vmem arena name from Tavor instance */
568 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
569 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
570
571 /*
572 * Do a vmem_create for the entire DDR range (not including the
573 * portion consumed by Tavor firmware). This creates the vmem arena
574 * from which all other DDR objects (specifically, tables of HW
575 * entries) will be allocated.
576 */
577 state->ts_ddrvmem = vmem_create(rsrc_name,
578 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
579 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
580 if (state->ts_ddrvmem == NULL) {
581 tavor_rsrc_fini(state, cleanup);
582 goto rsrcinitp2_fail;
583 }
584 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
585
586 /*
587 * Initialize the resource pools for all objects that exist in
588 * Tavor DDR memory. This includes ("In") mailboxes, context tables
589 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
590 */
591 cfgprof = state->ts_cfg_profile;
592
593 /*
594 * Initialize the resource pool for the MPT table entries. Notice
595 * that the number of MPTs is configurable. The configured value must
596 * be less that the maximum value (obtained from the QUERY_DEV_LIM
597 * command) or the initialization will fail. Note also that a certain
598 * number of MPTs must be set aside for Tavor firmware use.
599 */
600 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
601 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
602 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
603 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
604 rsrc_pool->rsrc_type = TAVOR_MPT;
605 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
606 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
607 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT;
608 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE;
609 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num);
610 rsrc_pool->rsrc_state = state;
611 rsrc_pool->rsrc_start = NULL;
612 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
613 entry_info.hwi_num = num;
614 entry_info.hwi_max = max;
615 entry_info.hwi_prealloc = num_prealloc;
616 entry_info.hwi_rsrcpool = rsrc_pool;
617 entry_info.hwi_rsrcname = rsrc_name;
618 status = tavor_rsrc_hw_entries_init(state, &entry_info);
619 if (status != DDI_SUCCESS) {
620 tavor_rsrc_fini(state, cleanup);
621 goto rsrcinitp2_fail;
622 }
623 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
624
625 /*
626 * Initialize the resource pool for the MTT table entries. Notice
627 * that the number of MTTs is configurable. The configured value must
628 * be less that the maximum value (obtained from the QUERY_DEV_LIM
629 * command) or the initialization will fail. Note also that a certain
630 * number of MTT segments must be set aside for Tavor firmware use.
631 */
632 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
633 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
634 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
635 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
636 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
637 rsrc_pool->rsrc_type = TAVOR_MTT;
638 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
639 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
640 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT;
641 rsrc_pool->rsrc_quantum = mttsegment_sz;
642 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num);
643 rsrc_pool->rsrc_state = state;
644 rsrc_pool->rsrc_start = NULL;
645 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
646 entry_info.hwi_num = num;
647 entry_info.hwi_max = max;
648 entry_info.hwi_prealloc = num_prealloc;
649 entry_info.hwi_rsrcpool = rsrc_pool;
650 entry_info.hwi_rsrcname = rsrc_name;
651 status = tavor_rsrc_hw_entries_init(state, &entry_info);
652 if (status != DDI_SUCCESS) {
653 tavor_rsrc_fini(state, cleanup);
654 goto rsrcinitp2_fail;
655 }
656 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
657
658 /*
659 * Initialize the resource pool for the QPC table entries. Notice
660 * that the number of QPs is configurable. The configured value must
661 * be less that the maximum value (obtained from the QUERY_DEV_LIM
662 * command) or the initialization will fail. Note also that a certain
663 * number of QP contexts must be set aside for Tavor firmware use.
664 */
665 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
666 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
667 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
668 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
669 rsrc_pool->rsrc_type = TAVOR_QPC;
670 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
671 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
672 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT;
673 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE;
674 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num);
675 rsrc_pool->rsrc_state = state;
676 rsrc_pool->rsrc_start = NULL;
677 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
678 entry_info.hwi_num = num;
679 entry_info.hwi_max = max;
680 entry_info.hwi_prealloc = num_prealloc;
681 entry_info.hwi_rsrcpool = rsrc_pool;
682 entry_info.hwi_rsrcname = rsrc_name;
683 status = tavor_rsrc_hw_entries_init(state, &entry_info);
684 if (status != DDI_SUCCESS) {
685 tavor_rsrc_fini(state, cleanup);
686 goto rsrcinitp2_fail;
687 }
688 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
689
690 /*
691 * Initialize the resource pool for the RDB table entries. Notice
692 * that the number of RDBs is configurable. The configured value must
693 * be less that the maximum value (obtained from the QUERY_DEV_LIM
694 * command) or the initialization will fail.
695 */
696 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
697 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
698 num_prealloc = 0;
699 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
700 rsrc_pool->rsrc_type = TAVOR_RDB;
701 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
702 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
703 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT;
704 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE;
705 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num);
706 rsrc_pool->rsrc_state = state;
707 rsrc_pool->rsrc_start = NULL;
708 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
709 entry_info.hwi_num = num;
710 entry_info.hwi_max = max;
711 entry_info.hwi_prealloc = num_prealloc;
712 entry_info.hwi_rsrcpool = rsrc_pool;
713 entry_info.hwi_rsrcname = rsrc_name;
714 status = tavor_rsrc_hw_entries_init(state, &entry_info);
715 if (status != DDI_SUCCESS) {
716 tavor_rsrc_fini(state, cleanup);
717 goto rsrcinitp2_fail;
718 }
719 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
720
721 /*
722 * Initialize the resource pool for the CQC table entries. Notice
723 * that the number of CQs is configurable. The configured value must
724 * be less that the maximum value (obtained from the QUERY_DEV_LIM
725 * command) or the initialization will fail. Note also that a certain
726 * number of CQ contexts must be set aside for Tavor firmware use.
727 */
728 num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
729 max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
730 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
731 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
732 rsrc_pool->rsrc_type = TAVOR_CQC;
733 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
734 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
735 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT;
736 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE;
737 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num);
738 rsrc_pool->rsrc_state = state;
739 rsrc_pool->rsrc_start = NULL;
740 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
741 entry_info.hwi_num = num;
742 entry_info.hwi_max = max;
743 entry_info.hwi_prealloc = num_prealloc;
744 entry_info.hwi_rsrcpool = rsrc_pool;
745 entry_info.hwi_rsrcname = rsrc_name;
746 status = tavor_rsrc_hw_entries_init(state, &entry_info);
747 if (status != DDI_SUCCESS) {
748 tavor_rsrc_fini(state, cleanup);
749 goto rsrcinitp2_fail;
750 }
751 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
752
753 /*
754 * Initialize the resource pool for the Extended QPC table entries.
755 * Notice that the number of EQPCs must be the same as the number
756 * of QP contexts. So the initialization is constructed in a
757 * similar way as above (for TAVOR_QPC). One notable difference
758 * here, however, is that by setting the rsrc_quantum field to
759 * zero (indicating a zero-sized object) we indicate that the
760 * object is not allocatable. The EQPC table is, in fact, managed
761 * internally by the hardware and it is, therefore, unnecessary to
762 * initialize an additional vmem_arena for this type of object.
763 */
764 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
765 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
766 num_prealloc = 0;
767 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
768 rsrc_pool->rsrc_type = TAVOR_EQPC;
769 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
770 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
771 rsrc_pool->rsrc_shift = 0;
772 rsrc_pool->rsrc_quantum = 0;
773 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE;
774 rsrc_pool->rsrc_state = state;
775 rsrc_pool->rsrc_start = NULL;
776 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
777 entry_info.hwi_num = num;
778 entry_info.hwi_max = max;
779 entry_info.hwi_prealloc = num_prealloc;
780 entry_info.hwi_rsrcpool = rsrc_pool;
781 entry_info.hwi_rsrcname = rsrc_name;
782 status = tavor_rsrc_hw_entries_init(state, &entry_info);
783 if (status != DDI_SUCCESS) {
784 tavor_rsrc_fini(state, cleanup);
785 goto rsrcinitp2_fail;
786 }
787 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
788
789 /*
790 * Initialize the resource pool for the UD address vector table
791 * entries. Notice that the number of UDAVs is configurable. The
792 * configured value must be less that the maximum value (obtained
793 * from the QUERY_DEV_LIM command) or the initialization will fail.
794 */
795 num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
796 max = ((uint64_t)1 << state->ts_devlim.log_max_av);
797 num_prealloc = 0;
798 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
799 rsrc_pool->rsrc_type = TAVOR_UDAV;
800 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
801 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
802 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT;
803 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE;
804 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE;
805 rsrc_pool->rsrc_state = state;
806 rsrc_pool->rsrc_start = NULL;
807 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
808 entry_info.hwi_num = num;
809 entry_info.hwi_max = max;
810 entry_info.hwi_prealloc = num_prealloc;
811 entry_info.hwi_rsrcpool = rsrc_pool;
812 entry_info.hwi_rsrcname = rsrc_name;
813 status = tavor_rsrc_hw_entries_init(state, &entry_info);
814 if (status != DDI_SUCCESS) {
815 tavor_rsrc_fini(state, cleanup);
816 goto rsrcinitp2_fail;
817 }
818 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
819
820 /*
821 * Initialize the resource pool for the UAR scratch table entries.
822 * Notice that the number of UARSCRs is configurable. The configured
823 * value must be less that the maximum value (obtained from the
824 * QUERY_DEV_LIM command) or the initialization will fail.
825 * Like the EQPCs above, UARSCR objects are not allocatable. The
826 * UARSCR table is also managed internally by the hardware and it
827 * is, therefore, unnecessary to initialize an additional vmem_arena
828 * for this type of object. We indicate this by setting the
829 * rsrc_quantum field to zero (indicating a zero-sized object).
830 */
831 uarscr_size = state->ts_devlim.uarscr_entry_sz;
832 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
833 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
834 PAGESHIFT));
835 num_prealloc = 0;
836 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
837 rsrc_pool->rsrc_type = TAVOR_UAR_SCR;
838 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
839 rsrc_pool->rsrc_pool_size = (uarscr_size * num);
840 rsrc_pool->rsrc_shift = 0;
841 rsrc_pool->rsrc_quantum = 0;
842 rsrc_pool->rsrc_align = uarscr_size;
843 rsrc_pool->rsrc_state = state;
844 rsrc_pool->rsrc_start = NULL;
845 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
846 entry_info.hwi_num = num;
847 entry_info.hwi_max = max;
848 entry_info.hwi_prealloc = num_prealloc;
849 entry_info.hwi_rsrcpool = rsrc_pool;
850 entry_info.hwi_rsrcname = rsrc_name;
851 status = tavor_rsrc_hw_entries_init(state, &entry_info);
852 if (status != DDI_SUCCESS) {
853 tavor_rsrc_fini(state, cleanup);
854 goto rsrcinitp2_fail;
855 }
856 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
857
858 /*
859 * Initialize the resource pool for the SRQC table entries. Notice
860 * that the number of SRQs is configurable. The configured value must
861 * be less that the maximum value (obtained from the QUERY_DEV_LIM
862 * command) or the initialization will fail. Note also that a certain
863 * number of SRQ contexts must be set aside for Tavor firmware use.
864 *
865 * Note: We only allocate these resources if SRQ is enabled in the
866 * config profile; see below.
867 */
868 num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
869 max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
870 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
871
872 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
873 rsrc_pool->rsrc_type = TAVOR_SRQC;
874 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
875 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
876 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT;
877 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE;
878 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num);
879 rsrc_pool->rsrc_state = state;
880 rsrc_pool->rsrc_start = NULL;
881 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
882 entry_info.hwi_num = num;
883 entry_info.hwi_max = max;
884 entry_info.hwi_prealloc = num_prealloc;
885 entry_info.hwi_rsrcpool = rsrc_pool;
886 entry_info.hwi_rsrcname = rsrc_name;
887
888 /*
889 * SRQ support is configurable. Only if SRQ is enabled (the default)
890 * do we actually try to configure these resources. Otherwise, we
891 * simply set the cleanup level and continue on to the next resource
892 */
893 if (state->ts_cfg_profile->cp_srq_enable != 0) {
894 status = tavor_rsrc_hw_entries_init(state, &entry_info);
895 if (status != DDI_SUCCESS) {
896 tavor_rsrc_fini(state, cleanup);
897 goto rsrcinitp2_fail;
898 }
899 }
900 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
901
902 /*
903 * Initialize the resource pool for "In" mailboxes. Notice that
904 * the number of "In" mailboxes, their size, and their location
905 * (DDR or system memory) is configurable. By default, however,
906 * all "In" mailboxes are located in system memory only (because
907 * they are primarily written to and rarely read from)
908 */
909 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
910 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
911 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
912 rsrc_pool->rsrc_type = TAVOR_IN_MBOX;
913 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
914 rsrc_pool->rsrc_pool_size = (size * num);
915 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
916 rsrc_pool->rsrc_quantum = size;
917 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
918 rsrc_pool->rsrc_state = state;
919 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
920 mbox_info.mbi_num = num;
921 mbox_info.mbi_size = size;
922 mbox_info.mbi_rsrcpool = rsrc_pool;
923 mbox_info.mbi_rsrcname = rsrc_name;
924 status = tavor_rsrc_mbox_init(state, &mbox_info);
925 if (status != DDI_SUCCESS) {
926 tavor_rsrc_fini(state, cleanup);
927 goto rsrcinitp2_fail;
928 }
929 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
930
931 /*
932 * Initialize the Tavor "In" mailbox list. This step actually uses
933 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
934 * "In" mailboxes, bind them for DMA access, and arrange them into
935 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
936 * for more details)
937 */
938 status = tavor_inmbox_list_init(state);
939 if (status != DDI_SUCCESS) {
940 tavor_rsrc_fini(state, cleanup);
941 goto rsrcinitp2_fail;
942 }
943 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
944
945 /*
946 * Initialize the resource pool for interrupt "In" mailboxes. Notice
947 * that the number of interrupt "In" mailboxes, their size, and their
948 * location (DDR or system memory) is configurable. By default,
949 * however, all interrupt "In" mailboxes are located in system memory
950 * only (because they are primarily written to and rarely read from)
951 */
952 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
953 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
954 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
955 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX;
956 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
957 rsrc_pool->rsrc_pool_size = (size * num);
958 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
959 rsrc_pool->rsrc_quantum = size;
960 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
961 rsrc_pool->rsrc_state = state;
962 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
963 mbox_info.mbi_num = num;
964 mbox_info.mbi_size = size;
965 mbox_info.mbi_rsrcpool = rsrc_pool;
966 mbox_info.mbi_rsrcname = rsrc_name;
967 status = tavor_rsrc_mbox_init(state, &mbox_info);
968 if (status != DDI_SUCCESS) {
969 tavor_rsrc_fini(state, cleanup);
970 goto rsrcinitp2_fail;
971 }
972 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
973
974 /*
975 * Initialize the Tavor interrupt "In" mailbox list. This step
976 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
977 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
978 * and arrange them into an easily accessed fast-allocation mechanism
979 * (see tavor_cmd.c for more details)
980 */
981 status = tavor_intr_inmbox_list_init(state);
982 if (status != DDI_SUCCESS) {
983 tavor_rsrc_fini(state, cleanup);
984 goto rsrcinitp2_fail;
985 }
986 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
987
988 /*
989 * Initialize the Tavor command handling interfaces. This step
990 * sets up the outstanding command tracking mechanism for easy access
991 * and fast allocation (see tavor_cmd.c for more details).
992 */
993 status = tavor_outstanding_cmdlist_init(state);
994 if (status != DDI_SUCCESS) {
995 tavor_rsrc_fini(state, cleanup);
996 goto rsrcinitp2_fail;
997 }
998 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
999
1000 /*
1001 * Calculate (and validate) the size of Multicast Group (MCG) entries
1002 */
1003 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1004 if (status != DDI_SUCCESS) {
1005 tavor_rsrc_fini(state, cleanup);
1006 goto rsrcinitp2_fail;
1007 }
1008 mcg_size = TAVOR_MCGMEM_SZ(state);
1009
1010 /*
1011 * Initialize the resource pool for the MCG table entries. Notice
1012 * that the number of MCGs is configurable. The configured value must
1013 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1014 * command) or the initialization will fail. Note also that a certain
1015 * number of MCGs must be set aside for Tavor firmware use (they
1016 * correspond to the number of MCGs used by the internal hash
1017 * function.
1018 */
1019 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1020 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1021 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1022 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1023 rsrc_pool->rsrc_type = TAVOR_MCG;
1024 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1025 rsrc_pool->rsrc_pool_size = (mcg_size * num);
1026 rsrc_pool->rsrc_shift = mcg_size_shift;
1027 rsrc_pool->rsrc_quantum = mcg_size;
1028 rsrc_pool->rsrc_align = mcg_size;
1029 rsrc_pool->rsrc_state = state;
1030 rsrc_pool->rsrc_start = NULL;
1031 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1032 entry_info.hwi_num = num;
1033 entry_info.hwi_max = max;
1034 entry_info.hwi_prealloc = num_prealloc;
1035 entry_info.hwi_rsrcpool = rsrc_pool;
1036 entry_info.hwi_rsrcname = rsrc_name;
1037 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1038 if (status != DDI_SUCCESS) {
1039 tavor_rsrc_fini(state, cleanup);
1040 goto rsrcinitp2_fail;
1041 }
1042 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1043
1044 /*
1045 * Initialize the resource pool for the EQC table entries. Notice
1046 * that the number of EQs is hardcoded. The hardcoded value should
1047 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1048 * command) or the initialization will fail.
1049 */
1050 num = TAVOR_NUM_EQ;
1051 max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1052 num_prealloc = 0;
1053 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1054 rsrc_pool->rsrc_type = TAVOR_EQC;
1055 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1056 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1057 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT;
1058 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE;
1059 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num);
1060 rsrc_pool->rsrc_state = state;
1061 rsrc_pool->rsrc_start = NULL;
1062 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1063 entry_info.hwi_num = num;
1064 entry_info.hwi_max = max;
1065 entry_info.hwi_prealloc = num_prealloc;
1066 entry_info.hwi_rsrcpool = rsrc_pool;
1067 entry_info.hwi_rsrcname = rsrc_name;
1068 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1069 if (status != DDI_SUCCESS) {
1070 tavor_rsrc_fini(state, cleanup);
1071 goto rsrcinitp2_fail;
1072 }
1073 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1074
1075 /*
1076 * Initialize the resource pools for all objects that exist in
1077 * system memory. This includes PD handles, MR handle, EQ handles,
1078 * QP handles, etc. These objects are almost entirely managed using
1079 * kmem_cache routines. (See comment above for more detail)
1080 */
1081
1082 /*
1083 * Initialize the resource pool for the PD handles. Notice
1084 * that the number of PDHDLs is configurable. The configured value
1085 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1086 * command) or the initialization will fail. Note also that the PD
1087 * handle has constructor and destructor methods associated with it.
1088 */
1089 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1090 rsrc_pool->rsrc_type = TAVOR_PDHDL;
1091 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1092 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s);
1093 rsrc_pool->rsrc_state = state;
1094 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1095 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1096 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1097 hdl_info.swi_rsrcpool = rsrc_pool;
1098 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1099 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor;
1100 hdl_info.swi_rsrcname = rsrc_name;
1101 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1102 status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1103 if (status != DDI_SUCCESS) {
1104 tavor_rsrc_fini(state, cleanup);
1105 goto rsrcinitp2_fail;
1106 }
1107 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1108
1109 /*
1110 * Initialize the resource pool for the MR handles. Notice
1111 * that the number of MRHDLs is configurable. The configured value
1112 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1113 * command) or the initialization will fail.
1114 */
1115 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1116 rsrc_pool->rsrc_type = TAVOR_MRHDL;
1117 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1118 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s);
1119 rsrc_pool->rsrc_state = state;
1120 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1121 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1122 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1123 hdl_info.swi_rsrcpool = rsrc_pool;
1124 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1125 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor;
1126 hdl_info.swi_rsrcname = rsrc_name;
1127 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1128 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1129 if (status != DDI_SUCCESS) {
1130 tavor_rsrc_fini(state, cleanup);
1131 goto rsrcinitp2_fail;
1132 }
1133 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1134
1135 /*
1136 * Initialize the resource pool for the EQ handles. Notice
1137 * that the number of EQHDLs is hardcoded. The hardcoded value
1138 * should be less that the maximum value (obtained from the
1139 * QUERY_DEV_LIM command) or the initialization will fail.
1140 */
1141 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1142 rsrc_pool->rsrc_type = TAVOR_EQHDL;
1143 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1144 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s);
1145 rsrc_pool->rsrc_state = state;
1146 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1147 hdl_info.swi_num = TAVOR_NUM_EQ;
1148 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1149 hdl_info.swi_rsrcpool = rsrc_pool;
1150 hdl_info.swi_constructor = NULL;
1151 hdl_info.swi_destructor = NULL;
1152 hdl_info.swi_rsrcname = rsrc_name;
1153 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1154 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1155 if (status != DDI_SUCCESS) {
1156 tavor_rsrc_fini(state, cleanup);
1157 goto rsrcinitp2_fail;
1158 }
1159 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1160
1161 /*
1162 * Initialize the resource pool for the CQ handles. Notice
1163 * that the number of CQHDLs is configurable. The configured value
1164 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1165 * command) or the initialization will fail. Note also that the CQ
1166 * handle has constructor and destructor methods associated with it.
1167 */
1168 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1169 rsrc_pool->rsrc_type = TAVOR_CQHDL;
1170 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1171 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s);
1172 rsrc_pool->rsrc_state = state;
1173 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1174 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1175 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1176 hdl_info.swi_rsrcpool = rsrc_pool;
1177 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1178 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor;
1179 hdl_info.swi_rsrcname = rsrc_name;
1180 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1181 TAVOR_SWHDL_TABLE_INIT);
1182 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1183 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1184 if (status != DDI_SUCCESS) {
1185 tavor_rsrc_fini(state, cleanup);
1186 goto rsrcinitp2_fail;
1187 }
1188
1189 /*
1190 * Save away the pointer to the central list of CQ handle pointers
1191 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1192 * lookup during EQ event processing. The table is a list of
1193 * tavor_cqhdl_t allocated by the above routine because of the
1194 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t
1195 * as the number of CQs.
1196 */
1197 state->ts_cqhdl = hdl_info.swi_table_ptr;
1198 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1199
1200 /*
1201 * Initialize the resource pool for the SRQ handles. Notice
1202 * that the number of SRQHDLs is configurable. The configured value
1203 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1204 * command) or the initialization will fail. Note also that the SRQ
1205 * handle has constructor and destructor methods associated with it.
1206 *
1207 * Note: We only allocate these resources if SRQ is enabled in the
1208 * config profile; see below.
1209 */
1210 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1211 rsrc_pool->rsrc_type = TAVOR_SRQHDL;
1212 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1213 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s);
1214 rsrc_pool->rsrc_state = state;
1215 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1216 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1217 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1218 hdl_info.swi_rsrcpool = rsrc_pool;
1219 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1220 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor;
1221 hdl_info.swi_rsrcname = rsrc_name;
1222 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1223 TAVOR_SWHDL_TABLE_INIT);
1224 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1225
1226 /*
1227 * SRQ support is configurable. Only if SRQ is enabled (the default)
1228 * do we actually try to configure these resources. Otherwise, we
1229 * simply set the cleanup level and continue on to the next resource
1230 */
1231 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1232 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1233 if (status != DDI_SUCCESS) {
1234 tavor_rsrc_fini(state, cleanup);
1235 goto rsrcinitp2_fail;
1236 }
1237
1238 /*
1239 * Save away the pointer to the central list of SRQ handle
1240 * pointers This this is used as a mechanism to enable fast
1241 * SRQnumber-to-SRQhandle lookup. The table is a list of
1242 * tavor_srqhdl_t allocated by the above routine because of the
1243 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many
1244 * tavor_srqhdl_t as the number of SRQs.
1245 */
1246 state->ts_srqhdl = hdl_info.swi_table_ptr;
1247 }
1248 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1249
1250 /*
1251 * Initialize the resource pool for the address handles. Notice
1252 * that the number of AHHDLs is configurable. The configured value
1253 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1254 * command) or the initialization will fail.
1255 */
1256 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1257 rsrc_pool->rsrc_type = TAVOR_AHHDL;
1258 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1259 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s);
1260 rsrc_pool->rsrc_state = state;
1261 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1262 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1263 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1264 hdl_info.swi_rsrcpool = rsrc_pool;
1265 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1266 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor;
1267 hdl_info.swi_rsrcname = rsrc_name;
1268 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1269 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1270 if (status != DDI_SUCCESS) {
1271 tavor_rsrc_fini(state, cleanup);
1272 goto rsrcinitp2_fail;
1273 }
1274 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1275
1276 /*
1277 * Initialize the resource pool for the QP handles. Notice
1278 * that the number of QPHDLs is configurable. The configured value
1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1280 * command) or the initialization will fail. Note also that the QP
1281 * handle has constructor and destructor methods associated with it.
1282 */
1283 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1284 rsrc_pool->rsrc_type = TAVOR_QPHDL;
1285 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1286 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s);
1287 rsrc_pool->rsrc_state = state;
1288 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1289 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1290 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1291 hdl_info.swi_rsrcpool = rsrc_pool;
1292 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1293 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor;
1294 hdl_info.swi_rsrcname = rsrc_name;
1295 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1296 TAVOR_SWHDL_TABLE_INIT);
1297 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1298 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1299 if (status != DDI_SUCCESS) {
1300 tavor_rsrc_fini(state, cleanup);
1301 goto rsrcinitp2_fail;
1302 }
1303
1304 /*
1305 * Save away the pointer to the central list of QP handle pointers
1306 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1307 * lookup during CQ event processing. The table is a list of
1308 * tavor_qphdl_t allocated by the above routine because of the
1309 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t
1310 * as the number of QPs.
1311 */
1312 state->ts_qphdl = hdl_info.swi_table_ptr;
1313 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1314
1315 /*
1316 * Initialize the resource pool for the reference count handles.
1317 * Notice that the number of REFCNTs is configurable, but it's value
1318 * is set to the number of MPTs. Since REFCNTs are used to support
1319 * shared memory regions, it is possible that we might require as
1320 * one REFCNT for every MPT.
1321 */
1322 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1323 rsrc_pool->rsrc_type = TAVOR_REFCNT;
1324 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1325 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t);
1326 rsrc_pool->rsrc_state = state;
1327 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1328 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1329 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1330 hdl_info.swi_rsrcpool = rsrc_pool;
1331 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1332 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor;
1333 hdl_info.swi_rsrcname = rsrc_name;
1334 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1335 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1336 if (status != DDI_SUCCESS) {
1337 tavor_rsrc_fini(state, cleanup);
1338 goto rsrcinitp2_fail;
1339 }
1340 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1341
1342 /*
1343 * Initialize the resource pool for the MCG handles. Notice that for
1344 * these MCG handles, we are allocating a table of structures (used to
1345 * keep track of the MCG entries that are being written to hardware
1346 * and to speed up multicast attach/detach operations).
1347 */
1348 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1349 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1350 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1351 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1352 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1353 if (status != DDI_SUCCESS) {
1354 tavor_rsrc_fini(state, cleanup);
1355 goto rsrcinitp2_fail;
1356 }
1357 state->ts_mcghdl = hdl_info.swi_table_ptr;
1358 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1359
1360 /*
1361 * Initialize the resource pools for all objects that exist in
1362 * UAR memory. The only objects that are allocated from UAR memory
1363 * are the UAR pages which are used for holding Tavor hardware's
1364 * doorbell registers.
1365 */
1366
1367 /*
1368 * Initialize the resource pool for the UAR pages. Notice
1369 * that the number of UARPGs is configurable. The configured value
1370 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1371 * command) or the initialization will fail. Note also that by
1372 * specifying the rsrc_start parameter in advance, we direct the
1373 * initialization routine not to attempt to allocated space from the
1374 * Tavor DDR vmem_arena.
1375 */
1376 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1377 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1378 PAGESHIFT));
1379 num_prealloc = 0;
1380 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1381 rsrc_pool->rsrc_type = TAVOR_UARPG;
1382 rsrc_pool->rsrc_loc = TAVOR_IN_UAR;
1383 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1384 rsrc_pool->rsrc_shift = PAGESHIFT;
1385 rsrc_pool->rsrc_quantum = PAGESIZE;
1386 rsrc_pool->rsrc_align = PAGESIZE;
1387 rsrc_pool->rsrc_state = state;
1388 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr;
1389 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1390 entry_info.hwi_num = num;
1391 entry_info.hwi_max = max;
1392 entry_info.hwi_prealloc = num_prealloc;
1393 entry_info.hwi_rsrcpool = rsrc_pool;
1394 entry_info.hwi_rsrcname = rsrc_name;
1395 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1396 if (status != DDI_SUCCESS) {
1397 tavor_rsrc_fini(state, cleanup);
1398 goto rsrcinitp2_fail;
1399 }
1400 cleanup = TAVOR_RSRC_CLEANUP_ALL;
1401
1402 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1403 return (DDI_SUCCESS);
1404
1405 rsrcinitp2_fail:
1406 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1407 return (status);
1408 }
1409
1410
1411 /*
1412 * tavor_rsrc_fini()
1413 * Context: Only called from attach() and/or detach() path contexts
1414 */
1415 void
tavor_rsrc_fini(tavor_state_t * state,tavor_rsrc_cleanup_level_t clean)1416 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1417 {
1418 tavor_rsrc_sw_hdl_info_t hdl_info;
1419 tavor_rsrc_hw_entry_info_t entry_info;
1420 tavor_rsrc_mbox_info_t mbox_info;
1421 tavor_cfg_profile_t *cfgprof;
1422
1423 ASSERT(state != NULL);
1424
1425 cfgprof = state->ts_cfg_profile;
1426
1427 switch (clean) {
1428 /*
1429 * If we add more resources that need to be cleaned up here, we should
1430 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1431 * corresponds to the last resource allocated).
1432 */
1433 case TAVOR_RSRC_CLEANUP_ALL:
1434 /* Cleanup the UAR page resource pool */
1435 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1436 tavor_rsrc_hw_entries_fini(state, &entry_info);
1437 /* FALLTHROUGH */
1438
1439 case TAVOR_RSRC_CLEANUP_LEVEL30:
1440 /* Cleanup the central MCG handle pointers list */
1441 hdl_info.swi_rsrcpool = NULL;
1442 hdl_info.swi_table_ptr = state->ts_mcghdl;
1443 hdl_info.swi_num =
1444 ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1445 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1446 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1447 /* FALLTHROUGH */
1448
1449 case TAVOR_RSRC_CLEANUP_LEVEL29:
1450 /* Cleanup the reference count resource pool */
1451 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1452 hdl_info.swi_table_ptr = NULL;
1453 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1454 /* FALLTHROUGH */
1455
1456 case TAVOR_RSRC_CLEANUP_LEVEL28:
1457 /* Cleanup the QP handle resource pool */
1458 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1459 hdl_info.swi_table_ptr = state->ts_qphdl;
1460 hdl_info.swi_num =
1461 ((uint64_t)1 << cfgprof->cp_log_num_qp);
1462 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1463 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1464 /* FALLTHROUGH */
1465
1466 case TAVOR_RSRC_CLEANUP_LEVEL27:
1467 /* Cleanup the address handle resource pool */
1468 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1469 hdl_info.swi_table_ptr = NULL;
1470 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1471 /* FALLTHROUGH */
1472
1473 case TAVOR_RSRC_CLEANUP_LEVEL26:
1474 /*
1475 * Cleanup the SRQ handle resource pool.
1476 *
1477 * Note: We only clean up if SRQ is enabled. Otherwise we
1478 * simply fallthrough to the next resource cleanup.
1479 */
1480 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1481 hdl_info.swi_rsrcpool =
1482 &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1483 hdl_info.swi_table_ptr = state->ts_srqhdl;
1484 hdl_info.swi_num =
1485 ((uint64_t)1 << cfgprof->cp_log_num_srq);
1486 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1487 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1488 }
1489 /* FALLTHROUGH */
1490
1491 case TAVOR_RSRC_CLEANUP_LEVEL25:
1492 /* Cleanup the CQ handle resource pool */
1493 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1494 hdl_info.swi_table_ptr = state->ts_cqhdl;
1495 hdl_info.swi_num =
1496 ((uint64_t)1 << cfgprof->cp_log_num_cq);
1497 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1498 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1499 /* FALLTHROUGH */
1500
1501 case TAVOR_RSRC_CLEANUP_LEVEL24:
1502 /* Cleanup the EQ handle resource pool */
1503 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1504 hdl_info.swi_table_ptr = NULL;
1505 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1506 /* FALLTHROUGH */
1507
1508 case TAVOR_RSRC_CLEANUP_LEVEL23:
1509 /* Cleanup the MR handle resource pool */
1510 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1511 hdl_info.swi_table_ptr = NULL;
1512 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1513 /* FALLTHROUGH */
1514
1515 case TAVOR_RSRC_CLEANUP_LEVEL22:
1516 /* Cleanup the PD handle resource pool */
1517 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1518 hdl_info.swi_table_ptr = NULL;
1519 tavor_rsrc_pd_handles_fini(state, &hdl_info);
1520 /* FALLTHROUGH */
1521
1522 case TAVOR_RSRC_CLEANUP_LEVEL21:
1523 /* Cleanup the EQC table resource pool */
1524 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1525 tavor_rsrc_hw_entries_fini(state, &entry_info);
1526 /* FALLTHROUGH */
1527
1528 case TAVOR_RSRC_CLEANUP_LEVEL20:
1529 /* Cleanup the MCG table resource pool */
1530 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1531 tavor_rsrc_hw_entries_fini(state, &entry_info);
1532 /* FALLTHROUGH */
1533
1534 case TAVOR_RSRC_CLEANUP_LEVEL19:
1535 /* Cleanup the outstanding command list */
1536 tavor_outstanding_cmdlist_fini(state);
1537 /* FALLTHROUGH */
1538
1539 case TAVOR_RSRC_CLEANUP_LEVEL18:
1540 /* Cleanup the "In" mailbox list */
1541 tavor_intr_inmbox_list_fini(state);
1542 /* FALLTHROUGH */
1543
1544 case TAVOR_RSRC_CLEANUP_LEVEL17:
1545 /* Cleanup the interrupt "In" mailbox resource pool */
1546 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1547 TAVOR_INTR_IN_MBOX];
1548 tavor_rsrc_mbox_fini(state, &mbox_info);
1549 /* FALLTHROUGH */
1550
1551 case TAVOR_RSRC_CLEANUP_LEVEL16:
1552 /* Cleanup the "In" mailbox list */
1553 tavor_inmbox_list_fini(state);
1554 /* FALLTHROUGH */
1555
1556 case TAVOR_RSRC_CLEANUP_LEVEL15:
1557 /* Cleanup the "In" mailbox resource pool */
1558 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1559 tavor_rsrc_mbox_fini(state, &mbox_info);
1560 /* FALLTHROUGH */
1561
1562 case TAVOR_RSRC_CLEANUP_LEVEL14:
1563 /*
1564 * Cleanup the SRQC table resource pool.
1565 *
1566 * Note: We only clean up if SRQ is enabled. Otherwise we
1567 * simply fallthrough to the next resource cleanup.
1568 */
1569 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1570 entry_info.hwi_rsrcpool =
1571 &state->ts_rsrc_hdl[TAVOR_SRQC];
1572 tavor_rsrc_hw_entries_fini(state, &entry_info);
1573 }
1574 /* FALLTHROUGH */
1575
1576 case TAVOR_RSRC_CLEANUP_LEVEL13:
1577 /* Cleanup the UAR scratch table resource pool */
1578 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1579 tavor_rsrc_hw_entries_fini(state, &entry_info);
1580 /* FALLTHROUGH */
1581
1582 case TAVOR_RSRC_CLEANUP_LEVEL12:
1583 /* Cleanup the UDAV table resource pool */
1584 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1585 tavor_rsrc_hw_entries_fini(state, &entry_info);
1586 /* FALLTHROUGH */
1587
1588 case TAVOR_RSRC_CLEANUP_LEVEL11:
1589 /* Cleanup the EQPC table resource pool */
1590 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1591 tavor_rsrc_hw_entries_fini(state, &entry_info);
1592 /* FALLTHROUGH */
1593
1594 case TAVOR_RSRC_CLEANUP_LEVEL10:
1595 /* Cleanup the CQC table resource pool */
1596 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1597 tavor_rsrc_hw_entries_fini(state, &entry_info);
1598 /* FALLTHROUGH */
1599
1600 case TAVOR_RSRC_CLEANUP_LEVEL9:
1601 /* Cleanup the RDB table resource pool */
1602 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1603 tavor_rsrc_hw_entries_fini(state, &entry_info);
1604 /* FALLTHROUGH */
1605
1606 case TAVOR_RSRC_CLEANUP_LEVEL8:
1607 /* Cleanup the QPC table resource pool */
1608 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1609 tavor_rsrc_hw_entries_fini(state, &entry_info);
1610 /* FALLTHROUGH */
1611
1612 case TAVOR_RSRC_CLEANUP_LEVEL7:
1613 /* Cleanup the MTT table resource pool */
1614 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1615 tavor_rsrc_hw_entries_fini(state, &entry_info);
1616 /* FALLTHROUGH */
1617
1618 case TAVOR_RSRC_CLEANUP_LEVEL6:
1619 /* Cleanup the MPT table resource pool */
1620 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1621 tavor_rsrc_hw_entries_fini(state, &entry_info);
1622 /* FALLTHROUGH */
1623
1624 case TAVOR_RSRC_CLEANUP_LEVEL5:
1625 /* Destroy the vmem arena for DDR memory */
1626 vmem_destroy(state->ts_ddrvmem);
1627 break;
1628
1629 /*
1630 * The cleanup below comes from the "Phase 1" initialization step.
1631 * (see tavor_rsrc_init_phase1() above)
1632 */
1633 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1634 /* Cleanup the interrupt "Out" mailbox list */
1635 tavor_intr_outmbox_list_fini(state);
1636 /* FALLTHROUGH */
1637
1638 case TAVOR_RSRC_CLEANUP_LEVEL3:
1639 /* Cleanup the "Out" mailbox resource pool */
1640 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1641 TAVOR_INTR_OUT_MBOX];
1642 tavor_rsrc_mbox_fini(state, &mbox_info);
1643 /* FALLTHROUGH */
1644
1645 case TAVOR_RSRC_CLEANUP_LEVEL2:
1646 /* Cleanup the "Out" mailbox list */
1647 tavor_outmbox_list_fini(state);
1648 /* FALLTHROUGH */
1649
1650 case TAVOR_RSRC_CLEANUP_LEVEL1:
1651 /* Cleanup the "Out" mailbox resource pool */
1652 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1653 tavor_rsrc_mbox_fini(state, &mbox_info);
1654 /* FALLTHROUGH */
1655
1656 case TAVOR_RSRC_CLEANUP_LEVEL0:
1657 /* Free the array of tavor_rsrc_pool_info_t's */
1658 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1659 sizeof (tavor_rsrc_pool_info_t));
1660 kmem_cache_destroy(state->ts_rsrc_cache);
1661 break;
1662
1663 default:
1664 TAVOR_WARNING(state, "unexpected resource cleanup level");
1665 }
1666 }
1667
1668
1669 /*
1670 * tavor_rsrc_mbox_init()
1671 * Context: Only called from attach() path context
1672 */
1673 static int
tavor_rsrc_mbox_init(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1674 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1675 {
1676 tavor_rsrc_pool_info_t *rsrc_pool;
1677 tavor_rsrc_priv_mbox_t *priv;
1678 vmem_t *vmp;
1679 uint64_t offset;
1680 uint_t dma_xfer_mode;
1681
1682 ASSERT(state != NULL);
1683 ASSERT(info != NULL);
1684
1685 rsrc_pool = info->mbi_rsrcpool;
1686 ASSERT(rsrc_pool != NULL);
1687
1688 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1689
1690 /* Allocate and initialize mailbox private structure */
1691 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1692 priv->pmb_dip = state->ts_dip;
1693 priv->pmb_acchdl = state->ts_reg_ddrhdl;
1694 priv->pmb_devaccattr = state->ts_reg_accattr;
1695 priv->pmb_xfer_mode = dma_xfer_mode;
1696
1697 /*
1698 * Initialize many of the default DMA attributes. Then set alignment
1699 * and scatter-gather restrictions specific for mailbox memory.
1700 */
1701 tavor_dma_attr_init(&priv->pmb_dmaattr);
1702 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN;
1703 priv->pmb_dmaattr.dma_attr_sgllen = 1;
1704
1705 rsrc_pool->rsrc_private = priv;
1706
1707 /* Is object in DDR memory or system memory? */
1708 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1709 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1710 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1711 0, 0, NULL, NULL, VM_SLEEP);
1712 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1713 /* Unable to alloc space for mailboxes */
1714 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1715 return (DDI_FAILURE);
1716 }
1717
1718 /* Calculate offset and starting point (in DDR) */
1719 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1720 state->ts_ddr.ddr_baseaddr);
1721 rsrc_pool->rsrc_start =
1722 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1723 offset);
1724
1725 /* Create new vmem arena for the mailboxes */
1726 vmp = vmem_create(info->mbi_rsrcname,
1727 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1728 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1729 if (vmp == NULL) {
1730 /* Unable to create vmem arena */
1731 vmem_xfree(state->ts_ddrvmem,
1732 rsrc_pool->rsrc_ddr_offset,
1733 rsrc_pool->rsrc_pool_size);
1734 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1735 return (DDI_FAILURE);
1736 }
1737 rsrc_pool->rsrc_vmp = vmp;
1738 } else {
1739 rsrc_pool->rsrc_ddr_offset = NULL;
1740 rsrc_pool->rsrc_start = NULL;
1741 rsrc_pool->rsrc_vmp = NULL;
1742 }
1743
1744 return (DDI_SUCCESS);
1745 }
1746
1747
1748 /*
1749 * tavor_rsrc_mbox_fini()
1750 * Context: Only called from attach() and/or detach() path contexts
1751 */
1752 static void
tavor_rsrc_mbox_fini(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1753 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1754 {
1755 tavor_rsrc_pool_info_t *rsrc_pool;
1756
1757 ASSERT(state != NULL);
1758 ASSERT(info != NULL);
1759
1760 rsrc_pool = info->mbi_rsrcpool;
1761 ASSERT(rsrc_pool != NULL);
1762
1763 /* If mailboxes are DDR memory, then destroy and free up vmem */
1764 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1765
1766 /* Destroy the specially created mbox vmem arena */
1767 vmem_destroy(rsrc_pool->rsrc_vmp);
1768
1769 /* Free up the region from the ddr_vmem arena */
1770 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1771 rsrc_pool->rsrc_pool_size);
1772 }
1773
1774 /* Free up the private struct */
1775 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1776 }
1777
1778
1779 /*
1780 * tavor_rsrc_hw_entries_init()
1781 * Context: Only called from attach() path context
1782 */
1783 static int
tavor_rsrc_hw_entries_init(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)1784 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1785 tavor_rsrc_hw_entry_info_t *info)
1786 {
1787 tavor_rsrc_pool_info_t *rsrc_pool;
1788 tavor_rsrc_t *rsvd_rsrc = NULL;
1789 vmem_t *vmp;
1790 uint64_t num_hwentry, max_hwentry, num_prealloc;
1791 uint64_t offset;
1792 int status;
1793
1794 ASSERT(state != NULL);
1795 ASSERT(info != NULL);
1796
1797 rsrc_pool = info->hwi_rsrcpool;
1798 ASSERT(rsrc_pool != NULL);
1799 num_hwentry = info->hwi_num;
1800 max_hwentry = info->hwi_max;
1801 num_prealloc = info->hwi_prealloc;
1802
1803 /* Make sure number of HW entries makes sense */
1804 if (num_hwentry > max_hwentry) {
1805 return (DDI_FAILURE);
1806 }
1807
1808 /*
1809 * Determine if we need to allocate DDR space to set up the
1810 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already
1811 * been initialized (as is the case for the UAR page init).
1812 */
1813 if (rsrc_pool->rsrc_start == NULL) {
1814 /* Make sure HW entries table is aligned as specified */
1815 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1816 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1817 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1818 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1819 /* Unable to alloc space for aligned HW table */
1820 return (DDI_FAILURE);
1821 }
1822
1823 /* Calculate offset and starting point (in DDR) */
1824 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1825 state->ts_ddr.ddr_baseaddr);
1826 rsrc_pool->rsrc_start =
1827 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1828 offset);
1829 } else {
1830 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1831 }
1832
1833 /*
1834 * Create new vmem arena for the HW entries table (if rsrc_quantum
1835 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW
1836 * entries are not going to be dynamically allocatable (i.e. they
1837 * won't be allocated/freed through tavor_rsrc_alloc/free). This
1838 * latter option is used for EQPC and UARSCR resource which are, in
1839 * fact, managed by the Tavor hardware.
1840 */
1841 if (rsrc_pool->rsrc_quantum != 0) {
1842 vmp = vmem_create(info->hwi_rsrcname,
1843 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1844 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1845 if (vmp == NULL) {
1846 /* Unable to create vmem arena */
1847 if (rsrc_pool->rsrc_ddr_offset !=
1848 rsrc_pool->rsrc_start) {
1849 vmem_xfree(state->ts_ddrvmem,
1850 rsrc_pool->rsrc_ddr_offset,
1851 rsrc_pool->rsrc_pool_size);
1852 }
1853 return (DDI_FAILURE);
1854 }
1855 rsrc_pool->rsrc_vmp = vmp;
1856 } else {
1857 rsrc_pool->rsrc_vmp = NULL;
1858 }
1859
1860 /* The first HW entries may be reserved by Tavor firmware */
1861 if (num_prealloc != 0) {
1862 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1863 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1864 if (status != DDI_SUCCESS) {
1865 /* Unable to preallocate the reserved HW entries */
1866 if (rsrc_pool->rsrc_vmp != NULL) {
1867 vmem_destroy(rsrc_pool->rsrc_vmp);
1868 }
1869 if (rsrc_pool->rsrc_ddr_offset !=
1870 rsrc_pool->rsrc_start) {
1871 vmem_xfree(state->ts_ddrvmem,
1872 rsrc_pool->rsrc_ddr_offset,
1873 rsrc_pool->rsrc_pool_size);
1874 }
1875 return (DDI_FAILURE);
1876 }
1877 }
1878 rsrc_pool->rsrc_private = rsvd_rsrc;
1879
1880 return (DDI_SUCCESS);
1881 }
1882
1883
1884 /*
1885 * tavor_rsrc_hw_entries_fini()
1886 * Context: Only called from attach() and/or detach() path contexts
1887 */
1888 static void
tavor_rsrc_hw_entries_fini(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)1889 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
1890 tavor_rsrc_hw_entry_info_t *info)
1891 {
1892 tavor_rsrc_pool_info_t *rsrc_pool;
1893 tavor_rsrc_t *rsvd_rsrc;
1894
1895 ASSERT(state != NULL);
1896 ASSERT(info != NULL);
1897
1898 rsrc_pool = info->hwi_rsrcpool;
1899 ASSERT(rsrc_pool != NULL);
1900
1901 /* Free up any "reserved" (i.e. preallocated) HW entries */
1902 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
1903 if (rsvd_rsrc != NULL) {
1904 tavor_rsrc_free(state, &rsvd_rsrc);
1905 }
1906
1907 /*
1908 * If we've actually setup a vmem arena for the HW entries, then
1909 * destroy it now
1910 */
1911 if (rsrc_pool->rsrc_vmp != NULL) {
1912 vmem_destroy(rsrc_pool->rsrc_vmp);
1913 }
1914
1915 /*
1916 * Determine if a region was allocated from the tavor_ddr_vmem
1917 * arena (and free it up if necessary)
1918 */
1919 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
1920 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1921 rsrc_pool->rsrc_pool_size);
1922 }
1923 }
1924
1925
1926 /*
1927 * tavor_rsrc_sw_handles_init()
1928 * Context: Only called from attach() path context
1929 */
1930 /* ARGSUSED */
1931 static int
tavor_rsrc_sw_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)1932 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
1933 {
1934 tavor_rsrc_pool_info_t *rsrc_pool;
1935 uint64_t num_swhdl, max_swhdl, prealloc_sz;
1936
1937 ASSERT(state != NULL);
1938 ASSERT(info != NULL);
1939
1940 rsrc_pool = info->swi_rsrcpool;
1941 ASSERT(rsrc_pool != NULL);
1942 num_swhdl = info->swi_num;
1943 max_swhdl = info->swi_max;
1944 prealloc_sz = info->swi_prealloc_sz;
1945
1946 /* Make sure number of SW handles makes sense */
1947 if (num_swhdl > max_swhdl) {
1948 return (DDI_FAILURE);
1949 }
1950
1951 /*
1952 * Depending on the flags parameter, create a kmem_cache for some
1953 * number of software handle structures. Note: kmem_cache_create()
1954 * will SLEEP until successful.
1955 */
1956 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
1957 rsrc_pool->rsrc_private = kmem_cache_create(
1958 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
1959 info->swi_constructor, info->swi_destructor, NULL,
1960 rsrc_pool->rsrc_state, NULL, 0);
1961 }
1962
1963 /* Allocate the central list of SW handle pointers */
1964 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
1965 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
1966 KM_SLEEP);
1967 }
1968
1969 return (DDI_SUCCESS);
1970 }
1971
1972
1973 /*
1974 * tavor_rsrc_sw_handles_fini()
1975 * Context: Only called from attach() and/or detach() path contexts
1976 */
1977 /* ARGSUSED */
1978 static void
tavor_rsrc_sw_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)1979 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
1980 {
1981 tavor_rsrc_pool_info_t *rsrc_pool;
1982 uint64_t num_swhdl, prealloc_sz;
1983
1984 ASSERT(state != NULL);
1985 ASSERT(info != NULL);
1986
1987 rsrc_pool = info->swi_rsrcpool;
1988 num_swhdl = info->swi_num;
1989 prealloc_sz = info->swi_prealloc_sz;
1990
1991 /*
1992 * If a "software handle" kmem_cache exists for this resource, then
1993 * destroy it now
1994 */
1995 if (rsrc_pool != NULL) {
1996 kmem_cache_destroy(rsrc_pool->rsrc_private);
1997 }
1998
1999 /* Free up this central list of SW handle pointers */
2000 if (info->swi_table_ptr != NULL) {
2001 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2002 }
2003 }
2004
2005
2006 /*
2007 * tavor_rsrc_pd_handles_init()
2008 * Context: Only called from attach() path context
2009 */
2010 static int
tavor_rsrc_pd_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2011 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2012 {
2013 tavor_rsrc_pool_info_t *rsrc_pool;
2014 vmem_t *vmp;
2015 char vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2016 int status;
2017
2018 ASSERT(state != NULL);
2019 ASSERT(info != NULL);
2020
2021 rsrc_pool = info->swi_rsrcpool;
2022 ASSERT(rsrc_pool != NULL);
2023
2024 /* Initialize the resource pool for software handle table */
2025 status = tavor_rsrc_sw_handles_init(state, info);
2026 if (status != DDI_SUCCESS) {
2027 return (DDI_FAILURE);
2028 }
2029
2030 /* Build vmem arena name from Tavor instance */
2031 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2032
2033 /* Create new vmem arena for PD numbers */
2034 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2035 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2036 if (vmp == NULL) {
2037 /* Unable to create vmem arena */
2038 info->swi_table_ptr = NULL;
2039 tavor_rsrc_sw_handles_fini(state, info);
2040 return (DDI_FAILURE);
2041 }
2042 rsrc_pool->rsrc_vmp = vmp;
2043
2044 return (DDI_SUCCESS);
2045 }
2046
2047
2048 /*
2049 * tavor_rsrc_pd_handles_fini()
2050 * Context: Only called from attach() and/or detach() path contexts
2051 */
2052 static void
tavor_rsrc_pd_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2053 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2054 {
2055 tavor_rsrc_pool_info_t *rsrc_pool;
2056
2057 ASSERT(state != NULL);
2058 ASSERT(info != NULL);
2059
2060 rsrc_pool = info->swi_rsrcpool;
2061
2062 /* Destroy the specially created UAR scratch table vmem arena */
2063 vmem_destroy(rsrc_pool->rsrc_vmp);
2064
2065 /* Destroy the "tavor_sw_pd_t" kmem_cache */
2066 tavor_rsrc_sw_handles_fini(state, info);
2067 }
2068
2069
2070 /*
2071 * tavor_rsrc_mbox_alloc()
2072 * Context: Only called from attach() path context
2073 */
2074 static int
tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,tavor_rsrc_t * hdl)2075 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2076 tavor_rsrc_t *hdl)
2077 {
2078 tavor_rsrc_priv_mbox_t *priv;
2079 void *addr;
2080 caddr_t kaddr;
2081 uint64_t offset;
2082 size_t real_len, temp_len;
2083 int status;
2084
2085 ASSERT(pool_info != NULL);
2086 ASSERT(hdl != NULL);
2087
2088 /* Get the private pointer for the mailboxes */
2089 priv = pool_info->rsrc_private;
2090 ASSERT(priv != NULL);
2091
2092 /*
2093 * Allocate a DMA handle for the mailbox. This will be used for
2094 * two purposes (potentially). First, it could be used below in
2095 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2096 * system memory. Second, it is definitely used later to bind
2097 * the mailbox for DMA access from/by the hardware.
2098 */
2099 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2100 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2101 if (status != DDI_SUCCESS) {
2102 return (DDI_FAILURE);
2103 }
2104
2105 /* Is mailbox in DDR memory or system memory? */
2106 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2107 /* Use vmem_alloc() to get DDR address of mbox */
2108 hdl->tr_len = (num * pool_info->rsrc_quantum);
2109 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2110 VM_SLEEP);
2111 if (addr == NULL) {
2112 /* No more DDR available for mailbox entries */
2113 ddi_dma_free_handle(&hdl->tr_dmahdl);
2114 return (DDI_FAILURE);
2115 }
2116 hdl->tr_acchdl = priv->pmb_acchdl;
2117
2118 /* Calculate kernel virtual address (from the DDR offset) */
2119 offset = ((uintptr_t)addr -
2120 (uintptr_t)pool_info->rsrc_ddr_offset);
2121 hdl->tr_addr = (void *)(uintptr_t)(offset +
2122 (uintptr_t)pool_info->rsrc_start);
2123
2124 } else { /* TAVOR_IN_SYSMEM */
2125
2126 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2127 temp_len = (num * pool_info->rsrc_quantum);
2128 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2129 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2130 NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2131 if (status != DDI_SUCCESS) {
2132 /* No more sys memory available for mailbox entries */
2133 ddi_dma_free_handle(&hdl->tr_dmahdl);
2134 return (DDI_FAILURE);
2135 }
2136 hdl->tr_addr = (void *)kaddr;
2137 hdl->tr_len = real_len;
2138 }
2139
2140 return (DDI_SUCCESS);
2141 }
2142
2143
2144 /*
2145 * tavor_rsrc_mbox_free()
2146 * Context: Can be called from interrupt or base context.
2147 */
2148 static void
tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2149 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2150 {
2151 void *addr;
2152 uint64_t offset;
2153
2154 ASSERT(pool_info != NULL);
2155 ASSERT(hdl != NULL);
2156
2157 /* Is mailbox in DDR memory or system memory? */
2158 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2159
2160 /* Calculate the allocated address (the mbox's DDR offset) */
2161 offset = ((uintptr_t)hdl->tr_addr -
2162 (uintptr_t)pool_info->rsrc_start);
2163 addr = (void *)(uintptr_t)(offset +
2164 (uintptr_t)pool_info->rsrc_ddr_offset);
2165
2166 /* Use vmem_free() to free up DDR memory for mailbox */
2167 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2168
2169 } else { /* TAVOR_IN_SYSMEM */
2170
2171 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2172 ddi_dma_mem_free(&hdl->tr_acchdl);
2173 }
2174
2175 /* Free the DMA handle for the mailbox */
2176 ddi_dma_free_handle(&hdl->tr_dmahdl);
2177 }
2178
2179
2180 /*
2181 * tavor_rsrc_hw_entry_alloc()
2182 * Context: Can be called from interrupt or base context.
2183 */
2184 static int
tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,uint_t num_align,ddi_acc_handle_t acc_handle,uint_t sleepflag,tavor_rsrc_t * hdl)2185 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2186 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2187 tavor_rsrc_t *hdl)
2188 {
2189 void *addr;
2190 uint64_t offset;
2191 uint32_t align;
2192 int flag;
2193
2194 ASSERT(pool_info != NULL);
2195 ASSERT(hdl != NULL);
2196
2197 /*
2198 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2199 * use dma_handle (because they are in Tavor locally attached DDR
2200 * memory) and, generally, don't use the acc_handle (because the
2201 * entries are not directly accessed by software). The exceptions
2202 * to this rule are the UARPG and UDAV entries.
2203 */
2204
2205 /*
2206 * Use vmem_xalloc() to get a properly aligned pointer (based on
2207 * the number requested) to the HW entry(ies). This handles the
2208 * cases (for special QPCs and for RDB entries) where we need more
2209 * than one and need to ensure that they are properly aligned.
2210 */
2211 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2212 hdl->tr_len = (num * pool_info->rsrc_quantum);
2213 align = (num_align * pool_info->rsrc_quantum);
2214 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2215 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2216 if (addr == NULL) {
2217 /* No more HW entries available */
2218 return (DDI_FAILURE);
2219 }
2220
2221 /* If an access handle was provided, fill it in */
2222 if (acc_handle != 0) {
2223 hdl->tr_acchdl = acc_handle;
2224 }
2225
2226 /* Calculate vaddr and HW table index (from the DDR offset) */
2227 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2228 hdl->tr_addr = (void *)(uintptr_t)(offset +
2229 (uintptr_t)pool_info->rsrc_start);
2230 hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2231
2232 return (DDI_SUCCESS);
2233 }
2234
2235
2236 /*
2237 * tavor_rsrc_hw_entry_free()
2238 * Context: Can be called from interrupt or base context.
2239 */
2240 static void
tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2241 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2242 {
2243 void *addr;
2244 uint64_t offset;
2245
2246 ASSERT(pool_info != NULL);
2247 ASSERT(hdl != NULL);
2248
2249 /* Calculate the allocated address (the entry's DDR offset) */
2250 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2251 addr = (void *)(uintptr_t)(offset +
2252 (uintptr_t)pool_info->rsrc_ddr_offset);
2253
2254 /* Use vmem_xfree() to free up the HW table entry */
2255 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2256 }
2257
2258
2259 /*
2260 * tavor_rsrc_swhdl_alloc()
2261 * Context: Can be called from interrupt or base context.
2262 */
2263 static int
tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2264 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2265 tavor_rsrc_t *hdl)
2266 {
2267 void *addr;
2268 int flag;
2269
2270 ASSERT(pool_info != NULL);
2271 ASSERT(hdl != NULL);
2272
2273 /* Allocate the software handle structure */
2274 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2275 addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2276 if (addr == NULL) {
2277 return (DDI_FAILURE);
2278 }
2279 hdl->tr_len = pool_info->rsrc_quantum;
2280 hdl->tr_addr = addr;
2281
2282 return (DDI_SUCCESS);
2283 }
2284
2285
2286 /*
2287 * tavor_rsrc_swhdl_free()
2288 * Context: Can be called from interrupt or base context.
2289 */
2290 static void
tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2291 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2292 {
2293 ASSERT(pool_info != NULL);
2294 ASSERT(hdl != NULL);
2295
2296 /* Free the software handle structure */
2297 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2298 }
2299
2300
2301 /*
2302 * tavor_rsrc_pdhdl_alloc()
2303 * Context: Can be called from interrupt or base context.
2304 */
2305 static int
tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2306 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2307 tavor_rsrc_t *hdl)
2308 {
2309 tavor_pdhdl_t addr;
2310 void *tmpaddr;
2311 int flag, status;
2312
2313 ASSERT(pool_info != NULL);
2314 ASSERT(hdl != NULL);
2315
2316 /* Allocate the software handle */
2317 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2318 if (status != DDI_SUCCESS) {
2319 return (DDI_FAILURE);
2320 }
2321 addr = (tavor_pdhdl_t)hdl->tr_addr;
2322 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2323
2324 /* Allocate a PD number for the handle */
2325 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2326 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2327 if (tmpaddr == NULL) {
2328 /* No more PD number entries available */
2329 tavor_rsrc_swhdl_free(pool_info, hdl);
2330 return (DDI_FAILURE);
2331 }
2332 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2333 addr->pd_rsrcp = hdl;
2334 hdl->tr_indx = addr->pd_pdnum;
2335
2336 return (DDI_SUCCESS);
2337 }
2338
2339
2340 /*
2341 * tavor_rsrc_pdhdl_free()
2342 * Context: Can be called from interrupt or base context.
2343 */
2344 static void
tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2345 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2346 {
2347 ASSERT(pool_info != NULL);
2348 ASSERT(hdl != NULL);
2349
2350 /* Use vmem_free() to free up the PD number */
2351 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2352
2353 /* Free the software handle structure */
2354 tavor_rsrc_swhdl_free(pool_info, hdl);
2355 }
2356
2357
2358 /*
2359 * tavor_rsrc_pdhdl_constructor()
2360 * Context: Can be called from interrupt or base context.
2361 */
2362 /* ARGSUSED */
2363 static int
tavor_rsrc_pdhdl_constructor(void * pd,void * priv,int flags)2364 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2365 {
2366 tavor_pdhdl_t pdhdl;
2367 tavor_state_t *state;
2368
2369 pdhdl = (tavor_pdhdl_t)pd;
2370 state = (tavor_state_t *)priv;
2371
2372 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2373 DDI_INTR_PRI(state->ts_intrmsi_pri));
2374
2375 return (DDI_SUCCESS);
2376 }
2377
2378
2379 /*
2380 * tavor_rsrc_pdhdl_destructor()
2381 * Context: Can be called from interrupt or base context.
2382 */
2383 /* ARGSUSED */
2384 static void
tavor_rsrc_pdhdl_destructor(void * pd,void * priv)2385 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2386 {
2387 tavor_pdhdl_t pdhdl;
2388
2389 pdhdl = (tavor_pdhdl_t)pd;
2390
2391 mutex_destroy(&pdhdl->pd_lock);
2392 }
2393
2394
2395 /*
2396 * tavor_rsrc_cqhdl_constructor()
2397 * Context: Can be called from interrupt or base context.
2398 */
2399 /* ARGSUSED */
2400 static int
tavor_rsrc_cqhdl_constructor(void * cq,void * priv,int flags)2401 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2402 {
2403 tavor_cqhdl_t cqhdl;
2404 tavor_state_t *state;
2405
2406 cqhdl = (tavor_cqhdl_t)cq;
2407 state = (tavor_state_t *)priv;
2408
2409 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2410 DDI_INTR_PRI(state->ts_intrmsi_pri));
2411 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2412 DDI_INTR_PRI(state->ts_intrmsi_pri));
2413
2414 return (DDI_SUCCESS);
2415 }
2416
2417
2418 /*
2419 * tavor_rsrc_cqhdl_destructor()
2420 * Context: Can be called from interrupt or base context.
2421 */
2422 /* ARGSUSED */
2423 static void
tavor_rsrc_cqhdl_destructor(void * cq,void * priv)2424 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2425 {
2426 tavor_cqhdl_t cqhdl;
2427
2428 cqhdl = (tavor_cqhdl_t)cq;
2429
2430 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2431 mutex_destroy(&cqhdl->cq_lock);
2432 }
2433
2434
2435 /*
2436 * tavor_rsrc_qphdl_constructor()
2437 * Context: Can be called from interrupt or base context.
2438 */
2439 /* ARGSUSED */
2440 static int
tavor_rsrc_qphdl_constructor(void * qp,void * priv,int flags)2441 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2442 {
2443 tavor_qphdl_t qphdl;
2444 tavor_state_t *state;
2445
2446 qphdl = (tavor_qphdl_t)qp;
2447 state = (tavor_state_t *)priv;
2448
2449 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2450 DDI_INTR_PRI(state->ts_intrmsi_pri));
2451
2452 return (DDI_SUCCESS);
2453 }
2454
2455
2456 /*
2457 * tavor_rsrc_qphdl_destructor()
2458 * Context: Can be called from interrupt or base context.
2459 */
2460 /* ARGSUSED */
2461 static void
tavor_rsrc_qphdl_destructor(void * qp,void * priv)2462 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2463 {
2464 tavor_qphdl_t qphdl;
2465
2466 qphdl = (tavor_qphdl_t)qp;
2467
2468 mutex_destroy(&qphdl->qp_lock);
2469 }
2470
2471
2472 /*
2473 * tavor_rsrc_srqhdl_constructor()
2474 * Context: Can be called from interrupt or base context.
2475 */
2476 /* ARGSUSED */
2477 static int
tavor_rsrc_srqhdl_constructor(void * srq,void * priv,int flags)2478 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2479 {
2480 tavor_srqhdl_t srqhdl;
2481 tavor_state_t *state;
2482
2483 srqhdl = (tavor_srqhdl_t)srq;
2484 state = (tavor_state_t *)priv;
2485
2486 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2487 DDI_INTR_PRI(state->ts_intrmsi_pri));
2488
2489 return (DDI_SUCCESS);
2490 }
2491
2492
2493 /*
2494 * tavor_rsrc_srqhdl_destructor()
2495 * Context: Can be called from interrupt or base context.
2496 */
2497 /* ARGSUSED */
2498 static void
tavor_rsrc_srqhdl_destructor(void * srq,void * priv)2499 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2500 {
2501 tavor_srqhdl_t srqhdl;
2502
2503 srqhdl = (tavor_srqhdl_t)srq;
2504
2505 mutex_destroy(&srqhdl->srq_lock);
2506 }
2507
2508
2509 /*
2510 * tavor_rsrc_refcnt_constructor()
2511 * Context: Can be called from interrupt or base context.
2512 */
2513 /* ARGSUSED */
2514 static int
tavor_rsrc_refcnt_constructor(void * rc,void * priv,int flags)2515 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2516 {
2517 tavor_sw_refcnt_t *refcnt;
2518 tavor_state_t *state;
2519
2520 refcnt = (tavor_sw_refcnt_t *)rc;
2521 state = (tavor_state_t *)priv;
2522
2523 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2524 DDI_INTR_PRI(state->ts_intrmsi_pri));
2525
2526 return (DDI_SUCCESS);
2527 }
2528
2529
2530 /*
2531 * tavor_rsrc_refcnt_destructor()
2532 * Context: Can be called from interrupt or base context.
2533 */
2534 /* ARGSUSED */
2535 static void
tavor_rsrc_refcnt_destructor(void * rc,void * priv)2536 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2537 {
2538 tavor_sw_refcnt_t *refcnt;
2539
2540 refcnt = (tavor_sw_refcnt_t *)rc;
2541
2542 mutex_destroy(&refcnt->swrc_lock);
2543 }
2544
2545
2546 /*
2547 * tavor_rsrc_ahhdl_constructor()
2548 * Context: Can be called from interrupt or base context.
2549 */
2550 /* ARGSUSED */
2551 static int
tavor_rsrc_ahhdl_constructor(void * ah,void * priv,int flags)2552 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2553 {
2554 tavor_ahhdl_t ahhdl;
2555 tavor_state_t *state;
2556
2557 ahhdl = (tavor_ahhdl_t)ah;
2558 state = (tavor_state_t *)priv;
2559
2560 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2561 DDI_INTR_PRI(state->ts_intrmsi_pri));
2562
2563 return (DDI_SUCCESS);
2564 }
2565
2566
2567 /*
2568 * tavor_rsrc_ahhdl_destructor()
2569 * Context: Can be called from interrupt or base context.
2570 */
2571 /* ARGSUSED */
2572 static void
tavor_rsrc_ahhdl_destructor(void * ah,void * priv)2573 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2574 {
2575 tavor_ahhdl_t ahhdl;
2576
2577 ahhdl = (tavor_ahhdl_t)ah;
2578
2579 mutex_destroy(&ahhdl->ah_lock);
2580 }
2581
2582
2583 /*
2584 * tavor_rsrc_mrhdl_constructor()
2585 * Context: Can be called from interrupt or base context.
2586 */
2587 /* ARGSUSED */
2588 static int
tavor_rsrc_mrhdl_constructor(void * mr,void * priv,int flags)2589 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2590 {
2591 tavor_mrhdl_t mrhdl;
2592 tavor_state_t *state;
2593
2594 mrhdl = (tavor_mrhdl_t)mr;
2595 state = (tavor_state_t *)priv;
2596
2597 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2598 DDI_INTR_PRI(state->ts_intrmsi_pri));
2599
2600 return (DDI_SUCCESS);
2601 }
2602
2603
2604 /*
2605 * tavor_rsrc_mrhdl_destructor()
2606 * Context: Can be called from interrupt or base context.
2607 */
2608 /* ARGSUSED */
2609 static void
tavor_rsrc_mrhdl_destructor(void * mr,void * priv)2610 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2611 {
2612 tavor_mrhdl_t mrhdl;
2613
2614 mrhdl = (tavor_mrhdl_t)mr;
2615
2616 mutex_destroy(&mrhdl->mr_lock);
2617 }
2618
2619
2620 /*
2621 * tavor_rsrc_mcg_entry_get_size()
2622 */
2623 static int
tavor_rsrc_mcg_entry_get_size(tavor_state_t * state,uint_t * mcg_size_shift)2624 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2625 {
2626 uint_t num_qp_per_mcg, max_qp_per_mcg, log2;
2627
2628 /*
2629 * Round the configured number of QP per MCG to next larger
2630 * power-of-2 size and update.
2631 */
2632 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2633 log2 = highbit(num_qp_per_mcg);
2634 if (ISP2(num_qp_per_mcg)) {
2635 log2 = log2 - 1;
2636 }
2637 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2638
2639 /* Now make sure number of QP per MCG makes sense */
2640 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2641 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2642 if (num_qp_per_mcg > max_qp_per_mcg) {
2643 return (DDI_FAILURE);
2644 }
2645
2646 /* Return the (shift) size of an individual MCG HW entry */
2647 *mcg_size_shift = log2 + 2;
2648
2649 return (DDI_SUCCESS);
2650 }
2651