1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_rsrc.c
29 * Tavor Resource Management Routines
30 *
31 * Implements all the routines necessary for setup, teardown, and
32 * alloc/free of all Tavor resources, including those that are managed
33 * by Tavor hardware or which live in Tavor's direct attached DDR memory.
34 */
35
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/vmem.h>
43 #include <sys/bitmap.h>
44
45 #include <sys/ib/adapters/tavor/tavor.h>
46
47 /*
48 * The following routines are used for initializing and destroying
49 * the resource pools used by the Tavor resource allocation routines.
50 * They consist of four classes of object:
51 *
52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor
53 * command interface routines. Mailboxes are used to pass information
54 * back and forth to the Tavor firmware. Either type of mailbox may
55 * be allocated from Tavor's direct attached DDR memory or from system
56 * memory (although currently all "In" mailboxes are in DDR and all "out"
57 * mailboxes come from system memory.
58 *
59 * HW entry objects: These objects represent resources required by the Tavor
60 * hardware. These objects include things like Queue Pair contexts (QPC),
61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
63 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
64 *
65 * What these objects all have in common is that they are each required
66 * to come from DDR memory, they are always allocated from tables, and
67 * they are not to be directly accessed (read or written) by driver
68 * software.
69 * One notable exceptions to this rule are the Extended QP contexts (EQPC),
70 * and the UAR scratch area (UAR_SCR), both of which are not directly
71 * accessible through the Tavor resource allocation routines, but both
72 * of which are also required to reside in DDR memory and are not to be
73 * manipulated by driver software (they are privately managed by Tavor
74 * hardware).
75 * The other notable exceptions are the UAR pages (UAR_PG) which are
76 * allocated from the UAR address space rather than DDR, and the UD
77 * address vectors (UDAV) which are similar to the common object types
78 * with the major difference being that UDAVs _are_ directly read and
79 * written by driver software.
80 *
81 * SW handle objects: These objects represent resources required by Tavor
82 * driver software. They are primarily software tracking structures,
83 * which are allocated from system memory (using kmem_cache). Several of
84 * the objects have both a "constructor" and "destructor" method
85 * associated with them (see below).
86 *
87 * Protection Domain (PD) handle objects: These objects are very much like
88 * a SW handle object with the notable difference that all PD handle
89 * objects have an actual Protection Domain number (PD) associated with
90 * them (and the PD number is allocated/managed through a separate
91 * vmem_arena specifically set aside for this purpose.
92 */
93
94 static int tavor_rsrc_mbox_init(tavor_state_t *state,
95 tavor_rsrc_mbox_info_t *info);
96 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
97 tavor_rsrc_mbox_info_t *info);
98
99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
100 tavor_rsrc_hw_entry_info_t *info);
101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
102 tavor_rsrc_hw_entry_info_t *info);
103
104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
105 tavor_rsrc_sw_hdl_info_t *info);
106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
107 tavor_rsrc_sw_hdl_info_t *info);
108
109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
110 tavor_rsrc_sw_hdl_info_t *info);
111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
112 tavor_rsrc_sw_hdl_info_t *info);
113
114 /*
115 * The following routines are used for allocating and freeing the specific
116 * types of objects described above from their associated resource pools.
117 */
118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
119 uint_t num, tavor_rsrc_t *hdl);
120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
121 tavor_rsrc_t *hdl);
122
123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
125 uint_t sleepflag, tavor_rsrc_t *hdl);
126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
127 tavor_rsrc_t *hdl);
128
129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
130 uint_t sleepflag, tavor_rsrc_t *hdl);
131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
132 tavor_rsrc_t *hdl);
133
134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
135 uint_t sleepflag, tavor_rsrc_t *hdl);
136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
137 tavor_rsrc_t *hdl);
138
139 /*
140 * The following routines are the constructors and destructors for several
141 * of the SW handle type objects. For certain types of SW handles objects
142 * (all of which are implemented using kmem_cache), we need to do some
143 * special field initialization (specifically, mutex_init/destroy). These
144 * routines enable that init and teardown.
145 */
146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
160
161 /*
162 * Special routine to calculate and return the size of a MCG object based
163 * on current driver configuration (specifically, the number of QP per MCG
164 * that has been configured.
165 */
166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
167 uint_t *mcg_size_shift);
168
169
170 /*
171 * tavor_rsrc_alloc()
172 *
173 * Context: Can be called from interrupt or base context.
174 * The "sleepflag" parameter is used by all object allocators to
175 * determine whether to SLEEP for resources or not.
176 */
177 int
tavor_rsrc_alloc(tavor_state_t * state,tavor_rsrc_type_t rsrc,uint_t num,uint_t sleepflag,tavor_rsrc_t ** hdl)178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
179 uint_t sleepflag, tavor_rsrc_t **hdl)
180 {
181 tavor_rsrc_pool_info_t *rsrc_pool;
182 tavor_rsrc_t *tmp_rsrc_hdl;
183 int flag, status = DDI_FAILURE;
184
185 TAVOR_TNF_ENTER(tavor_rsrc_alloc);
186
187 ASSERT(state != NULL);
188 ASSERT(hdl != NULL);
189
190 rsrc_pool = &state->ts_rsrc_hdl[rsrc];
191 ASSERT(rsrc_pool != NULL);
192
193 /*
194 * Allocate space for the object used to track the resource handle
195 */
196 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
197 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
198 flag);
199 if (tmp_rsrc_hdl == NULL) {
200 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, "");
201 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
202 return (DDI_FAILURE);
203 }
204 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
205
206 /*
207 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call
208 * to know what type of resource is being freed.
209 */
210 tmp_rsrc_hdl->rsrc_type = rsrc;
211
212 /*
213 * Depending on resource type, call the appropriate alloc routine
214 */
215 switch (rsrc_pool->rsrc_type) {
216 case TAVOR_IN_MBOX:
217 case TAVOR_OUT_MBOX:
218 case TAVOR_INTR_IN_MBOX:
219 case TAVOR_INTR_OUT_MBOX:
220 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
221 break;
222
223 case TAVOR_QPC:
224 case TAVOR_CQC:
225 case TAVOR_SRQC:
226 case TAVOR_EQC:
227 case TAVOR_RDB:
228 /*
229 * Because these objects are NOT accessed by Tavor driver
230 * software, we set the acc_handle parameter to zero. But
231 * if they are allocated in multiples, we specify here that
232 * they must be aligned on a more restrictive boundary.
233 */
234 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
235 sleepflag, tmp_rsrc_hdl);
236 break;
237
238 case TAVOR_MPT:
239 /*
240 * Because these MPT objects are sometimes accessed by Tavor
241 * driver software (FMR), we set the acc_handle parameter. But
242 * if they are allocated in multiples, we specify here that
243 * they must be aligned on a more restrictive boundary.
244 */
245 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
246 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
247 break;
248
249 case TAVOR_MCG:
250 /*
251 * Tavor MCG entries are also NOT accessed by Tavor driver
252 * software, but because MCG entries do not have the same
253 * alignnment restrictions we loosen the constraint here.
254 */
255 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
256 sleepflag, tmp_rsrc_hdl);
257 break;
258
259 case TAVOR_MTT:
260 case TAVOR_UDAV:
261 /*
262 * Because MTT segments are among the few HW resources that
263 * may be allocated in odd numbers, we specify a less
264 * restrictive alignment than for the above resources.
265 *
266 * Also because both UDAV and MTT segment objects are read
267 * and/or written by Tavor driver software, we set the
268 * acc_handle parameter to point to the ddi_acc_handle_t for
269 * the Tavor DDR memory.
270 */
271 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
272 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
273 break;
274
275 case TAVOR_UARPG:
276 /*
277 * Because UAR pages are written by Tavor driver software (for
278 * doorbells), we set the acc_handle parameter to point to
279 * the ddi_acc_handle_t for the Tavor UAR memory.
280 */
281 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
282 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
283 break;
284
285 case TAVOR_MRHDL:
286 case TAVOR_EQHDL:
287 case TAVOR_CQHDL:
288 case TAVOR_SRQHDL:
289 case TAVOR_AHHDL:
290 case TAVOR_QPHDL:
291 case TAVOR_REFCNT:
292 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
293 tmp_rsrc_hdl);
294 break;
295
296 case TAVOR_PDHDL:
297 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
298 tmp_rsrc_hdl);
299 break;
300
301 default:
302 TAVOR_WARNING(state, "unexpected resource type in alloc");
303 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail,
304 TAVOR_TNF_ERROR, "");
305 break;
306 }
307
308 /*
309 * If the resource allocation failed, then free the special resource
310 * tracking structure and return failure. Otherwise return the
311 * handle for the resource tracking structure.
312 */
313 if (status != DDI_SUCCESS) {
314 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
315 tmp_rsrc_hdl = NULL;
316 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "",
317 tnf_uint, rsrc_type, rsrc_pool->rsrc_type);
318 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
319 return (DDI_FAILURE);
320 } else {
321 *hdl = tmp_rsrc_hdl;
322 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
323 return (DDI_SUCCESS);
324 }
325 }
326
327
328 /*
329 * tavor_rsrc_free()
330 * Context: Can be called from interrupt or base context.
331 */
332 void
tavor_rsrc_free(tavor_state_t * state,tavor_rsrc_t ** hdl)333 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
334 {
335 tavor_rsrc_pool_info_t *rsrc_pool;
336
337 TAVOR_TNF_ENTER(tavor_rsrc_free);
338
339 ASSERT(state != NULL);
340 ASSERT(hdl != NULL);
341
342 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
343 ASSERT(rsrc_pool != NULL);
344
345 /*
346 * Depending on resource type, call the appropriate free routine
347 */
348 switch (rsrc_pool->rsrc_type) {
349 case TAVOR_IN_MBOX:
350 case TAVOR_OUT_MBOX:
351 case TAVOR_INTR_IN_MBOX:
352 case TAVOR_INTR_OUT_MBOX:
353 tavor_rsrc_mbox_free(rsrc_pool, *hdl);
354 break;
355
356 case TAVOR_QPC:
357 case TAVOR_CQC:
358 case TAVOR_SRQC:
359 case TAVOR_EQC:
360 case TAVOR_RDB:
361 case TAVOR_MCG:
362 case TAVOR_MPT:
363 case TAVOR_MTT:
364 case TAVOR_UDAV:
365 case TAVOR_UARPG:
366 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
367 break;
368
369 case TAVOR_MRHDL:
370 case TAVOR_EQHDL:
371 case TAVOR_CQHDL:
372 case TAVOR_SRQHDL:
373 case TAVOR_AHHDL:
374 case TAVOR_QPHDL:
375 case TAVOR_REFCNT:
376 tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
377 break;
378
379 case TAVOR_PDHDL:
380 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
381 break;
382
383 default:
384 TAVOR_WARNING(state, "unexpected resource type in free");
385 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail,
386 TAVOR_TNF_ERROR, "");
387 break;
388 }
389
390 /*
391 * Free the special resource tracking structure, set the handle to
392 * NULL, and return.
393 */
394 kmem_cache_free(state->ts_rsrc_cache, *hdl);
395 *hdl = NULL;
396
397 TAVOR_TNF_EXIT(tavor_rsrc_free);
398 }
399
400
401 /*
402 * tavor_rsrc_init_phase1()
403 *
404 * Completes the first phase of Tavor resource/configuration init.
405 * This involves creating the kmem_cache for the "tavor_rsrc_t"
406 * structs, allocating the space for the resource pool handles,
407 * and setting up the "Out" mailboxes.
408 *
409 * When this function completes, the Tavor driver is ready to
410 * post the following commands which return information only in the
411 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
412 * If any of these commands are to be posted at this time, they must be
413 * done so only when "spinning" (as the outstanding command list and
414 * EQ setup code has not yet run)
415 *
416 * Context: Only called from attach() path context
417 */
418 int
tavor_rsrc_init_phase1(tavor_state_t * state)419 tavor_rsrc_init_phase1(tavor_state_t *state)
420 {
421 tavor_rsrc_pool_info_t *rsrc_pool;
422 tavor_rsrc_mbox_info_t mbox_info;
423 tavor_rsrc_cleanup_level_t cleanup;
424 tavor_cfg_profile_t *cfgprof;
425 uint64_t num, size;
426 int status;
427 char *errormsg, *rsrc_name;
428
429 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1);
430
431 ASSERT(state != NULL);
432
433 /* This is where Phase 1 of resource initialization begins */
434 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
435
436 /* Build kmem cache name from Tavor instance */
437 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
438 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
439
440 /*
441 * Create the kmem_cache for "tavor_rsrc_t" structures
442 * (kmem_cache_create will SLEEP until successful)
443 */
444 state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
445 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
446
447 /*
448 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
449 * subsequent resource allocations)
450 */
451 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
452 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
453
454 cfgprof = state->ts_cfg_profile;
455
456 /*
457 * Initialize the resource pool for "Out" mailboxes. Notice that
458 * the number of "Out" mailboxes, their size, and their location
459 * (DDR or system memory) is configurable. By default, however,
460 * all "Out" mailboxes are located in system memory only (because
461 * they are primarily read from and never written to)
462 */
463 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
464 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
466 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX;
467 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
468 rsrc_pool->rsrc_pool_size = (size * num);
469 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
470 rsrc_pool->rsrc_quantum = size;
471 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
472 rsrc_pool->rsrc_state = state;
473 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
474 mbox_info.mbi_num = num;
475 mbox_info.mbi_size = size;
476 mbox_info.mbi_rsrcpool = rsrc_pool;
477 mbox_info.mbi_rsrcname = rsrc_name;
478 status = tavor_rsrc_mbox_init(state, &mbox_info);
479 if (status != DDI_SUCCESS) {
480 tavor_rsrc_fini(state, cleanup);
481 /* Set "status" and "errormsg" and goto failure */
482 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes");
483 goto rsrcinitp1_fail;
484 }
485 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
486
487 /*
488 * Initialize the Tavor "Out" mailbox list. This step actually uses
489 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
490 * "Out" mailboxes, bind them for DMA access, and arrange them into
491 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
492 * for more details)
493 */
494 status = tavor_outmbox_list_init(state);
495 if (status != DDI_SUCCESS) {
496 tavor_rsrc_fini(state, cleanup);
497 /* Set "status" and "errormsg" and goto failure */
498 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list");
499 goto rsrcinitp1_fail;
500 }
501 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
502
503 /*
504 * Initialize the resource pool for interrupt "Out" mailboxes. Notice
505 * that the number of interrupt "Out" mailboxes, their size, and their
506 * location (DDR or system memory) is configurable. By default,
507 * however, all interrupt "Out" mailboxes are located in system memory
508 * only (because they are primarily read from and never written to)
509 */
510 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
511 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
512 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
513 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX;
514 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
515 rsrc_pool->rsrc_pool_size = (size * num);
516 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
517 rsrc_pool->rsrc_quantum = size;
518 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
519 rsrc_pool->rsrc_state = state;
520 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
521 mbox_info.mbi_num = num;
522 mbox_info.mbi_size = size;
523 mbox_info.mbi_rsrcpool = rsrc_pool;
524 mbox_info.mbi_rsrcname = rsrc_name;
525 status = tavor_rsrc_mbox_init(state, &mbox_info);
526 if (status != DDI_SUCCESS) {
527 tavor_rsrc_fini(state, cleanup);
528 /* Set "status" and "errormsg" and goto failure */
529 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes");
530 goto rsrcinitp1_fail;
531 }
532 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
533
534 /*
535 * Initialize the Tavor "Out" mailbox list. This step actually uses
536 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
537 * "Out" mailboxes, bind them for DMA access, and arrange them into
538 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
539 * for more details)
540 */
541 status = tavor_intr_outmbox_list_init(state);
542 if (status != DDI_SUCCESS) {
543 tavor_rsrc_fini(state, cleanup);
544 /* Set "status" and "errormsg" and goto failure */
545 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list");
546 goto rsrcinitp1_fail;
547 }
548 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
549
550 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
551 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
552 return (DDI_SUCCESS);
553
554 rsrcinitp1_fail:
555 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
556 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "",
557 tnf_string, msg, errormsg);
558 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
559 return (status);
560 }
561
562
563 /*
564 * tavor_rsrc_init_phase2()
565 * Context: Only called from attach() path context
566 */
567 int
tavor_rsrc_init_phase2(tavor_state_t * state)568 tavor_rsrc_init_phase2(tavor_state_t *state)
569 {
570 tavor_rsrc_sw_hdl_info_t hdl_info;
571 tavor_rsrc_hw_entry_info_t entry_info;
572 tavor_rsrc_mbox_info_t mbox_info;
573 tavor_rsrc_pool_info_t *rsrc_pool;
574 tavor_rsrc_cleanup_level_t cleanup;
575 tavor_cfg_profile_t *cfgprof;
576 uint64_t num, max, size, num_prealloc;
577 uint64_t ddr_size, fw_size;
578 uint_t mcg_size, mcg_size_shift;
579 uint_t uarscr_size, mttsegment_sz;
580 int status;
581 char *errormsg, *rsrc_name;
582
583 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2);
584
585 ASSERT(state != NULL);
586
587 /* Phase 2 initialization begins where Phase 1 left off */
588 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
589
590 /*
591 * Calculate the extent of the DDR size and portion of which that
592 * is already reserved for Tavor firmware. (Note: this information
593 * is available because the QUERY_DDR and QUERY_FW commands have
594 * been posted to Tavor firmware prior to calling this routine)
595 */
596 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
597 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
598
599 /* Build the DDR vmem arena name from Tavor instance */
600 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
601 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
602
603 /*
604 * Do a vmem_create for the entire DDR range (not including the
605 * portion consumed by Tavor firmware). This creates the vmem arena
606 * from which all other DDR objects (specifically, tables of HW
607 * entries) will be allocated.
608 */
609 state->ts_ddrvmem = vmem_create(rsrc_name,
610 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
611 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
612 if (state->ts_ddrvmem == NULL) {
613 tavor_rsrc_fini(state, cleanup);
614 /* Set "status" and "errormsg" and goto failure */
615 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem");
616 goto rsrcinitp2_fail;
617 }
618 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
619
620 /*
621 * Initialize the resource pools for all objects that exist in
622 * Tavor DDR memory. This includes ("In") mailboxes, context tables
623 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
624 */
625 cfgprof = state->ts_cfg_profile;
626
627 /*
628 * Initialize the resource pool for the MPT table entries. Notice
629 * that the number of MPTs is configurable. The configured value must
630 * be less that the maximum value (obtained from the QUERY_DEV_LIM
631 * command) or the initialization will fail. Note also that a certain
632 * number of MPTs must be set aside for Tavor firmware use.
633 */
634 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
635 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
636 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
637 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
638 rsrc_pool->rsrc_type = TAVOR_MPT;
639 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
640 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
641 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT;
642 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE;
643 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num);
644 rsrc_pool->rsrc_state = state;
645 rsrc_pool->rsrc_start = NULL;
646 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
647 entry_info.hwi_num = num;
648 entry_info.hwi_max = max;
649 entry_info.hwi_prealloc = num_prealloc;
650 entry_info.hwi_rsrcpool = rsrc_pool;
651 entry_info.hwi_rsrcname = rsrc_name;
652 status = tavor_rsrc_hw_entries_init(state, &entry_info);
653 if (status != DDI_SUCCESS) {
654 tavor_rsrc_fini(state, cleanup);
655 /* Set "status" and "errormsg" and goto failure */
656 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table");
657 goto rsrcinitp2_fail;
658 }
659 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
660
661 /*
662 * Initialize the resource pool for the MTT table entries. Notice
663 * that the number of MTTs is configurable. The configured value must
664 * be less that the maximum value (obtained from the QUERY_DEV_LIM
665 * command) or the initialization will fail. Note also that a certain
666 * number of MTT segments must be set aside for Tavor firmware use.
667 */
668 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
669 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
670 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
671 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
672 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
673 rsrc_pool->rsrc_type = TAVOR_MTT;
674 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
675 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
676 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT;
677 rsrc_pool->rsrc_quantum = mttsegment_sz;
678 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num);
679 rsrc_pool->rsrc_state = state;
680 rsrc_pool->rsrc_start = NULL;
681 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
682 entry_info.hwi_num = num;
683 entry_info.hwi_max = max;
684 entry_info.hwi_prealloc = num_prealloc;
685 entry_info.hwi_rsrcpool = rsrc_pool;
686 entry_info.hwi_rsrcname = rsrc_name;
687 status = tavor_rsrc_hw_entries_init(state, &entry_info);
688 if (status != DDI_SUCCESS) {
689 tavor_rsrc_fini(state, cleanup);
690 /* Set "status" and "errormsg" and goto failure */
691 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table");
692 goto rsrcinitp2_fail;
693 }
694 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
695
696 /*
697 * Initialize the resource pool for the QPC table entries. Notice
698 * that the number of QPs is configurable. The configured value must
699 * be less that the maximum value (obtained from the QUERY_DEV_LIM
700 * command) or the initialization will fail. Note also that a certain
701 * number of QP contexts must be set aside for Tavor firmware use.
702 */
703 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
704 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
705 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
706 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
707 rsrc_pool->rsrc_type = TAVOR_QPC;
708 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
709 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
710 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT;
711 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE;
712 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num);
713 rsrc_pool->rsrc_state = state;
714 rsrc_pool->rsrc_start = NULL;
715 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
716 entry_info.hwi_num = num;
717 entry_info.hwi_max = max;
718 entry_info.hwi_prealloc = num_prealloc;
719 entry_info.hwi_rsrcpool = rsrc_pool;
720 entry_info.hwi_rsrcname = rsrc_name;
721 status = tavor_rsrc_hw_entries_init(state, &entry_info);
722 if (status != DDI_SUCCESS) {
723 tavor_rsrc_fini(state, cleanup);
724 /* Set "status" and "errormsg" and goto failure */
725 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table");
726 goto rsrcinitp2_fail;
727 }
728 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
729
730 /*
731 * Initialize the resource pool for the RDB table entries. Notice
732 * that the number of RDBs is configurable. The configured value must
733 * be less that the maximum value (obtained from the QUERY_DEV_LIM
734 * command) or the initialization will fail.
735 */
736 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
737 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
738 num_prealloc = 0;
739 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
740 rsrc_pool->rsrc_type = TAVOR_RDB;
741 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
742 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
743 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT;
744 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE;
745 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num);
746 rsrc_pool->rsrc_state = state;
747 rsrc_pool->rsrc_start = NULL;
748 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
749 entry_info.hwi_num = num;
750 entry_info.hwi_max = max;
751 entry_info.hwi_prealloc = num_prealloc;
752 entry_info.hwi_rsrcpool = rsrc_pool;
753 entry_info.hwi_rsrcname = rsrc_name;
754 status = tavor_rsrc_hw_entries_init(state, &entry_info);
755 if (status != DDI_SUCCESS) {
756 tavor_rsrc_fini(state, cleanup);
757 /* Set "status" and "errormsg" and goto failure */
758 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table");
759 goto rsrcinitp2_fail;
760 }
761 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
762
763 /*
764 * Initialize the resource pool for the CQC table entries. Notice
765 * that the number of CQs is configurable. The configured value must
766 * be less that the maximum value (obtained from the QUERY_DEV_LIM
767 * command) or the initialization will fail. Note also that a certain
768 * number of CQ contexts must be set aside for Tavor firmware use.
769 */
770 num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
771 max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
772 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
773 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
774 rsrc_pool->rsrc_type = TAVOR_CQC;
775 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
776 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
777 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT;
778 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE;
779 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num);
780 rsrc_pool->rsrc_state = state;
781 rsrc_pool->rsrc_start = NULL;
782 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
783 entry_info.hwi_num = num;
784 entry_info.hwi_max = max;
785 entry_info.hwi_prealloc = num_prealloc;
786 entry_info.hwi_rsrcpool = rsrc_pool;
787 entry_info.hwi_rsrcname = rsrc_name;
788 status = tavor_rsrc_hw_entries_init(state, &entry_info);
789 if (status != DDI_SUCCESS) {
790 tavor_rsrc_fini(state, cleanup);
791 /* Set "status" and "errormsg" and goto failure */
792 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table");
793 goto rsrcinitp2_fail;
794 }
795 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
796
797 /*
798 * Initialize the resource pool for the Extended QPC table entries.
799 * Notice that the number of EQPCs must be the same as the number
800 * of QP contexts. So the initialization is constructed in a
801 * similar way as above (for TAVOR_QPC). One notable difference
802 * here, however, is that by setting the rsrc_quantum field to
803 * zero (indicating a zero-sized object) we indicate that the
804 * object is not allocatable. The EQPC table is, in fact, managed
805 * internally by the hardware and it is, therefore, unnecessary to
806 * initialize an additional vmem_arena for this type of object.
807 */
808 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
809 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
810 num_prealloc = 0;
811 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
812 rsrc_pool->rsrc_type = TAVOR_EQPC;
813 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
814 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
815 rsrc_pool->rsrc_shift = 0;
816 rsrc_pool->rsrc_quantum = 0;
817 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE;
818 rsrc_pool->rsrc_state = state;
819 rsrc_pool->rsrc_start = NULL;
820 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
821 entry_info.hwi_num = num;
822 entry_info.hwi_max = max;
823 entry_info.hwi_prealloc = num_prealloc;
824 entry_info.hwi_rsrcpool = rsrc_pool;
825 entry_info.hwi_rsrcname = rsrc_name;
826 status = tavor_rsrc_hw_entries_init(state, &entry_info);
827 if (status != DDI_SUCCESS) {
828 tavor_rsrc_fini(state, cleanup);
829 /* Set "status" and "errormsg" and goto failure */
830 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table");
831 goto rsrcinitp2_fail;
832 }
833 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
834
835 /*
836 * Initialize the resource pool for the UD address vector table
837 * entries. Notice that the number of UDAVs is configurable. The
838 * configured value must be less that the maximum value (obtained
839 * from the QUERY_DEV_LIM command) or the initialization will fail.
840 */
841 num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
842 max = ((uint64_t)1 << state->ts_devlim.log_max_av);
843 num_prealloc = 0;
844 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
845 rsrc_pool->rsrc_type = TAVOR_UDAV;
846 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
847 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
848 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT;
849 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE;
850 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE;
851 rsrc_pool->rsrc_state = state;
852 rsrc_pool->rsrc_start = NULL;
853 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
854 entry_info.hwi_num = num;
855 entry_info.hwi_max = max;
856 entry_info.hwi_prealloc = num_prealloc;
857 entry_info.hwi_rsrcpool = rsrc_pool;
858 entry_info.hwi_rsrcname = rsrc_name;
859 status = tavor_rsrc_hw_entries_init(state, &entry_info);
860 if (status != DDI_SUCCESS) {
861 tavor_rsrc_fini(state, cleanup);
862 /* Set "status" and "errormsg" and goto failure */
863 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table");
864 goto rsrcinitp2_fail;
865 }
866 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
867
868 /*
869 * Initialize the resource pool for the UAR scratch table entries.
870 * Notice that the number of UARSCRs is configurable. The configured
871 * value must be less that the maximum value (obtained from the
872 * QUERY_DEV_LIM command) or the initialization will fail.
873 * Like the EQPCs above, UARSCR objects are not allocatable. The
874 * UARSCR table is also managed internally by the hardware and it
875 * is, therefore, unnecessary to initialize an additional vmem_arena
876 * for this type of object. We indicate this by setting the
877 * rsrc_quantum field to zero (indicating a zero-sized object).
878 */
879 uarscr_size = state->ts_devlim.uarscr_entry_sz;
880 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
881 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
882 PAGESHIFT));
883 num_prealloc = 0;
884 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
885 rsrc_pool->rsrc_type = TAVOR_UAR_SCR;
886 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
887 rsrc_pool->rsrc_pool_size = (uarscr_size * num);
888 rsrc_pool->rsrc_shift = 0;
889 rsrc_pool->rsrc_quantum = 0;
890 rsrc_pool->rsrc_align = uarscr_size;
891 rsrc_pool->rsrc_state = state;
892 rsrc_pool->rsrc_start = NULL;
893 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
894 entry_info.hwi_num = num;
895 entry_info.hwi_max = max;
896 entry_info.hwi_prealloc = num_prealloc;
897 entry_info.hwi_rsrcpool = rsrc_pool;
898 entry_info.hwi_rsrcname = rsrc_name;
899 status = tavor_rsrc_hw_entries_init(state, &entry_info);
900 if (status != DDI_SUCCESS) {
901 tavor_rsrc_fini(state, cleanup);
902 /* Set "status" and "errormsg" and goto failure */
903 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table");
904 goto rsrcinitp2_fail;
905 }
906 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
907
908 /*
909 * Initialize the resource pool for the SRQC table entries. Notice
910 * that the number of SRQs is configurable. The configured value must
911 * be less that the maximum value (obtained from the QUERY_DEV_LIM
912 * command) or the initialization will fail. Note also that a certain
913 * number of SRQ contexts must be set aside for Tavor firmware use.
914 *
915 * Note: We only allocate these resources if SRQ is enabled in the
916 * config profile; see below.
917 */
918 num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
919 max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
920 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
921
922 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
923 rsrc_pool->rsrc_type = TAVOR_SRQC;
924 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
925 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
926 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT;
927 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE;
928 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num);
929 rsrc_pool->rsrc_state = state;
930 rsrc_pool->rsrc_start = NULL;
931 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
932 entry_info.hwi_num = num;
933 entry_info.hwi_max = max;
934 entry_info.hwi_prealloc = num_prealloc;
935 entry_info.hwi_rsrcpool = rsrc_pool;
936 entry_info.hwi_rsrcname = rsrc_name;
937
938 /*
939 * SRQ support is configurable. Only if SRQ is enabled (the default)
940 * do we actually try to configure these resources. Otherwise, we
941 * simply set the cleanup level and continue on to the next resource
942 */
943 if (state->ts_cfg_profile->cp_srq_enable != 0) {
944 status = tavor_rsrc_hw_entries_init(state, &entry_info);
945 if (status != DDI_SUCCESS) {
946 tavor_rsrc_fini(state, cleanup);
947 /* Set "status" and "errormsg" and goto failure */
948 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table");
949 goto rsrcinitp2_fail;
950 }
951 }
952 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
953
954 /*
955 * Initialize the resource pool for "In" mailboxes. Notice that
956 * the number of "In" mailboxes, their size, and their location
957 * (DDR or system memory) is configurable. By default, however,
958 * all "In" mailboxes are located in system memory only (because
959 * they are primarily written to and rarely read from)
960 */
961 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
962 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
963 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
964 rsrc_pool->rsrc_type = TAVOR_IN_MBOX;
965 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
966 rsrc_pool->rsrc_pool_size = (size * num);
967 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
968 rsrc_pool->rsrc_quantum = size;
969 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
970 rsrc_pool->rsrc_state = state;
971 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
972 mbox_info.mbi_num = num;
973 mbox_info.mbi_size = size;
974 mbox_info.mbi_rsrcpool = rsrc_pool;
975 mbox_info.mbi_rsrcname = rsrc_name;
976 status = tavor_rsrc_mbox_init(state, &mbox_info);
977 if (status != DDI_SUCCESS) {
978 tavor_rsrc_fini(state, cleanup);
979 /* Set "status" and "errormsg" and goto failure */
980 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes");
981 goto rsrcinitp2_fail;
982 }
983 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
984
985 /*
986 * Initialize the Tavor "In" mailbox list. This step actually uses
987 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
988 * "In" mailboxes, bind them for DMA access, and arrange them into
989 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
990 * for more details)
991 */
992 status = tavor_inmbox_list_init(state);
993 if (status != DDI_SUCCESS) {
994 tavor_rsrc_fini(state, cleanup);
995 /* Set "status" and "errormsg" and goto failure */
996 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list");
997 goto rsrcinitp2_fail;
998 }
999 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
1000
1001 /*
1002 * Initialize the resource pool for interrupt "In" mailboxes. Notice
1003 * that the number of interrupt "In" mailboxes, their size, and their
1004 * location (DDR or system memory) is configurable. By default,
1005 * however, all interrupt "In" mailboxes are located in system memory
1006 * only (because they are primarily written to and rarely read from)
1007 */
1008 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
1009 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
1010 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
1011 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX;
1012 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1013 rsrc_pool->rsrc_pool_size = (size * num);
1014 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
1015 rsrc_pool->rsrc_quantum = size;
1016 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
1017 rsrc_pool->rsrc_state = state;
1018 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
1019 mbox_info.mbi_num = num;
1020 mbox_info.mbi_size = size;
1021 mbox_info.mbi_rsrcpool = rsrc_pool;
1022 mbox_info.mbi_rsrcname = rsrc_name;
1023 status = tavor_rsrc_mbox_init(state, &mbox_info);
1024 if (status != DDI_SUCCESS) {
1025 tavor_rsrc_fini(state, cleanup);
1026 /* Set "status" and "errormsg" and goto failure */
1027 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes");
1028 goto rsrcinitp2_fail;
1029 }
1030 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
1031
1032 /*
1033 * Initialize the Tavor interrupt "In" mailbox list. This step
1034 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
1035 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
1036 * and arrange them into an easily accessed fast-allocation mechanism
1037 * (see tavor_cmd.c for more details)
1038 */
1039 status = tavor_intr_inmbox_list_init(state);
1040 if (status != DDI_SUCCESS) {
1041 tavor_rsrc_fini(state, cleanup);
1042 /* Set "status" and "errormsg" and goto failure */
1043 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list");
1044 goto rsrcinitp2_fail;
1045 }
1046 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
1047
1048 /*
1049 * Initialize the Tavor command handling interfaces. This step
1050 * sets up the outstanding command tracking mechanism for easy access
1051 * and fast allocation (see tavor_cmd.c for more details).
1052 */
1053 status = tavor_outstanding_cmdlist_init(state);
1054 if (status != DDI_SUCCESS) {
1055 tavor_rsrc_fini(state, cleanup);
1056 /* Set "status" and "errormsg" and goto failure */
1057 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list");
1058 goto rsrcinitp2_fail;
1059 }
1060 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
1061
1062 /*
1063 * Calculate (and validate) the size of Multicast Group (MCG) entries
1064 */
1065 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1066 if (status != DDI_SUCCESS) {
1067 tavor_rsrc_fini(state, cleanup);
1068 /* Set "status" and "errormsg" and goto failure */
1069 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size");
1070 goto rsrcinitp2_fail;
1071 }
1072 mcg_size = TAVOR_MCGMEM_SZ(state);
1073
1074 /*
1075 * Initialize the resource pool for the MCG table entries. Notice
1076 * that the number of MCGs is configurable. The configured value must
1077 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1078 * command) or the initialization will fail. Note also that a certain
1079 * number of MCGs must be set aside for Tavor firmware use (they
1080 * correspond to the number of MCGs used by the internal hash
1081 * function.
1082 */
1083 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1084 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1085 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1086 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1087 rsrc_pool->rsrc_type = TAVOR_MCG;
1088 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1089 rsrc_pool->rsrc_pool_size = (mcg_size * num);
1090 rsrc_pool->rsrc_shift = mcg_size_shift;
1091 rsrc_pool->rsrc_quantum = mcg_size;
1092 rsrc_pool->rsrc_align = mcg_size;
1093 rsrc_pool->rsrc_state = state;
1094 rsrc_pool->rsrc_start = NULL;
1095 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1096 entry_info.hwi_num = num;
1097 entry_info.hwi_max = max;
1098 entry_info.hwi_prealloc = num_prealloc;
1099 entry_info.hwi_rsrcpool = rsrc_pool;
1100 entry_info.hwi_rsrcname = rsrc_name;
1101 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1102 if (status != DDI_SUCCESS) {
1103 tavor_rsrc_fini(state, cleanup);
1104 /* Set "status" and "errormsg" and goto failure */
1105 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table");
1106 goto rsrcinitp2_fail;
1107 }
1108 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1109
1110 /*
1111 * Initialize the resource pool for the EQC table entries. Notice
1112 * that the number of EQs is hardcoded. The hardcoded value should
1113 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1114 * command) or the initialization will fail.
1115 */
1116 num = TAVOR_NUM_EQ;
1117 max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1118 num_prealloc = 0;
1119 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1120 rsrc_pool->rsrc_type = TAVOR_EQC;
1121 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1122 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1123 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT;
1124 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE;
1125 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num);
1126 rsrc_pool->rsrc_state = state;
1127 rsrc_pool->rsrc_start = NULL;
1128 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1129 entry_info.hwi_num = num;
1130 entry_info.hwi_max = max;
1131 entry_info.hwi_prealloc = num_prealloc;
1132 entry_info.hwi_rsrcpool = rsrc_pool;
1133 entry_info.hwi_rsrcname = rsrc_name;
1134 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1135 if (status != DDI_SUCCESS) {
1136 tavor_rsrc_fini(state, cleanup);
1137 /* Set "status" and "errormsg" and goto failure */
1138 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table");
1139 goto rsrcinitp2_fail;
1140 }
1141 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1142
1143 /*
1144 * Initialize the resource pools for all objects that exist in
1145 * system memory. This includes PD handles, MR handle, EQ handles,
1146 * QP handles, etc. These objects are almost entirely managed using
1147 * kmem_cache routines. (See comment above for more detail)
1148 */
1149
1150 /*
1151 * Initialize the resource pool for the PD handles. Notice
1152 * that the number of PDHDLs is configurable. The configured value
1153 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1154 * command) or the initialization will fail. Note also that the PD
1155 * handle has constructor and destructor methods associated with it.
1156 */
1157 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1158 rsrc_pool->rsrc_type = TAVOR_PDHDL;
1159 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1160 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s);
1161 rsrc_pool->rsrc_state = state;
1162 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1163 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1164 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1165 hdl_info.swi_rsrcpool = rsrc_pool;
1166 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1167 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor;
1168 hdl_info.swi_rsrcname = rsrc_name;
1169 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1170 status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1171 if (status != DDI_SUCCESS) {
1172 tavor_rsrc_fini(state, cleanup);
1173 /* Set "status" and "errormsg" and goto failure */
1174 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle");
1175 goto rsrcinitp2_fail;
1176 }
1177 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1178
1179 /*
1180 * Initialize the resource pool for the MR handles. Notice
1181 * that the number of MRHDLs is configurable. The configured value
1182 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1183 * command) or the initialization will fail.
1184 */
1185 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1186 rsrc_pool->rsrc_type = TAVOR_MRHDL;
1187 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1188 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s);
1189 rsrc_pool->rsrc_state = state;
1190 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1191 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1192 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1193 hdl_info.swi_rsrcpool = rsrc_pool;
1194 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1195 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor;
1196 hdl_info.swi_rsrcname = rsrc_name;
1197 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1198 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1199 if (status != DDI_SUCCESS) {
1200 tavor_rsrc_fini(state, cleanup);
1201 /* Set "status" and "errormsg" and goto failure */
1202 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle");
1203 goto rsrcinitp2_fail;
1204 }
1205 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1206
1207 /*
1208 * Initialize the resource pool for the EQ handles. Notice
1209 * that the number of EQHDLs is hardcoded. The hardcoded value
1210 * should be less that the maximum value (obtained from the
1211 * QUERY_DEV_LIM command) or the initialization will fail.
1212 */
1213 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1214 rsrc_pool->rsrc_type = TAVOR_EQHDL;
1215 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1216 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s);
1217 rsrc_pool->rsrc_state = state;
1218 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1219 hdl_info.swi_num = TAVOR_NUM_EQ;
1220 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1221 hdl_info.swi_rsrcpool = rsrc_pool;
1222 hdl_info.swi_constructor = NULL;
1223 hdl_info.swi_destructor = NULL;
1224 hdl_info.swi_rsrcname = rsrc_name;
1225 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1226 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1227 if (status != DDI_SUCCESS) {
1228 tavor_rsrc_fini(state, cleanup);
1229 /* Set "status" and "errormsg" and goto failure */
1230 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle");
1231 goto rsrcinitp2_fail;
1232 }
1233 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1234
1235 /*
1236 * Initialize the resource pool for the CQ handles. Notice
1237 * that the number of CQHDLs is configurable. The configured value
1238 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1239 * command) or the initialization will fail. Note also that the CQ
1240 * handle has constructor and destructor methods associated with it.
1241 */
1242 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1243 rsrc_pool->rsrc_type = TAVOR_CQHDL;
1244 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1245 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s);
1246 rsrc_pool->rsrc_state = state;
1247 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1248 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1249 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1250 hdl_info.swi_rsrcpool = rsrc_pool;
1251 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1252 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor;
1253 hdl_info.swi_rsrcname = rsrc_name;
1254 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1255 TAVOR_SWHDL_TABLE_INIT);
1256 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1257 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1258 if (status != DDI_SUCCESS) {
1259 tavor_rsrc_fini(state, cleanup);
1260 /* Set "status" and "errormsg" and goto failure */
1261 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle");
1262 goto rsrcinitp2_fail;
1263 }
1264
1265 /*
1266 * Save away the pointer to the central list of CQ handle pointers
1267 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1268 * lookup during EQ event processing. The table is a list of
1269 * tavor_cqhdl_t allocated by the above routine because of the
1270 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t
1271 * as the number of CQs.
1272 */
1273 state->ts_cqhdl = hdl_info.swi_table_ptr;
1274 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1275
1276 /*
1277 * Initialize the resource pool for the SRQ handles. Notice
1278 * that the number of SRQHDLs is configurable. The configured value
1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1280 * command) or the initialization will fail. Note also that the SRQ
1281 * handle has constructor and destructor methods associated with it.
1282 *
1283 * Note: We only allocate these resources if SRQ is enabled in the
1284 * config profile; see below.
1285 */
1286 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1287 rsrc_pool->rsrc_type = TAVOR_SRQHDL;
1288 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1289 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s);
1290 rsrc_pool->rsrc_state = state;
1291 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1292 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1293 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1294 hdl_info.swi_rsrcpool = rsrc_pool;
1295 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1296 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor;
1297 hdl_info.swi_rsrcname = rsrc_name;
1298 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1299 TAVOR_SWHDL_TABLE_INIT);
1300 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1301
1302 /*
1303 * SRQ support is configurable. Only if SRQ is enabled (the default)
1304 * do we actually try to configure these resources. Otherwise, we
1305 * simply set the cleanup level and continue on to the next resource
1306 */
1307 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1308 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1309 if (status != DDI_SUCCESS) {
1310 tavor_rsrc_fini(state, cleanup);
1311 /* Set "status" and "errormsg" and goto failure */
1312 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle");
1313 goto rsrcinitp2_fail;
1314 }
1315
1316 /*
1317 * Save away the pointer to the central list of SRQ handle
1318 * pointers This this is used as a mechanism to enable fast
1319 * SRQnumber-to-SRQhandle lookup. The table is a list of
1320 * tavor_srqhdl_t allocated by the above routine because of the
1321 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many
1322 * tavor_srqhdl_t as the number of SRQs.
1323 */
1324 state->ts_srqhdl = hdl_info.swi_table_ptr;
1325 }
1326 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1327
1328 /*
1329 * Initialize the resource pool for the address handles. Notice
1330 * that the number of AHHDLs is configurable. The configured value
1331 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1332 * command) or the initialization will fail.
1333 */
1334 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1335 rsrc_pool->rsrc_type = TAVOR_AHHDL;
1336 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1337 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s);
1338 rsrc_pool->rsrc_state = state;
1339 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1340 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1341 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1342 hdl_info.swi_rsrcpool = rsrc_pool;
1343 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1344 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor;
1345 hdl_info.swi_rsrcname = rsrc_name;
1346 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1347 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1348 if (status != DDI_SUCCESS) {
1349 tavor_rsrc_fini(state, cleanup);
1350 /* Set "status" and "errormsg" and goto failure */
1351 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle");
1352 goto rsrcinitp2_fail;
1353 }
1354 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1355
1356 /*
1357 * Initialize the resource pool for the QP handles. Notice
1358 * that the number of QPHDLs is configurable. The configured value
1359 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1360 * command) or the initialization will fail. Note also that the QP
1361 * handle has constructor and destructor methods associated with it.
1362 */
1363 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1364 rsrc_pool->rsrc_type = TAVOR_QPHDL;
1365 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1366 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s);
1367 rsrc_pool->rsrc_state = state;
1368 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1369 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1370 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1371 hdl_info.swi_rsrcpool = rsrc_pool;
1372 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1373 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor;
1374 hdl_info.swi_rsrcname = rsrc_name;
1375 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1376 TAVOR_SWHDL_TABLE_INIT);
1377 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1378 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1379 if (status != DDI_SUCCESS) {
1380 tavor_rsrc_fini(state, cleanup);
1381 /* Set "status" and "errormsg" and goto failure */
1382 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle");
1383 goto rsrcinitp2_fail;
1384 }
1385
1386 /*
1387 * Save away the pointer to the central list of QP handle pointers
1388 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1389 * lookup during CQ event processing. The table is a list of
1390 * tavor_qphdl_t allocated by the above routine because of the
1391 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t
1392 * as the number of QPs.
1393 */
1394 state->ts_qphdl = hdl_info.swi_table_ptr;
1395 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1396
1397 /*
1398 * Initialize the resource pool for the reference count handles.
1399 * Notice that the number of REFCNTs is configurable, but it's value
1400 * is set to the number of MPTs. Since REFCNTs are used to support
1401 * shared memory regions, it is possible that we might require as
1402 * one REFCNT for every MPT.
1403 */
1404 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1405 rsrc_pool->rsrc_type = TAVOR_REFCNT;
1406 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1407 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t);
1408 rsrc_pool->rsrc_state = state;
1409 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1410 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1411 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1412 hdl_info.swi_rsrcpool = rsrc_pool;
1413 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1414 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor;
1415 hdl_info.swi_rsrcname = rsrc_name;
1416 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1417 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1418 if (status != DDI_SUCCESS) {
1419 tavor_rsrc_fini(state, cleanup);
1420 /* Set "status" and "errormsg" and goto failure */
1421 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle");
1422 goto rsrcinitp2_fail;
1423 }
1424 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1425
1426 /*
1427 * Initialize the resource pool for the MCG handles. Notice that for
1428 * these MCG handles, we are allocating a table of structures (used to
1429 * keep track of the MCG entries that are being written to hardware
1430 * and to speed up multicast attach/detach operations).
1431 */
1432 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1433 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1434 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1435 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1436 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1437 if (status != DDI_SUCCESS) {
1438 tavor_rsrc_fini(state, cleanup);
1439 /* Set "status" and "errormsg" and goto failure */
1440 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle");
1441 goto rsrcinitp2_fail;
1442 }
1443 state->ts_mcghdl = hdl_info.swi_table_ptr;
1444 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1445
1446 /*
1447 * Initialize the resource pools for all objects that exist in
1448 * UAR memory. The only objects that are allocated from UAR memory
1449 * are the UAR pages which are used for holding Tavor hardware's
1450 * doorbell registers.
1451 */
1452
1453 /*
1454 * Initialize the resource pool for the UAR pages. Notice
1455 * that the number of UARPGs is configurable. The configured value
1456 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1457 * command) or the initialization will fail. Note also that by
1458 * specifying the rsrc_start parameter in advance, we direct the
1459 * initialization routine not to attempt to allocated space from the
1460 * Tavor DDR vmem_arena.
1461 */
1462 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1463 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1464 PAGESHIFT));
1465 num_prealloc = 0;
1466 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1467 rsrc_pool->rsrc_type = TAVOR_UARPG;
1468 rsrc_pool->rsrc_loc = TAVOR_IN_UAR;
1469 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1470 rsrc_pool->rsrc_shift = PAGESHIFT;
1471 rsrc_pool->rsrc_quantum = PAGESIZE;
1472 rsrc_pool->rsrc_align = PAGESIZE;
1473 rsrc_pool->rsrc_state = state;
1474 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr;
1475 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1476 entry_info.hwi_num = num;
1477 entry_info.hwi_max = max;
1478 entry_info.hwi_prealloc = num_prealloc;
1479 entry_info.hwi_rsrcpool = rsrc_pool;
1480 entry_info.hwi_rsrcname = rsrc_name;
1481 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1482 if (status != DDI_SUCCESS) {
1483 tavor_rsrc_fini(state, cleanup);
1484 /* Set "status" and "errormsg" and goto failure */
1485 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table");
1486 goto rsrcinitp2_fail;
1487 }
1488 cleanup = TAVOR_RSRC_CLEANUP_ALL;
1489
1490 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1491 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1492 return (DDI_SUCCESS);
1493
1494 rsrcinitp2_fail:
1495 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1496 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "",
1497 tnf_string, msg, errormsg);
1498 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1499 return (status);
1500 }
1501
1502
1503 /*
1504 * tavor_rsrc_fini()
1505 * Context: Only called from attach() and/or detach() path contexts
1506 */
1507 void
tavor_rsrc_fini(tavor_state_t * state,tavor_rsrc_cleanup_level_t clean)1508 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1509 {
1510 tavor_rsrc_sw_hdl_info_t hdl_info;
1511 tavor_rsrc_hw_entry_info_t entry_info;
1512 tavor_rsrc_mbox_info_t mbox_info;
1513 tavor_cfg_profile_t *cfgprof;
1514
1515 TAVOR_TNF_ENTER(tavor_rsrc_fini);
1516
1517 ASSERT(state != NULL);
1518
1519 cfgprof = state->ts_cfg_profile;
1520
1521 switch (clean) {
1522 /*
1523 * If we add more resources that need to be cleaned up here, we should
1524 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1525 * corresponds to the last resource allocated).
1526 */
1527 case TAVOR_RSRC_CLEANUP_ALL:
1528 /* Cleanup the UAR page resource pool */
1529 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1530 tavor_rsrc_hw_entries_fini(state, &entry_info);
1531 /* FALLTHROUGH */
1532
1533 case TAVOR_RSRC_CLEANUP_LEVEL30:
1534 /* Cleanup the central MCG handle pointers list */
1535 hdl_info.swi_rsrcpool = NULL;
1536 hdl_info.swi_table_ptr = state->ts_mcghdl;
1537 hdl_info.swi_num =
1538 ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1539 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1540 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1541 /* FALLTHROUGH */
1542
1543 case TAVOR_RSRC_CLEANUP_LEVEL29:
1544 /* Cleanup the reference count resource pool */
1545 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1546 hdl_info.swi_table_ptr = NULL;
1547 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1548 /* FALLTHROUGH */
1549
1550 case TAVOR_RSRC_CLEANUP_LEVEL28:
1551 /* Cleanup the QP handle resource pool */
1552 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1553 hdl_info.swi_table_ptr = state->ts_qphdl;
1554 hdl_info.swi_num =
1555 ((uint64_t)1 << cfgprof->cp_log_num_qp);
1556 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1557 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1558 /* FALLTHROUGH */
1559
1560 case TAVOR_RSRC_CLEANUP_LEVEL27:
1561 /* Cleanup the address handle resource pool */
1562 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1563 hdl_info.swi_table_ptr = NULL;
1564 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1565 /* FALLTHROUGH */
1566
1567 case TAVOR_RSRC_CLEANUP_LEVEL26:
1568 /*
1569 * Cleanup the SRQ handle resource pool.
1570 *
1571 * Note: We only clean up if SRQ is enabled. Otherwise we
1572 * simply fallthrough to the next resource cleanup.
1573 */
1574 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1575 hdl_info.swi_rsrcpool =
1576 &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1577 hdl_info.swi_table_ptr = state->ts_srqhdl;
1578 hdl_info.swi_num =
1579 ((uint64_t)1 << cfgprof->cp_log_num_srq);
1580 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1581 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1582 }
1583 /* FALLTHROUGH */
1584
1585 case TAVOR_RSRC_CLEANUP_LEVEL25:
1586 /* Cleanup the CQ handle resource pool */
1587 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1588 hdl_info.swi_table_ptr = state->ts_cqhdl;
1589 hdl_info.swi_num =
1590 ((uint64_t)1 << cfgprof->cp_log_num_cq);
1591 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1592 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1593 /* FALLTHROUGH */
1594
1595 case TAVOR_RSRC_CLEANUP_LEVEL24:
1596 /* Cleanup the EQ handle resource pool */
1597 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1598 hdl_info.swi_table_ptr = NULL;
1599 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1600 /* FALLTHROUGH */
1601
1602 case TAVOR_RSRC_CLEANUP_LEVEL23:
1603 /* Cleanup the MR handle resource pool */
1604 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1605 hdl_info.swi_table_ptr = NULL;
1606 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1607 /* FALLTHROUGH */
1608
1609 case TAVOR_RSRC_CLEANUP_LEVEL22:
1610 /* Cleanup the PD handle resource pool */
1611 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1612 hdl_info.swi_table_ptr = NULL;
1613 tavor_rsrc_pd_handles_fini(state, &hdl_info);
1614 /* FALLTHROUGH */
1615
1616 case TAVOR_RSRC_CLEANUP_LEVEL21:
1617 /* Cleanup the EQC table resource pool */
1618 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1619 tavor_rsrc_hw_entries_fini(state, &entry_info);
1620 /* FALLTHROUGH */
1621
1622 case TAVOR_RSRC_CLEANUP_LEVEL20:
1623 /* Cleanup the MCG table resource pool */
1624 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1625 tavor_rsrc_hw_entries_fini(state, &entry_info);
1626 /* FALLTHROUGH */
1627
1628 case TAVOR_RSRC_CLEANUP_LEVEL19:
1629 /* Cleanup the outstanding command list */
1630 tavor_outstanding_cmdlist_fini(state);
1631 /* FALLTHROUGH */
1632
1633 case TAVOR_RSRC_CLEANUP_LEVEL18:
1634 /* Cleanup the "In" mailbox list */
1635 tavor_intr_inmbox_list_fini(state);
1636 /* FALLTHROUGH */
1637
1638 case TAVOR_RSRC_CLEANUP_LEVEL17:
1639 /* Cleanup the interrupt "In" mailbox resource pool */
1640 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1641 TAVOR_INTR_IN_MBOX];
1642 tavor_rsrc_mbox_fini(state, &mbox_info);
1643 /* FALLTHROUGH */
1644
1645 case TAVOR_RSRC_CLEANUP_LEVEL16:
1646 /* Cleanup the "In" mailbox list */
1647 tavor_inmbox_list_fini(state);
1648 /* FALLTHROUGH */
1649
1650 case TAVOR_RSRC_CLEANUP_LEVEL15:
1651 /* Cleanup the "In" mailbox resource pool */
1652 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1653 tavor_rsrc_mbox_fini(state, &mbox_info);
1654 /* FALLTHROUGH */
1655
1656 case TAVOR_RSRC_CLEANUP_LEVEL14:
1657 /*
1658 * Cleanup the SRQC table resource pool.
1659 *
1660 * Note: We only clean up if SRQ is enabled. Otherwise we
1661 * simply fallthrough to the next resource cleanup.
1662 */
1663 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1664 entry_info.hwi_rsrcpool =
1665 &state->ts_rsrc_hdl[TAVOR_SRQC];
1666 tavor_rsrc_hw_entries_fini(state, &entry_info);
1667 }
1668 /* FALLTHROUGH */
1669
1670 case TAVOR_RSRC_CLEANUP_LEVEL13:
1671 /* Cleanup the UAR scratch table resource pool */
1672 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1673 tavor_rsrc_hw_entries_fini(state, &entry_info);
1674 /* FALLTHROUGH */
1675
1676 case TAVOR_RSRC_CLEANUP_LEVEL12:
1677 /* Cleanup the UDAV table resource pool */
1678 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1679 tavor_rsrc_hw_entries_fini(state, &entry_info);
1680 /* FALLTHROUGH */
1681
1682 case TAVOR_RSRC_CLEANUP_LEVEL11:
1683 /* Cleanup the EQPC table resource pool */
1684 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1685 tavor_rsrc_hw_entries_fini(state, &entry_info);
1686 /* FALLTHROUGH */
1687
1688 case TAVOR_RSRC_CLEANUP_LEVEL10:
1689 /* Cleanup the CQC table resource pool */
1690 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1691 tavor_rsrc_hw_entries_fini(state, &entry_info);
1692 /* FALLTHROUGH */
1693
1694 case TAVOR_RSRC_CLEANUP_LEVEL9:
1695 /* Cleanup the RDB table resource pool */
1696 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1697 tavor_rsrc_hw_entries_fini(state, &entry_info);
1698 /* FALLTHROUGH */
1699
1700 case TAVOR_RSRC_CLEANUP_LEVEL8:
1701 /* Cleanup the QPC table resource pool */
1702 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1703 tavor_rsrc_hw_entries_fini(state, &entry_info);
1704 /* FALLTHROUGH */
1705
1706 case TAVOR_RSRC_CLEANUP_LEVEL7:
1707 /* Cleanup the MTT table resource pool */
1708 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1709 tavor_rsrc_hw_entries_fini(state, &entry_info);
1710 /* FALLTHROUGH */
1711
1712 case TAVOR_RSRC_CLEANUP_LEVEL6:
1713 /* Cleanup the MPT table resource pool */
1714 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1715 tavor_rsrc_hw_entries_fini(state, &entry_info);
1716 /* FALLTHROUGH */
1717
1718 case TAVOR_RSRC_CLEANUP_LEVEL5:
1719 /* Destroy the vmem arena for DDR memory */
1720 vmem_destroy(state->ts_ddrvmem);
1721 break;
1722
1723 /*
1724 * The cleanup below comes from the "Phase 1" initialization step.
1725 * (see tavor_rsrc_init_phase1() above)
1726 */
1727 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1728 /* Cleanup the interrupt "Out" mailbox list */
1729 tavor_intr_outmbox_list_fini(state);
1730 /* FALLTHROUGH */
1731
1732 case TAVOR_RSRC_CLEANUP_LEVEL3:
1733 /* Cleanup the "Out" mailbox resource pool */
1734 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1735 TAVOR_INTR_OUT_MBOX];
1736 tavor_rsrc_mbox_fini(state, &mbox_info);
1737 /* FALLTHROUGH */
1738
1739 case TAVOR_RSRC_CLEANUP_LEVEL2:
1740 /* Cleanup the "Out" mailbox list */
1741 tavor_outmbox_list_fini(state);
1742 /* FALLTHROUGH */
1743
1744 case TAVOR_RSRC_CLEANUP_LEVEL1:
1745 /* Cleanup the "Out" mailbox resource pool */
1746 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1747 tavor_rsrc_mbox_fini(state, &mbox_info);
1748 /* FALLTHROUGH */
1749
1750 case TAVOR_RSRC_CLEANUP_LEVEL0:
1751 /* Free the array of tavor_rsrc_pool_info_t's */
1752 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1753 sizeof (tavor_rsrc_pool_info_t));
1754 kmem_cache_destroy(state->ts_rsrc_cache);
1755 break;
1756
1757 default:
1758 TAVOR_WARNING(state, "unexpected resource cleanup level");
1759 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, "");
1760 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1761 return;
1762 }
1763
1764 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1765 }
1766
1767
1768 /*
1769 * tavor_rsrc_mbox_init()
1770 * Context: Only called from attach() path context
1771 */
1772 static int
tavor_rsrc_mbox_init(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1773 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1774 {
1775 tavor_rsrc_pool_info_t *rsrc_pool;
1776 tavor_rsrc_priv_mbox_t *priv;
1777 vmem_t *vmp;
1778 uint64_t offset;
1779 uint_t dma_xfer_mode;
1780
1781 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init);
1782
1783 ASSERT(state != NULL);
1784 ASSERT(info != NULL);
1785
1786 rsrc_pool = info->mbi_rsrcpool;
1787 ASSERT(rsrc_pool != NULL);
1788
1789 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1790
1791 /* Allocate and initialize mailbox private structure */
1792 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1793 priv->pmb_dip = state->ts_dip;
1794 priv->pmb_acchdl = state->ts_reg_ddrhdl;
1795 priv->pmb_devaccattr = state->ts_reg_accattr;
1796 priv->pmb_xfer_mode = dma_xfer_mode;
1797
1798 /*
1799 * Initialize many of the default DMA attributes. Then set alignment
1800 * and scatter-gather restrictions specific for mailbox memory.
1801 */
1802 tavor_dma_attr_init(&priv->pmb_dmaattr);
1803 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN;
1804 priv->pmb_dmaattr.dma_attr_sgllen = 1;
1805
1806 rsrc_pool->rsrc_private = priv;
1807
1808 /* Is object in DDR memory or system memory? */
1809 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1810 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1811 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1812 0, 0, NULL, NULL, VM_SLEEP);
1813 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1814 /* Unable to alloc space for mailboxes */
1815 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1816 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail,
1817 TAVOR_TNF_ERROR, "");
1818 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1819 return (DDI_FAILURE);
1820 }
1821
1822 /* Calculate offset and starting point (in DDR) */
1823 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1824 state->ts_ddr.ddr_baseaddr);
1825 rsrc_pool->rsrc_start =
1826 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1827 offset);
1828
1829 /* Create new vmem arena for the mailboxes */
1830 vmp = vmem_create(info->mbi_rsrcname,
1831 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1832 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1833 if (vmp == NULL) {
1834 /* Unable to create vmem arena */
1835 vmem_xfree(state->ts_ddrvmem,
1836 rsrc_pool->rsrc_ddr_offset,
1837 rsrc_pool->rsrc_pool_size);
1838 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1839 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail,
1840 TAVOR_TNF_ERROR, "");
1841 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1842 return (DDI_FAILURE);
1843 }
1844 rsrc_pool->rsrc_vmp = vmp;
1845 } else {
1846 rsrc_pool->rsrc_ddr_offset = NULL;
1847 rsrc_pool->rsrc_start = NULL;
1848 rsrc_pool->rsrc_vmp = NULL;
1849 }
1850
1851 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1852 return (DDI_SUCCESS);
1853 }
1854
1855
1856 /*
1857 * tavor_rsrc_mbox_fini()
1858 * Context: Only called from attach() and/or detach() path contexts
1859 */
1860 static void
tavor_rsrc_mbox_fini(tavor_state_t * state,tavor_rsrc_mbox_info_t * info)1861 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1862 {
1863 tavor_rsrc_pool_info_t *rsrc_pool;
1864
1865 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini);
1866
1867 ASSERT(state != NULL);
1868 ASSERT(info != NULL);
1869
1870 rsrc_pool = info->mbi_rsrcpool;
1871 ASSERT(rsrc_pool != NULL);
1872
1873 /* If mailboxes are DDR memory, then destroy and free up vmem */
1874 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1875
1876 /* Destroy the specially created mbox vmem arena */
1877 vmem_destroy(rsrc_pool->rsrc_vmp);
1878
1879 /* Free up the region from the ddr_vmem arena */
1880 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1881 rsrc_pool->rsrc_pool_size);
1882 }
1883
1884 /* Free up the private struct */
1885 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1886
1887 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini);
1888 }
1889
1890
1891 /*
1892 * tavor_rsrc_hw_entries_init()
1893 * Context: Only called from attach() path context
1894 */
1895 static int
tavor_rsrc_hw_entries_init(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)1896 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1897 tavor_rsrc_hw_entry_info_t *info)
1898 {
1899 tavor_rsrc_pool_info_t *rsrc_pool;
1900 tavor_rsrc_t *rsvd_rsrc = NULL;
1901 vmem_t *vmp;
1902 uint64_t num_hwentry, max_hwentry, num_prealloc;
1903 uint64_t offset;
1904 int status;
1905
1906 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init);
1907
1908 ASSERT(state != NULL);
1909 ASSERT(info != NULL);
1910
1911 rsrc_pool = info->hwi_rsrcpool;
1912 ASSERT(rsrc_pool != NULL);
1913 num_hwentry = info->hwi_num;
1914 max_hwentry = info->hwi_max;
1915 num_prealloc = info->hwi_prealloc;
1916
1917 /* Make sure number of HW entries makes sense */
1918 if (num_hwentry > max_hwentry) {
1919 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail,
1920 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW "
1921 "entries exceeds device maximum", tnf_uint, maxhw,
1922 max_hwentry);
1923 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1924 return (DDI_FAILURE);
1925 }
1926
1927 /*
1928 * Determine if we need to allocate DDR space to set up the
1929 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already
1930 * been initialized (as is the case for the UAR page init).
1931 */
1932 if (rsrc_pool->rsrc_start == NULL) {
1933 /* Make sure HW entries table is aligned as specified */
1934 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1935 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1936 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1937 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1938 /* Unable to alloc space for aligned HW table */
1939 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail,
1940 TAVOR_TNF_ERROR, "");
1941 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1942 return (DDI_FAILURE);
1943 }
1944
1945 /* Calculate offset and starting point (in DDR) */
1946 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1947 state->ts_ddr.ddr_baseaddr);
1948 rsrc_pool->rsrc_start =
1949 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1950 offset);
1951 } else {
1952 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1953 }
1954
1955 /*
1956 * Create new vmem arena for the HW entries table (if rsrc_quantum
1957 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW
1958 * entries are not going to be dynamically allocatable (i.e. they
1959 * won't be allocated/freed through tavor_rsrc_alloc/free). This
1960 * latter option is used for EQPC and UARSCR resource which are, in
1961 * fact, managed by the Tavor hardware.
1962 */
1963 if (rsrc_pool->rsrc_quantum != 0) {
1964 vmp = vmem_create(info->hwi_rsrcname,
1965 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1966 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1967 if (vmp == NULL) {
1968 /* Unable to create vmem arena */
1969 if (rsrc_pool->rsrc_ddr_offset !=
1970 rsrc_pool->rsrc_start) {
1971 vmem_xfree(state->ts_ddrvmem,
1972 rsrc_pool->rsrc_ddr_offset,
1973 rsrc_pool->rsrc_pool_size);
1974 }
1975 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail,
1976 TAVOR_TNF_ERROR, "");
1977 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1978 return (DDI_FAILURE);
1979 }
1980 rsrc_pool->rsrc_vmp = vmp;
1981 } else {
1982 rsrc_pool->rsrc_vmp = NULL;
1983 }
1984
1985 /* The first HW entries may be reserved by Tavor firmware */
1986 if (num_prealloc != 0) {
1987 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1988 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1989 if (status != DDI_SUCCESS) {
1990 /* Unable to preallocate the reserved HW entries */
1991 if (rsrc_pool->rsrc_vmp != NULL) {
1992 vmem_destroy(rsrc_pool->rsrc_vmp);
1993 }
1994 if (rsrc_pool->rsrc_ddr_offset !=
1995 rsrc_pool->rsrc_start) {
1996 vmem_xfree(state->ts_ddrvmem,
1997 rsrc_pool->rsrc_ddr_offset,
1998 rsrc_pool->rsrc_pool_size);
1999 }
2000 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail,
2001 TAVOR_TNF_ERROR, "");
2002 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2003 return (DDI_FAILURE);
2004 }
2005 }
2006 rsrc_pool->rsrc_private = rsvd_rsrc;
2007
2008 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2009 return (DDI_SUCCESS);
2010 }
2011
2012
2013 /*
2014 * tavor_rsrc_hw_entries_fini()
2015 * Context: Only called from attach() and/or detach() path contexts
2016 */
2017 static void
tavor_rsrc_hw_entries_fini(tavor_state_t * state,tavor_rsrc_hw_entry_info_t * info)2018 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
2019 tavor_rsrc_hw_entry_info_t *info)
2020 {
2021 tavor_rsrc_pool_info_t *rsrc_pool;
2022 tavor_rsrc_t *rsvd_rsrc;
2023
2024 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini);
2025
2026 ASSERT(state != NULL);
2027 ASSERT(info != NULL);
2028
2029 rsrc_pool = info->hwi_rsrcpool;
2030 ASSERT(rsrc_pool != NULL);
2031
2032 /* Free up any "reserved" (i.e. preallocated) HW entries */
2033 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
2034 if (rsvd_rsrc != NULL) {
2035 tavor_rsrc_free(state, &rsvd_rsrc);
2036 }
2037
2038 /*
2039 * If we've actually setup a vmem arena for the HW entries, then
2040 * destroy it now
2041 */
2042 if (rsrc_pool->rsrc_vmp != NULL) {
2043 vmem_destroy(rsrc_pool->rsrc_vmp);
2044 }
2045
2046 /*
2047 * Determine if a region was allocated from the tavor_ddr_vmem
2048 * arena (and free it up if necessary)
2049 */
2050 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
2051 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
2052 rsrc_pool->rsrc_pool_size);
2053 }
2054
2055 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini);
2056 }
2057
2058
2059 /*
2060 * tavor_rsrc_sw_handles_init()
2061 * Context: Only called from attach() path context
2062 */
2063 /* ARGSUSED */
2064 static int
tavor_rsrc_sw_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2065 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2066 {
2067 tavor_rsrc_pool_info_t *rsrc_pool;
2068 uint64_t num_swhdl, max_swhdl, prealloc_sz;
2069
2070 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init);
2071
2072 ASSERT(state != NULL);
2073 ASSERT(info != NULL);
2074
2075 rsrc_pool = info->swi_rsrcpool;
2076 ASSERT(rsrc_pool != NULL);
2077 num_swhdl = info->swi_num;
2078 max_swhdl = info->swi_max;
2079 prealloc_sz = info->swi_prealloc_sz;
2080
2081 /* Make sure number of SW handles makes sense */
2082 if (num_swhdl > max_swhdl) {
2083 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail,
2084 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW "
2085 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl);
2086 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2087 return (DDI_FAILURE);
2088 }
2089
2090 /*
2091 * Depending on the flags parameter, create a kmem_cache for some
2092 * number of software handle structures. Note: kmem_cache_create()
2093 * will SLEEP until successful.
2094 */
2095 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
2096 rsrc_pool->rsrc_private = kmem_cache_create(
2097 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
2098 info->swi_constructor, info->swi_destructor, NULL,
2099 rsrc_pool->rsrc_state, NULL, 0);
2100 }
2101
2102 /* Allocate the central list of SW handle pointers */
2103 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
2104 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
2105 KM_SLEEP);
2106 }
2107
2108 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2109 return (DDI_SUCCESS);
2110 }
2111
2112
2113 /*
2114 * tavor_rsrc_sw_handles_fini()
2115 * Context: Only called from attach() and/or detach() path contexts
2116 */
2117 /* ARGSUSED */
2118 static void
tavor_rsrc_sw_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2119 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2120 {
2121 tavor_rsrc_pool_info_t *rsrc_pool;
2122 uint64_t num_swhdl, prealloc_sz;
2123
2124 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini);
2125
2126 ASSERT(state != NULL);
2127 ASSERT(info != NULL);
2128
2129 rsrc_pool = info->swi_rsrcpool;
2130 num_swhdl = info->swi_num;
2131 prealloc_sz = info->swi_prealloc_sz;
2132
2133 /*
2134 * If a "software handle" kmem_cache exists for this resource, then
2135 * destroy it now
2136 */
2137 if (rsrc_pool != NULL) {
2138 kmem_cache_destroy(rsrc_pool->rsrc_private);
2139 }
2140
2141 /* Free up this central list of SW handle pointers */
2142 if (info->swi_table_ptr != NULL) {
2143 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2144 }
2145
2146 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini);
2147 }
2148
2149
2150 /*
2151 * tavor_rsrc_pd_handles_init()
2152 * Context: Only called from attach() path context
2153 */
2154 static int
tavor_rsrc_pd_handles_init(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2155 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2156 {
2157 tavor_rsrc_pool_info_t *rsrc_pool;
2158 vmem_t *vmp;
2159 char vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2160 int status;
2161
2162 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init);
2163
2164 ASSERT(state != NULL);
2165 ASSERT(info != NULL);
2166
2167 rsrc_pool = info->swi_rsrcpool;
2168 ASSERT(rsrc_pool != NULL);
2169
2170 /* Initialize the resource pool for software handle table */
2171 status = tavor_rsrc_sw_handles_init(state, info);
2172 if (status != DDI_SUCCESS) {
2173 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2174 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2175 return (DDI_FAILURE);
2176 }
2177
2178 /* Build vmem arena name from Tavor instance */
2179 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2180
2181 /* Create new vmem arena for PD numbers */
2182 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2183 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2184 if (vmp == NULL) {
2185 /* Unable to create vmem arena */
2186 info->swi_table_ptr = NULL;
2187 tavor_rsrc_sw_handles_fini(state, info);
2188 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail,
2189 TAVOR_TNF_ERROR, "");
2190 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2191 return (DDI_FAILURE);
2192 }
2193 rsrc_pool->rsrc_vmp = vmp;
2194
2195 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2196 return (DDI_SUCCESS);
2197 }
2198
2199
2200 /*
2201 * tavor_rsrc_pd_handles_fini()
2202 * Context: Only called from attach() and/or detach() path contexts
2203 */
2204 static void
tavor_rsrc_pd_handles_fini(tavor_state_t * state,tavor_rsrc_sw_hdl_info_t * info)2205 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2206 {
2207 tavor_rsrc_pool_info_t *rsrc_pool;
2208
2209 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini);
2210
2211 ASSERT(state != NULL);
2212 ASSERT(info != NULL);
2213
2214 rsrc_pool = info->swi_rsrcpool;
2215
2216 /* Destroy the specially created UAR scratch table vmem arena */
2217 vmem_destroy(rsrc_pool->rsrc_vmp);
2218
2219 /* Destroy the "tavor_sw_pd_t" kmem_cache */
2220 tavor_rsrc_sw_handles_fini(state, info);
2221
2222 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini);
2223 }
2224
2225
2226 /*
2227 * tavor_rsrc_mbox_alloc()
2228 * Context: Only called from attach() path context
2229 */
2230 static int
tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,tavor_rsrc_t * hdl)2231 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2232 tavor_rsrc_t *hdl)
2233 {
2234 tavor_rsrc_priv_mbox_t *priv;
2235 void *addr;
2236 caddr_t kaddr;
2237 uint64_t offset;
2238 size_t real_len, temp_len;
2239 int status;
2240
2241 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc);
2242
2243 ASSERT(pool_info != NULL);
2244 ASSERT(hdl != NULL);
2245
2246 /* Get the private pointer for the mailboxes */
2247 priv = pool_info->rsrc_private;
2248 ASSERT(priv != NULL);
2249
2250 /*
2251 * Allocate a DMA handle for the mailbox. This will be used for
2252 * two purposes (potentially). First, it could be used below in
2253 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2254 * system memory. Second, it is definitely used later to bind
2255 * the mailbox for DMA access from/by the hardware.
2256 */
2257 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2258 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2259 if (status != DDI_SUCCESS) {
2260 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR,
2261 "", tnf_uint, status, status);
2262 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2263 return (DDI_FAILURE);
2264 }
2265
2266 /* Is mailbox in DDR memory or system memory? */
2267 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2268 /* Use vmem_alloc() to get DDR address of mbox */
2269 hdl->tr_len = (num * pool_info->rsrc_quantum);
2270 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2271 VM_SLEEP);
2272 if (addr == NULL) {
2273 /* No more DDR available for mailbox entries */
2274 ddi_dma_free_handle(&hdl->tr_dmahdl);
2275 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail,
2276 TAVOR_TNF_ERROR, "");
2277 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2278 return (DDI_FAILURE);
2279 }
2280 hdl->tr_acchdl = priv->pmb_acchdl;
2281
2282 /* Calculate kernel virtual address (from the DDR offset) */
2283 offset = ((uintptr_t)addr -
2284 (uintptr_t)pool_info->rsrc_ddr_offset);
2285 hdl->tr_addr = (void *)(uintptr_t)(offset +
2286 (uintptr_t)pool_info->rsrc_start);
2287
2288 } else { /* TAVOR_IN_SYSMEM */
2289
2290 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2291 temp_len = (num * pool_info->rsrc_quantum);
2292 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2293 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2294 NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2295 if (status != DDI_SUCCESS) {
2296 /* No more sys memory available for mailbox entries */
2297 ddi_dma_free_handle(&hdl->tr_dmahdl);
2298 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail,
2299 TAVOR_TNF_ERROR, "");
2300 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2301 return (DDI_FAILURE);
2302 }
2303 hdl->tr_addr = (void *)kaddr;
2304 hdl->tr_len = real_len;
2305 }
2306
2307 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2308 return (DDI_SUCCESS);
2309 }
2310
2311
2312 /*
2313 * tavor_rsrc_mbox_free()
2314 * Context: Can be called from interrupt or base context.
2315 */
2316 static void
tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2317 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2318 {
2319 void *addr;
2320 uint64_t offset;
2321
2322 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free);
2323
2324 ASSERT(pool_info != NULL);
2325 ASSERT(hdl != NULL);
2326
2327 /* Is mailbox in DDR memory or system memory? */
2328 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2329
2330 /* Calculate the allocated address (the mbox's DDR offset) */
2331 offset = ((uintptr_t)hdl->tr_addr -
2332 (uintptr_t)pool_info->rsrc_start);
2333 addr = (void *)(uintptr_t)(offset +
2334 (uintptr_t)pool_info->rsrc_ddr_offset);
2335
2336 /* Use vmem_free() to free up DDR memory for mailbox */
2337 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2338
2339 } else { /* TAVOR_IN_SYSMEM */
2340
2341 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2342 ddi_dma_mem_free(&hdl->tr_acchdl);
2343 }
2344
2345 /* Free the DMA handle for the mailbox */
2346 ddi_dma_free_handle(&hdl->tr_dmahdl);
2347
2348 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free);
2349 }
2350
2351
2352 /*
2353 * tavor_rsrc_hw_entry_alloc()
2354 * Context: Can be called from interrupt or base context.
2355 */
2356 static int
tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t num,uint_t num_align,ddi_acc_handle_t acc_handle,uint_t sleepflag,tavor_rsrc_t * hdl)2357 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2358 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2359 tavor_rsrc_t *hdl)
2360 {
2361 void *addr;
2362 uint64_t offset;
2363 uint32_t align;
2364 int flag;
2365
2366 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc);
2367
2368 ASSERT(pool_info != NULL);
2369 ASSERT(hdl != NULL);
2370
2371 /*
2372 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2373 * use dma_handle (because they are in Tavor locally attached DDR
2374 * memory) and, generally, don't use the acc_handle (because the
2375 * entries are not directly accessed by software). The exceptions
2376 * to this rule are the UARPG and UDAV entries.
2377 */
2378
2379 /*
2380 * Use vmem_xalloc() to get a properly aligned pointer (based on
2381 * the number requested) to the HW entry(ies). This handles the
2382 * cases (for special QPCs and for RDB entries) where we need more
2383 * than one and need to ensure that they are properly aligned.
2384 */
2385 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2386 hdl->tr_len = (num * pool_info->rsrc_quantum);
2387 align = (num_align * pool_info->rsrc_quantum);
2388 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2389 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2390 if (addr == NULL) {
2391 /* No more HW entries available */
2392 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail,
2393 TAVOR_TNF_ERROR, "");
2394 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2395 return (DDI_FAILURE);
2396 }
2397
2398 /* If an access handle was provided, fill it in */
2399 if (acc_handle != 0) {
2400 hdl->tr_acchdl = acc_handle;
2401 }
2402
2403 /* Calculate vaddr and HW table index (from the DDR offset) */
2404 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2405 hdl->tr_addr = (void *)(uintptr_t)(offset +
2406 (uintptr_t)pool_info->rsrc_start);
2407 hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2408
2409 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2410 return (DDI_SUCCESS);
2411 }
2412
2413
2414 /*
2415 * tavor_rsrc_hw_entry_free()
2416 * Context: Can be called from interrupt or base context.
2417 */
2418 static void
tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2419 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2420 {
2421 void *addr;
2422 uint64_t offset;
2423
2424 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free);
2425
2426 ASSERT(pool_info != NULL);
2427 ASSERT(hdl != NULL);
2428
2429 /* Calculate the allocated address (the entry's DDR offset) */
2430 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2431 addr = (void *)(uintptr_t)(offset +
2432 (uintptr_t)pool_info->rsrc_ddr_offset);
2433
2434 /* Use vmem_xfree() to free up the HW table entry */
2435 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2436
2437 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free);
2438 }
2439
2440
2441 /*
2442 * tavor_rsrc_swhdl_alloc()
2443 * Context: Can be called from interrupt or base context.
2444 */
2445 static int
tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2446 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2447 tavor_rsrc_t *hdl)
2448 {
2449 void *addr;
2450 int flag;
2451
2452 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc);
2453
2454 ASSERT(pool_info != NULL);
2455 ASSERT(hdl != NULL);
2456
2457 /* Allocate the software handle structure */
2458 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2459 addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2460 if (addr == NULL) {
2461 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR,
2462 "");
2463 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2464 return (DDI_FAILURE);
2465 }
2466 hdl->tr_len = pool_info->rsrc_quantum;
2467 hdl->tr_addr = addr;
2468
2469 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2470 return (DDI_SUCCESS);
2471 }
2472
2473
2474 /*
2475 * tavor_rsrc_swhdl_free()
2476 * Context: Can be called from interrupt or base context.
2477 */
2478 static void
tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2479 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2480 {
2481 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free);
2482
2483 ASSERT(pool_info != NULL);
2484 ASSERT(hdl != NULL);
2485
2486 /* Free the software handle structure */
2487 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2488
2489 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free);
2490 }
2491
2492
2493 /*
2494 * tavor_rsrc_pdhdl_alloc()
2495 * Context: Can be called from interrupt or base context.
2496 */
2497 static int
tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t * pool_info,uint_t sleepflag,tavor_rsrc_t * hdl)2498 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2499 tavor_rsrc_t *hdl)
2500 {
2501 tavor_pdhdl_t addr;
2502 void *tmpaddr;
2503 int flag, status;
2504
2505 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc);
2506
2507 ASSERT(pool_info != NULL);
2508 ASSERT(hdl != NULL);
2509
2510 /* Allocate the software handle */
2511 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2512 if (status != DDI_SUCCESS) {
2513 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2514 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2515 return (DDI_FAILURE);
2516 }
2517 addr = (tavor_pdhdl_t)hdl->tr_addr;
2518 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2519
2520 /* Allocate a PD number for the handle */
2521 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2522 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2523 if (tmpaddr == NULL) {
2524 /* No more PD number entries available */
2525 tavor_rsrc_swhdl_free(pool_info, hdl);
2526 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail,
2527 TAVOR_TNF_ERROR, "");
2528 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2529 return (DDI_FAILURE);
2530 }
2531 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2532 addr->pd_rsrcp = hdl;
2533 hdl->tr_indx = addr->pd_pdnum;
2534
2535 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2536 return (DDI_SUCCESS);
2537 }
2538
2539
2540 /*
2541 * tavor_rsrc_pdhdl_free()
2542 * Context: Can be called from interrupt or base context.
2543 */
2544 static void
tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t * pool_info,tavor_rsrc_t * hdl)2545 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2546 {
2547 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free);
2548
2549 ASSERT(pool_info != NULL);
2550 ASSERT(hdl != NULL);
2551
2552 /* Use vmem_free() to free up the PD number */
2553 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2554
2555 /* Free the software handle structure */
2556 tavor_rsrc_swhdl_free(pool_info, hdl);
2557
2558 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free);
2559 }
2560
2561
2562 /*
2563 * tavor_rsrc_pdhdl_constructor()
2564 * Context: Can be called from interrupt or base context.
2565 */
2566 /* ARGSUSED */
2567 static int
tavor_rsrc_pdhdl_constructor(void * pd,void * priv,int flags)2568 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2569 {
2570 tavor_pdhdl_t pdhdl;
2571 tavor_state_t *state;
2572
2573 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor);
2574
2575 pdhdl = (tavor_pdhdl_t)pd;
2576 state = (tavor_state_t *)priv;
2577
2578 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2579 DDI_INTR_PRI(state->ts_intrmsi_pri));
2580
2581 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor);
2582 return (DDI_SUCCESS);
2583 }
2584
2585
2586 /*
2587 * tavor_rsrc_pdhdl_destructor()
2588 * Context: Can be called from interrupt or base context.
2589 */
2590 /* ARGSUSED */
2591 static void
tavor_rsrc_pdhdl_destructor(void * pd,void * priv)2592 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2593 {
2594 tavor_pdhdl_t pdhdl;
2595
2596 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor);
2597
2598 pdhdl = (tavor_pdhdl_t)pd;
2599
2600 mutex_destroy(&pdhdl->pd_lock);
2601
2602 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor);
2603 }
2604
2605
2606 /*
2607 * tavor_rsrc_cqhdl_constructor()
2608 * Context: Can be called from interrupt or base context.
2609 */
2610 /* ARGSUSED */
2611 static int
tavor_rsrc_cqhdl_constructor(void * cq,void * priv,int flags)2612 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2613 {
2614 tavor_cqhdl_t cqhdl;
2615 tavor_state_t *state;
2616
2617 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor);
2618
2619 cqhdl = (tavor_cqhdl_t)cq;
2620 state = (tavor_state_t *)priv;
2621
2622 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2623 DDI_INTR_PRI(state->ts_intrmsi_pri));
2624 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2625 DDI_INTR_PRI(state->ts_intrmsi_pri));
2626
2627 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor);
2628 return (DDI_SUCCESS);
2629 }
2630
2631
2632 /*
2633 * tavor_rsrc_cqhdl_destructor()
2634 * Context: Can be called from interrupt or base context.
2635 */
2636 /* ARGSUSED */
2637 static void
tavor_rsrc_cqhdl_destructor(void * cq,void * priv)2638 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2639 {
2640 tavor_cqhdl_t cqhdl;
2641
2642 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor);
2643
2644 cqhdl = (tavor_cqhdl_t)cq;
2645
2646 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2647 mutex_destroy(&cqhdl->cq_lock);
2648
2649 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor);
2650 }
2651
2652
2653 /*
2654 * tavor_rsrc_qphdl_constructor()
2655 * Context: Can be called from interrupt or base context.
2656 */
2657 /* ARGSUSED */
2658 static int
tavor_rsrc_qphdl_constructor(void * qp,void * priv,int flags)2659 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2660 {
2661 tavor_qphdl_t qphdl;
2662 tavor_state_t *state;
2663
2664 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor);
2665
2666 qphdl = (tavor_qphdl_t)qp;
2667 state = (tavor_state_t *)priv;
2668
2669 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2670 DDI_INTR_PRI(state->ts_intrmsi_pri));
2671
2672 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor);
2673 return (DDI_SUCCESS);
2674 }
2675
2676
2677 /*
2678 * tavor_rsrc_qphdl_destructor()
2679 * Context: Can be called from interrupt or base context.
2680 */
2681 /* ARGSUSED */
2682 static void
tavor_rsrc_qphdl_destructor(void * qp,void * priv)2683 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2684 {
2685 tavor_qphdl_t qphdl;
2686
2687 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2688
2689 qphdl = (tavor_qphdl_t)qp;
2690
2691 mutex_destroy(&qphdl->qp_lock);
2692
2693 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2694 }
2695
2696
2697 /*
2698 * tavor_rsrc_srqhdl_constructor()
2699 * Context: Can be called from interrupt or base context.
2700 */
2701 /* ARGSUSED */
2702 static int
tavor_rsrc_srqhdl_constructor(void * srq,void * priv,int flags)2703 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2704 {
2705 tavor_srqhdl_t srqhdl;
2706 tavor_state_t *state;
2707
2708 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor);
2709
2710 srqhdl = (tavor_srqhdl_t)srq;
2711 state = (tavor_state_t *)priv;
2712
2713 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2714 DDI_INTR_PRI(state->ts_intrmsi_pri));
2715
2716 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor);
2717 return (DDI_SUCCESS);
2718 }
2719
2720
2721 /*
2722 * tavor_rsrc_srqhdl_destructor()
2723 * Context: Can be called from interrupt or base context.
2724 */
2725 /* ARGSUSED */
2726 static void
tavor_rsrc_srqhdl_destructor(void * srq,void * priv)2727 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2728 {
2729 tavor_srqhdl_t srqhdl;
2730
2731 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor);
2732
2733 srqhdl = (tavor_srqhdl_t)srq;
2734
2735 mutex_destroy(&srqhdl->srq_lock);
2736
2737 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor);
2738 }
2739
2740
2741 /*
2742 * tavor_rsrc_refcnt_constructor()
2743 * Context: Can be called from interrupt or base context.
2744 */
2745 /* ARGSUSED */
2746 static int
tavor_rsrc_refcnt_constructor(void * rc,void * priv,int flags)2747 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2748 {
2749 tavor_sw_refcnt_t *refcnt;
2750 tavor_state_t *state;
2751
2752 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor);
2753
2754 refcnt = (tavor_sw_refcnt_t *)rc;
2755 state = (tavor_state_t *)priv;
2756
2757 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2758 DDI_INTR_PRI(state->ts_intrmsi_pri));
2759
2760 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor);
2761 return (DDI_SUCCESS);
2762 }
2763
2764
2765 /*
2766 * tavor_rsrc_refcnt_destructor()
2767 * Context: Can be called from interrupt or base context.
2768 */
2769 /* ARGSUSED */
2770 static void
tavor_rsrc_refcnt_destructor(void * rc,void * priv)2771 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2772 {
2773 tavor_sw_refcnt_t *refcnt;
2774
2775 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2776
2777 refcnt = (tavor_sw_refcnt_t *)rc;
2778
2779 mutex_destroy(&refcnt->swrc_lock);
2780
2781 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2782 }
2783
2784
2785 /*
2786 * tavor_rsrc_ahhdl_constructor()
2787 * Context: Can be called from interrupt or base context.
2788 */
2789 /* ARGSUSED */
2790 static int
tavor_rsrc_ahhdl_constructor(void * ah,void * priv,int flags)2791 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2792 {
2793 tavor_ahhdl_t ahhdl;
2794 tavor_state_t *state;
2795
2796 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor);
2797
2798 ahhdl = (tavor_ahhdl_t)ah;
2799 state = (tavor_state_t *)priv;
2800
2801 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2802 DDI_INTR_PRI(state->ts_intrmsi_pri));
2803
2804 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor);
2805 return (DDI_SUCCESS);
2806 }
2807
2808
2809 /*
2810 * tavor_rsrc_ahhdl_destructor()
2811 * Context: Can be called from interrupt or base context.
2812 */
2813 /* ARGSUSED */
2814 static void
tavor_rsrc_ahhdl_destructor(void * ah,void * priv)2815 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2816 {
2817 tavor_ahhdl_t ahhdl;
2818
2819 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2820
2821 ahhdl = (tavor_ahhdl_t)ah;
2822
2823 mutex_destroy(&ahhdl->ah_lock);
2824
2825 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2826 }
2827
2828
2829 /*
2830 * tavor_rsrc_mrhdl_constructor()
2831 * Context: Can be called from interrupt or base context.
2832 */
2833 /* ARGSUSED */
2834 static int
tavor_rsrc_mrhdl_constructor(void * mr,void * priv,int flags)2835 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2836 {
2837 tavor_mrhdl_t mrhdl;
2838 tavor_state_t *state;
2839
2840 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor);
2841
2842 mrhdl = (tavor_mrhdl_t)mr;
2843 state = (tavor_state_t *)priv;
2844
2845 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2846 DDI_INTR_PRI(state->ts_intrmsi_pri));
2847
2848 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor);
2849 return (DDI_SUCCESS);
2850 }
2851
2852
2853 /*
2854 * tavor_rsrc_mrhdl_destructor()
2855 * Context: Can be called from interrupt or base context.
2856 */
2857 /* ARGSUSED */
2858 static void
tavor_rsrc_mrhdl_destructor(void * mr,void * priv)2859 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2860 {
2861 tavor_mrhdl_t mrhdl;
2862
2863 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2864
2865 mrhdl = (tavor_mrhdl_t)mr;
2866
2867 mutex_destroy(&mrhdl->mr_lock);
2868
2869 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2870 }
2871
2872
2873 /*
2874 * tavor_rsrc_mcg_entry_get_size()
2875 */
2876 static int
tavor_rsrc_mcg_entry_get_size(tavor_state_t * state,uint_t * mcg_size_shift)2877 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2878 {
2879 uint_t num_qp_per_mcg, max_qp_per_mcg, log2;
2880
2881 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size);
2882
2883 /*
2884 * Round the configured number of QP per MCG to next larger
2885 * power-of-2 size and update.
2886 */
2887 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2888 log2 = highbit(num_qp_per_mcg);
2889 if (ISP2(num_qp_per_mcg)) {
2890 log2 = log2 - 1;
2891 }
2892 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2893
2894 /* Now make sure number of QP per MCG makes sense */
2895 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2896 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2897 if (num_qp_per_mcg > max_qp_per_mcg) {
2898 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail,
2899 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg);
2900 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2901 return (DDI_FAILURE);
2902 }
2903
2904 /* Return the (shift) size of an individual MCG HW entry */
2905 *mcg_size_shift = log2 + 2;
2906
2907 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2908 return (DDI_SUCCESS);
2909 }
2910