xref: /illumos-gate/usr/src/uts/common/sys/ib/adapters/tavor/tavor_rsrc.h (revision 9e39c5ba00a55fa05777cc94b148296af305e135)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_TAVOR_RSRC_H
28 #define	_SYS_IB_ADAPTERS_TAVOR_RSRC_H
29 
30 /*
31  * tavor_rsrc.h
32  *    Contains all of the prototypes, #defines, and structures necessary
33  *    for the Tavor Resource Management routines.
34  *    Specifically it contains the resource names, resource types, and
35  *    structures used for enabling both init/fini and alloc/free operations.
36  */
37 
38 #include <sys/types.h>
39 #include <sys/conf.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 #include <sys/disp.h>
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
48 /*
49  * The above extern and the following #defines and macro are used to determine
50  * the current context for purposes of setting the sleepflag.  If the calling
51  * thread is running in the interrupt context, then macro will return
52  * TAVOR_NOSLEEP (indicating that it is not appropriate to sleep in the current
53  * context.  In all other cases, this macro will return TAVOR_SLEEP.
54  *
55  * The TAVOR_CMD_SLEEP_NOSPIN and TAVOR_CMD_NOSLEEP_SPIN #defines from
56  * tavor_cmd.h are set to use and be compatible with the following SLEEP
57  * variables.  It is important that these remain in sync so that the
58  * TAVOR_SLEEPFLAG_FOR_CONTEXT() macro will work in all cases.
59  */
60 #define	TAVOR_SLEEP			0
61 #define	TAVOR_NOSLEEP			1
62 #define	TAVOR_SLEEPFLAG_FOR_CONTEXT()					\
63 	((servicing_interrupt() || ddi_in_panic()) ? TAVOR_NOSLEEP :	\
64 	    TAVOR_SLEEP)
65 
66 /*
67  * The following #defines are used as the names for various resource pools.
68  * They represent the kmem_cache and vmem_arena names, respectively.  In
69  * order to provide for unique naming when multiple Tavor drivers are present,
70  * the TAVOR_RSRC_NAME macro below is used to append the driver's instance
71  * number to the provided string.  Note: resource names should not be longer
72  * than TAVOR_RSRC_NAME_MAXLEN.
73  */
74 #define	TAVOR_RSRC_CACHE		"tavor_rsrc_cache"
75 #define	TAVOR_PDHDL_CACHE		"tavor_pdhdl_cache"
76 #define	TAVOR_MRHDL_CACHE		"tavor_mrhdl_cache"
77 #define	TAVOR_EQHDL_CACHE		"tavor_eqhdl_cache"
78 #define	TAVOR_CQHDL_CACHE		"tavor_cqhdl_cache"
79 #define	TAVOR_SRQHDL_CACHE		"tavor_srqhdl_cache"
80 #define	TAVOR_AHHDL_CACHE		"tavor_ahhdl_cache"
81 #define	TAVOR_QPHDL_CACHE		"tavor_qphdl_cache"
82 #define	TAVOR_REFCNT_CACHE		"tavor_refcnt_cache"
83 
84 #define	TAVOR_DDR_VMEM			"tavor_ddr_vmem"
85 #define	TAVOR_DDR_INMBOX_VMEM		"tavor_ddr_inmbox_vmem"
86 #define	TAVOR_DDR_OUTMBOX_VMEM		"tavor_ddr_outmbox_vmem"
87 #define	TAVOR_DDR_INTR_INMBOX_VMEM	"tavor_ddr_intr_inmbox_vmem"
88 #define	TAVOR_DDR_INTR_OUTMBOX_VMEM	"tavor_ddr_intr_outmbox_vmem"
89 #define	TAVOR_DDR_QPC_VMEM		"tavor_ddr_qpc_vmem"
90 #define	TAVOR_DDR_CQC_VMEM		"tavor_ddr_cqc_vmem"
91 #define	TAVOR_DDR_SRQC_VMEM		"tavor_ddr_srqc_vmem"
92 #define	TAVOR_DDR_EQC_VMEM		"tavor_ddr_eqc_vmem"
93 #define	TAVOR_DDR_EQPC_VMEM		"tavor_ddr_eqpc_vmem"
94 #define	TAVOR_DDR_RDB_VMEM		"tavor_ddr_rdb_vmem"
95 #define	TAVOR_DDR_MCG_VMEM		"tavor_ddr_mcg_vmem"
96 #define	TAVOR_DDR_MPT_VMEM		"tavor_ddr_mpt_vmem"
97 #define	TAVOR_DDR_MTT_VMEM		"tavor_ddr_mtt_vmem"
98 #define	TAVOR_DDR_UARSCR_VMEM		"tavor_ddr_uarscr_vmem"
99 #define	TAVOR_DDR_UDAV_VMEM		"tavor_ddr_udav_vmem"
100 #define	TAVOR_UAR_VMEM			"tavor_uar_vmem"
101 #define	TAVOR_PDHDL_VMEM		"tavor_pd_vmem"
102 
103 /* Macro provided for building unique naming for multiple instance  */
104 #define	TAVOR_RSRC_NAME(rsrc_name, string)		\
105 	(void) sprintf((rsrc_name), string"%08X",	\
106 	    state->ts_instance)
107 #define	TAVOR_RSRC_NAME_MAXLEN		0x80
108 
109 /*
110  * The following enumerated type is used to capture all the various types
111  * of Tavor resources.  Note: The TAVOR_NUM_RESOURCES type is used as a
112  * marker for the end of the resource types.  No additional resources should
113  * be added after TAVOR_NUM_RESOURCES.  Any addition resources should be
114  * added before it.
115  */
116 typedef enum {
117 	TAVOR_QPC,
118 	TAVOR_CQC,
119 	TAVOR_SRQC,
120 	TAVOR_EQC,
121 	TAVOR_EQPC,
122 	TAVOR_RDB,
123 	TAVOR_MCG,
124 	TAVOR_MPT,
125 	TAVOR_MTT,
126 	TAVOR_UAR_SCR,
127 	TAVOR_UDAV,
128 	TAVOR_IN_MBOX,
129 	TAVOR_OUT_MBOX,
130 	TAVOR_PDHDL,
131 	TAVOR_MRHDL,
132 	TAVOR_EQHDL,
133 	TAVOR_CQHDL,
134 	TAVOR_SRQHDL,
135 	TAVOR_AHHDL,
136 	TAVOR_QPHDL,
137 	TAVOR_REFCNT,
138 	TAVOR_UARPG,
139 	TAVOR_INTR_IN_MBOX,
140 	TAVOR_INTR_OUT_MBOX,
141 	/* No more resources types below this point! */
142 	TAVOR_NUM_RESOURCES
143 } tavor_rsrc_type_t;
144 
145 
146 /*
147  * The following enumerated type and structures are used during resource
148  * initialization.  Note: The TAVOR_RSRC_CLEANUP_ALL type is used as a marker
149  * for end of the cleanup steps.  No cleanup steps should be added after
150  * TAVOR_RSRC_CLEANUP_ALL.  Any addition steps should be added before it.
151  */
152 typedef enum {
153 	TAVOR_RSRC_CLEANUP_LEVEL0,
154 	TAVOR_RSRC_CLEANUP_LEVEL1,
155 	TAVOR_RSRC_CLEANUP_LEVEL2,
156 	TAVOR_RSRC_CLEANUP_LEVEL3,
157 	TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE,
158 	TAVOR_RSRC_CLEANUP_LEVEL5,
159 	TAVOR_RSRC_CLEANUP_LEVEL6,
160 	TAVOR_RSRC_CLEANUP_LEVEL7,
161 	TAVOR_RSRC_CLEANUP_LEVEL8,
162 	TAVOR_RSRC_CLEANUP_LEVEL9,
163 	TAVOR_RSRC_CLEANUP_LEVEL10,
164 	TAVOR_RSRC_CLEANUP_LEVEL11,
165 	TAVOR_RSRC_CLEANUP_LEVEL12,
166 	TAVOR_RSRC_CLEANUP_LEVEL13,
167 	TAVOR_RSRC_CLEANUP_LEVEL14,
168 	TAVOR_RSRC_CLEANUP_LEVEL15,
169 	TAVOR_RSRC_CLEANUP_LEVEL16,
170 	TAVOR_RSRC_CLEANUP_LEVEL17,
171 	TAVOR_RSRC_CLEANUP_LEVEL18,
172 	TAVOR_RSRC_CLEANUP_LEVEL19,
173 	TAVOR_RSRC_CLEANUP_LEVEL20,
174 	TAVOR_RSRC_CLEANUP_LEVEL21,
175 	TAVOR_RSRC_CLEANUP_LEVEL22,
176 	TAVOR_RSRC_CLEANUP_LEVEL23,
177 	TAVOR_RSRC_CLEANUP_LEVEL24,
178 	TAVOR_RSRC_CLEANUP_LEVEL25,
179 	TAVOR_RSRC_CLEANUP_LEVEL26,
180 	TAVOR_RSRC_CLEANUP_LEVEL27,
181 	TAVOR_RSRC_CLEANUP_LEVEL28,
182 	TAVOR_RSRC_CLEANUP_LEVEL29,
183 	TAVOR_RSRC_CLEANUP_LEVEL30,
184 	/* No more cleanup steps below this point! */
185 	TAVOR_RSRC_CLEANUP_ALL
186 } tavor_rsrc_cleanup_level_t;
187 
188 /*
189  * The tavor_rsrc_mbox_info_t structure is used when initializing the two
190  * Tavor mailbox types ("In" and "Out").  This structure contains the
191  * requested number and size of the mailboxes, the resource pool from which
192  * the other relevant properties will come, and the name of the resource
193  */
194 typedef struct tavor_rsrc_mbox_info_s {
195 	uint64_t		mbi_num;
196 	uint64_t		mbi_size;
197 	tavor_rsrc_pool_info_t 	*mbi_rsrcpool;
198 	char			*mbi_rsrcname;
199 } tavor_rsrc_mbox_info_t;
200 
201 /*
202  * The tavor_rsrc_hw_entry_info_t structure is used when initializing the
203  * Tavor HW entry types.  This structure contains the requested number of
204  * entries for the resource.  That value is compared against the maximum
205  * number (usually determined as a result of the Tavor QUERY_DEV_LIM command).
206  * In addition is contains a number of requested entries to be "pre-allocated"
207  * (this is generally because the Tavor hardware requires a certain number
208  * for its own purposes).  Lastly the resource pool and resource name
209  * information.
210  */
211 typedef struct tavor_rsrc_hw_entry_info_s {
212 	uint64_t		hwi_num;
213 	uint64_t		hwi_max;
214 	uint64_t		hwi_prealloc;
215 	tavor_rsrc_pool_info_t 	*hwi_rsrcpool;
216 	char			*hwi_rsrcname;
217 } tavor_rsrc_hw_entry_info_t;
218 
219 /*
220  * The tavor_rsrc_sw_hdl_info_t structure is used when initializing the
221  * Tavor software handle types.  This structure also contains the requested
222  * number of handles for the resource.  That value is compared against a
223  * maximum number passed in.  Because many of the software handle resource
224  * types are managed through the use of kmem_cache, fields are provided for
225  * specifying cache constructor and destructor methods.  Just like above,
226  * there is space for resource pool and resource name information.  And,
227  * somewhat like above, there is space to provide information (size, type,
228  * pointer to table, etc). about any "pre-allocated" resources that need to
229  * be set aside.
230  * Note specifically that the "swi_flags" field may contain any of the flags
231  * #define'd below.  The TAVOR_SWHDL_KMEMCACHE_INIT flag indicates that the
232  * given resource should have a kmem_cache setup for it, and the
233  * TAVOR_SWHDL_TABLE_INIT flag indicates that some preallocation (as defined
234  * by the "swi_num" and "swi_prealloc_sz" fields) should be done, with the
235  * resulting table pointer passed back in "swi_table_ptr".
236  */
237 typedef struct tavor_rsrc_sw_hdl_info_s {
238 	uint64_t		swi_num;
239 	uint64_t		swi_max;
240 	uint64_t		swi_prealloc_sz;
241 	tavor_rsrc_pool_info_t 	*swi_rsrcpool;
242 	int (*swi_constructor)(void *, void *, int);
243 	void (*swi_destructor)(void *, void *);
244 	char			*swi_rsrcname;
245 	uint_t			swi_flags;
246 	void			*swi_table_ptr;
247 } tavor_rsrc_sw_hdl_info_t;
248 #define	TAVOR_SWHDL_NOFLAGS		0
249 #define	TAVOR_SWHDL_KMEMCACHE_INIT	(1 << 0)
250 #define	TAVOR_SWHDL_TABLE_INIT		(1 << 1)
251 
252 
253 /*
254  * The following structure is used to specify (at init time) and to track
255  * (during allocation and freeing) all the useful information regarding a
256  * particular resource type.  An array of these resources (indexed by
257  * resource type) is allocated at driver startup time.  It is available
258  * through the driver's soft state structure.
259  * Each resource has an indication of its type and its location.  Resources
260  * may be located in one of three possible places - in Tavor DDR memory,
261  * in system memory, or in Tavor UAR memory.
262  * Each resource pool also has properties associated with it and the object
263  * that make up the pool.  These include the pool's size, the size of the
264  * individual objects (rsrc_quantum), any alignment restrictions placed on
265  * the pool of objects, and the shift size (log2) of each object.
266  * In addition (depending on object type) the "rsrc_ddr_offset" field may
267  * indicate where in DDR memory a given resource pool is located (e.g. a
268  * QP context table).  It may have a pointer to a vmem_arena for that table
269  * and/or it may point to some other private information (rsrc_private)
270  * specific to the given object type.
271  * Always, though, the resource pool pointer provides a pointer back to the
272  * soft state structure of the Tavor driver instance with which it is
273  * associated.
274  */
275 struct tavor_rsrc_pool_info_s {
276 	tavor_rsrc_type_t	rsrc_type;
277 	uint_t			rsrc_loc;
278 	uint64_t		rsrc_pool_size;
279 	uint64_t		rsrc_align;
280 	uint_t			rsrc_shift;
281 	uint_t			rsrc_quantum;
282 	void			*rsrc_start;
283 	void			*rsrc_ddr_offset;
284 	vmem_t			*rsrc_vmp;
285 	tavor_state_t		*rsrc_state;
286 	void			*rsrc_private;
287 };
288 #define	TAVOR_IN_DDR			0x0
289 #define	TAVOR_IN_SYSMEM			0x1
290 #define	TAVOR_IN_UAR			0x2
291 
292 /*
293  * The tavor_rsrc_priv_mbox_t structure is used to pass along additional
294  * information about the mailbox types.  Specifically, by containing the
295  * DMA attributes, access handle, dev access handle, etc., it provides enough
296  * information that each mailbox can be later by bound/unbound/etc. for
297  * DMA access by the hardware.  Note: we can also specify (using the
298  * "pmb_xfer_mode" field), whether a given mailbox type should be bound for
299  * DDI_DMA_STREAMING or DDI_DMA_CONSISTENT operations.
300  */
301 typedef struct tavor_rsrc_priv_mbox_s {
302 	dev_info_t		*pmb_dip;
303 	ddi_dma_attr_t		pmb_dmaattr;
304 	ddi_acc_handle_t	pmb_acchdl;
305 	ddi_device_acc_attr_t	pmb_devaccattr;
306 	uint_t			pmb_xfer_mode;
307 } tavor_rsrc_priv_mbox_t;
308 
309 /*
310  * The tavor_rsrc_t structure is the structure returned by the Tavor resource
311  * allocation routines.  It contains all the necessary information about the
312  * allocated object.  Specifically, it provides an address where the object
313  * can be accessed.  It also provides the length and index (specifically, for
314  * those resources that are accessed from tables).  In addition it can provide
315  * an access handles and DMA handle to be used when accessing or setting DMA
316  * to a specific object.  Note: not all of this information is valid for all
317  * object types.  See the consumers of each object for more explanation of
318  * which fields are used (and for what purpose).
319  */
320 struct tavor_rsrc_s {
321 	tavor_rsrc_type_t	rsrc_type;
322 	void			*tr_addr;
323 	uint32_t		tr_len;
324 	uint32_t		tr_indx;
325 	ddi_acc_handle_t	tr_acchdl;
326 	ddi_dma_handle_t	tr_dmahdl;
327 };
328 
329 
330 /*
331  * The following are the Tavor Resource Management routines that accessible
332  * externally (i.e. throughout the rest of the Tavor driver software).
333  * These include the alloc/free routines, the initialization routines, which
334  * are broken into two phases (see tavor_rsrc.c for further explanation),
335  * and the Tavor resource cleanup routines (which are used at driver detach()
336  * time.
337  */
338 int tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc,
339     uint_t num, uint_t sleepflag, tavor_rsrc_t **hdl);
340 void tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl);
341 int tavor_rsrc_init_phase1(tavor_state_t *state);
342 int tavor_rsrc_init_phase2(tavor_state_t *state);
343 void tavor_rsrc_fini(tavor_state_t *state,
344     tavor_rsrc_cleanup_level_t clean);
345 
346 
347 #ifdef __cplusplus
348 }
349 #endif
350 
351 #endif	/* _SYS_IB_ADAPTERS_TAVOR_RSRC_H */
352