xref: /titanic_51/usr/src/uts/i86pc/sys/rootnex.h (revision 6bbe05905a1c10a2703f95fb4912eb14b87f6670)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef	_SYS_ROOTNEX_H
27 #define	_SYS_ROOTNEX_H
28 
29 /*
30  * x86 root nexus implementation specific state
31  */
32 
33 #include <sys/types.h>
34 #include <sys/conf.h>
35 #include <sys/modctl.h>
36 #include <sys/sunddi.h>
37 #include <sys/iommulib.h>
38 #include <sys/sdt.h>
39 
40 #ifdef	__cplusplus
41 extern "C" {
42 #endif
43 
44 
45 /* size of buffer used for ctlop reportdev */
46 #define	REPORTDEV_BUFSIZE	1024
47 
48 /* min and max interrupt vectors */
49 #define	VEC_MIN			1
50 #define	VEC_MAX			255
51 
52 /* atomic increment/decrement to keep track of outstanding binds, etc */
53 #ifdef DEBUG
54 #define	ROOTNEX_DPROF_INC(addr)		atomic_inc_64(addr)
55 #define	ROOTNEX_DPROF_DEC(addr)		atomic_add_64(addr, -1)
56 #define	ROOTNEX_DPROBE1(name, type1, arg1) \
57 	DTRACE_PROBE1(name, type1, arg1)
58 #define	ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \
59 	DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3)
60 #else
61 #define	ROOTNEX_DPROF_INC(addr)
62 #define	ROOTNEX_DPROF_DEC(addr)
63 #define	ROOTNEX_DPROBE1(name, type1, arg1)
64 #define	ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3)
65 #endif
66 #define	ROOTNEX_PROF_INC(addr)		atomic_inc_64(addr)
67 #define	ROOTNEX_PROF_DEC(addr)		atomic_add_64(addr, -1)
68 
69 /* set in dmac_type to signify that this cookie uses the copy buffer */
70 #define	ROOTNEX_USES_COPYBUF		0x80000000
71 
72 /*
73  * integer or boolean property name and value. A few static rootnex properties
74  * are created during rootnex attach from an array of rootnex_intprop_t..
75  */
76 typedef struct rootnex_intprop_s {
77 	char	*prop_name;
78 	int	prop_value;
79 } rootnex_intprop_t;
80 
81 /*
82  * sgl related information which is visible to rootnex_get_sgl(). Trying to
83  * isolate get_sgl() as much as possible so it can be easily replaced.
84  */
85 typedef struct rootnex_sglinfo_s {
86 	/*
87 	 * These are passed into rootnex_get_sgl().
88 	 *
89 	 * si_min_addr - the minimum physical address
90 	 * si_max_addr - the maximum physical address
91 	 * si_max_cookie_size - the maximum size of a physically contiguous
92 	 *    piece of memory that we can handle in a sgl.
93 	 * si_segmask - segment mask to determine if we cross a segment boundary
94 	 * si_max_pages - max number of pages this sgl could occupy (which
95 	 *    is also the maximum number of cookies we might see.
96 	 */
97 	uint64_t	si_min_addr;
98 	uint64_t	si_max_addr;
99 	uint64_t	si_max_cookie_size;
100 	uint64_t	si_segmask;
101 	uint_t		si_max_pages;
102 
103 	/*
104 	 * these are returned by rootnex_get_sgl()
105 	 *
106 	 * si_copybuf_req - amount of copy buffer needed by the buffer.
107 	 * si_buf_offset - The initial offset into the first page of the buffer.
108 	 *    It's set in get sgl and used in the bind slow path to help
109 	 *    calculate the current page index & offset from the current offset
110 	 *    which is relative to the start of the buffer.
111 	 * si_asp - address space of buffer passed in.
112 	 * si_sgl_size - The actual number of cookies in the sgl. This does
113 	 *    not reflect and sharing that we might do on window boundaries.
114 	 */
115 	size_t		si_copybuf_req;
116 	off_t		si_buf_offset;
117 	struct as	*si_asp;
118 	uint_t		si_sgl_size;
119 } rootnex_sglinfo_t;
120 
121 /*
122  * When we have to use the copy buffer, we allocate one of these structures per
123  * buffer page to track which pages need the copy buffer, what the kernel
124  * virtual address is (which the device can't reach), and what the copy buffer
125  * virtual address is (where the device dma's to/from). For 32-bit kernels,
126  * since we can't use seg kpm, we also need to keep the page_t around and state
127  * if we've currently mapped in the page into KVA space for buffers which don't
128  * have kva already and when we have multiple windows because we used up all our
129  * copy buffer space.
130  */
131 typedef struct rootnex_pgmap_s {
132 	boolean_t	pm_uses_copybuf;
133 #if !defined(__amd64)
134 	boolean_t	pm_mapped;
135 	page_t		*pm_pp;
136 	caddr_t		pm_vaddr;
137 #endif
138 	caddr_t		pm_kaddr;
139 	caddr_t		pm_cbaddr;
140 } rootnex_pgmap_t;
141 
142 /*
143  * We only need to trim a buffer when we have multiple windows. Each window has
144  * trim state. We might have trimmed the end of the previous window, leaving the
145  * first cookie of this window trimmed[tr_trim_first] (which basically means we
146  * won't start with a new cookie), or we might need to trim the end of the
147  * current window [tr_trim_last] (which basically means we won't end with a
148  * complete cookie). We keep the same state for the first & last cookie in a
149  * window (a window can have one or more cookies). However, when we trim the
150  * last cookie, we keep a pointer to the last cookie in the trim state since we
151  * only need this info when we trim. The pointer to the first cookie in the
152  * window is in the window state since we need to know what the first cookie in
153  * the window is in various places.
154  *
155  * If we do trim a cookie, we save away the physical address and size of the
156  * cookie so that we can over write the cookie when we switch windows (the
157  * space for a cookie which is in two windows is shared between the windows.
158  * We keep around the same information for the last page in a window.
159  *
160  * if we happened to trim on a page that uses the copy buffer, and that page
161  * is also in the middle of a window boundary because we have filled up the
162  * copy buffer, we need to remember the copy buffer address for both windows
163  * since the same page will have different copy buffer addresses in the two
164  * windows. We need to due the same for kaddr in the 32-bit kernel since we
165  * have a limited kva space which we map to.
166  */
167 typedef struct rootnex_trim_s {
168 	boolean_t		tr_trim_first;
169 	boolean_t		tr_trim_last;
170 	ddi_dma_cookie_t	*tr_last_cookie;
171 	uint64_t		tr_first_paddr;
172 	uint64_t		tr_last_paddr;
173 	size_t			tr_first_size;
174 	size_t			tr_last_size;
175 
176 	boolean_t		tr_first_copybuf_win;
177 	boolean_t		tr_last_copybuf_win;
178 	uint_t			tr_first_pidx;
179 	uint_t			tr_last_pidx;
180 	caddr_t			tr_first_cbaddr;
181 	caddr_t			tr_last_cbaddr;
182 #if !defined(__amd64)
183 	caddr_t			tr_first_kaddr;
184 	caddr_t			tr_last_kaddr;
185 #endif
186 } rootnex_trim_t;
187 
188 /*
189  * per window state. A bound DMA handle can have multiple windows. Each window
190  * will have the following state. We track if this window needs to sync,
191  * the offset into the buffer where the window starts, the size of the window.
192  * a pointer to the first cookie in the window, the number of cookies in the
193  * window, and the trim state for the window. For the 32-bit kernel, we keep
194  * track of if we need to remap the copy buffer when we switch to a this window
195  */
196 typedef struct rootnex_window_s {
197 	boolean_t		wd_dosync;
198 	uint_t			wd_cookie_cnt;
199 	off_t			wd_offset;
200 	size_t			wd_size;
201 	ddi_dma_cookie_t	*wd_first_cookie;
202 	rootnex_trim_t		wd_trim;
203 #if !defined(__amd64)
204 	boolean_t		wd_remap_copybuf;
205 #endif
206 } rootnex_window_t;
207 
208 /* per dma handle private state */
209 typedef struct rootnex_dma_s {
210 	/*
211 	 * sgl related state used to build and describe the sgl.
212 	 *
213 	 * dp_partial_required - used in the bind slow path to identify if we
214 	 *    need to do a partial mapping or not.
215 	 * dp_trim_required - used in the bind slow path to identify if we
216 	 *    need to trim when switching to a new window. This should only be
217 	 *    set when partial is set.
218 	 * dp_granularity_power_2 - set in alloc handle and used in bind slow
219 	 *    path to determine if we & or % to calculate the trim.
220 	 * dp_dma - copy of dma "object" passed in during bind
221 	 * dp_maxxfer - trimmed dma_attr_maxxfer so that it is a whole
222 	 *    multiple of granularity
223 	 * dp_sglinfo - See rootnex_sglinfo_t above.
224 	 */
225 	boolean_t		dp_partial_required;
226 	boolean_t		dp_trim_required;
227 	boolean_t		dp_granularity_power_2;
228 	uint64_t		dp_maxxfer;
229 	ddi_dma_obj_t		dp_dma;
230 	rootnex_sglinfo_t	dp_sglinfo;
231 
232 	/*
233 	 * Copy buffer related state
234 	 *
235 	 * dp_copybuf_size - the actual size of the copy buffer that we are
236 	 *    using. This can be smaller that dp_copybuf_req, i.e. bind size >
237 	 *    max copy buffer size.
238 	 * dp_cbaddr - kernel address of copy buffer. Used to determine where
239 	 *    where to copy to/from.
240 	 * dp_cbsize - the "real" size returned from the copy buffer alloc.
241 	 *    Set in the copybuf alloc and used to free copybuf.
242 	 * dp_pgmap - page map used in sync to determine which pages in the
243 	 *    buffer use the copy buffer and what addresses to use to copy to/
244 	 *    from.
245 	 * dp_cb_remaping - status if this bind causes us to have to remap
246 	 *    the copybuf when switching to new windows. This is only used in
247 	 *    the 32-bit kernel since we use seg kpm in the 64-bit kernel for
248 	 *    this case.
249 	 * dp_kva - kernel heap arena vmem space for mapping to buffers which
250 	 *    we don't have a kernel VA to bcopy to/from. This is only used in
251 	 *    the 32-bit kernel since we use seg kpm in the 64-bit kernel for
252 	 *    this case.
253 	 */
254 	size_t			dp_copybuf_size;
255 	caddr_t			dp_cbaddr;
256 	size_t			dp_cbsize;
257 	rootnex_pgmap_t		*dp_pgmap;
258 #if !defined(__amd64)
259 	boolean_t		dp_cb_remaping;
260 	caddr_t			dp_kva;
261 #endif
262 
263 	/*
264 	 * window related state. The pointer to the window state array which may
265 	 * be a pointer into the pre allocated state, or we may have had to
266 	 * allocate the window array on the fly because it wouldn't fit. If
267 	 * we allocate it, we'll use dp_need_to_free_window and dp_window_size
268 	 * during cleanup. dp_current_win keeps track of the current window.
269 	 * dp_max_win is the maximum number of windows we could have.
270 	 */
271 	uint_t			dp_current_win;
272 	rootnex_window_t	*dp_window;
273 	boolean_t		dp_need_to_free_window;
274 	uint_t			dp_window_size;
275 	uint_t			dp_max_win;
276 
277 	/* dip of driver which "owns" handle. set to rdip in alloc_handle() */
278 	dev_info_t		*dp_dip;
279 
280 	/*
281 	 * dp_mutex and dp_inuse are only used to see if a driver is trying to
282 	 * bind to an already bound dma handle. dp_mutex only used for dp_inuse
283 	 */
284 	kmutex_t		dp_mutex;
285 	boolean_t		dp_inuse;
286 
287 	/*
288 	 * cookie related state. The pointer to the cookies (dp_cookies) may
289 	 * be a pointer into the pre allocated state, or we may have had to
290 	 * allocate the cookie array on the fly because it wouldn't fit. If
291 	 * we allocate it, we'll use dp_need_to_free_cookie and dp_cookie_size
292 	 * during cleanup. dp_current_cookie is only used in the obsoleted
293 	 * interfaces to determine when we've used up all the cookies in a
294 	 * window during nextseg()..
295 	 */
296 	size_t			dp_cookie_size;
297 	ddi_dma_cookie_t	*dp_cookies;
298 	boolean_t		dp_need_to_free_cookie;
299 	uint_t			dp_current_cookie; /* for obsoleted I/Fs */
300 	ddi_dma_cookie_t	*dp_saved_cookies;
301 	boolean_t		dp_need_to_switch_cookies;
302 
303 	/*
304 	 * pre allocated space for the bind state, allocated during alloc
305 	 * handle. For a lot of devices, this will save us from having to do
306 	 * kmem_alloc's during the bind most of the time. kmem_alloc's can be
307 	 * expensive on x86 when the cpu count goes up since xcalls are
308 	 * expensive on x86.
309 	 */
310 	uchar_t			*dp_prealloc_buffer;
311 
312 	/*
313 	 * intel iommu related state
314 	 * dvma_cookies saves the dvma allocated for this handler, it has the
315 	 * size of si_max_pages, set when bind handler and freed when unbind
316 	 */
317 	void			*dp_dvma_cookies;
318 
319 	/*
320 	 * sleep flags set on bind and unset on unbind
321 	 */
322 	int			dp_sleep_flags;
323 } rootnex_dma_t;
324 
325 /*
326  * profile/performance counters. Most things will be dtrace probes, but there
327  * are a couple of things we want to keep track all the time. We track the
328  * total number of active handles and binds (i.e. an alloc without a free or
329  * a bind without an unbind) since rootnex attach. We also track the total
330  * number of binds which have failed since rootnex attach.
331  */
332 typedef enum {
333 	ROOTNEX_CNT_ACTIVE_HDLS = 0,
334 	ROOTNEX_CNT_ACTIVE_BINDS = 1,
335 	ROOTNEX_CNT_ALLOC_FAIL = 2,
336 	ROOTNEX_CNT_BIND_FAIL = 3,
337 	ROOTNEX_CNT_SYNC_FAIL = 4,
338 	ROOTNEX_CNT_GETWIN_FAIL = 5,
339 
340 	/* This one must be last */
341 	ROOTNEX_CNT_LAST
342 } rootnex_cnt_t;
343 
344 /*
345  * global driver state.
346  *   r_dmahdl_cache - dma_handle kmem_cache
347  *   r_dvma_call_list_id - ddi_set_callback() id
348  *   r_peekpoke_mutex - serialize peeks and pokes.
349  *   r_dip - rootnex dip
350  *   r_reserved_msg_printed - ctlops reserve message threshold
351  *   r_counters - profile/performance counters
352  *   r_intel_iommu_enabled - intel iommu enabled
353  */
354 typedef struct rootnex_state_s {
355 	uint_t			r_prealloc_cookies;
356 	uint_t			r_prealloc_size;
357 	kmem_cache_t		*r_dmahdl_cache;
358 	uintptr_t		r_dvma_call_list_id;
359 	kmutex_t		r_peekpoke_mutex;
360 	dev_info_t		*r_dip;
361 	ddi_iblock_cookie_t	r_err_ibc;
362 	boolean_t		r_reserved_msg_printed;
363 	uint64_t		r_counters[ROOTNEX_CNT_LAST];
364 	boolean_t		r_intel_iommu_enabled;
365 	iommulib_nexhandle_t    r_iommulib_handle;
366 } rootnex_state_t;
367 
368 
369 #ifdef	__cplusplus
370 }
371 #endif
372 
373 #endif	/* _SYS_ROOTNEX_H */
374