xref: /freebsd/sys/dev/sfxge/common/efsys.h (revision 0c848230f72ce4108cfc3fc6904fb55b8d66b614)
1e948693eSPhilip Paeps /*-
23c838a9fSAndrew Rybchenko  * Copyright (c) 2010-2015 Solarflare Communications Inc.
3e948693eSPhilip Paeps  * All rights reserved.
4e948693eSPhilip Paeps  *
5e948693eSPhilip Paeps  * This software was developed in part by Philip Paeps under contract for
6e948693eSPhilip Paeps  * Solarflare Communications, Inc.
7e948693eSPhilip Paeps  *
8e948693eSPhilip Paeps  * Redistribution and use in source and binary forms, with or without
93c838a9fSAndrew Rybchenko  * modification, are permitted provided that the following conditions are met:
10e948693eSPhilip Paeps  *
113c838a9fSAndrew Rybchenko  * 1. Redistributions of source code must retain the above copyright notice,
123c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer.
133c838a9fSAndrew Rybchenko  * 2. Redistributions in binary form must reproduce the above copyright notice,
143c838a9fSAndrew Rybchenko  *    this list of conditions and the following disclaimer in the documentation
153c838a9fSAndrew Rybchenko  *    and/or other materials provided with the distribution.
163c838a9fSAndrew Rybchenko  *
173c838a9fSAndrew Rybchenko  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
183c838a9fSAndrew Rybchenko  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
193c838a9fSAndrew Rybchenko  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
203c838a9fSAndrew Rybchenko  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
213c838a9fSAndrew Rybchenko  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
223c838a9fSAndrew Rybchenko  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
233c838a9fSAndrew Rybchenko  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
243c838a9fSAndrew Rybchenko  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
253c838a9fSAndrew Rybchenko  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
263c838a9fSAndrew Rybchenko  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
273c838a9fSAndrew Rybchenko  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
283c838a9fSAndrew Rybchenko  *
293c838a9fSAndrew Rybchenko  * The views and conclusions contained in the software and documentation are
303c838a9fSAndrew Rybchenko  * those of the authors and should not be interpreted as representing official
313c838a9fSAndrew Rybchenko  * policies, either expressed or implied, of the FreeBSD Project.
325dee87d7SPhilip Paeps  *
335dee87d7SPhilip Paeps  * $FreeBSD$
34e948693eSPhilip Paeps  */
35e948693eSPhilip Paeps 
36e948693eSPhilip Paeps #ifndef	_SYS_EFSYS_H
37e948693eSPhilip Paeps #define	_SYS_EFSYS_H
38e948693eSPhilip Paeps 
39e948693eSPhilip Paeps #ifdef	__cplusplus
40e948693eSPhilip Paeps extern "C" {
41e948693eSPhilip Paeps #endif
42e948693eSPhilip Paeps 
43e948693eSPhilip Paeps #include <sys/param.h>
44e948693eSPhilip Paeps #include <sys/bus.h>
45e948693eSPhilip Paeps #include <sys/endian.h>
46e948693eSPhilip Paeps #include <sys/lock.h>
47e948693eSPhilip Paeps #include <sys/malloc.h>
48e948693eSPhilip Paeps #include <sys/mbuf.h>
49e948693eSPhilip Paeps #include <sys/mutex.h>
50e948693eSPhilip Paeps #include <sys/rwlock.h>
51e948693eSPhilip Paeps #include <sys/sdt.h>
52e948693eSPhilip Paeps #include <sys/systm.h>
53e948693eSPhilip Paeps 
54e948693eSPhilip Paeps #include <machine/bus.h>
55e948693eSPhilip Paeps #include <machine/endian.h>
56e948693eSPhilip Paeps 
57e948693eSPhilip Paeps #define	EFSYS_HAS_UINT64 1
580f999687SAndrew Rybchenko #if defined(__x86_64__)
590f999687SAndrew Rybchenko #define	EFSYS_USE_UINT64 1
600f999687SAndrew Rybchenko #else
61e948693eSPhilip Paeps #define	EFSYS_USE_UINT64 0
620f999687SAndrew Rybchenko #endif
633c838a9fSAndrew Rybchenko #define	EFSYS_HAS_SSE2_M128 0
64e948693eSPhilip Paeps #if _BYTE_ORDER == _BIG_ENDIAN
65e948693eSPhilip Paeps #define	EFSYS_IS_BIG_ENDIAN 1
66e948693eSPhilip Paeps #define	EFSYS_IS_LITTLE_ENDIAN 0
67e948693eSPhilip Paeps #elif _BYTE_ORDER == _LITTLE_ENDIAN
68e948693eSPhilip Paeps #define	EFSYS_IS_BIG_ENDIAN 0
69e948693eSPhilip Paeps #define	EFSYS_IS_LITTLE_ENDIAN 1
70e948693eSPhilip Paeps #endif
71e948693eSPhilip Paeps #include "efx_types.h"
72e948693eSPhilip Paeps 
73e948693eSPhilip Paeps /* Common code requires this */
74e948693eSPhilip Paeps #if __FreeBSD_version < 800068
75e948693eSPhilip Paeps #define	memmove(d, s, l) bcopy(s, d, l)
76e948693eSPhilip Paeps #endif
77e948693eSPhilip Paeps 
78e948693eSPhilip Paeps /* FreeBSD equivalents of Solaris things */
79e948693eSPhilip Paeps #ifndef _NOTE
80e948693eSPhilip Paeps #define	_NOTE(s)
81e948693eSPhilip Paeps #endif
82e948693eSPhilip Paeps 
83e948693eSPhilip Paeps #ifndef B_FALSE
84e948693eSPhilip Paeps #define	B_FALSE	FALSE
85e948693eSPhilip Paeps #endif
86e948693eSPhilip Paeps #ifndef B_TRUE
87e948693eSPhilip Paeps #define	B_TRUE	TRUE
88e948693eSPhilip Paeps #endif
89e948693eSPhilip Paeps 
90e948693eSPhilip Paeps #ifndef IS_P2ALIGNED
91e948693eSPhilip Paeps #define	IS_P2ALIGNED(v, a)	((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
92e948693eSPhilip Paeps #endif
93e948693eSPhilip Paeps 
94e948693eSPhilip Paeps #ifndef P2ROUNDUP
95e948693eSPhilip Paeps #define	P2ROUNDUP(x, align)	(-(-(x) & -(align)))
96e948693eSPhilip Paeps #endif
97e948693eSPhilip Paeps 
983c838a9fSAndrew Rybchenko #ifndef P2ALIGN
993c838a9fSAndrew Rybchenko #define	P2ALIGN(_x, _a)		((_x) & -(_a))
1003c838a9fSAndrew Rybchenko #endif
1013c838a9fSAndrew Rybchenko 
102e948693eSPhilip Paeps #ifndef IS2P
103e948693eSPhilip Paeps #define	ISP2(x)			(((x) & ((x) - 1)) == 0)
104e948693eSPhilip Paeps #endif
105e948693eSPhilip Paeps 
1065dadbb64SAndrew Rybchenko #if defined(__x86_64__) && __FreeBSD_version >= 1000000
1075dadbb64SAndrew Rybchenko 
1085dadbb64SAndrew Rybchenko #define	SFXGE_USE_BUS_SPACE_8		1
1095dadbb64SAndrew Rybchenko 
110d9ff48abSAndrew Rybchenko #if !defined(bus_space_read_stream_8)
1115dadbb64SAndrew Rybchenko 
112d9ff48abSAndrew Rybchenko #define	bus_space_read_stream_8(t, h, o)				\
113d9ff48abSAndrew Rybchenko 	bus_space_read_8((t), (h), (o))
1145dadbb64SAndrew Rybchenko 
115d9ff48abSAndrew Rybchenko #define	bus_space_write_stream_8(t, h, o, v)				\
116d9ff48abSAndrew Rybchenko 	bus_space_write_8((t), (h), (o), (v))
1175dadbb64SAndrew Rybchenko 
118d9ff48abSAndrew Rybchenko #endif
1195dadbb64SAndrew Rybchenko 
120d9ff48abSAndrew Rybchenko #endif
121d9ff48abSAndrew Rybchenko 
122e948693eSPhilip Paeps #define	ENOTACTIVE EINVAL
123e948693eSPhilip Paeps 
124e948693eSPhilip Paeps /* Memory type to use on FreeBSD */
125e948693eSPhilip Paeps MALLOC_DECLARE(M_SFXGE);
126e948693eSPhilip Paeps 
127e948693eSPhilip Paeps /* Machine dependend prefetch wrappers */
128dc1788e8SMarius Strobl #if defined(__i386__) || defined(__amd64__)
129e948693eSPhilip Paeps static __inline void
130e948693eSPhilip Paeps prefetch_read_many(void *addr)
131e948693eSPhilip Paeps {
132dc1788e8SMarius Strobl 
133e948693eSPhilip Paeps 	__asm__(
134e948693eSPhilip Paeps 	    "prefetcht0 (%0)"
135e948693eSPhilip Paeps 	    :
136e948693eSPhilip Paeps 	    : "r" (addr));
137e948693eSPhilip Paeps }
138e948693eSPhilip Paeps 
139e948693eSPhilip Paeps static __inline void
140e948693eSPhilip Paeps prefetch_read_once(void *addr)
141e948693eSPhilip Paeps {
142dc1788e8SMarius Strobl 
143e948693eSPhilip Paeps 	__asm__(
144e948693eSPhilip Paeps 	    "prefetchnta (%0)"
145e948693eSPhilip Paeps 	    :
146e948693eSPhilip Paeps 	    : "r" (addr));
147e948693eSPhilip Paeps }
148dc1788e8SMarius Strobl #elif defined(__sparc64__)
149dc1788e8SMarius Strobl static __inline void
150dc1788e8SMarius Strobl prefetch_read_many(void *addr)
151dc1788e8SMarius Strobl {
152dc1788e8SMarius Strobl 
153dc1788e8SMarius Strobl 	__asm__(
154dc1788e8SMarius Strobl 	    "prefetch [%0], 0"
155dc1788e8SMarius Strobl 	    :
156dc1788e8SMarius Strobl 	    : "r" (addr));
157dc1788e8SMarius Strobl }
158dc1788e8SMarius Strobl 
159dc1788e8SMarius Strobl static __inline void
160dc1788e8SMarius Strobl prefetch_read_once(void *addr)
161dc1788e8SMarius Strobl {
162dc1788e8SMarius Strobl 
163dc1788e8SMarius Strobl 	__asm__(
164dc1788e8SMarius Strobl 	    "prefetch [%0], 1"
165dc1788e8SMarius Strobl 	    :
166dc1788e8SMarius Strobl 	    : "r" (addr));
167dc1788e8SMarius Strobl }
168dc1788e8SMarius Strobl #else
169dc1788e8SMarius Strobl static __inline void
170dc1788e8SMarius Strobl prefetch_read_many(void *addr)
171dc1788e8SMarius Strobl {
172dc1788e8SMarius Strobl 
173dc1788e8SMarius Strobl }
174dc1788e8SMarius Strobl 
175dc1788e8SMarius Strobl static __inline void
176dc1788e8SMarius Strobl prefetch_read_once(void *addr)
177dc1788e8SMarius Strobl {
178dc1788e8SMarius Strobl 
179dc1788e8SMarius Strobl }
180e948693eSPhilip Paeps #endif
181e948693eSPhilip Paeps 
182e948693eSPhilip Paeps #if defined(__i386__) || defined(__amd64__)
183e948693eSPhilip Paeps #include <vm/vm.h>
184e948693eSPhilip Paeps #include <vm/pmap.h>
185e948693eSPhilip Paeps #endif
186e948693eSPhilip Paeps static __inline void
187e948693eSPhilip Paeps sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188e948693eSPhilip Paeps 		    struct mbuf *m, bus_dma_segment_t *seg)
189e948693eSPhilip Paeps {
190e948693eSPhilip Paeps #if defined(__i386__) || defined(__amd64__)
191e948693eSPhilip Paeps 	seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192e948693eSPhilip Paeps 	seg->ds_len = m->m_len;
193e948693eSPhilip Paeps #else
194e948693eSPhilip Paeps 	int nsegstmp;
195e948693eSPhilip Paeps 
196e948693eSPhilip Paeps 	bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
197e948693eSPhilip Paeps #endif
198e948693eSPhilip Paeps }
199e948693eSPhilip Paeps 
200e948693eSPhilip Paeps /* Modifiers used for Windows builds */
201e948693eSPhilip Paeps #define	__in
202e948693eSPhilip Paeps #define	__in_opt
203e948693eSPhilip Paeps #define	__in_ecount(_n)
204e948693eSPhilip Paeps #define	__in_ecount_opt(_n)
205e948693eSPhilip Paeps #define	__in_bcount(_n)
206e948693eSPhilip Paeps #define	__in_bcount_opt(_n)
207e948693eSPhilip Paeps 
208e948693eSPhilip Paeps #define	__out
209e948693eSPhilip Paeps #define	__out_opt
210e948693eSPhilip Paeps #define	__out_ecount(_n)
211e948693eSPhilip Paeps #define	__out_ecount_opt(_n)
212e948693eSPhilip Paeps #define	__out_bcount(_n)
213e948693eSPhilip Paeps #define	__out_bcount_opt(_n)
214e948693eSPhilip Paeps 
215e948693eSPhilip Paeps #define	__deref_out
216e948693eSPhilip Paeps 
217e948693eSPhilip Paeps #define	__inout
218e948693eSPhilip Paeps #define	__inout_opt
219e948693eSPhilip Paeps #define	__inout_ecount(_n)
220e948693eSPhilip Paeps #define	__inout_ecount_opt(_n)
221e948693eSPhilip Paeps #define	__inout_bcount(_n)
222e948693eSPhilip Paeps #define	__inout_bcount_opt(_n)
223e948693eSPhilip Paeps #define	__inout_bcount_full_opt(_n)
224e948693eSPhilip Paeps 
225e948693eSPhilip Paeps #define	__deref_out_bcount_opt(n)
226e948693eSPhilip Paeps 
227e948693eSPhilip Paeps #define	__checkReturn
228460cb568SAndrew Rybchenko #define	__success(_x)
229e948693eSPhilip Paeps 
230e948693eSPhilip Paeps #define	__drv_when(_p, _c)
231e948693eSPhilip Paeps 
232e948693eSPhilip Paeps /* Code inclusion options */
233e948693eSPhilip Paeps 
234e948693eSPhilip Paeps 
235e948693eSPhilip Paeps #define	EFSYS_OPT_NAMES 1
236e948693eSPhilip Paeps 
237e948693eSPhilip Paeps #define	EFSYS_OPT_FALCON 0
238e948693eSPhilip Paeps #define	EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
239e948693eSPhilip Paeps #define	EFSYS_OPT_SIENA 1
2403c838a9fSAndrew Rybchenko #define	EFSYS_OPT_HUNTINGTON 1
241b53f4a64SAndrew Rybchenko #define	EFSYS_OPT_MEDFORD 0
242e948693eSPhilip Paeps #ifdef DEBUG
243e948693eSPhilip Paeps #define	EFSYS_OPT_CHECK_REG 1
244e948693eSPhilip Paeps #else
245e948693eSPhilip Paeps #define	EFSYS_OPT_CHECK_REG 0
246e948693eSPhilip Paeps #endif
247e948693eSPhilip Paeps 
248e948693eSPhilip Paeps #define	EFSYS_OPT_MCDI 1
249870b1186SAndrew Rybchenko #define	EFSYS_OPT_MCDI_LOGGING 0
2503edad197SAndrew Rybchenko #define	EFSYS_OPT_MCDI_PROXY_AUTH 0
251e948693eSPhilip Paeps 
252e948693eSPhilip Paeps #define	EFSYS_OPT_MAC_FALCON_GMAC 0
253e948693eSPhilip Paeps #define	EFSYS_OPT_MAC_FALCON_XMAC 0
254e948693eSPhilip Paeps #define	EFSYS_OPT_MAC_STATS 1
255e948693eSPhilip Paeps 
256e948693eSPhilip Paeps #define	EFSYS_OPT_LOOPBACK 0
257e948693eSPhilip Paeps 
258e948693eSPhilip Paeps #define	EFSYS_OPT_MON_NULL 0
259e948693eSPhilip Paeps #define	EFSYS_OPT_MON_LM87 0
260e948693eSPhilip Paeps #define	EFSYS_OPT_MON_MAX6647 0
2613c838a9fSAndrew Rybchenko #define	EFSYS_OPT_MON_MCDI 0
262e948693eSPhilip Paeps #define	EFSYS_OPT_MON_STATS 0
263e948693eSPhilip Paeps 
264e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_NULL 0
265e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_QT2022C2 0
266e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_SFX7101 0
267e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_TXC43128 0
268e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_SFT9001 0
269e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_QT2025C 0
270e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_STATS 1
271e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_PROPS 0
2723c838a9fSAndrew Rybchenko #define	EFSYS_OPT_PHY_BIST 0
2733c838a9fSAndrew Rybchenko #define	EFSYS_OPT_BIST 1
274e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_LED_CONTROL 1
275e948693eSPhilip Paeps #define	EFSYS_OPT_PHY_FLAGS 0
276e948693eSPhilip Paeps 
277e948693eSPhilip Paeps #define	EFSYS_OPT_VPD 1
278e948693eSPhilip Paeps #define	EFSYS_OPT_NVRAM 1
279e948693eSPhilip Paeps #define	EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
280e948693eSPhilip Paeps #define	EFSYS_OPT_NVRAM_SFT9001	0
281e948693eSPhilip Paeps #define	EFSYS_OPT_NVRAM_SFX7101	0
282e948693eSPhilip Paeps #define	EFSYS_OPT_BOOTCFG 0
283e948693eSPhilip Paeps 
284e948693eSPhilip Paeps #define	EFSYS_OPT_PCIE_TUNE 0
285e948693eSPhilip Paeps #define	EFSYS_OPT_DIAG 0
286e948693eSPhilip Paeps #define	EFSYS_OPT_WOL 1
287e948693eSPhilip Paeps #define	EFSYS_OPT_RX_SCALE 1
288e948693eSPhilip Paeps #define	EFSYS_OPT_QSTATS 1
2893c838a9fSAndrew Rybchenko #define	EFSYS_OPT_FILTER 1
290e948693eSPhilip Paeps #define	EFSYS_OPT_RX_SCATTER 0
291e948693eSPhilip Paeps 
292e948693eSPhilip Paeps #define	EFSYS_OPT_EV_PREFETCH 0
293e948693eSPhilip Paeps 
294e948693eSPhilip Paeps #define	EFSYS_OPT_DECODE_INTR_FATAL 1
295e948693eSPhilip Paeps 
296*0c848230SAndrew Rybchenko #define	EFSYS_OPT_LICENSING 0
297*0c848230SAndrew Rybchenko 
298e948693eSPhilip Paeps /* ID */
299e948693eSPhilip Paeps 
300e948693eSPhilip Paeps typedef struct __efsys_identifier_s	efsys_identifier_t;
301e948693eSPhilip Paeps 
302e948693eSPhilip Paeps /* PROBE */
303e948693eSPhilip Paeps 
30454366c0bSAttilio Rao #ifndef DTRACE_PROBE
305e948693eSPhilip Paeps 
306e948693eSPhilip Paeps #define	EFSYS_PROBE(_name)
307e948693eSPhilip Paeps 
308e948693eSPhilip Paeps #define	EFSYS_PROBE1(_name, _type1, _arg1)
309e948693eSPhilip Paeps 
310e948693eSPhilip Paeps #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
311e948693eSPhilip Paeps 
312e948693eSPhilip Paeps #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
313e948693eSPhilip Paeps 	    _type3, _arg3)
314e948693eSPhilip Paeps 
315e948693eSPhilip Paeps #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
316e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4)
317e948693eSPhilip Paeps 
318e948693eSPhilip Paeps #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
319e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
320e948693eSPhilip Paeps 
321e948693eSPhilip Paeps #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
322e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
323e948693eSPhilip Paeps 	    _type6, _arg6)
324e948693eSPhilip Paeps 
325e948693eSPhilip Paeps #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
326e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
327e948693eSPhilip Paeps 	    _type6, _arg6, _type7, _arg7)
328e948693eSPhilip Paeps 
32954366c0bSAttilio Rao #else /* DTRACE_PROBE */
330e948693eSPhilip Paeps 
331e948693eSPhilip Paeps #define	EFSYS_PROBE(_name)						\
332e948693eSPhilip Paeps 	DTRACE_PROBE(_name)
333e948693eSPhilip Paeps 
334e948693eSPhilip Paeps #define	EFSYS_PROBE1(_name, _type1, _arg1)				\
335e948693eSPhilip Paeps 	DTRACE_PROBE1(_name, _type1, _arg1)
336e948693eSPhilip Paeps 
337e948693eSPhilip Paeps #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)		\
338e948693eSPhilip Paeps 	DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
339e948693eSPhilip Paeps 
340e948693eSPhilip Paeps #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
341e948693eSPhilip Paeps 	    _type3, _arg3)						\
342e948693eSPhilip Paeps 	DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
343e948693eSPhilip Paeps 	    _type3, _arg3)
344e948693eSPhilip Paeps 
345e948693eSPhilip Paeps #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
346e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4)				\
347e948693eSPhilip Paeps 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
348e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4)
349e948693eSPhilip Paeps 
350e948693eSPhilip Paeps #ifdef DTRACE_PROBE5
351e948693eSPhilip Paeps #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
352e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
353e948693eSPhilip Paeps 	DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
354e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
355e948693eSPhilip Paeps #else
356e948693eSPhilip Paeps #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
357e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
358e948693eSPhilip Paeps 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
359e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4)
360e948693eSPhilip Paeps #endif
361e948693eSPhilip Paeps 
362e948693eSPhilip Paeps #ifdef DTRACE_PROBE6
363e948693eSPhilip Paeps #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
364e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
365e948693eSPhilip Paeps 	    _type6, _arg6)						\
366e948693eSPhilip Paeps 	DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
367e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
368e948693eSPhilip Paeps 	    _type6, _arg6)
369e948693eSPhilip Paeps #else
370e948693eSPhilip Paeps #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
371e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
372e948693eSPhilip Paeps 	    _type6, _arg6)						\
373e948693eSPhilip Paeps 	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
374e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
375e948693eSPhilip Paeps #endif
376e948693eSPhilip Paeps 
377e948693eSPhilip Paeps #ifdef DTRACE_PROBE7
378e948693eSPhilip Paeps #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
379e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
380e948693eSPhilip Paeps 	    _type6, _arg6, _type7, _arg7)				\
381e948693eSPhilip Paeps 	DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
382e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
383e948693eSPhilip Paeps 	    _type6, _arg6, _type7, _arg7)
384e948693eSPhilip Paeps #else
385e948693eSPhilip Paeps #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
386e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
387e948693eSPhilip Paeps 	    _type6, _arg6, _type7, _arg7)				\
388e948693eSPhilip Paeps 	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
389e948693eSPhilip Paeps 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
390e948693eSPhilip Paeps 	    _type6, _arg6)
391e948693eSPhilip Paeps #endif
392e948693eSPhilip Paeps 
39354366c0bSAttilio Rao #endif /* DTRACE_PROBE */
394e948693eSPhilip Paeps 
395e948693eSPhilip Paeps /* DMA */
396e948693eSPhilip Paeps 
397e948693eSPhilip Paeps typedef uint64_t		efsys_dma_addr_t;
398e948693eSPhilip Paeps 
399e948693eSPhilip Paeps typedef struct efsys_mem_s {
400e948693eSPhilip Paeps 	bus_dma_tag_t		esm_tag;
401e948693eSPhilip Paeps 	bus_dmamap_t		esm_map;
402e948693eSPhilip Paeps 	caddr_t			esm_base;
403e948693eSPhilip Paeps 	efsys_dma_addr_t	esm_addr;
404e948693eSPhilip Paeps } efsys_mem_t;
405e948693eSPhilip Paeps 
406e948693eSPhilip Paeps 
407e948693eSPhilip Paeps #define	EFSYS_MEM_ZERO(_esmp, _size)					\
408e948693eSPhilip Paeps 	do {								\
409e948693eSPhilip Paeps 		(void) memset((_esmp)->esm_base, 0, (_size));		\
410e948693eSPhilip Paeps 									\
411e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
412e948693eSPhilip Paeps 	} while (B_FALSE)
413e948693eSPhilip Paeps 
414e948693eSPhilip Paeps #define	EFSYS_MEM_READD(_esmp, _offset, _edp)				\
415e948693eSPhilip Paeps 	do {								\
416e948693eSPhilip Paeps 		uint32_t *addr;						\
417e948693eSPhilip Paeps 									\
418e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
419e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
420e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
421e948693eSPhilip Paeps 									\
422e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
423e948693eSPhilip Paeps 									\
424e948693eSPhilip Paeps 		(_edp)->ed_u32[0] = *addr;				\
425e948693eSPhilip Paeps 									\
426e948693eSPhilip Paeps 		EFSYS_PROBE2(mem_readd, unsigned int, (_offset),	\
427e948693eSPhilip Paeps 		    uint32_t, (_edp)->ed_u32[0]);			\
428e948693eSPhilip Paeps 									\
429e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
430e948693eSPhilip Paeps 	} while (B_FALSE)
431e948693eSPhilip Paeps 
4320f999687SAndrew Rybchenko #if defined(__x86_64__)
4330f999687SAndrew Rybchenko #define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
4340f999687SAndrew Rybchenko 	do {								\
4350f999687SAndrew Rybchenko 		uint64_t *addr;						\
4360f999687SAndrew Rybchenko 									\
4370f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
4380f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
4390f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
4400f999687SAndrew Rybchenko 									\
4410f999687SAndrew Rybchenko 		addr = (void *)((_esmp)->esm_base + (_offset));		\
4420f999687SAndrew Rybchenko 									\
4430f999687SAndrew Rybchenko 		(_eqp)->eq_u64[0] = *addr;				\
4440f999687SAndrew Rybchenko 									\
4450f999687SAndrew Rybchenko 		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
4460f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[1],			\
4470f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[0]);			\
4480f999687SAndrew Rybchenko 									\
4490f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
4500f999687SAndrew Rybchenko 	} while (B_FALSE)
4510f999687SAndrew Rybchenko #else
452e948693eSPhilip Paeps #define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
453e948693eSPhilip Paeps 	do {								\
454e948693eSPhilip Paeps 		uint32_t *addr;						\
455e948693eSPhilip Paeps 									\
456e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
457e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
458e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
459e948693eSPhilip Paeps 									\
460e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
461e948693eSPhilip Paeps 									\
462e948693eSPhilip Paeps 		(_eqp)->eq_u32[0] = *addr++;				\
463e948693eSPhilip Paeps 		(_eqp)->eq_u32[1] = *addr;				\
464e948693eSPhilip Paeps 									\
465e948693eSPhilip Paeps 		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
466e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[1],			\
467e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[0]);			\
468e948693eSPhilip Paeps 									\
469e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
470e948693eSPhilip Paeps 	} while (B_FALSE)
4710f999687SAndrew Rybchenko #endif
472e948693eSPhilip Paeps 
4730f999687SAndrew Rybchenko #if defined(__x86_64__)
4740f999687SAndrew Rybchenko #define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
4750f999687SAndrew Rybchenko 	do {								\
4760f999687SAndrew Rybchenko 		uint64_t *addr;						\
4770f999687SAndrew Rybchenko 									\
4780f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
4790f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
4800f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
4810f999687SAndrew Rybchenko 									\
4820f999687SAndrew Rybchenko 		addr = (void *)((_esmp)->esm_base + (_offset));		\
4830f999687SAndrew Rybchenko 									\
4840f999687SAndrew Rybchenko 		(_eop)->eo_u64[0] = *addr++;				\
4850f999687SAndrew Rybchenko 		(_eop)->eo_u64[1] = *addr;				\
4860f999687SAndrew Rybchenko 									\
4870f999687SAndrew Rybchenko 		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
4880f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[3],			\
4890f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[2],			\
4900f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[1],			\
4910f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[0]);			\
4920f999687SAndrew Rybchenko 									\
4930f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
4940f999687SAndrew Rybchenko 	} while (B_FALSE)
4950f999687SAndrew Rybchenko #else
496e948693eSPhilip Paeps #define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
497e948693eSPhilip Paeps 	do {								\
498e948693eSPhilip Paeps 		uint32_t *addr;						\
499e948693eSPhilip Paeps 									\
500e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
501e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
502e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
503e948693eSPhilip Paeps 									\
504e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
505e948693eSPhilip Paeps 									\
506e948693eSPhilip Paeps 		(_eop)->eo_u32[0] = *addr++;				\
507e948693eSPhilip Paeps 		(_eop)->eo_u32[1] = *addr++;				\
508e948693eSPhilip Paeps 		(_eop)->eo_u32[2] = *addr++;				\
509e948693eSPhilip Paeps 		(_eop)->eo_u32[3] = *addr;				\
510e948693eSPhilip Paeps 									\
511e948693eSPhilip Paeps 		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
512e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[3],			\
513e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[2],			\
514e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[1],			\
515e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[0]);			\
516e948693eSPhilip Paeps 									\
517e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
518e948693eSPhilip Paeps 	} while (B_FALSE)
5190f999687SAndrew Rybchenko #endif
520e948693eSPhilip Paeps 
521e948693eSPhilip Paeps #define	EFSYS_MEM_WRITED(_esmp, _offset, _edp)				\
522e948693eSPhilip Paeps 	do {								\
523e948693eSPhilip Paeps 		uint32_t *addr;						\
524e948693eSPhilip Paeps 									\
525e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
526e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
527e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
528e948693eSPhilip Paeps 									\
529e948693eSPhilip Paeps 		EFSYS_PROBE2(mem_writed, unsigned int, (_offset),	\
530e948693eSPhilip Paeps 		    uint32_t, (_edp)->ed_u32[0]);			\
531e948693eSPhilip Paeps 									\
532e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
533e948693eSPhilip Paeps 									\
534e948693eSPhilip Paeps 		*addr = (_edp)->ed_u32[0];				\
535e948693eSPhilip Paeps 									\
536e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
537e948693eSPhilip Paeps 	} while (B_FALSE)
538e948693eSPhilip Paeps 
5390f999687SAndrew Rybchenko #if defined(__x86_64__)
5400f999687SAndrew Rybchenko #define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
5410f999687SAndrew Rybchenko 	do {								\
5420f999687SAndrew Rybchenko 		uint64_t *addr;						\
5430f999687SAndrew Rybchenko 									\
5440f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
5450f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
5460f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
5470f999687SAndrew Rybchenko 									\
5480f999687SAndrew Rybchenko 		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
5490f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[1],			\
5500f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[0]);			\
5510f999687SAndrew Rybchenko 									\
5520f999687SAndrew Rybchenko 		addr = (void *)((_esmp)->esm_base + (_offset));		\
5530f999687SAndrew Rybchenko 									\
5540f999687SAndrew Rybchenko 		*addr   = (_eqp)->eq_u64[0];				\
5550f999687SAndrew Rybchenko 									\
5560f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
5570f999687SAndrew Rybchenko 	} while (B_FALSE)
5580f999687SAndrew Rybchenko 
5590f999687SAndrew Rybchenko #else
560e948693eSPhilip Paeps #define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
561e948693eSPhilip Paeps 	do {								\
562e948693eSPhilip Paeps 		uint32_t *addr;						\
563e948693eSPhilip Paeps 									\
564e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
565e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
566e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
567e948693eSPhilip Paeps 									\
568e948693eSPhilip Paeps 		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
569e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[1],			\
570e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[0]);			\
571e948693eSPhilip Paeps 									\
572e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
573e948693eSPhilip Paeps 									\
574e948693eSPhilip Paeps 		*addr++ = (_eqp)->eq_u32[0];				\
575e948693eSPhilip Paeps 		*addr   = (_eqp)->eq_u32[1];				\
576e948693eSPhilip Paeps 									\
577e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
578e948693eSPhilip Paeps 	} while (B_FALSE)
5790f999687SAndrew Rybchenko #endif
580e948693eSPhilip Paeps 
5810f999687SAndrew Rybchenko #if defined(__x86_64__)
5820f999687SAndrew Rybchenko #define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
5830f999687SAndrew Rybchenko 	do {								\
5840f999687SAndrew Rybchenko 		uint64_t *addr;						\
5850f999687SAndrew Rybchenko 									\
5860f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
5870f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
5880f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
5890f999687SAndrew Rybchenko 									\
5900f999687SAndrew Rybchenko 		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
5910f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[3],			\
5920f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[2],			\
5930f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[1],			\
5940f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[0]);			\
5950f999687SAndrew Rybchenko 									\
5960f999687SAndrew Rybchenko 		addr = (void *)((_esmp)->esm_base + (_offset));		\
5970f999687SAndrew Rybchenko 									\
5980f999687SAndrew Rybchenko 		*addr++ = (_eop)->eo_u64[0];				\
5990f999687SAndrew Rybchenko 		*addr   = (_eop)->eo_u64[1];				\
6000f999687SAndrew Rybchenko 									\
6010f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
6020f999687SAndrew Rybchenko 	} while (B_FALSE)
6030f999687SAndrew Rybchenko #else
604e948693eSPhilip Paeps #define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
605e948693eSPhilip Paeps 	do {								\
606e948693eSPhilip Paeps 		uint32_t *addr;						\
607e948693eSPhilip Paeps 									\
608e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
609e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
610e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
611e948693eSPhilip Paeps 									\
612e948693eSPhilip Paeps 		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
613e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[3],			\
614e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[2],			\
615e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[1],			\
616e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[0]);			\
617e948693eSPhilip Paeps 									\
618e948693eSPhilip Paeps 		addr = (void *)((_esmp)->esm_base + (_offset));		\
619e948693eSPhilip Paeps 									\
620e948693eSPhilip Paeps 		*addr++ = (_eop)->eo_u32[0];				\
621e948693eSPhilip Paeps 		*addr++ = (_eop)->eo_u32[1];				\
622e948693eSPhilip Paeps 		*addr++ = (_eop)->eo_u32[2];				\
623e948693eSPhilip Paeps 		*addr   = (_eop)->eo_u32[3];				\
624e948693eSPhilip Paeps 									\
625e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
626e948693eSPhilip Paeps 	} while (B_FALSE)
6270f999687SAndrew Rybchenko #endif
628e948693eSPhilip Paeps 
629e948693eSPhilip Paeps #define	EFSYS_MEM_ADDR(_esmp)						\
630e948693eSPhilip Paeps 	((_esmp)->esm_addr)
631e948693eSPhilip Paeps 
6323c838a9fSAndrew Rybchenko #define	EFSYS_MEM_IS_NULL(_esmp)					\
6333c838a9fSAndrew Rybchenko 	((_esmp)->esm_base == NULL)
6343c838a9fSAndrew Rybchenko 
635e948693eSPhilip Paeps /* BAR */
636e948693eSPhilip Paeps 
63733d45dc5SAndrew Rybchenko #define	SFXGE_LOCK_NAME_MAX	16
63833d45dc5SAndrew Rybchenko 
639e948693eSPhilip Paeps typedef struct efsys_bar_s {
640e948693eSPhilip Paeps 	struct mtx		esb_lock;
64133d45dc5SAndrew Rybchenko 	char			esb_lock_name[SFXGE_LOCK_NAME_MAX];
642e948693eSPhilip Paeps 	bus_space_tag_t		esb_tag;
643e948693eSPhilip Paeps 	bus_space_handle_t	esb_handle;
644e948693eSPhilip Paeps 	int			esb_rid;
645e948693eSPhilip Paeps 	struct resource		*esb_res;
646e948693eSPhilip Paeps } efsys_bar_t;
647e948693eSPhilip Paeps 
64833d45dc5SAndrew Rybchenko #define	SFXGE_BAR_LOCK_INIT(_esbp, _ifname)				\
64933d45dc5SAndrew Rybchenko 	do {								\
65033d45dc5SAndrew Rybchenko 		snprintf((_esbp)->esb_lock_name,			\
65133d45dc5SAndrew Rybchenko 			 sizeof((_esbp)->esb_lock_name),		\
65233d45dc5SAndrew Rybchenko 			 "%s:bar", (_ifname));				\
65333d45dc5SAndrew Rybchenko 		mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name,	\
65433d45dc5SAndrew Rybchenko 			 NULL, MTX_DEF);				\
65533d45dc5SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
65633d45dc5SAndrew Rybchenko 	} while (B_FALSE)
657763cab71SAndrew Rybchenko #define	SFXGE_BAR_LOCK_DESTROY(_esbp)					\
658763cab71SAndrew Rybchenko 	mtx_destroy(&(_esbp)->esb_lock)
659763cab71SAndrew Rybchenko #define	SFXGE_BAR_LOCK(_esbp)						\
660763cab71SAndrew Rybchenko 	mtx_lock(&(_esbp)->esb_lock)
661763cab71SAndrew Rybchenko #define	SFXGE_BAR_UNLOCK(_esbp)						\
662763cab71SAndrew Rybchenko 	mtx_unlock(&(_esbp)->esb_lock)
663763cab71SAndrew Rybchenko 
664e948693eSPhilip Paeps #define	EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)			\
665e948693eSPhilip Paeps 	do {								\
666e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
667e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
668e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
669e948693eSPhilip Paeps 									\
670e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
671e948693eSPhilip Paeps 		if (_lock)						\
672763cab71SAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
673e948693eSPhilip Paeps 									\
674d9ff48abSAndrew Rybchenko 		(_edp)->ed_u32[0] = bus_space_read_stream_4(		\
675d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
676d9ff48abSAndrew Rybchenko 		    (_offset));						\
677e948693eSPhilip Paeps 									\
678e948693eSPhilip Paeps 		EFSYS_PROBE2(bar_readd, unsigned int, (_offset),	\
679e948693eSPhilip Paeps 		    uint32_t, (_edp)->ed_u32[0]);			\
680e948693eSPhilip Paeps 									\
681e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
682e948693eSPhilip Paeps 		if (_lock)						\
683763cab71SAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
684e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
685e948693eSPhilip Paeps 	} while (B_FALSE)
686e948693eSPhilip Paeps 
6875dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8)
688e948693eSPhilip Paeps #define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
689e948693eSPhilip Paeps 	do {								\
690e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
691e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
692e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
693e948693eSPhilip Paeps 									\
694763cab71SAndrew Rybchenko 		SFXGE_BAR_LOCK(_esbp);					\
695e948693eSPhilip Paeps 									\
696d9ff48abSAndrew Rybchenko 		(_eqp)->eq_u64[0] = bus_space_read_stream_8(		\
697d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
698d9ff48abSAndrew Rybchenko 		    (_offset));						\
6990f999687SAndrew Rybchenko 									\
7000f999687SAndrew Rybchenko 		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
7010f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[1],			\
7020f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[0]);			\
7030f999687SAndrew Rybchenko 									\
704687adefaSAndrew Rybchenko 		SFXGE_BAR_UNLOCK(_esbp);				\
7050f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
7060f999687SAndrew Rybchenko 	} while (B_FALSE)
7070f999687SAndrew Rybchenko 
7080f999687SAndrew Rybchenko #define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
7090f999687SAndrew Rybchenko 	do {								\
7100f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
7110f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
7120f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
7130f999687SAndrew Rybchenko 									\
7140f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
7150f999687SAndrew Rybchenko 		if (_lock)						\
716687adefaSAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
7170f999687SAndrew Rybchenko 									\
718d9ff48abSAndrew Rybchenko 		(_eop)->eo_u64[0] = bus_space_read_stream_8(		\
719d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
720d9ff48abSAndrew Rybchenko 		    (_offset));						\
721d9ff48abSAndrew Rybchenko 		(_eop)->eo_u64[1] = bus_space_read_stream_8(		\
722d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
723d9ff48abSAndrew Rybchenko 		    (_offset) + 8);					\
7240f999687SAndrew Rybchenko 									\
7250f999687SAndrew Rybchenko 		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
7260f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[3],			\
7270f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[2],			\
7280f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[1],			\
7290f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[0]);			\
7300f999687SAndrew Rybchenko 									\
7310f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
7320f999687SAndrew Rybchenko 		if (_lock)						\
733687adefaSAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
7340f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
7350f999687SAndrew Rybchenko 	} while (B_FALSE)
7360f999687SAndrew Rybchenko 
7370f999687SAndrew Rybchenko #else
7380f999687SAndrew Rybchenko #define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
7390f999687SAndrew Rybchenko 	do {								\
7400f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
7410f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
7420f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
7430f999687SAndrew Rybchenko 									\
744687adefaSAndrew Rybchenko 		SFXGE_BAR_LOCK(_esbp);					\
7450f999687SAndrew Rybchenko 									\
746d9ff48abSAndrew Rybchenko 		(_eqp)->eq_u32[0] = bus_space_read_stream_4(		\
747d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
748d9ff48abSAndrew Rybchenko 		    (_offset));						\
749d9ff48abSAndrew Rybchenko 		(_eqp)->eq_u32[1] = bus_space_read_stream_4(		\
750d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
751d9ff48abSAndrew Rybchenko 		    (_offset) + 4);					\
752e948693eSPhilip Paeps 									\
753e948693eSPhilip Paeps 		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
754e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[1],			\
755e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[0]);			\
756e948693eSPhilip Paeps 									\
757763cab71SAndrew Rybchenko 		SFXGE_BAR_UNLOCK(_esbp);				\
758e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
759e948693eSPhilip Paeps 	} while (B_FALSE)
760e948693eSPhilip Paeps 
761e948693eSPhilip Paeps #define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
762e948693eSPhilip Paeps 	do {								\
763e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
764e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
765e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
766e948693eSPhilip Paeps 									\
767e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
768e948693eSPhilip Paeps 		if (_lock)						\
769763cab71SAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
770e948693eSPhilip Paeps 									\
771d9ff48abSAndrew Rybchenko 		(_eop)->eo_u32[0] = bus_space_read_stream_4(		\
772d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
773d9ff48abSAndrew Rybchenko 		    (_offset));						\
774d9ff48abSAndrew Rybchenko 		(_eop)->eo_u32[1] = bus_space_read_stream_4(		\
775d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
776d9ff48abSAndrew Rybchenko 		    (_offset) + 4);					\
777d9ff48abSAndrew Rybchenko 		(_eop)->eo_u32[2] = bus_space_read_stream_4(		\
778d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
779d9ff48abSAndrew Rybchenko 		    (_offset) + 8);					\
780d9ff48abSAndrew Rybchenko 		(_eop)->eo_u32[3] = bus_space_read_stream_4(		\
781d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
782d9ff48abSAndrew Rybchenko 		    (_offset) + 12);					\
783e948693eSPhilip Paeps 									\
784e948693eSPhilip Paeps 		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
785e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[3],			\
786e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[2],			\
787e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[1],			\
788e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[0]);			\
789e948693eSPhilip Paeps 									\
790e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
791e948693eSPhilip Paeps 		if (_lock)						\
792763cab71SAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
793e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
794e948693eSPhilip Paeps 	} while (B_FALSE)
7950f999687SAndrew Rybchenko #endif
796e948693eSPhilip Paeps 
797e948693eSPhilip Paeps #define	EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)			\
798e948693eSPhilip Paeps 	do {								\
799e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
800e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
801e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
802e948693eSPhilip Paeps 									\
803e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
804e948693eSPhilip Paeps 		if (_lock)						\
805763cab71SAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
806e948693eSPhilip Paeps 									\
807e948693eSPhilip Paeps 		EFSYS_PROBE2(bar_writed, unsigned int, (_offset),	\
808e948693eSPhilip Paeps 		    uint32_t, (_edp)->ed_u32[0]);			\
809e948693eSPhilip Paeps 									\
8101cc8febfSAndrew Rybchenko 		/*							\
8111cc8febfSAndrew Rybchenko 		 * Make sure that previous writes to the dword have	\
8121cc8febfSAndrew Rybchenko 		 * been done. It should be cheaper than barrier just	\
8131cc8febfSAndrew Rybchenko 		 * after the write below.				\
8141cc8febfSAndrew Rybchenko 		 */							\
8151cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
8161cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_dword_t),			\
8171cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
818d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
819d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
820e948693eSPhilip Paeps 		    (_offset), (_edp)->ed_u32[0]);			\
821e948693eSPhilip Paeps 									\
822e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
823e948693eSPhilip Paeps 		if (_lock)						\
824763cab71SAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
825e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
826e948693eSPhilip Paeps 	} while (B_FALSE)
827e948693eSPhilip Paeps 
8285dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8)
829e948693eSPhilip Paeps #define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
830e948693eSPhilip Paeps 	do {								\
831e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
832e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
833e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
834e948693eSPhilip Paeps 									\
835763cab71SAndrew Rybchenko 		SFXGE_BAR_LOCK(_esbp);					\
836e948693eSPhilip Paeps 									\
837e948693eSPhilip Paeps 		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
838e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[1],			\
839e948693eSPhilip Paeps 		    uint32_t, (_eqp)->eq_u32[0]);			\
840e948693eSPhilip Paeps 									\
8411cc8febfSAndrew Rybchenko 		/*							\
8421cc8febfSAndrew Rybchenko 		 * Make sure that previous writes to the qword have	\
8431cc8febfSAndrew Rybchenko 		 * been done. It should be cheaper than barrier just	\
8441cc8febfSAndrew Rybchenko 		 * after the write below.				\
8451cc8febfSAndrew Rybchenko 		 */							\
8461cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
8471cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_qword_t),			\
8481cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
849d9ff48abSAndrew Rybchenko 		bus_space_write_stream_8((_esbp)->esb_tag,		\
850d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
8510f999687SAndrew Rybchenko 		    (_offset), (_eqp)->eq_u64[0]);			\
8520f999687SAndrew Rybchenko 									\
853687adefaSAndrew Rybchenko 		SFXGE_BAR_UNLOCK(_esbp);				\
8540f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
8550f999687SAndrew Rybchenko 	} while (B_FALSE)
8560f999687SAndrew Rybchenko #else
8570f999687SAndrew Rybchenko #define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
8580f999687SAndrew Rybchenko 	do {								\
8590f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
8600f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
8610f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
8620f999687SAndrew Rybchenko 									\
863687adefaSAndrew Rybchenko 		SFXGE_BAR_LOCK(_esbp);					\
8640f999687SAndrew Rybchenko 									\
8650f999687SAndrew Rybchenko 		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
8660f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[1],			\
8670f999687SAndrew Rybchenko 		    uint32_t, (_eqp)->eq_u32[0]);			\
8680f999687SAndrew Rybchenko 									\
8691cc8febfSAndrew Rybchenko 		/*							\
8701cc8febfSAndrew Rybchenko 		 * Make sure that previous writes to the qword have	\
8711cc8febfSAndrew Rybchenko 		 * been done. It should be cheaper than barrier just	\
8721cc8febfSAndrew Rybchenko 		 * after the last write below.				\
8731cc8febfSAndrew Rybchenko 		 */							\
8741cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
8751cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_qword_t),			\
8761cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
877d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
878d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
879e948693eSPhilip Paeps 		    (_offset), (_eqp)->eq_u32[0]);			\
8801cc8febfSAndrew Rybchenko 		/*							\
8811cc8febfSAndrew Rybchenko 		 * It should be guaranteed that the last dword comes	\
8821cc8febfSAndrew Rybchenko 		 * the last, so barrier entire qword to be sure that	\
8831cc8febfSAndrew Rybchenko 		 * neither above nor below writes are reordered.	\
8841cc8febfSAndrew Rybchenko 		 */							\
8851cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
8861cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_qword_t),			\
8871cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
888d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
889d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
890d9ff48abSAndrew Rybchenko 		    (_offset) + 4, (_eqp)->eq_u32[1]);			\
891e948693eSPhilip Paeps 									\
892763cab71SAndrew Rybchenko 		SFXGE_BAR_UNLOCK(_esbp);				\
893e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
894e948693eSPhilip Paeps 	} while (B_FALSE)
8950f999687SAndrew Rybchenko #endif
896e948693eSPhilip Paeps 
8973c838a9fSAndrew Rybchenko /*
8983c838a9fSAndrew Rybchenko  * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
8993c838a9fSAndrew Rybchenko  * (required by PIO hardware)
9003c838a9fSAndrew Rybchenko  */
9013c838a9fSAndrew Rybchenko #define	EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp)			\
9023c838a9fSAndrew Rybchenko 	do {								\
9033c838a9fSAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
9043c838a9fSAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
9053c838a9fSAndrew Rybchenko 		    ("not power of 2 aligned"));			\
9063c838a9fSAndrew Rybchenko 									\
9073c838a9fSAndrew Rybchenko 		(void) (_esbp);						\
9083c838a9fSAndrew Rybchenko 									\
9093c838a9fSAndrew Rybchenko 		/* FIXME: Perform a 64-bit write */			\
9103c838a9fSAndrew Rybchenko 		KASSERT(0, ("not implemented"));			\
9113c838a9fSAndrew Rybchenko 									\
9123c838a9fSAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
9133c838a9fSAndrew Rybchenko 	} while (B_FALSE)
9143c838a9fSAndrew Rybchenko 
9155dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8)
916e948693eSPhilip Paeps #define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
917e948693eSPhilip Paeps 	do {								\
918e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
919e948693eSPhilip Paeps 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
920e948693eSPhilip Paeps 		    ("not power of 2 aligned"));			\
921e948693eSPhilip Paeps 									\
922e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
923e948693eSPhilip Paeps 		if (_lock)						\
924763cab71SAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
925e948693eSPhilip Paeps 									\
926e948693eSPhilip Paeps 		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
927e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[3],			\
928e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[2],			\
929e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[1],			\
930e948693eSPhilip Paeps 		    uint32_t, (_eop)->eo_u32[0]);			\
931e948693eSPhilip Paeps 									\
9321cc8febfSAndrew Rybchenko 		/*							\
9331cc8febfSAndrew Rybchenko 		 * Make sure that previous writes to the oword have	\
9341cc8febfSAndrew Rybchenko 		 * been done. It should be cheaper than barrier just	\
9351cc8febfSAndrew Rybchenko 		 * after the last write below.				\
9361cc8febfSAndrew Rybchenko 		 */							\
9371cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
9381cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_oword_t),			\
9391cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
940d9ff48abSAndrew Rybchenko 		bus_space_write_stream_8((_esbp)->esb_tag,		\
941d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
9420f999687SAndrew Rybchenko 		    (_offset), (_eop)->eo_u64[0]);			\
9431cc8febfSAndrew Rybchenko 		/*							\
9441cc8febfSAndrew Rybchenko 		 * It should be guaranteed that the last qword comes	\
9451cc8febfSAndrew Rybchenko 		 * the last, so barrier entire oword to be sure that	\
9461cc8febfSAndrew Rybchenko 		 * neither above nor below writes are reordered.	\
9471cc8febfSAndrew Rybchenko 		 */							\
9481cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
9491cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_oword_t),			\
9501cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
951d9ff48abSAndrew Rybchenko 		bus_space_write_stream_8((_esbp)->esb_tag,		\
952d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
953d9ff48abSAndrew Rybchenko 		    (_offset) + 8, (_eop)->eo_u64[1]);			\
9540f999687SAndrew Rybchenko 									\
9550f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
9560f999687SAndrew Rybchenko 		if (_lock)						\
957687adefaSAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
9580f999687SAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
9590f999687SAndrew Rybchenko 	} while (B_FALSE)
9600f999687SAndrew Rybchenko 
9610f999687SAndrew Rybchenko #else
9620f999687SAndrew Rybchenko #define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
9630f999687SAndrew Rybchenko 	do {								\
9640f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
9650f999687SAndrew Rybchenko 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
9660f999687SAndrew Rybchenko 		    ("not power of 2 aligned"));			\
9670f999687SAndrew Rybchenko 									\
9680f999687SAndrew Rybchenko 		_NOTE(CONSTANTCONDITION)				\
9690f999687SAndrew Rybchenko 		if (_lock)						\
970687adefaSAndrew Rybchenko 			SFXGE_BAR_LOCK(_esbp);				\
9710f999687SAndrew Rybchenko 									\
9720f999687SAndrew Rybchenko 		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
9730f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[3],			\
9740f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[2],			\
9750f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[1],			\
9760f999687SAndrew Rybchenko 		    uint32_t, (_eop)->eo_u32[0]);			\
9770f999687SAndrew Rybchenko 									\
9781cc8febfSAndrew Rybchenko 		/*							\
9791cc8febfSAndrew Rybchenko 		 * Make sure that previous writes to the oword have	\
9801cc8febfSAndrew Rybchenko 		 * been done. It should be cheaper than barrier just	\
9811cc8febfSAndrew Rybchenko 		 * after the last write below.				\
9821cc8febfSAndrew Rybchenko 		 */							\
9831cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
9841cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_oword_t),			\
9851cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
986d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
987d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
988e948693eSPhilip Paeps 		    (_offset), (_eop)->eo_u32[0]);			\
989d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
990d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
991d9ff48abSAndrew Rybchenko 		    (_offset) + 4, (_eop)->eo_u32[1]);			\
992d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
993d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
994d9ff48abSAndrew Rybchenko 		    (_offset) + 8, (_eop)->eo_u32[2]);			\
9951cc8febfSAndrew Rybchenko 		/*							\
9961cc8febfSAndrew Rybchenko 		 * It should be guaranteed that the last dword comes	\
9971cc8febfSAndrew Rybchenko 		 * the last, so barrier entire oword to be sure that	\
9981cc8febfSAndrew Rybchenko 		 * neither above nor below writes are reordered.	\
9991cc8febfSAndrew Rybchenko 		 */							\
10001cc8febfSAndrew Rybchenko 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
10011cc8febfSAndrew Rybchenko 		    (_offset), sizeof (efx_oword_t),			\
10021cc8febfSAndrew Rybchenko 		    BUS_SPACE_BARRIER_WRITE);				\
1003d9ff48abSAndrew Rybchenko 		bus_space_write_stream_4((_esbp)->esb_tag,		\
1004d9ff48abSAndrew Rybchenko 		    (_esbp)->esb_handle,				\
1005d9ff48abSAndrew Rybchenko 		    (_offset) + 12, (_eop)->eo_u32[3]);			\
1006e948693eSPhilip Paeps 									\
1007e948693eSPhilip Paeps 		_NOTE(CONSTANTCONDITION)				\
1008e948693eSPhilip Paeps 		if (_lock)						\
1009763cab71SAndrew Rybchenko 			SFXGE_BAR_UNLOCK(_esbp);			\
1010e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1011e948693eSPhilip Paeps 	} while (B_FALSE)
10120f999687SAndrew Rybchenko #endif
1013e948693eSPhilip Paeps 
10143c838a9fSAndrew Rybchenko /* Use the standard octo-word write for doorbell writes */
10153c838a9fSAndrew Rybchenko #define	EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop)			\
10163c838a9fSAndrew Rybchenko 	do {								\
10173c838a9fSAndrew Rybchenko 		EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE);	\
10183c838a9fSAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
10193c838a9fSAndrew Rybchenko 	} while (B_FALSE)
10203c838a9fSAndrew Rybchenko 
1021e948693eSPhilip Paeps /* SPIN */
1022e948693eSPhilip Paeps 
1023e948693eSPhilip Paeps #define	EFSYS_SPIN(_us)							\
1024e948693eSPhilip Paeps 	do {								\
1025e948693eSPhilip Paeps 		DELAY(_us);						\
1026e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1027e948693eSPhilip Paeps 	} while (B_FALSE)
1028e948693eSPhilip Paeps 
1029e948693eSPhilip Paeps #define	EFSYS_SLEEP	EFSYS_SPIN
1030e948693eSPhilip Paeps 
1031e948693eSPhilip Paeps /* BARRIERS */
1032e948693eSPhilip Paeps 
10337c00963aSAndrew Rybchenko #define	EFSYS_MEM_READ_BARRIER()	rmb()
1034e948693eSPhilip Paeps #define	EFSYS_PIO_WRITE_BARRIER()
1035e948693eSPhilip Paeps 
10363c838a9fSAndrew Rybchenko /* DMA SYNC */
10373c838a9fSAndrew Rybchenko #define	EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size)		\
10383c838a9fSAndrew Rybchenko 	do {								\
10393c838a9fSAndrew Rybchenko 		bus_dmamap_sync((_esmp)->esm_tag,			\
10403c838a9fSAndrew Rybchenko 		    (_esmp)->esm_map,					\
10413c838a9fSAndrew Rybchenko 		    BUS_DMASYNC_POSTREAD);				\
10423c838a9fSAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
10433c838a9fSAndrew Rybchenko 	} while (B_FALSE)
10443c838a9fSAndrew Rybchenko 
10453c838a9fSAndrew Rybchenko #define	EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size)		\
10463c838a9fSAndrew Rybchenko 	do {								\
10473c838a9fSAndrew Rybchenko 		bus_dmamap_sync((_esmp)->esm_tag,			\
10483c838a9fSAndrew Rybchenko 		    (_esmp)->esm_map,					\
10493c838a9fSAndrew Rybchenko 		    BUS_DMASYNC_PREWRITE);				\
10503c838a9fSAndrew Rybchenko 	_NOTE(CONSTANTCONDITION)					\
10513c838a9fSAndrew Rybchenko 	} while (B_FALSE)
10523c838a9fSAndrew Rybchenko 
1053e948693eSPhilip Paeps /* TIMESTAMP */
1054e948693eSPhilip Paeps 
1055e948693eSPhilip Paeps typedef	clock_t	efsys_timestamp_t;
1056e948693eSPhilip Paeps 
1057e948693eSPhilip Paeps #define	EFSYS_TIMESTAMP(_usp)						\
1058e948693eSPhilip Paeps 	do {								\
1059e948693eSPhilip Paeps 		clock_t now;						\
1060e948693eSPhilip Paeps 									\
1061e948693eSPhilip Paeps 		now = ticks;						\
1062e948693eSPhilip Paeps 		*(_usp) = now * hz / 1000000;				\
1063e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1064e948693eSPhilip Paeps 	} while (B_FALSE)
1065e948693eSPhilip Paeps 
1066e948693eSPhilip Paeps /* KMEM */
1067e948693eSPhilip Paeps 
1068e948693eSPhilip Paeps #define	EFSYS_KMEM_ALLOC(_esip, _size, _p)				\
1069e948693eSPhilip Paeps 	do {								\
1070e948693eSPhilip Paeps 		(_esip) = (_esip);					\
107183488e18SAndrew Rybchenko 		/*							\
107283488e18SAndrew Rybchenko 		 * The macro is used in non-sleepable contexts, for	\
107383488e18SAndrew Rybchenko 		 * example, holding a mutex.				\
107483488e18SAndrew Rybchenko 		 */							\
107583488e18SAndrew Rybchenko 		(_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO);	\
1076e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1077e948693eSPhilip Paeps 	} while (B_FALSE)
1078e948693eSPhilip Paeps 
1079e948693eSPhilip Paeps #define	EFSYS_KMEM_FREE(_esip, _size, _p)				\
1080e948693eSPhilip Paeps 	do {								\
1081e948693eSPhilip Paeps 		(void) (_esip);						\
1082e948693eSPhilip Paeps 		(void) (_size);						\
1083e948693eSPhilip Paeps 		free((_p), M_SFXGE);					\
1084e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1085e948693eSPhilip Paeps 	} while (B_FALSE)
1086e948693eSPhilip Paeps 
1087e948693eSPhilip Paeps /* LOCK */
1088e948693eSPhilip Paeps 
108933d45dc5SAndrew Rybchenko typedef struct efsys_lock_s {
109033d45dc5SAndrew Rybchenko 	struct mtx	lock;
109133d45dc5SAndrew Rybchenko 	char		lock_name[SFXGE_LOCK_NAME_MAX];
109233d45dc5SAndrew Rybchenko } efsys_lock_t;
109333d45dc5SAndrew Rybchenko 
109433d45dc5SAndrew Rybchenko #define	SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label)			\
109533d45dc5SAndrew Rybchenko 	do {								\
109633d45dc5SAndrew Rybchenko 		efsys_lock_t *__eslp = (_eslp);				\
109733d45dc5SAndrew Rybchenko 									\
109833d45dc5SAndrew Rybchenko 		snprintf((__eslp)->lock_name,				\
109933d45dc5SAndrew Rybchenko 			 sizeof((__eslp)->lock_name),			\
110033d45dc5SAndrew Rybchenko 			 "%s:%s", (_ifname), (_label));			\
110133d45dc5SAndrew Rybchenko 		mtx_init(&(__eslp)->lock, (__eslp)->lock_name,		\
110233d45dc5SAndrew Rybchenko 			 NULL, MTX_DEF);				\
110333d45dc5SAndrew Rybchenko 	} while (B_FALSE)
110433d45dc5SAndrew Rybchenko #define	SFXGE_EFSYS_LOCK_DESTROY(_eslp)					\
110533d45dc5SAndrew Rybchenko 	mtx_destroy(&(_eslp)->lock)
110633d45dc5SAndrew Rybchenko #define	SFXGE_EFSYS_LOCK(_eslp)						\
110733d45dc5SAndrew Rybchenko 	mtx_lock(&(_eslp)->lock)
110833d45dc5SAndrew Rybchenko #define	SFXGE_EFSYS_UNLOCK(_eslp)					\
110933d45dc5SAndrew Rybchenko 	mtx_unlock(&(_eslp)->lock)
111033d45dc5SAndrew Rybchenko #define	SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp)				\
111133d45dc5SAndrew Rybchenko 	mtx_assert(&(_eslp)->lock, MA_OWNED)
1112e948693eSPhilip Paeps 
1113e948693eSPhilip Paeps #define	EFSYS_LOCK_MAGIC	0x000010c4
1114e948693eSPhilip Paeps 
1115e948693eSPhilip Paeps #define	EFSYS_LOCK(_lockp, _state)					\
1116e948693eSPhilip Paeps 	do {								\
111733d45dc5SAndrew Rybchenko 		SFXGE_EFSYS_LOCK(_lockp);				\
1118e948693eSPhilip Paeps 		(_state) = EFSYS_LOCK_MAGIC;				\
1119e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1120e948693eSPhilip Paeps 	} while (B_FALSE)
1121e948693eSPhilip Paeps 
1122e948693eSPhilip Paeps #define	EFSYS_UNLOCK(_lockp, _state)					\
1123e948693eSPhilip Paeps 	do {								\
1124e948693eSPhilip Paeps 		if ((_state) != EFSYS_LOCK_MAGIC)			\
1125e948693eSPhilip Paeps 			KASSERT(B_FALSE, ("not locked"));		\
112633d45dc5SAndrew Rybchenko 		SFXGE_EFSYS_UNLOCK(_lockp);				\
1127e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1128e948693eSPhilip Paeps 	} while (B_FALSE)
1129e948693eSPhilip Paeps 
1130e948693eSPhilip Paeps /* PREEMPT */
1131e948693eSPhilip Paeps 
1132e948693eSPhilip Paeps #define	EFSYS_PREEMPT_DISABLE(_state)					\
1133e948693eSPhilip Paeps 	do {								\
1134e948693eSPhilip Paeps 		(_state) = (_state);					\
1135e948693eSPhilip Paeps 		critical_enter();					\
1136e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1137e948693eSPhilip Paeps 	} while (B_FALSE)
1138e948693eSPhilip Paeps 
1139e948693eSPhilip Paeps #define	EFSYS_PREEMPT_ENABLE(_state)					\
1140e948693eSPhilip Paeps 	do {								\
1141e948693eSPhilip Paeps 		(_state) = (_state);					\
1142e948693eSPhilip Paeps 		critical_exit(_state);					\
1143e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1144e948693eSPhilip Paeps 	} while (B_FALSE)
1145e948693eSPhilip Paeps 
1146e948693eSPhilip Paeps /* STAT */
1147e948693eSPhilip Paeps 
1148e948693eSPhilip Paeps typedef uint64_t		efsys_stat_t;
1149e948693eSPhilip Paeps 
1150e948693eSPhilip Paeps #define	EFSYS_STAT_INCR(_knp, _delta) 					\
1151e948693eSPhilip Paeps 	do {								\
1152e948693eSPhilip Paeps 		*(_knp) += (_delta);					\
1153e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1154e948693eSPhilip Paeps 	} while (B_FALSE)
1155e948693eSPhilip Paeps 
1156e948693eSPhilip Paeps #define	EFSYS_STAT_DECR(_knp, _delta) 					\
1157e948693eSPhilip Paeps 	do {								\
1158e948693eSPhilip Paeps 		*(_knp) -= (_delta);					\
1159e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1160e948693eSPhilip Paeps 	} while (B_FALSE)
1161e948693eSPhilip Paeps 
1162e948693eSPhilip Paeps #define	EFSYS_STAT_SET(_knp, _val)					\
1163e948693eSPhilip Paeps 	do {								\
1164e948693eSPhilip Paeps 		*(_knp) = (_val);					\
1165e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1166e948693eSPhilip Paeps 	} while (B_FALSE)
1167e948693eSPhilip Paeps 
1168e948693eSPhilip Paeps #define	EFSYS_STAT_SET_QWORD(_knp, _valp)				\
1169e948693eSPhilip Paeps 	do {								\
1170e948693eSPhilip Paeps 		*(_knp) = le64toh((_valp)->eq_u64[0]);			\
1171e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1172e948693eSPhilip Paeps 	} while (B_FALSE)
1173e948693eSPhilip Paeps 
1174e948693eSPhilip Paeps #define	EFSYS_STAT_SET_DWORD(_knp, _valp)				\
1175e948693eSPhilip Paeps 	do {								\
1176e948693eSPhilip Paeps 		*(_knp) = le32toh((_valp)->ed_u32[0]);			\
1177e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1178e948693eSPhilip Paeps 	} while (B_FALSE)
1179e948693eSPhilip Paeps 
1180e948693eSPhilip Paeps #define	EFSYS_STAT_INCR_QWORD(_knp, _valp)				\
1181e948693eSPhilip Paeps 	do {								\
1182e948693eSPhilip Paeps 		*(_knp) += le64toh((_valp)->eq_u64[0]);			\
1183e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1184e948693eSPhilip Paeps 	} while (B_FALSE)
1185e948693eSPhilip Paeps 
1186e948693eSPhilip Paeps #define	EFSYS_STAT_SUBR_QWORD(_knp, _valp)				\
1187e948693eSPhilip Paeps 	do {								\
1188e948693eSPhilip Paeps 		*(_knp) -= le64toh((_valp)->eq_u64[0]);			\
1189e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1190e948693eSPhilip Paeps 	} while (B_FALSE)
1191e948693eSPhilip Paeps 
1192e948693eSPhilip Paeps /* ERR */
1193e948693eSPhilip Paeps 
1194e948693eSPhilip Paeps extern void	sfxge_err(efsys_identifier_t *, unsigned int,
1195e948693eSPhilip Paeps 		    uint32_t, uint32_t);
1196e948693eSPhilip Paeps 
1197e948693eSPhilip Paeps #if EFSYS_OPT_DECODE_INTR_FATAL
1198e948693eSPhilip Paeps #define	EFSYS_ERR(_esip, _code, _dword0, _dword1)			\
1199e948693eSPhilip Paeps 	do {								\
1200e948693eSPhilip Paeps 		sfxge_err((_esip), (_code), (_dword0), (_dword1));	\
1201e948693eSPhilip Paeps 	_NOTE(CONSTANTCONDITION)					\
1202e948693eSPhilip Paeps 	} while (B_FALSE)
1203e948693eSPhilip Paeps #endif
1204e948693eSPhilip Paeps 
1205e948693eSPhilip Paeps /* ASSERT */
1206e948693eSPhilip Paeps 
1207e948693eSPhilip Paeps #define	EFSYS_ASSERT(_exp) do {						\
1208e948693eSPhilip Paeps 	if (!(_exp))							\
12093c838a9fSAndrew Rybchenko 		panic("%s", #_exp);					\
1210e948693eSPhilip Paeps 	} while (0)
1211e948693eSPhilip Paeps 
1212e948693eSPhilip Paeps #define	EFSYS_ASSERT3(_x, _op, _y, _t) do {				\
1213e948693eSPhilip Paeps 	const _t __x = (_t)(_x);					\
1214e948693eSPhilip Paeps 	const _t __y = (_t)(_y);					\
1215e948693eSPhilip Paeps 	if (!(__x _op __y))						\
1216e948693eSPhilip Paeps 		panic("assertion failed at %s:%u", __FILE__, __LINE__);	\
1217e948693eSPhilip Paeps 	} while(0)
1218e948693eSPhilip Paeps 
1219e948693eSPhilip Paeps #define	EFSYS_ASSERT3U(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1220e948693eSPhilip Paeps #define	EFSYS_ASSERT3S(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, int64_t)
1221e948693eSPhilip Paeps #define	EFSYS_ASSERT3P(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1222e948693eSPhilip Paeps 
12233c838a9fSAndrew Rybchenko /* ROTATE */
12243c838a9fSAndrew Rybchenko 
12253c838a9fSAndrew Rybchenko #define	EFSYS_HAS_ROTL_DWORD 0
12263c838a9fSAndrew Rybchenko 
1227e948693eSPhilip Paeps #ifdef	__cplusplus
1228e948693eSPhilip Paeps }
1229e948693eSPhilip Paeps #endif
1230e948693eSPhilip Paeps 
1231e948693eSPhilip Paeps #endif	/* _SYS_EFSYS_H */
1232