xref: /freebsd/sys/dev/sfxge/common/efsys.h (revision fc7284da06408923f7850a0e4e954a903b8ee038)
1 /*-
2  * Copyright (c) 2010-2015 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * This software was developed in part by Philip Paeps under contract for
6  * Solarflare Communications, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright notice,
12  *    this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright notice,
14  *    this list of conditions and the following disclaimer in the documentation
15  *    and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  *
29  * The views and conclusions contained in the software and documentation are
30  * those of the authors and should not be interpreted as representing official
31  * policies, either expressed or implied, of the FreeBSD Project.
32  *
33  * $FreeBSD$
34  */
35 
36 #ifndef	_SYS_EFSYS_H
37 #define	_SYS_EFSYS_H
38 
39 #ifdef	__cplusplus
40 extern "C" {
41 #endif
42 
43 #include <sys/param.h>
44 #include <sys/bus.h>
45 #include <sys/endian.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/mbuf.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
51 #include <sys/sdt.h>
52 #include <sys/systm.h>
53 
54 #include <machine/bus.h>
55 #include <machine/endian.h>
56 
57 #define	EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define	EFSYS_USE_UINT64 1
60 #else
61 #define	EFSYS_USE_UINT64 0
62 #endif
63 #define	EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define	EFSYS_IS_BIG_ENDIAN 1
66 #define	EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define	EFSYS_IS_BIG_ENDIAN 0
69 #define	EFSYS_IS_LITTLE_ENDIAN 1
70 #endif
71 #include "efx_types.h"
72 
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define	memmove(d, s, l) bcopy(s, d, l)
76 #endif
77 
78 /* FreeBSD equivalents of Solaris things */
79 #ifndef _NOTE
80 #define	_NOTE(s)
81 #endif
82 
83 #ifndef B_FALSE
84 #define	B_FALSE	FALSE
85 #endif
86 #ifndef B_TRUE
87 #define	B_TRUE	TRUE
88 #endif
89 
90 #ifndef IS_P2ALIGNED
91 #define	IS_P2ALIGNED(v, a)	((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
92 #endif
93 
94 #ifndef P2ROUNDUP
95 #define	P2ROUNDUP(x, align)	(-(-(x) & -(align)))
96 #endif
97 
98 #ifndef P2ALIGN
99 #define	P2ALIGN(_x, _a)		((_x) & -(_a))
100 #endif
101 
102 #ifndef IS2P
103 #define	ISP2(x)			(((x) & ((x) - 1)) == 0)
104 #endif
105 
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
107 
108 #define	SFXGE_USE_BUS_SPACE_8		1
109 
110 #if !defined(bus_space_read_stream_8)
111 
112 #define	bus_space_read_stream_8(t, h, o)				\
113 	bus_space_read_8((t), (h), (o))
114 
115 #define	bus_space_write_stream_8(t, h, o, v)				\
116 	bus_space_write_8((t), (h), (o), (v))
117 
118 #endif
119 
120 #endif
121 
122 #define	ENOTACTIVE EINVAL
123 
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
126 
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
129 static __inline void
130 prefetch_read_many(void *addr)
131 {
132 
133 	__asm__(
134 	    "prefetcht0 (%0)"
135 	    :
136 	    : "r" (addr));
137 }
138 
139 static __inline void
140 prefetch_read_once(void *addr)
141 {
142 
143 	__asm__(
144 	    "prefetchnta (%0)"
145 	    :
146 	    : "r" (addr));
147 }
148 #elif defined(__sparc64__)
149 static __inline void
150 prefetch_read_many(void *addr)
151 {
152 
153 	__asm__(
154 	    "prefetch [%0], 0"
155 	    :
156 	    : "r" (addr));
157 }
158 
159 static __inline void
160 prefetch_read_once(void *addr)
161 {
162 
163 	__asm__(
164 	    "prefetch [%0], 1"
165 	    :
166 	    : "r" (addr));
167 }
168 #else
169 static __inline void
170 prefetch_read_many(void *addr)
171 {
172 
173 }
174 
175 static __inline void
176 prefetch_read_once(void *addr)
177 {
178 
179 }
180 #endif
181 
182 #if defined(__i386__) || defined(__amd64__)
183 #include <vm/vm.h>
184 #include <vm/pmap.h>
185 #endif
186 static __inline void
187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 		    struct mbuf *m, bus_dma_segment_t *seg)
189 {
190 #if defined(__i386__) || defined(__amd64__)
191 	seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 	seg->ds_len = m->m_len;
193 #else
194 	int nsegstmp;
195 
196 	bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
197 #endif
198 }
199 
200 /* Modifiers used for Windows builds */
201 #define	__in
202 #define	__in_opt
203 #define	__in_ecount(_n)
204 #define	__in_ecount_opt(_n)
205 #define	__in_bcount(_n)
206 #define	__in_bcount_opt(_n)
207 
208 #define	__out
209 #define	__out_opt
210 #define	__out_ecount(_n)
211 #define	__out_ecount_opt(_n)
212 #define	__out_bcount(_n)
213 #define	__out_bcount_opt(_n)
214 
215 #define	__deref_out
216 
217 #define	__inout
218 #define	__inout_opt
219 #define	__inout_ecount(_n)
220 #define	__inout_ecount_opt(_n)
221 #define	__inout_bcount(_n)
222 #define	__inout_bcount_opt(_n)
223 #define	__inout_bcount_full_opt(_n)
224 
225 #define	__deref_out_bcount_opt(n)
226 
227 #define	__checkReturn
228 #define	__success(_x)
229 
230 #define	__drv_when(_p, _c)
231 
232 /* Code inclusion options */
233 
234 
235 #define	EFSYS_OPT_NAMES 1
236 
237 #define	EFSYS_OPT_FALCON 0
238 #define	EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
239 #define	EFSYS_OPT_SIENA 1
240 #define	EFSYS_OPT_HUNTINGTON 1
241 #ifdef DEBUG
242 #define	EFSYS_OPT_CHECK_REG 1
243 #else
244 #define	EFSYS_OPT_CHECK_REG 0
245 #endif
246 
247 #define	EFSYS_OPT_MCDI 1
248 #define	EFSYS_OPT_MCDI_LOGGING 0
249 #define	EFSYS_OPT_MCDI_PROXY_AUTH 0
250 
251 #define	EFSYS_OPT_MAC_FALCON_GMAC 0
252 #define	EFSYS_OPT_MAC_FALCON_XMAC 0
253 #define	EFSYS_OPT_MAC_STATS 1
254 
255 #define	EFSYS_OPT_LOOPBACK 0
256 
257 #define	EFSYS_OPT_MON_NULL 0
258 #define	EFSYS_OPT_MON_LM87 0
259 #define	EFSYS_OPT_MON_MAX6647 0
260 #define	EFSYS_OPT_MON_MCDI 0
261 #define	EFSYS_OPT_MON_STATS 0
262 
263 #define	EFSYS_OPT_PHY_NULL 0
264 #define	EFSYS_OPT_PHY_QT2022C2 0
265 #define	EFSYS_OPT_PHY_SFX7101 0
266 #define	EFSYS_OPT_PHY_TXC43128 0
267 #define	EFSYS_OPT_PHY_SFT9001 0
268 #define	EFSYS_OPT_PHY_QT2025C 0
269 #define	EFSYS_OPT_PHY_STATS 1
270 #define	EFSYS_OPT_PHY_PROPS 0
271 #define	EFSYS_OPT_PHY_BIST 0
272 #define	EFSYS_OPT_BIST 1
273 #define	EFSYS_OPT_PHY_LED_CONTROL 1
274 #define	EFSYS_OPT_PHY_FLAGS 0
275 
276 #define	EFSYS_OPT_VPD 1
277 #define	EFSYS_OPT_NVRAM 1
278 #define	EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
279 #define	EFSYS_OPT_NVRAM_SFT9001	0
280 #define	EFSYS_OPT_NVRAM_SFX7101	0
281 #define	EFSYS_OPT_BOOTCFG 0
282 
283 #define	EFSYS_OPT_PCIE_TUNE 0
284 #define	EFSYS_OPT_DIAG 0
285 #define	EFSYS_OPT_WOL 1
286 #define	EFSYS_OPT_RX_SCALE 1
287 #define	EFSYS_OPT_QSTATS 1
288 #define	EFSYS_OPT_FILTER 1
289 #define	EFSYS_OPT_MCAST_FILTER_LIST 1
290 #define	EFSYS_OPT_RX_SCATTER 0
291 #define	EFSYS_OPT_RX_HDR_SPLIT 0
292 
293 #define	EFSYS_OPT_EV_PREFETCH 0
294 
295 #define	EFSYS_OPT_DECODE_INTR_FATAL 1
296 
297 /* ID */
298 
299 typedef struct __efsys_identifier_s	efsys_identifier_t;
300 
301 /* PROBE */
302 
303 #ifndef DTRACE_PROBE
304 
305 #define	EFSYS_PROBE(_name)
306 
307 #define	EFSYS_PROBE1(_name, _type1, _arg1)
308 
309 #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
310 
311 #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
312 	    _type3, _arg3)
313 
314 #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
315 	    _type3, _arg3, _type4, _arg4)
316 
317 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
318 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
319 
320 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
321 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
322 	    _type6, _arg6)
323 
324 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
325 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
326 	    _type6, _arg6, _type7, _arg7)
327 
328 #else /* DTRACE_PROBE */
329 
330 #define	EFSYS_PROBE(_name)						\
331 	DTRACE_PROBE(_name)
332 
333 #define	EFSYS_PROBE1(_name, _type1, _arg1)				\
334 	DTRACE_PROBE1(_name, _type1, _arg1)
335 
336 #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)		\
337 	DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
338 
339 #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
340 	    _type3, _arg3)						\
341 	DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
342 	    _type3, _arg3)
343 
344 #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
345 	    _type3, _arg3, _type4, _arg4)				\
346 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
347 	    _type3, _arg3, _type4, _arg4)
348 
349 #ifdef DTRACE_PROBE5
350 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
351 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
352 	DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
353 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
354 #else
355 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
356 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
357 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
358 	    _type3, _arg3, _type4, _arg4)
359 #endif
360 
361 #ifdef DTRACE_PROBE6
362 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
363 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
364 	    _type6, _arg6)						\
365 	DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
366 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
367 	    _type6, _arg6)
368 #else
369 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
370 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
371 	    _type6, _arg6)						\
372 	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
373 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
374 #endif
375 
376 #ifdef DTRACE_PROBE7
377 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
378 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
379 	    _type6, _arg6, _type7, _arg7)				\
380 	DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
381 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
382 	    _type6, _arg6, _type7, _arg7)
383 #else
384 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
385 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
386 	    _type6, _arg6, _type7, _arg7)				\
387 	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
388 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
389 	    _type6, _arg6)
390 #endif
391 
392 #endif /* DTRACE_PROBE */
393 
394 /* DMA */
395 
396 typedef uint64_t		efsys_dma_addr_t;
397 
398 typedef struct efsys_mem_s {
399 	bus_dma_tag_t		esm_tag;
400 	bus_dmamap_t		esm_map;
401 	caddr_t			esm_base;
402 	efsys_dma_addr_t	esm_addr;
403 } efsys_mem_t;
404 
405 
406 #define	EFSYS_MEM_ZERO(_esmp, _size)					\
407 	do {								\
408 		(void) memset((_esmp)->esm_base, 0, (_size));		\
409 									\
410 	_NOTE(CONSTANTCONDITION)					\
411 	} while (B_FALSE)
412 
413 #define	EFSYS_MEM_READD(_esmp, _offset, _edp)				\
414 	do {								\
415 		uint32_t *addr;						\
416 									\
417 		_NOTE(CONSTANTCONDITION)				\
418 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
419 		    ("not power of 2 aligned"));			\
420 									\
421 		addr = (void *)((_esmp)->esm_base + (_offset));		\
422 									\
423 		(_edp)->ed_u32[0] = *addr;				\
424 									\
425 		EFSYS_PROBE2(mem_readd, unsigned int, (_offset),	\
426 		    uint32_t, (_edp)->ed_u32[0]);			\
427 									\
428 	_NOTE(CONSTANTCONDITION)					\
429 	} while (B_FALSE)
430 
431 #if defined(__x86_64__)
432 #define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
433 	do {								\
434 		uint64_t *addr;						\
435 									\
436 		_NOTE(CONSTANTCONDITION)				\
437 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
438 		    ("not power of 2 aligned"));			\
439 									\
440 		addr = (void *)((_esmp)->esm_base + (_offset));		\
441 									\
442 		(_eqp)->eq_u64[0] = *addr;				\
443 									\
444 		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
445 		    uint32_t, (_eqp)->eq_u32[1],			\
446 		    uint32_t, (_eqp)->eq_u32[0]);			\
447 									\
448 	_NOTE(CONSTANTCONDITION)					\
449 	} while (B_FALSE)
450 #else
451 #define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
452 	do {								\
453 		uint32_t *addr;						\
454 									\
455 		_NOTE(CONSTANTCONDITION)				\
456 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
457 		    ("not power of 2 aligned"));			\
458 									\
459 		addr = (void *)((_esmp)->esm_base + (_offset));		\
460 									\
461 		(_eqp)->eq_u32[0] = *addr++;				\
462 		(_eqp)->eq_u32[1] = *addr;				\
463 									\
464 		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
465 		    uint32_t, (_eqp)->eq_u32[1],			\
466 		    uint32_t, (_eqp)->eq_u32[0]);			\
467 									\
468 	_NOTE(CONSTANTCONDITION)					\
469 	} while (B_FALSE)
470 #endif
471 
472 #if defined(__x86_64__)
473 #define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
474 	do {								\
475 		uint64_t *addr;						\
476 									\
477 		_NOTE(CONSTANTCONDITION)				\
478 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
479 		    ("not power of 2 aligned"));			\
480 									\
481 		addr = (void *)((_esmp)->esm_base + (_offset));		\
482 									\
483 		(_eop)->eo_u64[0] = *addr++;				\
484 		(_eop)->eo_u64[1] = *addr;				\
485 									\
486 		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
487 		    uint32_t, (_eop)->eo_u32[3],			\
488 		    uint32_t, (_eop)->eo_u32[2],			\
489 		    uint32_t, (_eop)->eo_u32[1],			\
490 		    uint32_t, (_eop)->eo_u32[0]);			\
491 									\
492 	_NOTE(CONSTANTCONDITION)					\
493 	} while (B_FALSE)
494 #else
495 #define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
496 	do {								\
497 		uint32_t *addr;						\
498 									\
499 		_NOTE(CONSTANTCONDITION)				\
500 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
501 		    ("not power of 2 aligned"));			\
502 									\
503 		addr = (void *)((_esmp)->esm_base + (_offset));		\
504 									\
505 		(_eop)->eo_u32[0] = *addr++;				\
506 		(_eop)->eo_u32[1] = *addr++;				\
507 		(_eop)->eo_u32[2] = *addr++;				\
508 		(_eop)->eo_u32[3] = *addr;				\
509 									\
510 		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
511 		    uint32_t, (_eop)->eo_u32[3],			\
512 		    uint32_t, (_eop)->eo_u32[2],			\
513 		    uint32_t, (_eop)->eo_u32[1],			\
514 		    uint32_t, (_eop)->eo_u32[0]);			\
515 									\
516 	_NOTE(CONSTANTCONDITION)					\
517 	} while (B_FALSE)
518 #endif
519 
520 #define	EFSYS_MEM_WRITED(_esmp, _offset, _edp)				\
521 	do {								\
522 		uint32_t *addr;						\
523 									\
524 		_NOTE(CONSTANTCONDITION)				\
525 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
526 		    ("not power of 2 aligned"));			\
527 									\
528 		EFSYS_PROBE2(mem_writed, unsigned int, (_offset),	\
529 		    uint32_t, (_edp)->ed_u32[0]);			\
530 									\
531 		addr = (void *)((_esmp)->esm_base + (_offset));		\
532 									\
533 		*addr = (_edp)->ed_u32[0];				\
534 									\
535 	_NOTE(CONSTANTCONDITION)					\
536 	} while (B_FALSE)
537 
538 #if defined(__x86_64__)
539 #define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
540 	do {								\
541 		uint64_t *addr;						\
542 									\
543 		_NOTE(CONSTANTCONDITION)				\
544 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
545 		    ("not power of 2 aligned"));			\
546 									\
547 		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
548 		    uint32_t, (_eqp)->eq_u32[1],			\
549 		    uint32_t, (_eqp)->eq_u32[0]);			\
550 									\
551 		addr = (void *)((_esmp)->esm_base + (_offset));		\
552 									\
553 		*addr   = (_eqp)->eq_u64[0];				\
554 									\
555 	_NOTE(CONSTANTCONDITION)					\
556 	} while (B_FALSE)
557 
558 #else
559 #define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
560 	do {								\
561 		uint32_t *addr;						\
562 									\
563 		_NOTE(CONSTANTCONDITION)				\
564 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
565 		    ("not power of 2 aligned"));			\
566 									\
567 		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
568 		    uint32_t, (_eqp)->eq_u32[1],			\
569 		    uint32_t, (_eqp)->eq_u32[0]);			\
570 									\
571 		addr = (void *)((_esmp)->esm_base + (_offset));		\
572 									\
573 		*addr++ = (_eqp)->eq_u32[0];				\
574 		*addr   = (_eqp)->eq_u32[1];				\
575 									\
576 	_NOTE(CONSTANTCONDITION)					\
577 	} while (B_FALSE)
578 #endif
579 
580 #if defined(__x86_64__)
581 #define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
582 	do {								\
583 		uint64_t *addr;						\
584 									\
585 		_NOTE(CONSTANTCONDITION)				\
586 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
587 		    ("not power of 2 aligned"));			\
588 									\
589 		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
590 		    uint32_t, (_eop)->eo_u32[3],			\
591 		    uint32_t, (_eop)->eo_u32[2],			\
592 		    uint32_t, (_eop)->eo_u32[1],			\
593 		    uint32_t, (_eop)->eo_u32[0]);			\
594 									\
595 		addr = (void *)((_esmp)->esm_base + (_offset));		\
596 									\
597 		*addr++ = (_eop)->eo_u64[0];				\
598 		*addr   = (_eop)->eo_u64[1];				\
599 									\
600 	_NOTE(CONSTANTCONDITION)					\
601 	} while (B_FALSE)
602 #else
603 #define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
604 	do {								\
605 		uint32_t *addr;						\
606 									\
607 		_NOTE(CONSTANTCONDITION)				\
608 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
609 		    ("not power of 2 aligned"));			\
610 									\
611 		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
612 		    uint32_t, (_eop)->eo_u32[3],			\
613 		    uint32_t, (_eop)->eo_u32[2],			\
614 		    uint32_t, (_eop)->eo_u32[1],			\
615 		    uint32_t, (_eop)->eo_u32[0]);			\
616 									\
617 		addr = (void *)((_esmp)->esm_base + (_offset));		\
618 									\
619 		*addr++ = (_eop)->eo_u32[0];				\
620 		*addr++ = (_eop)->eo_u32[1];				\
621 		*addr++ = (_eop)->eo_u32[2];				\
622 		*addr   = (_eop)->eo_u32[3];				\
623 									\
624 	_NOTE(CONSTANTCONDITION)					\
625 	} while (B_FALSE)
626 #endif
627 
628 #define	EFSYS_MEM_ADDR(_esmp)						\
629 	((_esmp)->esm_addr)
630 
631 #define	EFSYS_MEM_IS_NULL(_esmp)					\
632 	((_esmp)->esm_base == NULL)
633 
634 /* BAR */
635 
636 #define	SFXGE_LOCK_NAME_MAX	16
637 
638 typedef struct efsys_bar_s {
639 	struct mtx		esb_lock;
640 	char			esb_lock_name[SFXGE_LOCK_NAME_MAX];
641 	bus_space_tag_t		esb_tag;
642 	bus_space_handle_t	esb_handle;
643 	int			esb_rid;
644 	struct resource		*esb_res;
645 } efsys_bar_t;
646 
647 #define	SFXGE_BAR_LOCK_INIT(_esbp, _ifname)				\
648 	do {								\
649 		snprintf((_esbp)->esb_lock_name,			\
650 			 sizeof((_esbp)->esb_lock_name),		\
651 			 "%s:bar", (_ifname));				\
652 		mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name,	\
653 			 NULL, MTX_DEF);				\
654 	_NOTE(CONSTANTCONDITION)					\
655 	} while (B_FALSE)
656 #define	SFXGE_BAR_LOCK_DESTROY(_esbp)					\
657 	mtx_destroy(&(_esbp)->esb_lock)
658 #define	SFXGE_BAR_LOCK(_esbp)						\
659 	mtx_lock(&(_esbp)->esb_lock)
660 #define	SFXGE_BAR_UNLOCK(_esbp)						\
661 	mtx_unlock(&(_esbp)->esb_lock)
662 
663 #define	EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)			\
664 	do {								\
665 		_NOTE(CONSTANTCONDITION)				\
666 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
667 		    ("not power of 2 aligned"));			\
668 									\
669 		_NOTE(CONSTANTCONDITION)				\
670 		if (_lock)						\
671 			SFXGE_BAR_LOCK(_esbp);				\
672 									\
673 		(_edp)->ed_u32[0] = bus_space_read_stream_4(		\
674 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
675 		    (_offset));						\
676 									\
677 		EFSYS_PROBE2(bar_readd, unsigned int, (_offset),	\
678 		    uint32_t, (_edp)->ed_u32[0]);			\
679 									\
680 		_NOTE(CONSTANTCONDITION)				\
681 		if (_lock)						\
682 			SFXGE_BAR_UNLOCK(_esbp);			\
683 	_NOTE(CONSTANTCONDITION)					\
684 	} while (B_FALSE)
685 
686 #if defined(SFXGE_USE_BUS_SPACE_8)
687 #define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
688 	do {								\
689 		_NOTE(CONSTANTCONDITION)				\
690 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
691 		    ("not power of 2 aligned"));			\
692 									\
693 		SFXGE_BAR_LOCK(_esbp);					\
694 									\
695 		(_eqp)->eq_u64[0] = bus_space_read_stream_8(		\
696 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
697 		    (_offset));						\
698 									\
699 		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
700 		    uint32_t, (_eqp)->eq_u32[1],			\
701 		    uint32_t, (_eqp)->eq_u32[0]);			\
702 									\
703 		SFXGE_BAR_UNLOCK(_esbp);				\
704 	_NOTE(CONSTANTCONDITION)					\
705 	} while (B_FALSE)
706 
707 #define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
708 	do {								\
709 		_NOTE(CONSTANTCONDITION)				\
710 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
711 		    ("not power of 2 aligned"));			\
712 									\
713 		_NOTE(CONSTANTCONDITION)				\
714 		if (_lock)						\
715 			SFXGE_BAR_LOCK(_esbp);				\
716 									\
717 		(_eop)->eo_u64[0] = bus_space_read_stream_8(		\
718 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
719 		    (_offset));						\
720 		(_eop)->eo_u64[1] = bus_space_read_stream_8(		\
721 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
722 		    (_offset) + 8);					\
723 									\
724 		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
725 		    uint32_t, (_eop)->eo_u32[3],			\
726 		    uint32_t, (_eop)->eo_u32[2],			\
727 		    uint32_t, (_eop)->eo_u32[1],			\
728 		    uint32_t, (_eop)->eo_u32[0]);			\
729 									\
730 		_NOTE(CONSTANTCONDITION)				\
731 		if (_lock)						\
732 			SFXGE_BAR_UNLOCK(_esbp);			\
733 	_NOTE(CONSTANTCONDITION)					\
734 	} while (B_FALSE)
735 
736 #else
737 #define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
738 	do {								\
739 		_NOTE(CONSTANTCONDITION)				\
740 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
741 		    ("not power of 2 aligned"));			\
742 									\
743 		SFXGE_BAR_LOCK(_esbp);					\
744 									\
745 		(_eqp)->eq_u32[0] = bus_space_read_stream_4(		\
746 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
747 		    (_offset));						\
748 		(_eqp)->eq_u32[1] = bus_space_read_stream_4(		\
749 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
750 		    (_offset) + 4);					\
751 									\
752 		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
753 		    uint32_t, (_eqp)->eq_u32[1],			\
754 		    uint32_t, (_eqp)->eq_u32[0]);			\
755 									\
756 		SFXGE_BAR_UNLOCK(_esbp);				\
757 	_NOTE(CONSTANTCONDITION)					\
758 	} while (B_FALSE)
759 
760 #define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
761 	do {								\
762 		_NOTE(CONSTANTCONDITION)				\
763 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
764 		    ("not power of 2 aligned"));			\
765 									\
766 		_NOTE(CONSTANTCONDITION)				\
767 		if (_lock)						\
768 			SFXGE_BAR_LOCK(_esbp);				\
769 									\
770 		(_eop)->eo_u32[0] = bus_space_read_stream_4(		\
771 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
772 		    (_offset));						\
773 		(_eop)->eo_u32[1] = bus_space_read_stream_4(		\
774 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
775 		    (_offset) + 4);					\
776 		(_eop)->eo_u32[2] = bus_space_read_stream_4(		\
777 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
778 		    (_offset) + 8);					\
779 		(_eop)->eo_u32[3] = bus_space_read_stream_4(		\
780 		    (_esbp)->esb_tag, (_esbp)->esb_handle,		\
781 		    (_offset) + 12);					\
782 									\
783 		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
784 		    uint32_t, (_eop)->eo_u32[3],			\
785 		    uint32_t, (_eop)->eo_u32[2],			\
786 		    uint32_t, (_eop)->eo_u32[1],			\
787 		    uint32_t, (_eop)->eo_u32[0]);			\
788 									\
789 		_NOTE(CONSTANTCONDITION)				\
790 		if (_lock)						\
791 			SFXGE_BAR_UNLOCK(_esbp);			\
792 	_NOTE(CONSTANTCONDITION)					\
793 	} while (B_FALSE)
794 #endif
795 
796 #define	EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)			\
797 	do {								\
798 		_NOTE(CONSTANTCONDITION)				\
799 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
800 		    ("not power of 2 aligned"));			\
801 									\
802 		_NOTE(CONSTANTCONDITION)				\
803 		if (_lock)						\
804 			SFXGE_BAR_LOCK(_esbp);				\
805 									\
806 		EFSYS_PROBE2(bar_writed, unsigned int, (_offset),	\
807 		    uint32_t, (_edp)->ed_u32[0]);			\
808 									\
809 		/*							\
810 		 * Make sure that previous writes to the dword have	\
811 		 * been done. It should be cheaper than barrier just	\
812 		 * after the write below.				\
813 		 */							\
814 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
815 		    (_offset), sizeof (efx_dword_t),			\
816 		    BUS_SPACE_BARRIER_WRITE);				\
817 		bus_space_write_stream_4((_esbp)->esb_tag,		\
818 		    (_esbp)->esb_handle,				\
819 		    (_offset), (_edp)->ed_u32[0]);			\
820 									\
821 		_NOTE(CONSTANTCONDITION)				\
822 		if (_lock)						\
823 			SFXGE_BAR_UNLOCK(_esbp);			\
824 	_NOTE(CONSTANTCONDITION)					\
825 	} while (B_FALSE)
826 
827 #if defined(SFXGE_USE_BUS_SPACE_8)
828 #define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
829 	do {								\
830 		_NOTE(CONSTANTCONDITION)				\
831 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
832 		    ("not power of 2 aligned"));			\
833 									\
834 		SFXGE_BAR_LOCK(_esbp);					\
835 									\
836 		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
837 		    uint32_t, (_eqp)->eq_u32[1],			\
838 		    uint32_t, (_eqp)->eq_u32[0]);			\
839 									\
840 		/*							\
841 		 * Make sure that previous writes to the qword have	\
842 		 * been done. It should be cheaper than barrier just	\
843 		 * after the write below.				\
844 		 */							\
845 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
846 		    (_offset), sizeof (efx_qword_t),			\
847 		    BUS_SPACE_BARRIER_WRITE);				\
848 		bus_space_write_stream_8((_esbp)->esb_tag,		\
849 		    (_esbp)->esb_handle,				\
850 		    (_offset), (_eqp)->eq_u64[0]);			\
851 									\
852 		SFXGE_BAR_UNLOCK(_esbp);				\
853 	_NOTE(CONSTANTCONDITION)					\
854 	} while (B_FALSE)
855 #else
856 #define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
857 	do {								\
858 		_NOTE(CONSTANTCONDITION)				\
859 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
860 		    ("not power of 2 aligned"));			\
861 									\
862 		SFXGE_BAR_LOCK(_esbp);					\
863 									\
864 		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
865 		    uint32_t, (_eqp)->eq_u32[1],			\
866 		    uint32_t, (_eqp)->eq_u32[0]);			\
867 									\
868 		/*							\
869 		 * Make sure that previous writes to the qword have	\
870 		 * been done. It should be cheaper than barrier just	\
871 		 * after the last write below.				\
872 		 */							\
873 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
874 		    (_offset), sizeof (efx_qword_t),			\
875 		    BUS_SPACE_BARRIER_WRITE);				\
876 		bus_space_write_stream_4((_esbp)->esb_tag,		\
877 		    (_esbp)->esb_handle,				\
878 		    (_offset), (_eqp)->eq_u32[0]);			\
879 		/*							\
880 		 * It should be guaranteed that the last dword comes	\
881 		 * the last, so barrier entire qword to be sure that	\
882 		 * neither above nor below writes are reordered.	\
883 		 */							\
884 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
885 		    (_offset), sizeof (efx_qword_t),			\
886 		    BUS_SPACE_BARRIER_WRITE);				\
887 		bus_space_write_stream_4((_esbp)->esb_tag,		\
888 		    (_esbp)->esb_handle,				\
889 		    (_offset) + 4, (_eqp)->eq_u32[1]);			\
890 									\
891 		SFXGE_BAR_UNLOCK(_esbp);				\
892 	_NOTE(CONSTANTCONDITION)					\
893 	} while (B_FALSE)
894 #endif
895 
896 /*
897  * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
898  * (required by PIO hardware)
899  */
900 #define	EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp)			\
901 	do {								\
902 		_NOTE(CONSTANTCONDITION)				\
903 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
904 		    ("not power of 2 aligned"));			\
905 									\
906 		(void) (_esbp);						\
907 									\
908 		/* FIXME: Perform a 64-bit write */			\
909 		KASSERT(0, ("not implemented"));			\
910 									\
911 	_NOTE(CONSTANTCONDITION)					\
912 	} while (B_FALSE)
913 
914 #if defined(SFXGE_USE_BUS_SPACE_8)
915 #define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
916 	do {								\
917 		_NOTE(CONSTANTCONDITION)				\
918 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
919 		    ("not power of 2 aligned"));			\
920 									\
921 		_NOTE(CONSTANTCONDITION)				\
922 		if (_lock)						\
923 			SFXGE_BAR_LOCK(_esbp);				\
924 									\
925 		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
926 		    uint32_t, (_eop)->eo_u32[3],			\
927 		    uint32_t, (_eop)->eo_u32[2],			\
928 		    uint32_t, (_eop)->eo_u32[1],			\
929 		    uint32_t, (_eop)->eo_u32[0]);			\
930 									\
931 		/*							\
932 		 * Make sure that previous writes to the oword have	\
933 		 * been done. It should be cheaper than barrier just	\
934 		 * after the last write below.				\
935 		 */							\
936 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
937 		    (_offset), sizeof (efx_oword_t),			\
938 		    BUS_SPACE_BARRIER_WRITE);				\
939 		bus_space_write_stream_8((_esbp)->esb_tag,		\
940 		    (_esbp)->esb_handle,				\
941 		    (_offset), (_eop)->eo_u64[0]);			\
942 		/*							\
943 		 * It should be guaranteed that the last qword comes	\
944 		 * the last, so barrier entire oword to be sure that	\
945 		 * neither above nor below writes are reordered.	\
946 		 */							\
947 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
948 		    (_offset), sizeof (efx_oword_t),			\
949 		    BUS_SPACE_BARRIER_WRITE);				\
950 		bus_space_write_stream_8((_esbp)->esb_tag,		\
951 		    (_esbp)->esb_handle,				\
952 		    (_offset) + 8, (_eop)->eo_u64[1]);			\
953 									\
954 		_NOTE(CONSTANTCONDITION)				\
955 		if (_lock)						\
956 			SFXGE_BAR_UNLOCK(_esbp);			\
957 	_NOTE(CONSTANTCONDITION)					\
958 	} while (B_FALSE)
959 
960 #else
961 #define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
962 	do {								\
963 		_NOTE(CONSTANTCONDITION)				\
964 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
965 		    ("not power of 2 aligned"));			\
966 									\
967 		_NOTE(CONSTANTCONDITION)				\
968 		if (_lock)						\
969 			SFXGE_BAR_LOCK(_esbp);				\
970 									\
971 		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
972 		    uint32_t, (_eop)->eo_u32[3],			\
973 		    uint32_t, (_eop)->eo_u32[2],			\
974 		    uint32_t, (_eop)->eo_u32[1],			\
975 		    uint32_t, (_eop)->eo_u32[0]);			\
976 									\
977 		/*							\
978 		 * Make sure that previous writes to the oword have	\
979 		 * been done. It should be cheaper than barrier just	\
980 		 * after the last write below.				\
981 		 */							\
982 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
983 		    (_offset), sizeof (efx_oword_t),			\
984 		    BUS_SPACE_BARRIER_WRITE);				\
985 		bus_space_write_stream_4((_esbp)->esb_tag,		\
986 		    (_esbp)->esb_handle,				\
987 		    (_offset), (_eop)->eo_u32[0]);			\
988 		bus_space_write_stream_4((_esbp)->esb_tag,		\
989 		    (_esbp)->esb_handle,				\
990 		    (_offset) + 4, (_eop)->eo_u32[1]);			\
991 		bus_space_write_stream_4((_esbp)->esb_tag,		\
992 		    (_esbp)->esb_handle,				\
993 		    (_offset) + 8, (_eop)->eo_u32[2]);			\
994 		/*							\
995 		 * It should be guaranteed that the last dword comes	\
996 		 * the last, so barrier entire oword to be sure that	\
997 		 * neither above nor below writes are reordered.	\
998 		 */							\
999 		bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
1000 		    (_offset), sizeof (efx_oword_t),			\
1001 		    BUS_SPACE_BARRIER_WRITE);				\
1002 		bus_space_write_stream_4((_esbp)->esb_tag,		\
1003 		    (_esbp)->esb_handle,				\
1004 		    (_offset) + 12, (_eop)->eo_u32[3]);			\
1005 									\
1006 		_NOTE(CONSTANTCONDITION)				\
1007 		if (_lock)						\
1008 			SFXGE_BAR_UNLOCK(_esbp);			\
1009 	_NOTE(CONSTANTCONDITION)					\
1010 	} while (B_FALSE)
1011 #endif
1012 
1013 /* Use the standard octo-word write for doorbell writes */
1014 #define	EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop)			\
1015 	do {								\
1016 		EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE);	\
1017 	_NOTE(CONSTANTCONDITION)					\
1018 	} while (B_FALSE)
1019 
1020 /* SPIN */
1021 
1022 #define	EFSYS_SPIN(_us)							\
1023 	do {								\
1024 		DELAY(_us);						\
1025 	_NOTE(CONSTANTCONDITION)					\
1026 	} while (B_FALSE)
1027 
1028 #define	EFSYS_SLEEP	EFSYS_SPIN
1029 
1030 /* BARRIERS */
1031 
1032 #define	EFSYS_MEM_READ_BARRIER()	rmb()
1033 #define	EFSYS_PIO_WRITE_BARRIER()
1034 
1035 /* DMA SYNC */
1036 #define	EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size)		\
1037 	do {								\
1038 		bus_dmamap_sync((_esmp)->esm_tag,			\
1039 		    (_esmp)->esm_map,					\
1040 		    BUS_DMASYNC_POSTREAD);				\
1041 	_NOTE(CONSTANTCONDITION)					\
1042 	} while (B_FALSE)
1043 
1044 #define	EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size)		\
1045 	do {								\
1046 		bus_dmamap_sync((_esmp)->esm_tag,			\
1047 		    (_esmp)->esm_map,					\
1048 		    BUS_DMASYNC_PREWRITE);				\
1049 	_NOTE(CONSTANTCONDITION)					\
1050 	} while (B_FALSE)
1051 
1052 /* TIMESTAMP */
1053 
1054 typedef	clock_t	efsys_timestamp_t;
1055 
1056 #define	EFSYS_TIMESTAMP(_usp)						\
1057 	do {								\
1058 		clock_t now;						\
1059 									\
1060 		now = ticks;						\
1061 		*(_usp) = now * hz / 1000000;				\
1062 	_NOTE(CONSTANTCONDITION)					\
1063 	} while (B_FALSE)
1064 
1065 /* KMEM */
1066 
1067 #define	EFSYS_KMEM_ALLOC(_esip, _size, _p)				\
1068 	do {								\
1069 		(_esip) = (_esip);					\
1070 		/*							\
1071 		 * The macro is used in non-sleepable contexts, for	\
1072 		 * example, holding a mutex.				\
1073 		 */							\
1074 		(_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO);	\
1075 	_NOTE(CONSTANTCONDITION)					\
1076 	} while (B_FALSE)
1077 
1078 #define	EFSYS_KMEM_FREE(_esip, _size, _p)				\
1079 	do {								\
1080 		(void) (_esip);						\
1081 		(void) (_size);						\
1082 		free((_p), M_SFXGE);					\
1083 	_NOTE(CONSTANTCONDITION)					\
1084 	} while (B_FALSE)
1085 
1086 /* LOCK */
1087 
1088 typedef struct efsys_lock_s {
1089 	struct mtx	lock;
1090 	char		lock_name[SFXGE_LOCK_NAME_MAX];
1091 } efsys_lock_t;
1092 
1093 #define	SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label)			\
1094 	do {								\
1095 		efsys_lock_t *__eslp = (_eslp);				\
1096 									\
1097 		snprintf((__eslp)->lock_name,				\
1098 			 sizeof((__eslp)->lock_name),			\
1099 			 "%s:%s", (_ifname), (_label));			\
1100 		mtx_init(&(__eslp)->lock, (__eslp)->lock_name,		\
1101 			 NULL, MTX_DEF);				\
1102 	} while (B_FALSE)
1103 #define	SFXGE_EFSYS_LOCK_DESTROY(_eslp)					\
1104 	mtx_destroy(&(_eslp)->lock)
1105 #define	SFXGE_EFSYS_LOCK(_eslp)						\
1106 	mtx_lock(&(_eslp)->lock)
1107 #define	SFXGE_EFSYS_UNLOCK(_eslp)					\
1108 	mtx_unlock(&(_eslp)->lock)
1109 #define	SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp)				\
1110 	mtx_assert(&(_eslp)->lock, MA_OWNED)
1111 
1112 #define	EFSYS_LOCK_MAGIC	0x000010c4
1113 
1114 #define	EFSYS_LOCK(_lockp, _state)					\
1115 	do {								\
1116 		SFXGE_EFSYS_LOCK(_lockp);				\
1117 		(_state) = EFSYS_LOCK_MAGIC;				\
1118 	_NOTE(CONSTANTCONDITION)					\
1119 	} while (B_FALSE)
1120 
1121 #define	EFSYS_UNLOCK(_lockp, _state)					\
1122 	do {								\
1123 		if ((_state) != EFSYS_LOCK_MAGIC)			\
1124 			KASSERT(B_FALSE, ("not locked"));		\
1125 		SFXGE_EFSYS_UNLOCK(_lockp);				\
1126 	_NOTE(CONSTANTCONDITION)					\
1127 	} while (B_FALSE)
1128 
1129 /* PREEMPT */
1130 
1131 #define	EFSYS_PREEMPT_DISABLE(_state)					\
1132 	do {								\
1133 		(_state) = (_state);					\
1134 		critical_enter();					\
1135 	_NOTE(CONSTANTCONDITION)					\
1136 	} while (B_FALSE)
1137 
1138 #define	EFSYS_PREEMPT_ENABLE(_state)					\
1139 	do {								\
1140 		(_state) = (_state);					\
1141 		critical_exit(_state);					\
1142 	_NOTE(CONSTANTCONDITION)					\
1143 	} while (B_FALSE)
1144 
1145 /* STAT */
1146 
1147 typedef uint64_t		efsys_stat_t;
1148 
1149 #define	EFSYS_STAT_INCR(_knp, _delta) 					\
1150 	do {								\
1151 		*(_knp) += (_delta);					\
1152 	_NOTE(CONSTANTCONDITION)					\
1153 	} while (B_FALSE)
1154 
1155 #define	EFSYS_STAT_DECR(_knp, _delta) 					\
1156 	do {								\
1157 		*(_knp) -= (_delta);					\
1158 	_NOTE(CONSTANTCONDITION)					\
1159 	} while (B_FALSE)
1160 
1161 #define	EFSYS_STAT_SET(_knp, _val)					\
1162 	do {								\
1163 		*(_knp) = (_val);					\
1164 	_NOTE(CONSTANTCONDITION)					\
1165 	} while (B_FALSE)
1166 
1167 #define	EFSYS_STAT_SET_QWORD(_knp, _valp)				\
1168 	do {								\
1169 		*(_knp) = le64toh((_valp)->eq_u64[0]);			\
1170 	_NOTE(CONSTANTCONDITION)					\
1171 	} while (B_FALSE)
1172 
1173 #define	EFSYS_STAT_SET_DWORD(_knp, _valp)				\
1174 	do {								\
1175 		*(_knp) = le32toh((_valp)->ed_u32[0]);			\
1176 	_NOTE(CONSTANTCONDITION)					\
1177 	} while (B_FALSE)
1178 
1179 #define	EFSYS_STAT_INCR_QWORD(_knp, _valp)				\
1180 	do {								\
1181 		*(_knp) += le64toh((_valp)->eq_u64[0]);			\
1182 	_NOTE(CONSTANTCONDITION)					\
1183 	} while (B_FALSE)
1184 
1185 #define	EFSYS_STAT_SUBR_QWORD(_knp, _valp)				\
1186 	do {								\
1187 		*(_knp) -= le64toh((_valp)->eq_u64[0]);			\
1188 	_NOTE(CONSTANTCONDITION)					\
1189 	} while (B_FALSE)
1190 
1191 /* ERR */
1192 
1193 extern void	sfxge_err(efsys_identifier_t *, unsigned int,
1194 		    uint32_t, uint32_t);
1195 
1196 #if EFSYS_OPT_DECODE_INTR_FATAL
1197 #define	EFSYS_ERR(_esip, _code, _dword0, _dword1)			\
1198 	do {								\
1199 		sfxge_err((_esip), (_code), (_dword0), (_dword1));	\
1200 	_NOTE(CONSTANTCONDITION)					\
1201 	} while (B_FALSE)
1202 #endif
1203 
1204 /* ASSERT */
1205 
1206 #define	EFSYS_ASSERT(_exp) do {						\
1207 	if (!(_exp))							\
1208 		panic("%s", #_exp);					\
1209 	} while (0)
1210 
1211 #define	EFSYS_ASSERT3(_x, _op, _y, _t) do {				\
1212 	const _t __x = (_t)(_x);					\
1213 	const _t __y = (_t)(_y);					\
1214 	if (!(__x _op __y))						\
1215 		panic("assertion failed at %s:%u", __FILE__, __LINE__);	\
1216 	} while(0)
1217 
1218 #define	EFSYS_ASSERT3U(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1219 #define	EFSYS_ASSERT3S(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, int64_t)
1220 #define	EFSYS_ASSERT3P(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1221 
1222 /* ROTATE */
1223 
1224 #define	EFSYS_HAS_ROTL_DWORD 0
1225 
1226 #ifdef	__cplusplus
1227 }
1228 #endif
1229 
1230 #endif	/* _SYS_EFSYS_H */
1231