xref: /freebsd/sys/dev/sfxge/common/efsys.h (revision bb15ca603fa442c72dde3f3cb8b46db6970e3950)
1 /*-
2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
3  * All rights reserved.
4  *
5  * This software was developed in part by Philip Paeps under contract for
6  * Solarflare Communications, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #ifndef	_SYS_EFSYS_H
33 #define	_SYS_EFSYS_H
34 
35 #ifdef	__cplusplus
36 extern "C" {
37 #endif
38 
39 #include <sys/param.h>
40 #include <sys/bus.h>
41 #include <sys/endian.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/mbuf.h>
45 #include <sys/mutex.h>
46 #include <sys/rwlock.h>
47 #include <sys/sdt.h>
48 #include <sys/systm.h>
49 
50 #include <machine/bus.h>
51 #include <machine/endian.h>
52 
53 #define	EFSYS_HAS_UINT64 1
54 #define	EFSYS_USE_UINT64 0
55 #if _BYTE_ORDER == _BIG_ENDIAN
56 #define EFSYS_IS_BIG_ENDIAN 1
57 #define EFSYS_IS_LITTLE_ENDIAN 0
58 #elif _BYTE_ORDER == _LITTLE_ENDIAN
59 #define EFSYS_IS_BIG_ENDIAN 0
60 #define EFSYS_IS_LITTLE_ENDIAN 1
61 #endif
62 #include "efx_types.h"
63 
64 /* Common code requires this */
65 #if __FreeBSD_version < 800068
66 #define memmove(d, s, l) bcopy(s, d, l)
67 #endif
68 
69 /* FreeBSD equivalents of Solaris things */
70 #ifndef _NOTE
71 #define _NOTE(s)
72 #endif
73 
74 #ifndef B_FALSE
75 #define B_FALSE FALSE
76 #endif
77 #ifndef B_TRUE
78 #define B_TRUE TRUE
79 #endif
80 
81 #ifndef IS_P2ALIGNED
82 #define	IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
83 #endif
84 
85 #ifndef P2ROUNDUP
86 #define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
87 #endif
88 
89 #ifndef IS2P
90 #define ISP2(x)         (((x) & ((x) - 1)) == 0)
91 #endif
92 
93 #define ENOTACTIVE EINVAL
94 
95 /* Memory type to use on FreeBSD */
96 MALLOC_DECLARE(M_SFXGE);
97 
98 /* Machine dependend prefetch wrappers */
99 #if defined(__i386__) || defined(__amd64__)
100 static __inline void
101 prefetch_read_many(void *addr)
102 {
103 
104 	__asm__(
105 	    "prefetcht0 (%0)"
106 	    :
107 	    : "r" (addr));
108 }
109 
110 static __inline void
111 prefetch_read_once(void *addr)
112 {
113 
114 	__asm__(
115 	    "prefetchnta (%0)"
116 	    :
117 	    : "r" (addr));
118 }
119 #elif defined(__sparc64__)
120 static __inline void
121 prefetch_read_many(void *addr)
122 {
123 
124 	__asm__(
125 	    "prefetch [%0], 0"
126 	    :
127 	    : "r" (addr));
128 }
129 
130 static __inline void
131 prefetch_read_once(void *addr)
132 {
133 
134 	__asm__(
135 	    "prefetch [%0], 1"
136 	    :
137 	    : "r" (addr));
138 }
139 #else
140 static __inline void
141 prefetch_read_many(void *addr)
142 {
143 
144 }
145 
146 static __inline void
147 prefetch_read_once(void *addr)
148 {
149 
150 }
151 #endif
152 
153 #if defined(__i386__) || defined(__amd64__)
154 #include <vm/vm.h>
155 #include <vm/pmap.h>
156 #endif
157 static __inline void
158 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
159     struct mbuf *m, bus_dma_segment_t *seg)
160 {
161 #if defined(__i386__) || defined(__amd64__)
162 	seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
163 	seg->ds_len = m->m_len;
164 #else
165 	int nsegstmp;
166 
167 	bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
168 #endif
169 }
170 
171 /* Modifiers used for DOS builds */
172 #define	__cs
173 #define	__far
174 
175 /* Modifiers used for Windows builds */
176 #define	__in
177 #define	__in_opt
178 #define	__in_ecount(_n)
179 #define	__in_ecount_opt(_n)
180 #define	__in_bcount(_n)
181 #define	__in_bcount_opt(_n)
182 
183 #define	__out
184 #define	__out_opt
185 #define	__out_ecount(_n)
186 #define	__out_ecount_opt(_n)
187 #define	__out_bcount(_n)
188 #define	__out_bcount_opt(_n)
189 
190 #define	__deref_out
191 
192 #define	__inout
193 #define	__inout_opt
194 #define	__inout_ecount(_n)
195 #define	__inout_ecount_opt(_n)
196 #define	__inout_bcount(_n)
197 #define	__inout_bcount_opt(_n)
198 #define	__inout_bcount_full_opt(_n)
199 
200 #define	__deref_out_bcount_opt(n)
201 
202 #define	__checkReturn
203 
204 #define	__drv_when(_p, _c)
205 
206 /* Code inclusion options */
207 
208 
209 #define	EFSYS_OPT_NAMES 1
210 
211 #define	EFSYS_OPT_FALCON 0
212 #define	EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
213 #define	EFSYS_OPT_SIENA 1
214 #ifdef DEBUG
215 #define	EFSYS_OPT_CHECK_REG 1
216 #else
217 #define	EFSYS_OPT_CHECK_REG 0
218 #endif
219 
220 #define	EFSYS_OPT_MCDI 1
221 
222 #define	EFSYS_OPT_MAC_FALCON_GMAC 0
223 #define	EFSYS_OPT_MAC_FALCON_XMAC 0
224 #define	EFSYS_OPT_MAC_STATS 1
225 
226 #define	EFSYS_OPT_LOOPBACK 0
227 
228 #define	EFSYS_OPT_MON_NULL 0
229 #define	EFSYS_OPT_MON_LM87 0
230 #define	EFSYS_OPT_MON_MAX6647 0
231 #define	EFSYS_OPT_MON_SIENA 0
232 #define	EFSYS_OPT_MON_STATS 0
233 
234 #define	EFSYS_OPT_PHY_NULL 0
235 #define	EFSYS_OPT_PHY_QT2022C2 0
236 #define	EFSYS_OPT_PHY_SFX7101 0
237 #define	EFSYS_OPT_PHY_TXC43128 0
238 #define	EFSYS_OPT_PHY_PM8358 0
239 #define	EFSYS_OPT_PHY_SFT9001 0
240 #define	EFSYS_OPT_PHY_QT2025C 0
241 #define	EFSYS_OPT_PHY_STATS 1
242 #define	EFSYS_OPT_PHY_PROPS 0
243 #define	EFSYS_OPT_PHY_BIST 1
244 #define	EFSYS_OPT_PHY_LED_CONTROL 1
245 #define EFSYS_OPT_PHY_FLAGS 0
246 
247 #define	EFSYS_OPT_VPD 1
248 #define	EFSYS_OPT_NVRAM 1
249 #define	EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
250 #define	EFSYS_OPT_NVRAM_SFT9001	0
251 #define	EFSYS_OPT_NVRAM_SFX7101	0
252 #define	EFSYS_OPT_BOOTCFG 0
253 
254 #define	EFSYS_OPT_PCIE_TUNE 0
255 #define	EFSYS_OPT_DIAG 0
256 #define	EFSYS_OPT_WOL 1
257 #define	EFSYS_OPT_RX_SCALE 1
258 #define	EFSYS_OPT_QSTATS 1
259 #define EFSYS_OPT_FILTER 0
260 #define EFSYS_OPT_RX_SCATTER 0
261 #define	EFSYS_OPT_RX_HDR_SPLIT 0
262 
263 #define	EFSYS_OPT_EV_PREFETCH 0
264 
265 #define	EFSYS_OPT_DECODE_INTR_FATAL 1
266 
267 /* ID */
268 
269 typedef struct __efsys_identifier_s	efsys_identifier_t;
270 
271 /* PROBE */
272 
273 #ifndef KDTRACE_HOOKS
274 
275 #define EFSYS_PROBE(_name)
276 
277 #define	EFSYS_PROBE1(_name, _type1, _arg1)
278 
279 #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
280 
281 #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
282 	    _type3, _arg3)
283 
284 #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
285 	    _type3, _arg3, _type4, _arg4)
286 
287 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
288 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
289 
290 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
291 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
292 	    _type6, _arg6)
293 
294 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
295 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
296 	    _type6, _arg6, _type7, _arg7)
297 
298 #else /* KDTRACE_HOOKS */
299 
300 #define	EFSYS_PROBE(_name)						\
301 	DTRACE_PROBE(_name)
302 
303 #define	EFSYS_PROBE1(_name, _type1, _arg1)				\
304 	DTRACE_PROBE1(_name, _type1, _arg1)
305 
306 #define	EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)		\
307 	DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
308 
309 #define	EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
310 	    _type3, _arg3)						\
311 	DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,		\
312 	    _type3, _arg3)
313 
314 #define	EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
315 	    _type3, _arg3, _type4, _arg4)				\
316 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
317 	    _type3, _arg3, _type4, _arg4)
318 
319 #ifdef DTRACE_PROBE5
320 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
321 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
322 	DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
323 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
324 #else
325 #define	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
326 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)		\
327 	DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,		\
328 	    _type3, _arg3, _type4, _arg4)
329 #endif
330 
331 #ifdef DTRACE_PROBE6
332 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
333 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
334 	    _type6, _arg6)						\
335 	DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
336 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
337 	    _type6, _arg6)
338 #else
339 #define	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
340 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
341 	    _type6, _arg6)						\
342 	EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,		\
343 	    _type3, _arg3, _type4, _arg4, _type5, _arg5)
344 #endif
345 
346 #ifdef DTRACE_PROBE7
347 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
348 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
349 	    _type6, _arg6, _type7, _arg7)				\
350 	DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
351 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
352 	    _type6, _arg6, _type7, _arg7)
353 #else
354 #define	EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,		\
355 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
356 	    _type6, _arg6, _type7, _arg7)				\
357 	EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,		\
358 	    _type3, _arg3, _type4, _arg4, _type5, _arg5,		\
359 	    _type6, _arg6)
360 #endif
361 
362 #endif /* KDTRACE_HOOKS */
363 
364 /* DMA */
365 
366 typedef uint64_t		efsys_dma_addr_t;
367 
368 typedef struct efsys_mem_s {
369 	bus_dma_tag_t		esm_tag;
370 	bus_dmamap_t		esm_map;
371 	caddr_t			esm_base;
372 	efsys_dma_addr_t	esm_addr;
373 	size_t			esm_size;
374 } efsys_mem_t;
375 
376 
377 #define	EFSYS_MEM_ZERO(_esmp, _size)					\
378 	do {								\
379 		(void) memset((_esmp)->esm_base, 0, (_size));		\
380 									\
381 	_NOTE(CONSTANTCONDITION)					\
382 	} while (B_FALSE)
383 
384 #define	EFSYS_MEM_READD(_esmp, _offset, _edp)				\
385 	do {								\
386 		uint32_t *addr;						\
387 									\
388 		_NOTE(CONSTANTCONDITION)				\
389 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
390 		    ("not power of 2 aligned"));			\
391 									\
392 		addr = (void *)((_esmp)->esm_base + (_offset));		\
393 									\
394 		(_edp)->ed_u32[0] = *addr;				\
395 									\
396 		EFSYS_PROBE2(mem_readd, unsigned int, (_offset),	\
397 		    uint32_t, (_edp)->ed_u32[0]);			\
398 									\
399 	_NOTE(CONSTANTCONDITION)					\
400 	} while (B_FALSE)
401 
402 #define	EFSYS_MEM_READQ(_esmp, _offset, _eqp)				\
403 	do {								\
404 		uint32_t *addr;						\
405 									\
406 		_NOTE(CONSTANTCONDITION)				\
407 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
408 		    ("not power of 2 aligned"));			\
409 									\
410 		addr = (void *)((_esmp)->esm_base + (_offset));		\
411 									\
412 		(_eqp)->eq_u32[0] = *addr++;				\
413 		(_eqp)->eq_u32[1] = *addr;				\
414 									\
415 		EFSYS_PROBE3(mem_readq, unsigned int, (_offset),	\
416 		    uint32_t, (_eqp)->eq_u32[1],			\
417 		    uint32_t, (_eqp)->eq_u32[0]);			\
418 									\
419 	_NOTE(CONSTANTCONDITION)					\
420 	} while (B_FALSE)
421 
422 #define	EFSYS_MEM_READO(_esmp, _offset, _eop)				\
423 	do {								\
424 		uint32_t *addr;						\
425 									\
426 		_NOTE(CONSTANTCONDITION)				\
427 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
428 		    ("not power of 2 aligned"));			\
429 									\
430 		addr = (void *)((_esmp)->esm_base + (_offset));		\
431 									\
432 		(_eop)->eo_u32[0] = *addr++;				\
433 		(_eop)->eo_u32[1] = *addr++;				\
434 		(_eop)->eo_u32[2] = *addr++;				\
435 		(_eop)->eo_u32[3] = *addr;				\
436 									\
437 		EFSYS_PROBE5(mem_reado, unsigned int, (_offset),	\
438 		    uint32_t, (_eop)->eo_u32[3],			\
439 		    uint32_t, (_eop)->eo_u32[2],			\
440 		    uint32_t, (_eop)->eo_u32[1],			\
441 		    uint32_t, (_eop)->eo_u32[0]);			\
442 									\
443 	_NOTE(CONSTANTCONDITION)					\
444 	} while (B_FALSE)
445 
446 #define	EFSYS_MEM_WRITED(_esmp, _offset, _edp)				\
447 	do {								\
448 		uint32_t *addr;						\
449 									\
450 		_NOTE(CONSTANTCONDITION)				\
451 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
452 		    ("not power of 2 aligned"));			\
453 									\
454 		EFSYS_PROBE2(mem_writed, unsigned int, (_offset),	\
455 		    uint32_t, (_edp)->ed_u32[0]);			\
456 									\
457 		addr = (void *)((_esmp)->esm_base + (_offset));		\
458 									\
459 		*addr = (_edp)->ed_u32[0];				\
460 									\
461 	_NOTE(CONSTANTCONDITION)					\
462 	} while (B_FALSE)
463 
464 #define	EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)				\
465 	do {								\
466 		uint32_t *addr;						\
467 									\
468 		_NOTE(CONSTANTCONDITION)				\
469 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
470 		    ("not power of 2 aligned"));			\
471 									\
472 		EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),	\
473 		    uint32_t, (_eqp)->eq_u32[1],			\
474 		    uint32_t, (_eqp)->eq_u32[0]);			\
475 									\
476 		addr = (void *)((_esmp)->esm_base + (_offset));		\
477 									\
478 		*addr++ = (_eqp)->eq_u32[0];				\
479 		*addr   = (_eqp)->eq_u32[1];				\
480 									\
481 	_NOTE(CONSTANTCONDITION)					\
482 	} while (B_FALSE)
483 
484 #define	EFSYS_MEM_WRITEO(_esmp, _offset, _eop)				\
485 	do {								\
486 		uint32_t *addr;						\
487 									\
488 		_NOTE(CONSTANTCONDITION)				\
489 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
490 		    ("not power of 2 aligned"));			\
491 									\
492 		EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),	\
493 		    uint32_t, (_eop)->eo_u32[3],			\
494 		    uint32_t, (_eop)->eo_u32[2],			\
495 		    uint32_t, (_eop)->eo_u32[1],			\
496 		    uint32_t, (_eop)->eo_u32[0]);			\
497 									\
498 		addr = (void *)((_esmp)->esm_base + (_offset));		\
499 									\
500 		*addr++ = (_eop)->eo_u32[0];				\
501 		*addr++ = (_eop)->eo_u32[1];				\
502 		*addr++ = (_eop)->eo_u32[2];				\
503 		*addr   = (_eop)->eo_u32[3];				\
504 									\
505 	_NOTE(CONSTANTCONDITION)					\
506 	} while (B_FALSE)
507 
508 #define	EFSYS_MEM_ADDR(_esmp)						\
509 	((_esmp)->esm_addr)
510 
511 /* BAR */
512 
513 typedef struct efsys_bar_s {
514 	struct mtx		esb_lock;
515 	bus_space_tag_t		esb_tag;
516 	bus_space_handle_t	esb_handle;
517 	int			esb_rid;
518 	struct resource		*esb_res;
519 } efsys_bar_t;
520 
521 #define	EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)			\
522 	do {								\
523 		_NOTE(CONSTANTCONDITION)				\
524 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
525 		    ("not power of 2 aligned"));			\
526 									\
527 		_NOTE(CONSTANTCONDITION)				\
528 		if (_lock)						\
529 			mtx_lock(&((_esbp)->esb_lock));			\
530 									\
531 		(_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
532 		    (_esbp)->esb_handle, (_offset));			\
533 									\
534 		EFSYS_PROBE2(bar_readd, unsigned int, (_offset),	\
535 		    uint32_t, (_edp)->ed_u32[0]);			\
536 									\
537 		_NOTE(CONSTANTCONDITION)				\
538 		if (_lock)						\
539 			mtx_unlock(&((_esbp)->esb_lock));		\
540 	_NOTE(CONSTANTCONDITION)					\
541 	} while (B_FALSE)
542 
543 #define	EFSYS_BAR_READQ(_esbp, _offset, _eqp)				\
544 	do {								\
545 		_NOTE(CONSTANTCONDITION)				\
546 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
547 		    ("not power of 2 aligned"));			\
548 									\
549 		mtx_lock(&((_esbp)->esb_lock));				\
550 									\
551 		(_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
552 		    (_esbp)->esb_handle, (_offset));			\
553 		(_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag,	\
554 		    (_esbp)->esb_handle, (_offset+4));			\
555 									\
556 		EFSYS_PROBE3(bar_readq, unsigned int, (_offset),	\
557 		    uint32_t, (_eqp)->eq_u32[1],			\
558 		    uint32_t, (_eqp)->eq_u32[0]);			\
559 									\
560 		mtx_unlock(&((_esbp)->esb_lock));			\
561 	_NOTE(CONSTANTCONDITION)					\
562 	} while (B_FALSE)
563 
564 #define	EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)			\
565 	do {								\
566 		_NOTE(CONSTANTCONDITION)				\
567 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
568 		    ("not power of 2 aligned"));			\
569 									\
570 		_NOTE(CONSTANTCONDITION)				\
571 		if (_lock)						\
572 			mtx_lock(&((_esbp)->esb_lock));			\
573 									\
574 		(_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag,	\
575 		    (_esbp)->esb_handle, (_offset));			\
576 		(_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag,	\
577 		    (_esbp)->esb_handle, (_offset+4));			\
578 		(_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag,	\
579 		    (_esbp)->esb_handle, (_offset+8));			\
580 		(_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag,	\
581 		    (_esbp)->esb_handle, (_offset+12));			\
582 									\
583 		EFSYS_PROBE5(bar_reado, unsigned int, (_offset),	\
584 		    uint32_t, (_eop)->eo_u32[3],			\
585 		    uint32_t, (_eop)->eo_u32[2],			\
586 		    uint32_t, (_eop)->eo_u32[1],			\
587 		    uint32_t, (_eop)->eo_u32[0]);			\
588 									\
589 		_NOTE(CONSTANTCONDITION)				\
590 		if (_lock)						\
591 			mtx_unlock(&((_esbp)->esb_lock));		\
592 	_NOTE(CONSTANTCONDITION)					\
593 	} while (B_FALSE)
594 
595 #define	EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)			\
596 	do {								\
597 		_NOTE(CONSTANTCONDITION)				\
598 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),	\
599 		    ("not power of 2 aligned"));			\
600 									\
601 		_NOTE(CONSTANTCONDITION)				\
602 		if (_lock)						\
603 			mtx_lock(&((_esbp)->esb_lock));			\
604 									\
605 		EFSYS_PROBE2(bar_writed, unsigned int, (_offset),	\
606 		    uint32_t, (_edp)->ed_u32[0]);			\
607 									\
608 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
609 		    (_offset), (_edp)->ed_u32[0]);			\
610 									\
611 		_NOTE(CONSTANTCONDITION)				\
612 		if (_lock)						\
613 			mtx_unlock(&((_esbp)->esb_lock));		\
614 	_NOTE(CONSTANTCONDITION)					\
615 	} while (B_FALSE)
616 
617 #define	EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)				\
618 	do {								\
619 		_NOTE(CONSTANTCONDITION)				\
620 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),	\
621 		    ("not power of 2 aligned"));			\
622 									\
623 		mtx_lock(&((_esbp)->esb_lock));				\
624 									\
625 		EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),	\
626 		    uint32_t, (_eqp)->eq_u32[1],			\
627 		    uint32_t, (_eqp)->eq_u32[0]);			\
628 									\
629 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
630 		    (_offset), (_eqp)->eq_u32[0]);			\
631 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
632 		    (_offset+4), (_eqp)->eq_u32[1]);			\
633 									\
634 		mtx_unlock(&((_esbp)->esb_lock));			\
635 	_NOTE(CONSTANTCONDITION)					\
636 	} while (B_FALSE)
637 
638 #define	EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)			\
639 	do {								\
640 		_NOTE(CONSTANTCONDITION)				\
641 		KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),	\
642 		    ("not power of 2 aligned"));			\
643 									\
644 		_NOTE(CONSTANTCONDITION)				\
645 		if (_lock)						\
646 			mtx_lock(&((_esbp)->esb_lock));			\
647 									\
648 		EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),	\
649 		    uint32_t, (_eop)->eo_u32[3],			\
650 		    uint32_t, (_eop)->eo_u32[2],			\
651 		    uint32_t, (_eop)->eo_u32[1],			\
652 		    uint32_t, (_eop)->eo_u32[0]);			\
653 									\
654 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
655 		    (_offset), (_eop)->eo_u32[0]);			\
656 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
657 		    (_offset+4), (_eop)->eo_u32[1]);			\
658 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
659 		    (_offset+8), (_eop)->eo_u32[2]);			\
660 		bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
661 		    (_offset+12), (_eop)->eo_u32[3]);			\
662 									\
663 		_NOTE(CONSTANTCONDITION)				\
664 		if (_lock)						\
665 			mtx_unlock(&((_esbp)->esb_lock));		\
666 	_NOTE(CONSTANTCONDITION)					\
667 	} while (B_FALSE)
668 
669 /* SPIN */
670 
671 #define	EFSYS_SPIN(_us)							\
672 	do {								\
673 		DELAY(_us);						\
674 	_NOTE(CONSTANTCONDITION)					\
675 	} while (B_FALSE)
676 
677 #define	EFSYS_SLEEP	EFSYS_SPIN
678 
679 /* BARRIERS */
680 
681 /* Strict ordering guaranteed by devacc.devacc_attr_dataorder */
682 #define	EFSYS_MEM_READ_BARRIER()
683 #define	EFSYS_PIO_WRITE_BARRIER()
684 
685 /* TIMESTAMP */
686 
687 typedef	clock_t	efsys_timestamp_t;
688 
689 #define	EFSYS_TIMESTAMP(_usp)						\
690 	do {								\
691 		clock_t now;						\
692 									\
693 		now = ticks;						\
694 		*(_usp) = now * hz / 1000000;				\
695 	_NOTE(CONSTANTCONDITION)					\
696 	} while (B_FALSE)
697 
698 /* KMEM */
699 
700 #define	EFSYS_KMEM_ALLOC(_esip, _size, _p)				\
701 	do {								\
702 		(_esip) = (_esip);					\
703 		(_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO);	\
704 	_NOTE(CONSTANTCONDITION)					\
705 	} while (B_FALSE)
706 
707 #define	EFSYS_KMEM_FREE(_esip, _size, _p)				\
708 	do {								\
709 		(void) (_esip);						\
710 		(void) (_size);						\
711 		free((_p), M_SFXGE);					\
712 	_NOTE(CONSTANTCONDITION)					\
713 	} while (B_FALSE)
714 
715 /* LOCK */
716 
717 typedef struct mtx	efsys_lock_t;
718 
719 #define	EFSYS_LOCK_MAGIC	0x000010c4
720 
721 #define	EFSYS_LOCK(_lockp, _state)					\
722 	do {								\
723 		mtx_lock(_lockp);					\
724 		(_state) = EFSYS_LOCK_MAGIC;				\
725 	_NOTE(CONSTANTCONDITION)					\
726 	} while (B_FALSE)
727 
728 #define	EFSYS_UNLOCK(_lockp, _state)					\
729 	do {								\
730 		if ((_state) != EFSYS_LOCK_MAGIC)			\
731 			KASSERT(B_FALSE, ("not locked"));		\
732 		mtx_unlock(_lockp);					\
733 	_NOTE(CONSTANTCONDITION)					\
734 	} while (B_FALSE)
735 
736 /* PREEMPT */
737 
738 #define	EFSYS_PREEMPT_DISABLE(_state)					\
739 	do {								\
740 		(_state) = (_state);					\
741 		critical_enter();					\
742 	_NOTE(CONSTANTCONDITION)					\
743 	} while (B_FALSE)
744 
745 #define	EFSYS_PREEMPT_ENABLE(_state)					\
746 	do {								\
747 		(_state) = (_state);					\
748 		critical_exit(_state);					\
749 	_NOTE(CONSTANTCONDITION)					\
750 	} while (B_FALSE)
751 
752 /* STAT */
753 
754 typedef uint64_t		efsys_stat_t;
755 
756 #define	EFSYS_STAT_INCR(_knp, _delta) 					\
757 	do {								\
758 		*(_knp) += (_delta);					\
759 	_NOTE(CONSTANTCONDITION)					\
760 	} while (B_FALSE)
761 
762 #define	EFSYS_STAT_DECR(_knp, _delta) 					\
763 	do {								\
764 		*(_knp) -= (_delta);					\
765 	_NOTE(CONSTANTCONDITION)					\
766 	} while (B_FALSE)
767 
768 #define	EFSYS_STAT_SET(_knp, _val)					\
769 	do {								\
770 		*(_knp) = (_val);					\
771 	_NOTE(CONSTANTCONDITION)					\
772 	} while (B_FALSE)
773 
774 #define	EFSYS_STAT_SET_QWORD(_knp, _valp)				\
775 	do {								\
776 		*(_knp) = le64toh((_valp)->eq_u64[0]);			\
777 	_NOTE(CONSTANTCONDITION)					\
778 	} while (B_FALSE)
779 
780 #define	EFSYS_STAT_SET_DWORD(_knp, _valp)				\
781 	do {								\
782 		*(_knp) = le32toh((_valp)->ed_u32[0]);			\
783 	_NOTE(CONSTANTCONDITION)					\
784 	} while (B_FALSE)
785 
786 #define	EFSYS_STAT_INCR_QWORD(_knp, _valp)				\
787 	do {								\
788 		*(_knp) += le64toh((_valp)->eq_u64[0]);			\
789 	_NOTE(CONSTANTCONDITION)					\
790 	} while (B_FALSE)
791 
792 #define	EFSYS_STAT_SUBR_QWORD(_knp, _valp)				\
793 	do {								\
794 		*(_knp) -= le64toh((_valp)->eq_u64[0]);			\
795 	_NOTE(CONSTANTCONDITION)					\
796 	} while (B_FALSE)
797 
798 /* ERR */
799 
800 extern void	sfxge_err(efsys_identifier_t *, unsigned int,
801 		    uint32_t, uint32_t);
802 
803 #if EFSYS_OPT_DECODE_INTR_FATAL
804 #define	EFSYS_ERR(_esip, _code, _dword0, _dword1)			\
805 	do {								\
806 		sfxge_err((_esip), (_code), (_dword0), (_dword1));	\
807 	_NOTE(CONSTANTCONDITION)					\
808 	} while (B_FALSE)
809 #endif
810 
811 /* ASSERT */
812 
813 #define	EFSYS_ASSERT(_exp) do {						\
814 	if (!(_exp))							\
815 		panic(#_exp);						\
816 	} while (0)
817 
818 #define EFSYS_ASSERT3(_x, _op, _y, _t) do {				\
819 	const _t __x = (_t)(_x);					\
820 	const _t __y = (_t)(_y);					\
821 	if (!(__x _op __y))						\
822 	        panic("assertion failed at %s:%u", __FILE__, __LINE__);	\
823 	} while(0)
824 
825 #define EFSYS_ASSERT3U(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uint64_t)
826 #define EFSYS_ASSERT3S(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, int64_t)
827 #define EFSYS_ASSERT3P(_x, _op, _y)	EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
828 
829 #ifdef	__cplusplus
830 }
831 #endif
832 
833 #endif	/* _SYS_EFSYS_H */
834