1e948693eSPhilip Paeps /*- 23c838a9fSAndrew Rybchenko * Copyright (c) 2010-2015 Solarflare Communications Inc. 3e948693eSPhilip Paeps * All rights reserved. 4e948693eSPhilip Paeps * 5e948693eSPhilip Paeps * This software was developed in part by Philip Paeps under contract for 6e948693eSPhilip Paeps * Solarflare Communications, Inc. 7e948693eSPhilip Paeps * 8e948693eSPhilip Paeps * Redistribution and use in source and binary forms, with or without 93c838a9fSAndrew Rybchenko * modification, are permitted provided that the following conditions are met: 10e948693eSPhilip Paeps * 113c838a9fSAndrew Rybchenko * 1. Redistributions of source code must retain the above copyright notice, 123c838a9fSAndrew Rybchenko * this list of conditions and the following disclaimer. 133c838a9fSAndrew Rybchenko * 2. Redistributions in binary form must reproduce the above copyright notice, 143c838a9fSAndrew Rybchenko * this list of conditions and the following disclaimer in the documentation 153c838a9fSAndrew Rybchenko * and/or other materials provided with the distribution. 163c838a9fSAndrew Rybchenko * 173c838a9fSAndrew Rybchenko * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 183c838a9fSAndrew Rybchenko * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 193c838a9fSAndrew Rybchenko * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 203c838a9fSAndrew Rybchenko * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 213c838a9fSAndrew Rybchenko * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 223c838a9fSAndrew Rybchenko * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 233c838a9fSAndrew Rybchenko * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 243c838a9fSAndrew Rybchenko * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 253c838a9fSAndrew Rybchenko * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 263c838a9fSAndrew Rybchenko * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 273c838a9fSAndrew Rybchenko * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 283c838a9fSAndrew Rybchenko * 293c838a9fSAndrew Rybchenko * The views and conclusions contained in the software and documentation are 303c838a9fSAndrew Rybchenko * those of the authors and should not be interpreted as representing official 313c838a9fSAndrew Rybchenko * policies, either expressed or implied, of the FreeBSD Project. 325dee87d7SPhilip Paeps * 335dee87d7SPhilip Paeps * $FreeBSD$ 34e948693eSPhilip Paeps */ 35e948693eSPhilip Paeps 36e948693eSPhilip Paeps #ifndef _SYS_EFSYS_H 37e948693eSPhilip Paeps #define _SYS_EFSYS_H 38e948693eSPhilip Paeps 39e948693eSPhilip Paeps #ifdef __cplusplus 40e948693eSPhilip Paeps extern "C" { 41e948693eSPhilip Paeps #endif 42e948693eSPhilip Paeps 43e948693eSPhilip Paeps #include <sys/param.h> 44e948693eSPhilip Paeps #include <sys/bus.h> 45e948693eSPhilip Paeps #include <sys/endian.h> 46e948693eSPhilip Paeps #include <sys/lock.h> 47e948693eSPhilip Paeps #include <sys/malloc.h> 48e948693eSPhilip Paeps #include <sys/mbuf.h> 49e948693eSPhilip Paeps #include <sys/mutex.h> 50e948693eSPhilip Paeps #include <sys/rwlock.h> 51e948693eSPhilip Paeps #include <sys/sdt.h> 52e948693eSPhilip Paeps #include <sys/systm.h> 53e948693eSPhilip Paeps 54e948693eSPhilip Paeps #include <machine/bus.h> 55e948693eSPhilip Paeps #include <machine/endian.h> 56e948693eSPhilip Paeps 57e948693eSPhilip Paeps #define EFSYS_HAS_UINT64 1 580f999687SAndrew Rybchenko #if defined(__x86_64__) 590f999687SAndrew Rybchenko #define EFSYS_USE_UINT64 1 600f999687SAndrew Rybchenko #else 61e948693eSPhilip Paeps #define EFSYS_USE_UINT64 0 620f999687SAndrew Rybchenko #endif 633c838a9fSAndrew Rybchenko #define EFSYS_HAS_SSE2_M128 0 64e948693eSPhilip Paeps #if _BYTE_ORDER == _BIG_ENDIAN 65e948693eSPhilip Paeps #define EFSYS_IS_BIG_ENDIAN 1 66e948693eSPhilip Paeps #define EFSYS_IS_LITTLE_ENDIAN 0 67e948693eSPhilip Paeps #elif _BYTE_ORDER == _LITTLE_ENDIAN 68e948693eSPhilip Paeps #define EFSYS_IS_BIG_ENDIAN 0 69e948693eSPhilip Paeps #define EFSYS_IS_LITTLE_ENDIAN 1 70e948693eSPhilip Paeps #endif 71e948693eSPhilip Paeps #include "efx_types.h" 72e948693eSPhilip Paeps 73e948693eSPhilip Paeps /* Common code requires this */ 74e948693eSPhilip Paeps #if __FreeBSD_version < 800068 75e948693eSPhilip Paeps #define memmove(d, s, l) bcopy(s, d, l) 76e948693eSPhilip Paeps #endif 77e948693eSPhilip Paeps 78e948693eSPhilip Paeps /* FreeBSD equivalents of Solaris things */ 79e948693eSPhilip Paeps #ifndef _NOTE 80e948693eSPhilip Paeps #define _NOTE(s) 81e948693eSPhilip Paeps #endif 82e948693eSPhilip Paeps 83e948693eSPhilip Paeps #ifndef B_FALSE 84e948693eSPhilip Paeps #define B_FALSE FALSE 85e948693eSPhilip Paeps #endif 86e948693eSPhilip Paeps #ifndef B_TRUE 87e948693eSPhilip Paeps #define B_TRUE TRUE 88e948693eSPhilip Paeps #endif 89e948693eSPhilip Paeps 90e948693eSPhilip Paeps #ifndef IS_P2ALIGNED 91e948693eSPhilip Paeps #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92e948693eSPhilip Paeps #endif 93e948693eSPhilip Paeps 94e948693eSPhilip Paeps #ifndef P2ROUNDUP 95e948693eSPhilip Paeps #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96e948693eSPhilip Paeps #endif 97e948693eSPhilip Paeps 983c838a9fSAndrew Rybchenko #ifndef P2ALIGN 993c838a9fSAndrew Rybchenko #define P2ALIGN(_x, _a) ((_x) & -(_a)) 1003c838a9fSAndrew Rybchenko #endif 1013c838a9fSAndrew Rybchenko 102e948693eSPhilip Paeps #ifndef IS2P 103e948693eSPhilip Paeps #define ISP2(x) (((x) & ((x) - 1)) == 0) 104e948693eSPhilip Paeps #endif 105e948693eSPhilip Paeps 1065dadbb64SAndrew Rybchenko #if defined(__x86_64__) && __FreeBSD_version >= 1000000 1075dadbb64SAndrew Rybchenko 1085dadbb64SAndrew Rybchenko #define SFXGE_USE_BUS_SPACE_8 1 1095dadbb64SAndrew Rybchenko 110d9ff48abSAndrew Rybchenko #if !defined(bus_space_read_stream_8) 1115dadbb64SAndrew Rybchenko 112d9ff48abSAndrew Rybchenko #define bus_space_read_stream_8(t, h, o) \ 113d9ff48abSAndrew Rybchenko bus_space_read_8((t), (h), (o)) 1145dadbb64SAndrew Rybchenko 115d9ff48abSAndrew Rybchenko #define bus_space_write_stream_8(t, h, o, v) \ 116d9ff48abSAndrew Rybchenko bus_space_write_8((t), (h), (o), (v)) 1175dadbb64SAndrew Rybchenko 118d9ff48abSAndrew Rybchenko #endif 1195dadbb64SAndrew Rybchenko 120d9ff48abSAndrew Rybchenko #endif 121d9ff48abSAndrew Rybchenko 122e948693eSPhilip Paeps #define ENOTACTIVE EINVAL 123e948693eSPhilip Paeps 124e948693eSPhilip Paeps /* Memory type to use on FreeBSD */ 125e948693eSPhilip Paeps MALLOC_DECLARE(M_SFXGE); 126e948693eSPhilip Paeps 127e948693eSPhilip Paeps /* Machine dependend prefetch wrappers */ 128dc1788e8SMarius Strobl #if defined(__i386__) || defined(__amd64__) 129e948693eSPhilip Paeps static __inline void 130e948693eSPhilip Paeps prefetch_read_many(void *addr) 131e948693eSPhilip Paeps { 132dc1788e8SMarius Strobl 133e948693eSPhilip Paeps __asm__( 134e948693eSPhilip Paeps "prefetcht0 (%0)" 135e948693eSPhilip Paeps : 136e948693eSPhilip Paeps : "r" (addr)); 137e948693eSPhilip Paeps } 138e948693eSPhilip Paeps 139e948693eSPhilip Paeps static __inline void 140e948693eSPhilip Paeps prefetch_read_once(void *addr) 141e948693eSPhilip Paeps { 142dc1788e8SMarius Strobl 143e948693eSPhilip Paeps __asm__( 144e948693eSPhilip Paeps "prefetchnta (%0)" 145e948693eSPhilip Paeps : 146e948693eSPhilip Paeps : "r" (addr)); 147e948693eSPhilip Paeps } 148dc1788e8SMarius Strobl #elif defined(__sparc64__) 149dc1788e8SMarius Strobl static __inline void 150dc1788e8SMarius Strobl prefetch_read_many(void *addr) 151dc1788e8SMarius Strobl { 152dc1788e8SMarius Strobl 153dc1788e8SMarius Strobl __asm__( 154dc1788e8SMarius Strobl "prefetch [%0], 0" 155dc1788e8SMarius Strobl : 156dc1788e8SMarius Strobl : "r" (addr)); 157dc1788e8SMarius Strobl } 158dc1788e8SMarius Strobl 159dc1788e8SMarius Strobl static __inline void 160dc1788e8SMarius Strobl prefetch_read_once(void *addr) 161dc1788e8SMarius Strobl { 162dc1788e8SMarius Strobl 163dc1788e8SMarius Strobl __asm__( 164dc1788e8SMarius Strobl "prefetch [%0], 1" 165dc1788e8SMarius Strobl : 166dc1788e8SMarius Strobl : "r" (addr)); 167dc1788e8SMarius Strobl } 168dc1788e8SMarius Strobl #else 169dc1788e8SMarius Strobl static __inline void 170dc1788e8SMarius Strobl prefetch_read_many(void *addr) 171dc1788e8SMarius Strobl { 172dc1788e8SMarius Strobl 173dc1788e8SMarius Strobl } 174dc1788e8SMarius Strobl 175dc1788e8SMarius Strobl static __inline void 176dc1788e8SMarius Strobl prefetch_read_once(void *addr) 177dc1788e8SMarius Strobl { 178dc1788e8SMarius Strobl 179dc1788e8SMarius Strobl } 180e948693eSPhilip Paeps #endif 181e948693eSPhilip Paeps 182e948693eSPhilip Paeps #if defined(__i386__) || defined(__amd64__) 183e948693eSPhilip Paeps #include <vm/vm.h> 184e948693eSPhilip Paeps #include <vm/pmap.h> 185e948693eSPhilip Paeps #endif 186e948693eSPhilip Paeps static __inline void 187e948693eSPhilip Paeps sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188e948693eSPhilip Paeps struct mbuf *m, bus_dma_segment_t *seg) 189e948693eSPhilip Paeps { 190e948693eSPhilip Paeps #if defined(__i386__) || defined(__amd64__) 191e948693eSPhilip Paeps seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192e948693eSPhilip Paeps seg->ds_len = m->m_len; 193e948693eSPhilip Paeps #else 194e948693eSPhilip Paeps int nsegstmp; 195e948693eSPhilip Paeps 196e948693eSPhilip Paeps bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197e948693eSPhilip Paeps #endif 198e948693eSPhilip Paeps } 199e948693eSPhilip Paeps 200e948693eSPhilip Paeps /* Modifiers used for Windows builds */ 201e948693eSPhilip Paeps #define __in 202e948693eSPhilip Paeps #define __in_opt 203e948693eSPhilip Paeps #define __in_ecount(_n) 204e948693eSPhilip Paeps #define __in_ecount_opt(_n) 205e948693eSPhilip Paeps #define __in_bcount(_n) 206e948693eSPhilip Paeps #define __in_bcount_opt(_n) 207e948693eSPhilip Paeps 208e948693eSPhilip Paeps #define __out 209e948693eSPhilip Paeps #define __out_opt 210e948693eSPhilip Paeps #define __out_ecount(_n) 211e948693eSPhilip Paeps #define __out_ecount_opt(_n) 212e948693eSPhilip Paeps #define __out_bcount(_n) 213e948693eSPhilip Paeps #define __out_bcount_opt(_n) 214e948693eSPhilip Paeps 215e948693eSPhilip Paeps #define __deref_out 216e948693eSPhilip Paeps 217e948693eSPhilip Paeps #define __inout 218e948693eSPhilip Paeps #define __inout_opt 219e948693eSPhilip Paeps #define __inout_ecount(_n) 220e948693eSPhilip Paeps #define __inout_ecount_opt(_n) 221e948693eSPhilip Paeps #define __inout_bcount(_n) 222e948693eSPhilip Paeps #define __inout_bcount_opt(_n) 223e948693eSPhilip Paeps #define __inout_bcount_full_opt(_n) 224e948693eSPhilip Paeps 225e948693eSPhilip Paeps #define __deref_out_bcount_opt(n) 226e948693eSPhilip Paeps 227e948693eSPhilip Paeps #define __checkReturn 228460cb568SAndrew Rybchenko #define __success(_x) 229e948693eSPhilip Paeps 230e948693eSPhilip Paeps #define __drv_when(_p, _c) 231e948693eSPhilip Paeps 232e948693eSPhilip Paeps /* Code inclusion options */ 233e948693eSPhilip Paeps 234e948693eSPhilip Paeps 235e948693eSPhilip Paeps #define EFSYS_OPT_NAMES 1 236e948693eSPhilip Paeps 237e948693eSPhilip Paeps #define EFSYS_OPT_FALCON 0 238e948693eSPhilip Paeps #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 239e948693eSPhilip Paeps #define EFSYS_OPT_SIENA 1 2403c838a9fSAndrew Rybchenko #define EFSYS_OPT_HUNTINGTON 1 241b53f4a64SAndrew Rybchenko #define EFSYS_OPT_MEDFORD 0 242e948693eSPhilip Paeps #ifdef DEBUG 243e948693eSPhilip Paeps #define EFSYS_OPT_CHECK_REG 1 244e948693eSPhilip Paeps #else 245e948693eSPhilip Paeps #define EFSYS_OPT_CHECK_REG 0 246e948693eSPhilip Paeps #endif 247e948693eSPhilip Paeps 248e948693eSPhilip Paeps #define EFSYS_OPT_MCDI 1 249870b1186SAndrew Rybchenko #define EFSYS_OPT_MCDI_LOGGING 0 2503edad197SAndrew Rybchenko #define EFSYS_OPT_MCDI_PROXY_AUTH 0 251e948693eSPhilip Paeps 252e948693eSPhilip Paeps #define EFSYS_OPT_MAC_FALCON_GMAC 0 253e948693eSPhilip Paeps #define EFSYS_OPT_MAC_FALCON_XMAC 0 254e948693eSPhilip Paeps #define EFSYS_OPT_MAC_STATS 1 255e948693eSPhilip Paeps 256e948693eSPhilip Paeps #define EFSYS_OPT_LOOPBACK 0 257e948693eSPhilip Paeps 258e948693eSPhilip Paeps #define EFSYS_OPT_MON_NULL 0 259e948693eSPhilip Paeps #define EFSYS_OPT_MON_LM87 0 260e948693eSPhilip Paeps #define EFSYS_OPT_MON_MAX6647 0 2613c838a9fSAndrew Rybchenko #define EFSYS_OPT_MON_MCDI 0 262e948693eSPhilip Paeps #define EFSYS_OPT_MON_STATS 0 263e948693eSPhilip Paeps 264e948693eSPhilip Paeps #define EFSYS_OPT_PHY_NULL 0 265e948693eSPhilip Paeps #define EFSYS_OPT_PHY_QT2022C2 0 266e948693eSPhilip Paeps #define EFSYS_OPT_PHY_SFX7101 0 267e948693eSPhilip Paeps #define EFSYS_OPT_PHY_TXC43128 0 268e948693eSPhilip Paeps #define EFSYS_OPT_PHY_SFT9001 0 269e948693eSPhilip Paeps #define EFSYS_OPT_PHY_QT2025C 0 270e948693eSPhilip Paeps #define EFSYS_OPT_PHY_STATS 1 271e948693eSPhilip Paeps #define EFSYS_OPT_PHY_PROPS 0 2723c838a9fSAndrew Rybchenko #define EFSYS_OPT_PHY_BIST 0 2733c838a9fSAndrew Rybchenko #define EFSYS_OPT_BIST 1 274e948693eSPhilip Paeps #define EFSYS_OPT_PHY_LED_CONTROL 1 275e948693eSPhilip Paeps #define EFSYS_OPT_PHY_FLAGS 0 276e948693eSPhilip Paeps 277e948693eSPhilip Paeps #define EFSYS_OPT_VPD 1 278e948693eSPhilip Paeps #define EFSYS_OPT_NVRAM 1 279e948693eSPhilip Paeps #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 280e948693eSPhilip Paeps #define EFSYS_OPT_NVRAM_SFT9001 0 281e948693eSPhilip Paeps #define EFSYS_OPT_NVRAM_SFX7101 0 282e948693eSPhilip Paeps #define EFSYS_OPT_BOOTCFG 0 283e948693eSPhilip Paeps 284e948693eSPhilip Paeps #define EFSYS_OPT_PCIE_TUNE 0 285e948693eSPhilip Paeps #define EFSYS_OPT_DIAG 0 286e948693eSPhilip Paeps #define EFSYS_OPT_WOL 1 287e948693eSPhilip Paeps #define EFSYS_OPT_RX_SCALE 1 288e948693eSPhilip Paeps #define EFSYS_OPT_QSTATS 1 2893c838a9fSAndrew Rybchenko #define EFSYS_OPT_FILTER 1 290e948693eSPhilip Paeps #define EFSYS_OPT_RX_SCATTER 0 291e948693eSPhilip Paeps 292e948693eSPhilip Paeps #define EFSYS_OPT_EV_PREFETCH 0 293e948693eSPhilip Paeps 294e948693eSPhilip Paeps #define EFSYS_OPT_DECODE_INTR_FATAL 1 295e948693eSPhilip Paeps 296*0c848230SAndrew Rybchenko #define EFSYS_OPT_LICENSING 0 297*0c848230SAndrew Rybchenko 298e948693eSPhilip Paeps /* ID */ 299e948693eSPhilip Paeps 300e948693eSPhilip Paeps typedef struct __efsys_identifier_s efsys_identifier_t; 301e948693eSPhilip Paeps 302e948693eSPhilip Paeps /* PROBE */ 303e948693eSPhilip Paeps 30454366c0bSAttilio Rao #ifndef DTRACE_PROBE 305e948693eSPhilip Paeps 306e948693eSPhilip Paeps #define EFSYS_PROBE(_name) 307e948693eSPhilip Paeps 308e948693eSPhilip Paeps #define EFSYS_PROBE1(_name, _type1, _arg1) 309e948693eSPhilip Paeps 310e948693eSPhilip Paeps #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 311e948693eSPhilip Paeps 312e948693eSPhilip Paeps #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 313e948693eSPhilip Paeps _type3, _arg3) 314e948693eSPhilip Paeps 315e948693eSPhilip Paeps #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 316e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4) 317e948693eSPhilip Paeps 318e948693eSPhilip Paeps #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 319e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5) 320e948693eSPhilip Paeps 321e948693eSPhilip Paeps #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 322e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 323e948693eSPhilip Paeps _type6, _arg6) 324e948693eSPhilip Paeps 325e948693eSPhilip Paeps #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 326e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 327e948693eSPhilip Paeps _type6, _arg6, _type7, _arg7) 328e948693eSPhilip Paeps 32954366c0bSAttilio Rao #else /* DTRACE_PROBE */ 330e948693eSPhilip Paeps 331e948693eSPhilip Paeps #define EFSYS_PROBE(_name) \ 332e948693eSPhilip Paeps DTRACE_PROBE(_name) 333e948693eSPhilip Paeps 334e948693eSPhilip Paeps #define EFSYS_PROBE1(_name, _type1, _arg1) \ 335e948693eSPhilip Paeps DTRACE_PROBE1(_name, _type1, _arg1) 336e948693eSPhilip Paeps 337e948693eSPhilip Paeps #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 338e948693eSPhilip Paeps DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 339e948693eSPhilip Paeps 340e948693eSPhilip Paeps #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 341e948693eSPhilip Paeps _type3, _arg3) \ 342e948693eSPhilip Paeps DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 343e948693eSPhilip Paeps _type3, _arg3) 344e948693eSPhilip Paeps 345e948693eSPhilip Paeps #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 346e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4) \ 347e948693eSPhilip Paeps DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 348e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4) 349e948693eSPhilip Paeps 350e948693eSPhilip Paeps #ifdef DTRACE_PROBE5 351e948693eSPhilip Paeps #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 352e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 353e948693eSPhilip Paeps DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 354e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5) 355e948693eSPhilip Paeps #else 356e948693eSPhilip Paeps #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 357e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 358e948693eSPhilip Paeps DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 359e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4) 360e948693eSPhilip Paeps #endif 361e948693eSPhilip Paeps 362e948693eSPhilip Paeps #ifdef DTRACE_PROBE6 363e948693eSPhilip Paeps #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 364e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 365e948693eSPhilip Paeps _type6, _arg6) \ 366e948693eSPhilip Paeps DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 367e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 368e948693eSPhilip Paeps _type6, _arg6) 369e948693eSPhilip Paeps #else 370e948693eSPhilip Paeps #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 371e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 372e948693eSPhilip Paeps _type6, _arg6) \ 373e948693eSPhilip Paeps EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 374e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5) 375e948693eSPhilip Paeps #endif 376e948693eSPhilip Paeps 377e948693eSPhilip Paeps #ifdef DTRACE_PROBE7 378e948693eSPhilip Paeps #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 379e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 380e948693eSPhilip Paeps _type6, _arg6, _type7, _arg7) \ 381e948693eSPhilip Paeps DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 382e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 383e948693eSPhilip Paeps _type6, _arg6, _type7, _arg7) 384e948693eSPhilip Paeps #else 385e948693eSPhilip Paeps #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 386e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 387e948693eSPhilip Paeps _type6, _arg6, _type7, _arg7) \ 388e948693eSPhilip Paeps EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 389e948693eSPhilip Paeps _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 390e948693eSPhilip Paeps _type6, _arg6) 391e948693eSPhilip Paeps #endif 392e948693eSPhilip Paeps 39354366c0bSAttilio Rao #endif /* DTRACE_PROBE */ 394e948693eSPhilip Paeps 395e948693eSPhilip Paeps /* DMA */ 396e948693eSPhilip Paeps 397e948693eSPhilip Paeps typedef uint64_t efsys_dma_addr_t; 398e948693eSPhilip Paeps 399e948693eSPhilip Paeps typedef struct efsys_mem_s { 400e948693eSPhilip Paeps bus_dma_tag_t esm_tag; 401e948693eSPhilip Paeps bus_dmamap_t esm_map; 402e948693eSPhilip Paeps caddr_t esm_base; 403e948693eSPhilip Paeps efsys_dma_addr_t esm_addr; 404e948693eSPhilip Paeps } efsys_mem_t; 405e948693eSPhilip Paeps 406e948693eSPhilip Paeps 407e948693eSPhilip Paeps #define EFSYS_MEM_ZERO(_esmp, _size) \ 408e948693eSPhilip Paeps do { \ 409e948693eSPhilip Paeps (void) memset((_esmp)->esm_base, 0, (_size)); \ 410e948693eSPhilip Paeps \ 411e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 412e948693eSPhilip Paeps } while (B_FALSE) 413e948693eSPhilip Paeps 414e948693eSPhilip Paeps #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 415e948693eSPhilip Paeps do { \ 416e948693eSPhilip Paeps uint32_t *addr; \ 417e948693eSPhilip Paeps \ 418e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 419e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 420e948693eSPhilip Paeps ("not power of 2 aligned")); \ 421e948693eSPhilip Paeps \ 422e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 423e948693eSPhilip Paeps \ 424e948693eSPhilip Paeps (_edp)->ed_u32[0] = *addr; \ 425e948693eSPhilip Paeps \ 426e948693eSPhilip Paeps EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 427e948693eSPhilip Paeps uint32_t, (_edp)->ed_u32[0]); \ 428e948693eSPhilip Paeps \ 429e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 430e948693eSPhilip Paeps } while (B_FALSE) 431e948693eSPhilip Paeps 4320f999687SAndrew Rybchenko #if defined(__x86_64__) 4330f999687SAndrew Rybchenko #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 4340f999687SAndrew Rybchenko do { \ 4350f999687SAndrew Rybchenko uint64_t *addr; \ 4360f999687SAndrew Rybchenko \ 4370f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 4380f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 4390f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 4400f999687SAndrew Rybchenko \ 4410f999687SAndrew Rybchenko addr = (void *)((_esmp)->esm_base + (_offset)); \ 4420f999687SAndrew Rybchenko \ 4430f999687SAndrew Rybchenko (_eqp)->eq_u64[0] = *addr; \ 4440f999687SAndrew Rybchenko \ 4450f999687SAndrew Rybchenko EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 4460f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[1], \ 4470f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[0]); \ 4480f999687SAndrew Rybchenko \ 4490f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 4500f999687SAndrew Rybchenko } while (B_FALSE) 4510f999687SAndrew Rybchenko #else 452e948693eSPhilip Paeps #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 453e948693eSPhilip Paeps do { \ 454e948693eSPhilip Paeps uint32_t *addr; \ 455e948693eSPhilip Paeps \ 456e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 457e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 458e948693eSPhilip Paeps ("not power of 2 aligned")); \ 459e948693eSPhilip Paeps \ 460e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 461e948693eSPhilip Paeps \ 462e948693eSPhilip Paeps (_eqp)->eq_u32[0] = *addr++; \ 463e948693eSPhilip Paeps (_eqp)->eq_u32[1] = *addr; \ 464e948693eSPhilip Paeps \ 465e948693eSPhilip Paeps EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 466e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[1], \ 467e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[0]); \ 468e948693eSPhilip Paeps \ 469e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 470e948693eSPhilip Paeps } while (B_FALSE) 4710f999687SAndrew Rybchenko #endif 472e948693eSPhilip Paeps 4730f999687SAndrew Rybchenko #if defined(__x86_64__) 4740f999687SAndrew Rybchenko #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 4750f999687SAndrew Rybchenko do { \ 4760f999687SAndrew Rybchenko uint64_t *addr; \ 4770f999687SAndrew Rybchenko \ 4780f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 4790f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 4800f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 4810f999687SAndrew Rybchenko \ 4820f999687SAndrew Rybchenko addr = (void *)((_esmp)->esm_base + (_offset)); \ 4830f999687SAndrew Rybchenko \ 4840f999687SAndrew Rybchenko (_eop)->eo_u64[0] = *addr++; \ 4850f999687SAndrew Rybchenko (_eop)->eo_u64[1] = *addr; \ 4860f999687SAndrew Rybchenko \ 4870f999687SAndrew Rybchenko EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 4880f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[3], \ 4890f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[2], \ 4900f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[1], \ 4910f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[0]); \ 4920f999687SAndrew Rybchenko \ 4930f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 4940f999687SAndrew Rybchenko } while (B_FALSE) 4950f999687SAndrew Rybchenko #else 496e948693eSPhilip Paeps #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 497e948693eSPhilip Paeps do { \ 498e948693eSPhilip Paeps uint32_t *addr; \ 499e948693eSPhilip Paeps \ 500e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 501e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 502e948693eSPhilip Paeps ("not power of 2 aligned")); \ 503e948693eSPhilip Paeps \ 504e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 505e948693eSPhilip Paeps \ 506e948693eSPhilip Paeps (_eop)->eo_u32[0] = *addr++; \ 507e948693eSPhilip Paeps (_eop)->eo_u32[1] = *addr++; \ 508e948693eSPhilip Paeps (_eop)->eo_u32[2] = *addr++; \ 509e948693eSPhilip Paeps (_eop)->eo_u32[3] = *addr; \ 510e948693eSPhilip Paeps \ 511e948693eSPhilip Paeps EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 512e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[3], \ 513e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[2], \ 514e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[1], \ 515e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[0]); \ 516e948693eSPhilip Paeps \ 517e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 518e948693eSPhilip Paeps } while (B_FALSE) 5190f999687SAndrew Rybchenko #endif 520e948693eSPhilip Paeps 521e948693eSPhilip Paeps #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 522e948693eSPhilip Paeps do { \ 523e948693eSPhilip Paeps uint32_t *addr; \ 524e948693eSPhilip Paeps \ 525e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 526e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 527e948693eSPhilip Paeps ("not power of 2 aligned")); \ 528e948693eSPhilip Paeps \ 529e948693eSPhilip Paeps EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 530e948693eSPhilip Paeps uint32_t, (_edp)->ed_u32[0]); \ 531e948693eSPhilip Paeps \ 532e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 533e948693eSPhilip Paeps \ 534e948693eSPhilip Paeps *addr = (_edp)->ed_u32[0]; \ 535e948693eSPhilip Paeps \ 536e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 537e948693eSPhilip Paeps } while (B_FALSE) 538e948693eSPhilip Paeps 5390f999687SAndrew Rybchenko #if defined(__x86_64__) 5400f999687SAndrew Rybchenko #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 5410f999687SAndrew Rybchenko do { \ 5420f999687SAndrew Rybchenko uint64_t *addr; \ 5430f999687SAndrew Rybchenko \ 5440f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 5450f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 5460f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 5470f999687SAndrew Rybchenko \ 5480f999687SAndrew Rybchenko EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 5490f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[1], \ 5500f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[0]); \ 5510f999687SAndrew Rybchenko \ 5520f999687SAndrew Rybchenko addr = (void *)((_esmp)->esm_base + (_offset)); \ 5530f999687SAndrew Rybchenko \ 5540f999687SAndrew Rybchenko *addr = (_eqp)->eq_u64[0]; \ 5550f999687SAndrew Rybchenko \ 5560f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 5570f999687SAndrew Rybchenko } while (B_FALSE) 5580f999687SAndrew Rybchenko 5590f999687SAndrew Rybchenko #else 560e948693eSPhilip Paeps #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 561e948693eSPhilip Paeps do { \ 562e948693eSPhilip Paeps uint32_t *addr; \ 563e948693eSPhilip Paeps \ 564e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 565e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 566e948693eSPhilip Paeps ("not power of 2 aligned")); \ 567e948693eSPhilip Paeps \ 568e948693eSPhilip Paeps EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 569e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[1], \ 570e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[0]); \ 571e948693eSPhilip Paeps \ 572e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 573e948693eSPhilip Paeps \ 574e948693eSPhilip Paeps *addr++ = (_eqp)->eq_u32[0]; \ 575e948693eSPhilip Paeps *addr = (_eqp)->eq_u32[1]; \ 576e948693eSPhilip Paeps \ 577e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 578e948693eSPhilip Paeps } while (B_FALSE) 5790f999687SAndrew Rybchenko #endif 580e948693eSPhilip Paeps 5810f999687SAndrew Rybchenko #if defined(__x86_64__) 5820f999687SAndrew Rybchenko #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 5830f999687SAndrew Rybchenko do { \ 5840f999687SAndrew Rybchenko uint64_t *addr; \ 5850f999687SAndrew Rybchenko \ 5860f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 5870f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 5880f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 5890f999687SAndrew Rybchenko \ 5900f999687SAndrew Rybchenko EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 5910f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[3], \ 5920f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[2], \ 5930f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[1], \ 5940f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[0]); \ 5950f999687SAndrew Rybchenko \ 5960f999687SAndrew Rybchenko addr = (void *)((_esmp)->esm_base + (_offset)); \ 5970f999687SAndrew Rybchenko \ 5980f999687SAndrew Rybchenko *addr++ = (_eop)->eo_u64[0]; \ 5990f999687SAndrew Rybchenko *addr = (_eop)->eo_u64[1]; \ 6000f999687SAndrew Rybchenko \ 6010f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 6020f999687SAndrew Rybchenko } while (B_FALSE) 6030f999687SAndrew Rybchenko #else 604e948693eSPhilip Paeps #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 605e948693eSPhilip Paeps do { \ 606e948693eSPhilip Paeps uint32_t *addr; \ 607e948693eSPhilip Paeps \ 608e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 609e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 610e948693eSPhilip Paeps ("not power of 2 aligned")); \ 611e948693eSPhilip Paeps \ 612e948693eSPhilip Paeps EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 613e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[3], \ 614e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[2], \ 615e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[1], \ 616e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[0]); \ 617e948693eSPhilip Paeps \ 618e948693eSPhilip Paeps addr = (void *)((_esmp)->esm_base + (_offset)); \ 619e948693eSPhilip Paeps \ 620e948693eSPhilip Paeps *addr++ = (_eop)->eo_u32[0]; \ 621e948693eSPhilip Paeps *addr++ = (_eop)->eo_u32[1]; \ 622e948693eSPhilip Paeps *addr++ = (_eop)->eo_u32[2]; \ 623e948693eSPhilip Paeps *addr = (_eop)->eo_u32[3]; \ 624e948693eSPhilip Paeps \ 625e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 626e948693eSPhilip Paeps } while (B_FALSE) 6270f999687SAndrew Rybchenko #endif 628e948693eSPhilip Paeps 629e948693eSPhilip Paeps #define EFSYS_MEM_ADDR(_esmp) \ 630e948693eSPhilip Paeps ((_esmp)->esm_addr) 631e948693eSPhilip Paeps 6323c838a9fSAndrew Rybchenko #define EFSYS_MEM_IS_NULL(_esmp) \ 6333c838a9fSAndrew Rybchenko ((_esmp)->esm_base == NULL) 6343c838a9fSAndrew Rybchenko 635e948693eSPhilip Paeps /* BAR */ 636e948693eSPhilip Paeps 63733d45dc5SAndrew Rybchenko #define SFXGE_LOCK_NAME_MAX 16 63833d45dc5SAndrew Rybchenko 639e948693eSPhilip Paeps typedef struct efsys_bar_s { 640e948693eSPhilip Paeps struct mtx esb_lock; 64133d45dc5SAndrew Rybchenko char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 642e948693eSPhilip Paeps bus_space_tag_t esb_tag; 643e948693eSPhilip Paeps bus_space_handle_t esb_handle; 644e948693eSPhilip Paeps int esb_rid; 645e948693eSPhilip Paeps struct resource *esb_res; 646e948693eSPhilip Paeps } efsys_bar_t; 647e948693eSPhilip Paeps 64833d45dc5SAndrew Rybchenko #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 64933d45dc5SAndrew Rybchenko do { \ 65033d45dc5SAndrew Rybchenko snprintf((_esbp)->esb_lock_name, \ 65133d45dc5SAndrew Rybchenko sizeof((_esbp)->esb_lock_name), \ 65233d45dc5SAndrew Rybchenko "%s:bar", (_ifname)); \ 65333d45dc5SAndrew Rybchenko mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 65433d45dc5SAndrew Rybchenko NULL, MTX_DEF); \ 65533d45dc5SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 65633d45dc5SAndrew Rybchenko } while (B_FALSE) 657763cab71SAndrew Rybchenko #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 658763cab71SAndrew Rybchenko mtx_destroy(&(_esbp)->esb_lock) 659763cab71SAndrew Rybchenko #define SFXGE_BAR_LOCK(_esbp) \ 660763cab71SAndrew Rybchenko mtx_lock(&(_esbp)->esb_lock) 661763cab71SAndrew Rybchenko #define SFXGE_BAR_UNLOCK(_esbp) \ 662763cab71SAndrew Rybchenko mtx_unlock(&(_esbp)->esb_lock) 663763cab71SAndrew Rybchenko 664e948693eSPhilip Paeps #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 665e948693eSPhilip Paeps do { \ 666e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 667e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 668e948693eSPhilip Paeps ("not power of 2 aligned")); \ 669e948693eSPhilip Paeps \ 670e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 671e948693eSPhilip Paeps if (_lock) \ 672763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 673e948693eSPhilip Paeps \ 674d9ff48abSAndrew Rybchenko (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 675d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 676d9ff48abSAndrew Rybchenko (_offset)); \ 677e948693eSPhilip Paeps \ 678e948693eSPhilip Paeps EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 679e948693eSPhilip Paeps uint32_t, (_edp)->ed_u32[0]); \ 680e948693eSPhilip Paeps \ 681e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 682e948693eSPhilip Paeps if (_lock) \ 683763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 684e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 685e948693eSPhilip Paeps } while (B_FALSE) 686e948693eSPhilip Paeps 6875dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8) 688e948693eSPhilip Paeps #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 689e948693eSPhilip Paeps do { \ 690e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 691e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 692e948693eSPhilip Paeps ("not power of 2 aligned")); \ 693e948693eSPhilip Paeps \ 694763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 695e948693eSPhilip Paeps \ 696d9ff48abSAndrew Rybchenko (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 697d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 698d9ff48abSAndrew Rybchenko (_offset)); \ 6990f999687SAndrew Rybchenko \ 7000f999687SAndrew Rybchenko EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 7010f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[1], \ 7020f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[0]); \ 7030f999687SAndrew Rybchenko \ 704687adefaSAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 7050f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7060f999687SAndrew Rybchenko } while (B_FALSE) 7070f999687SAndrew Rybchenko 7080f999687SAndrew Rybchenko #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 7090f999687SAndrew Rybchenko do { \ 7100f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7110f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 7120f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 7130f999687SAndrew Rybchenko \ 7140f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7150f999687SAndrew Rybchenko if (_lock) \ 716687adefaSAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 7170f999687SAndrew Rybchenko \ 718d9ff48abSAndrew Rybchenko (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 719d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 720d9ff48abSAndrew Rybchenko (_offset)); \ 721d9ff48abSAndrew Rybchenko (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 722d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 723d9ff48abSAndrew Rybchenko (_offset) + 8); \ 7240f999687SAndrew Rybchenko \ 7250f999687SAndrew Rybchenko EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 7260f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[3], \ 7270f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[2], \ 7280f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[1], \ 7290f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[0]); \ 7300f999687SAndrew Rybchenko \ 7310f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7320f999687SAndrew Rybchenko if (_lock) \ 733687adefaSAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 7340f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7350f999687SAndrew Rybchenko } while (B_FALSE) 7360f999687SAndrew Rybchenko 7370f999687SAndrew Rybchenko #else 7380f999687SAndrew Rybchenko #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 7390f999687SAndrew Rybchenko do { \ 7400f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 7410f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 7420f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 7430f999687SAndrew Rybchenko \ 744687adefaSAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 7450f999687SAndrew Rybchenko \ 746d9ff48abSAndrew Rybchenko (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 747d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 748d9ff48abSAndrew Rybchenko (_offset)); \ 749d9ff48abSAndrew Rybchenko (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 750d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 751d9ff48abSAndrew Rybchenko (_offset) + 4); \ 752e948693eSPhilip Paeps \ 753e948693eSPhilip Paeps EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 754e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[1], \ 755e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[0]); \ 756e948693eSPhilip Paeps \ 757763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 758e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 759e948693eSPhilip Paeps } while (B_FALSE) 760e948693eSPhilip Paeps 761e948693eSPhilip Paeps #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 762e948693eSPhilip Paeps do { \ 763e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 764e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 765e948693eSPhilip Paeps ("not power of 2 aligned")); \ 766e948693eSPhilip Paeps \ 767e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 768e948693eSPhilip Paeps if (_lock) \ 769763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 770e948693eSPhilip Paeps \ 771d9ff48abSAndrew Rybchenko (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 772d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 773d9ff48abSAndrew Rybchenko (_offset)); \ 774d9ff48abSAndrew Rybchenko (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 775d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 776d9ff48abSAndrew Rybchenko (_offset) + 4); \ 777d9ff48abSAndrew Rybchenko (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 778d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 779d9ff48abSAndrew Rybchenko (_offset) + 8); \ 780d9ff48abSAndrew Rybchenko (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 781d9ff48abSAndrew Rybchenko (_esbp)->esb_tag, (_esbp)->esb_handle, \ 782d9ff48abSAndrew Rybchenko (_offset) + 12); \ 783e948693eSPhilip Paeps \ 784e948693eSPhilip Paeps EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 785e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[3], \ 786e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[2], \ 787e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[1], \ 788e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[0]); \ 789e948693eSPhilip Paeps \ 790e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 791e948693eSPhilip Paeps if (_lock) \ 792763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 793e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 794e948693eSPhilip Paeps } while (B_FALSE) 7950f999687SAndrew Rybchenko #endif 796e948693eSPhilip Paeps 797e948693eSPhilip Paeps #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 798e948693eSPhilip Paeps do { \ 799e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 800e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 801e948693eSPhilip Paeps ("not power of 2 aligned")); \ 802e948693eSPhilip Paeps \ 803e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 804e948693eSPhilip Paeps if (_lock) \ 805763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 806e948693eSPhilip Paeps \ 807e948693eSPhilip Paeps EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 808e948693eSPhilip Paeps uint32_t, (_edp)->ed_u32[0]); \ 809e948693eSPhilip Paeps \ 8101cc8febfSAndrew Rybchenko /* \ 8111cc8febfSAndrew Rybchenko * Make sure that previous writes to the dword have \ 8121cc8febfSAndrew Rybchenko * been done. It should be cheaper than barrier just \ 8131cc8febfSAndrew Rybchenko * after the write below. \ 8141cc8febfSAndrew Rybchenko */ \ 8151cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 8161cc8febfSAndrew Rybchenko (_offset), sizeof (efx_dword_t), \ 8171cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 818d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 819d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 820e948693eSPhilip Paeps (_offset), (_edp)->ed_u32[0]); \ 821e948693eSPhilip Paeps \ 822e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 823e948693eSPhilip Paeps if (_lock) \ 824763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 825e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 826e948693eSPhilip Paeps } while (B_FALSE) 827e948693eSPhilip Paeps 8285dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8) 829e948693eSPhilip Paeps #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 830e948693eSPhilip Paeps do { \ 831e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 832e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 833e948693eSPhilip Paeps ("not power of 2 aligned")); \ 834e948693eSPhilip Paeps \ 835763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 836e948693eSPhilip Paeps \ 837e948693eSPhilip Paeps EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 838e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[1], \ 839e948693eSPhilip Paeps uint32_t, (_eqp)->eq_u32[0]); \ 840e948693eSPhilip Paeps \ 8411cc8febfSAndrew Rybchenko /* \ 8421cc8febfSAndrew Rybchenko * Make sure that previous writes to the qword have \ 8431cc8febfSAndrew Rybchenko * been done. It should be cheaper than barrier just \ 8441cc8febfSAndrew Rybchenko * after the write below. \ 8451cc8febfSAndrew Rybchenko */ \ 8461cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 8471cc8febfSAndrew Rybchenko (_offset), sizeof (efx_qword_t), \ 8481cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 849d9ff48abSAndrew Rybchenko bus_space_write_stream_8((_esbp)->esb_tag, \ 850d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 8510f999687SAndrew Rybchenko (_offset), (_eqp)->eq_u64[0]); \ 8520f999687SAndrew Rybchenko \ 853687adefaSAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 8540f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 8550f999687SAndrew Rybchenko } while (B_FALSE) 8560f999687SAndrew Rybchenko #else 8570f999687SAndrew Rybchenko #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 8580f999687SAndrew Rybchenko do { \ 8590f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 8600f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 8610f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 8620f999687SAndrew Rybchenko \ 863687adefaSAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 8640f999687SAndrew Rybchenko \ 8650f999687SAndrew Rybchenko EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 8660f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[1], \ 8670f999687SAndrew Rybchenko uint32_t, (_eqp)->eq_u32[0]); \ 8680f999687SAndrew Rybchenko \ 8691cc8febfSAndrew Rybchenko /* \ 8701cc8febfSAndrew Rybchenko * Make sure that previous writes to the qword have \ 8711cc8febfSAndrew Rybchenko * been done. It should be cheaper than barrier just \ 8721cc8febfSAndrew Rybchenko * after the last write below. \ 8731cc8febfSAndrew Rybchenko */ \ 8741cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 8751cc8febfSAndrew Rybchenko (_offset), sizeof (efx_qword_t), \ 8761cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 877d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 878d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 879e948693eSPhilip Paeps (_offset), (_eqp)->eq_u32[0]); \ 8801cc8febfSAndrew Rybchenko /* \ 8811cc8febfSAndrew Rybchenko * It should be guaranteed that the last dword comes \ 8821cc8febfSAndrew Rybchenko * the last, so barrier entire qword to be sure that \ 8831cc8febfSAndrew Rybchenko * neither above nor below writes are reordered. \ 8841cc8febfSAndrew Rybchenko */ \ 8851cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 8861cc8febfSAndrew Rybchenko (_offset), sizeof (efx_qword_t), \ 8871cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 888d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 889d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 890d9ff48abSAndrew Rybchenko (_offset) + 4, (_eqp)->eq_u32[1]); \ 891e948693eSPhilip Paeps \ 892763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 893e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 894e948693eSPhilip Paeps } while (B_FALSE) 8950f999687SAndrew Rybchenko #endif 896e948693eSPhilip Paeps 8973c838a9fSAndrew Rybchenko /* 8983c838a9fSAndrew Rybchenko * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 8993c838a9fSAndrew Rybchenko * (required by PIO hardware) 9003c838a9fSAndrew Rybchenko */ 9013c838a9fSAndrew Rybchenko #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 9023c838a9fSAndrew Rybchenko do { \ 9033c838a9fSAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9043c838a9fSAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 9053c838a9fSAndrew Rybchenko ("not power of 2 aligned")); \ 9063c838a9fSAndrew Rybchenko \ 9073c838a9fSAndrew Rybchenko (void) (_esbp); \ 9083c838a9fSAndrew Rybchenko \ 9093c838a9fSAndrew Rybchenko /* FIXME: Perform a 64-bit write */ \ 9103c838a9fSAndrew Rybchenko KASSERT(0, ("not implemented")); \ 9113c838a9fSAndrew Rybchenko \ 9123c838a9fSAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9133c838a9fSAndrew Rybchenko } while (B_FALSE) 9143c838a9fSAndrew Rybchenko 9155dadbb64SAndrew Rybchenko #if defined(SFXGE_USE_BUS_SPACE_8) 916e948693eSPhilip Paeps #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 917e948693eSPhilip Paeps do { \ 918e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 919e948693eSPhilip Paeps KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 920e948693eSPhilip Paeps ("not power of 2 aligned")); \ 921e948693eSPhilip Paeps \ 922e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 923e948693eSPhilip Paeps if (_lock) \ 924763cab71SAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 925e948693eSPhilip Paeps \ 926e948693eSPhilip Paeps EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 927e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[3], \ 928e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[2], \ 929e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[1], \ 930e948693eSPhilip Paeps uint32_t, (_eop)->eo_u32[0]); \ 931e948693eSPhilip Paeps \ 9321cc8febfSAndrew Rybchenko /* \ 9331cc8febfSAndrew Rybchenko * Make sure that previous writes to the oword have \ 9341cc8febfSAndrew Rybchenko * been done. It should be cheaper than barrier just \ 9351cc8febfSAndrew Rybchenko * after the last write below. \ 9361cc8febfSAndrew Rybchenko */ \ 9371cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 9381cc8febfSAndrew Rybchenko (_offset), sizeof (efx_oword_t), \ 9391cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 940d9ff48abSAndrew Rybchenko bus_space_write_stream_8((_esbp)->esb_tag, \ 941d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 9420f999687SAndrew Rybchenko (_offset), (_eop)->eo_u64[0]); \ 9431cc8febfSAndrew Rybchenko /* \ 9441cc8febfSAndrew Rybchenko * It should be guaranteed that the last qword comes \ 9451cc8febfSAndrew Rybchenko * the last, so barrier entire oword to be sure that \ 9461cc8febfSAndrew Rybchenko * neither above nor below writes are reordered. \ 9471cc8febfSAndrew Rybchenko */ \ 9481cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 9491cc8febfSAndrew Rybchenko (_offset), sizeof (efx_oword_t), \ 9501cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 951d9ff48abSAndrew Rybchenko bus_space_write_stream_8((_esbp)->esb_tag, \ 952d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 953d9ff48abSAndrew Rybchenko (_offset) + 8, (_eop)->eo_u64[1]); \ 9540f999687SAndrew Rybchenko \ 9550f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9560f999687SAndrew Rybchenko if (_lock) \ 957687adefaSAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 9580f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9590f999687SAndrew Rybchenko } while (B_FALSE) 9600f999687SAndrew Rybchenko 9610f999687SAndrew Rybchenko #else 9620f999687SAndrew Rybchenko #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 9630f999687SAndrew Rybchenko do { \ 9640f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9650f999687SAndrew Rybchenko KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 9660f999687SAndrew Rybchenko ("not power of 2 aligned")); \ 9670f999687SAndrew Rybchenko \ 9680f999687SAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 9690f999687SAndrew Rybchenko if (_lock) \ 970687adefaSAndrew Rybchenko SFXGE_BAR_LOCK(_esbp); \ 9710f999687SAndrew Rybchenko \ 9720f999687SAndrew Rybchenko EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 9730f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[3], \ 9740f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[2], \ 9750f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[1], \ 9760f999687SAndrew Rybchenko uint32_t, (_eop)->eo_u32[0]); \ 9770f999687SAndrew Rybchenko \ 9781cc8febfSAndrew Rybchenko /* \ 9791cc8febfSAndrew Rybchenko * Make sure that previous writes to the oword have \ 9801cc8febfSAndrew Rybchenko * been done. It should be cheaper than barrier just \ 9811cc8febfSAndrew Rybchenko * after the last write below. \ 9821cc8febfSAndrew Rybchenko */ \ 9831cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 9841cc8febfSAndrew Rybchenko (_offset), sizeof (efx_oword_t), \ 9851cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 986d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 987d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 988e948693eSPhilip Paeps (_offset), (_eop)->eo_u32[0]); \ 989d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 990d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 991d9ff48abSAndrew Rybchenko (_offset) + 4, (_eop)->eo_u32[1]); \ 992d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 993d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 994d9ff48abSAndrew Rybchenko (_offset) + 8, (_eop)->eo_u32[2]); \ 9951cc8febfSAndrew Rybchenko /* \ 9961cc8febfSAndrew Rybchenko * It should be guaranteed that the last dword comes \ 9971cc8febfSAndrew Rybchenko * the last, so barrier entire oword to be sure that \ 9981cc8febfSAndrew Rybchenko * neither above nor below writes are reordered. \ 9991cc8febfSAndrew Rybchenko */ \ 10001cc8febfSAndrew Rybchenko bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 10011cc8febfSAndrew Rybchenko (_offset), sizeof (efx_oword_t), \ 10021cc8febfSAndrew Rybchenko BUS_SPACE_BARRIER_WRITE); \ 1003d9ff48abSAndrew Rybchenko bus_space_write_stream_4((_esbp)->esb_tag, \ 1004d9ff48abSAndrew Rybchenko (_esbp)->esb_handle, \ 1005d9ff48abSAndrew Rybchenko (_offset) + 12, (_eop)->eo_u32[3]); \ 1006e948693eSPhilip Paeps \ 1007e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1008e948693eSPhilip Paeps if (_lock) \ 1009763cab71SAndrew Rybchenko SFXGE_BAR_UNLOCK(_esbp); \ 1010e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1011e948693eSPhilip Paeps } while (B_FALSE) 10120f999687SAndrew Rybchenko #endif 1013e948693eSPhilip Paeps 10143c838a9fSAndrew Rybchenko /* Use the standard octo-word write for doorbell writes */ 10153c838a9fSAndrew Rybchenko #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 10163c838a9fSAndrew Rybchenko do { \ 10173c838a9fSAndrew Rybchenko EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 10183c838a9fSAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 10193c838a9fSAndrew Rybchenko } while (B_FALSE) 10203c838a9fSAndrew Rybchenko 1021e948693eSPhilip Paeps /* SPIN */ 1022e948693eSPhilip Paeps 1023e948693eSPhilip Paeps #define EFSYS_SPIN(_us) \ 1024e948693eSPhilip Paeps do { \ 1025e948693eSPhilip Paeps DELAY(_us); \ 1026e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1027e948693eSPhilip Paeps } while (B_FALSE) 1028e948693eSPhilip Paeps 1029e948693eSPhilip Paeps #define EFSYS_SLEEP EFSYS_SPIN 1030e948693eSPhilip Paeps 1031e948693eSPhilip Paeps /* BARRIERS */ 1032e948693eSPhilip Paeps 10337c00963aSAndrew Rybchenko #define EFSYS_MEM_READ_BARRIER() rmb() 1034e948693eSPhilip Paeps #define EFSYS_PIO_WRITE_BARRIER() 1035e948693eSPhilip Paeps 10363c838a9fSAndrew Rybchenko /* DMA SYNC */ 10373c838a9fSAndrew Rybchenko #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 10383c838a9fSAndrew Rybchenko do { \ 10393c838a9fSAndrew Rybchenko bus_dmamap_sync((_esmp)->esm_tag, \ 10403c838a9fSAndrew Rybchenko (_esmp)->esm_map, \ 10413c838a9fSAndrew Rybchenko BUS_DMASYNC_POSTREAD); \ 10423c838a9fSAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 10433c838a9fSAndrew Rybchenko } while (B_FALSE) 10443c838a9fSAndrew Rybchenko 10453c838a9fSAndrew Rybchenko #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 10463c838a9fSAndrew Rybchenko do { \ 10473c838a9fSAndrew Rybchenko bus_dmamap_sync((_esmp)->esm_tag, \ 10483c838a9fSAndrew Rybchenko (_esmp)->esm_map, \ 10493c838a9fSAndrew Rybchenko BUS_DMASYNC_PREWRITE); \ 10503c838a9fSAndrew Rybchenko _NOTE(CONSTANTCONDITION) \ 10513c838a9fSAndrew Rybchenko } while (B_FALSE) 10523c838a9fSAndrew Rybchenko 1053e948693eSPhilip Paeps /* TIMESTAMP */ 1054e948693eSPhilip Paeps 1055e948693eSPhilip Paeps typedef clock_t efsys_timestamp_t; 1056e948693eSPhilip Paeps 1057e948693eSPhilip Paeps #define EFSYS_TIMESTAMP(_usp) \ 1058e948693eSPhilip Paeps do { \ 1059e948693eSPhilip Paeps clock_t now; \ 1060e948693eSPhilip Paeps \ 1061e948693eSPhilip Paeps now = ticks; \ 1062e948693eSPhilip Paeps *(_usp) = now * hz / 1000000; \ 1063e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1064e948693eSPhilip Paeps } while (B_FALSE) 1065e948693eSPhilip Paeps 1066e948693eSPhilip Paeps /* KMEM */ 1067e948693eSPhilip Paeps 1068e948693eSPhilip Paeps #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1069e948693eSPhilip Paeps do { \ 1070e948693eSPhilip Paeps (_esip) = (_esip); \ 107183488e18SAndrew Rybchenko /* \ 107283488e18SAndrew Rybchenko * The macro is used in non-sleepable contexts, for \ 107383488e18SAndrew Rybchenko * example, holding a mutex. \ 107483488e18SAndrew Rybchenko */ \ 107583488e18SAndrew Rybchenko (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1076e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1077e948693eSPhilip Paeps } while (B_FALSE) 1078e948693eSPhilip Paeps 1079e948693eSPhilip Paeps #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1080e948693eSPhilip Paeps do { \ 1081e948693eSPhilip Paeps (void) (_esip); \ 1082e948693eSPhilip Paeps (void) (_size); \ 1083e948693eSPhilip Paeps free((_p), M_SFXGE); \ 1084e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1085e948693eSPhilip Paeps } while (B_FALSE) 1086e948693eSPhilip Paeps 1087e948693eSPhilip Paeps /* LOCK */ 1088e948693eSPhilip Paeps 108933d45dc5SAndrew Rybchenko typedef struct efsys_lock_s { 109033d45dc5SAndrew Rybchenko struct mtx lock; 109133d45dc5SAndrew Rybchenko char lock_name[SFXGE_LOCK_NAME_MAX]; 109233d45dc5SAndrew Rybchenko } efsys_lock_t; 109333d45dc5SAndrew Rybchenko 109433d45dc5SAndrew Rybchenko #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 109533d45dc5SAndrew Rybchenko do { \ 109633d45dc5SAndrew Rybchenko efsys_lock_t *__eslp = (_eslp); \ 109733d45dc5SAndrew Rybchenko \ 109833d45dc5SAndrew Rybchenko snprintf((__eslp)->lock_name, \ 109933d45dc5SAndrew Rybchenko sizeof((__eslp)->lock_name), \ 110033d45dc5SAndrew Rybchenko "%s:%s", (_ifname), (_label)); \ 110133d45dc5SAndrew Rybchenko mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 110233d45dc5SAndrew Rybchenko NULL, MTX_DEF); \ 110333d45dc5SAndrew Rybchenko } while (B_FALSE) 110433d45dc5SAndrew Rybchenko #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 110533d45dc5SAndrew Rybchenko mtx_destroy(&(_eslp)->lock) 110633d45dc5SAndrew Rybchenko #define SFXGE_EFSYS_LOCK(_eslp) \ 110733d45dc5SAndrew Rybchenko mtx_lock(&(_eslp)->lock) 110833d45dc5SAndrew Rybchenko #define SFXGE_EFSYS_UNLOCK(_eslp) \ 110933d45dc5SAndrew Rybchenko mtx_unlock(&(_eslp)->lock) 111033d45dc5SAndrew Rybchenko #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 111133d45dc5SAndrew Rybchenko mtx_assert(&(_eslp)->lock, MA_OWNED) 1112e948693eSPhilip Paeps 1113e948693eSPhilip Paeps #define EFSYS_LOCK_MAGIC 0x000010c4 1114e948693eSPhilip Paeps 1115e948693eSPhilip Paeps #define EFSYS_LOCK(_lockp, _state) \ 1116e948693eSPhilip Paeps do { \ 111733d45dc5SAndrew Rybchenko SFXGE_EFSYS_LOCK(_lockp); \ 1118e948693eSPhilip Paeps (_state) = EFSYS_LOCK_MAGIC; \ 1119e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1120e948693eSPhilip Paeps } while (B_FALSE) 1121e948693eSPhilip Paeps 1122e948693eSPhilip Paeps #define EFSYS_UNLOCK(_lockp, _state) \ 1123e948693eSPhilip Paeps do { \ 1124e948693eSPhilip Paeps if ((_state) != EFSYS_LOCK_MAGIC) \ 1125e948693eSPhilip Paeps KASSERT(B_FALSE, ("not locked")); \ 112633d45dc5SAndrew Rybchenko SFXGE_EFSYS_UNLOCK(_lockp); \ 1127e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1128e948693eSPhilip Paeps } while (B_FALSE) 1129e948693eSPhilip Paeps 1130e948693eSPhilip Paeps /* PREEMPT */ 1131e948693eSPhilip Paeps 1132e948693eSPhilip Paeps #define EFSYS_PREEMPT_DISABLE(_state) \ 1133e948693eSPhilip Paeps do { \ 1134e948693eSPhilip Paeps (_state) = (_state); \ 1135e948693eSPhilip Paeps critical_enter(); \ 1136e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1137e948693eSPhilip Paeps } while (B_FALSE) 1138e948693eSPhilip Paeps 1139e948693eSPhilip Paeps #define EFSYS_PREEMPT_ENABLE(_state) \ 1140e948693eSPhilip Paeps do { \ 1141e948693eSPhilip Paeps (_state) = (_state); \ 1142e948693eSPhilip Paeps critical_exit(_state); \ 1143e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1144e948693eSPhilip Paeps } while (B_FALSE) 1145e948693eSPhilip Paeps 1146e948693eSPhilip Paeps /* STAT */ 1147e948693eSPhilip Paeps 1148e948693eSPhilip Paeps typedef uint64_t efsys_stat_t; 1149e948693eSPhilip Paeps 1150e948693eSPhilip Paeps #define EFSYS_STAT_INCR(_knp, _delta) \ 1151e948693eSPhilip Paeps do { \ 1152e948693eSPhilip Paeps *(_knp) += (_delta); \ 1153e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1154e948693eSPhilip Paeps } while (B_FALSE) 1155e948693eSPhilip Paeps 1156e948693eSPhilip Paeps #define EFSYS_STAT_DECR(_knp, _delta) \ 1157e948693eSPhilip Paeps do { \ 1158e948693eSPhilip Paeps *(_knp) -= (_delta); \ 1159e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1160e948693eSPhilip Paeps } while (B_FALSE) 1161e948693eSPhilip Paeps 1162e948693eSPhilip Paeps #define EFSYS_STAT_SET(_knp, _val) \ 1163e948693eSPhilip Paeps do { \ 1164e948693eSPhilip Paeps *(_knp) = (_val); \ 1165e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1166e948693eSPhilip Paeps } while (B_FALSE) 1167e948693eSPhilip Paeps 1168e948693eSPhilip Paeps #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1169e948693eSPhilip Paeps do { \ 1170e948693eSPhilip Paeps *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1171e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1172e948693eSPhilip Paeps } while (B_FALSE) 1173e948693eSPhilip Paeps 1174e948693eSPhilip Paeps #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1175e948693eSPhilip Paeps do { \ 1176e948693eSPhilip Paeps *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1177e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1178e948693eSPhilip Paeps } while (B_FALSE) 1179e948693eSPhilip Paeps 1180e948693eSPhilip Paeps #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1181e948693eSPhilip Paeps do { \ 1182e948693eSPhilip Paeps *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1183e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1184e948693eSPhilip Paeps } while (B_FALSE) 1185e948693eSPhilip Paeps 1186e948693eSPhilip Paeps #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1187e948693eSPhilip Paeps do { \ 1188e948693eSPhilip Paeps *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1189e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1190e948693eSPhilip Paeps } while (B_FALSE) 1191e948693eSPhilip Paeps 1192e948693eSPhilip Paeps /* ERR */ 1193e948693eSPhilip Paeps 1194e948693eSPhilip Paeps extern void sfxge_err(efsys_identifier_t *, unsigned int, 1195e948693eSPhilip Paeps uint32_t, uint32_t); 1196e948693eSPhilip Paeps 1197e948693eSPhilip Paeps #if EFSYS_OPT_DECODE_INTR_FATAL 1198e948693eSPhilip Paeps #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1199e948693eSPhilip Paeps do { \ 1200e948693eSPhilip Paeps sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1201e948693eSPhilip Paeps _NOTE(CONSTANTCONDITION) \ 1202e948693eSPhilip Paeps } while (B_FALSE) 1203e948693eSPhilip Paeps #endif 1204e948693eSPhilip Paeps 1205e948693eSPhilip Paeps /* ASSERT */ 1206e948693eSPhilip Paeps 1207e948693eSPhilip Paeps #define EFSYS_ASSERT(_exp) do { \ 1208e948693eSPhilip Paeps if (!(_exp)) \ 12093c838a9fSAndrew Rybchenko panic("%s", #_exp); \ 1210e948693eSPhilip Paeps } while (0) 1211e948693eSPhilip Paeps 1212e948693eSPhilip Paeps #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1213e948693eSPhilip Paeps const _t __x = (_t)(_x); \ 1214e948693eSPhilip Paeps const _t __y = (_t)(_y); \ 1215e948693eSPhilip Paeps if (!(__x _op __y)) \ 1216e948693eSPhilip Paeps panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1217e948693eSPhilip Paeps } while(0) 1218e948693eSPhilip Paeps 1219e948693eSPhilip Paeps #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1220e948693eSPhilip Paeps #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1221e948693eSPhilip Paeps #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1222e948693eSPhilip Paeps 12233c838a9fSAndrew Rybchenko /* ROTATE */ 12243c838a9fSAndrew Rybchenko 12253c838a9fSAndrew Rybchenko #define EFSYS_HAS_ROTL_DWORD 0 12263c838a9fSAndrew Rybchenko 1227e948693eSPhilip Paeps #ifdef __cplusplus 1228e948693eSPhilip Paeps } 1229e948693eSPhilip Paeps #endif 1230e948693eSPhilip Paeps 1231e948693eSPhilip Paeps #endif /* _SYS_EFSYS_H */ 1232