1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #ifndef _SYS_EFSYS_H 31 #define _SYS_EFSYS_H 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #ifdef __cplusplus 37 extern "C" { 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/bus.h> 42 #include <sys/endian.h> 43 #include <sys/lock.h> 44 #include <sys/malloc.h> 45 #include <sys/mbuf.h> 46 #include <sys/mutex.h> 47 #include <sys/rwlock.h> 48 #include <sys/sdt.h> 49 #include <sys/systm.h> 50 51 #include <machine/bus.h> 52 #include <machine/endian.h> 53 54 #define EFSYS_HAS_UINT64 1 55 #define EFSYS_USE_UINT64 0 56 #if _BYTE_ORDER == _BIG_ENDIAN 57 #define EFSYS_IS_BIG_ENDIAN 1 58 #define EFSYS_IS_LITTLE_ENDIAN 0 59 #elif _BYTE_ORDER == _LITTLE_ENDIAN 60 #define EFSYS_IS_BIG_ENDIAN 0 61 #define EFSYS_IS_LITTLE_ENDIAN 1 62 #endif 63 #include "efx_types.h" 64 65 /* Common code requires this */ 66 #if __FreeBSD_version < 800068 67 #define memmove(d, s, l) bcopy(s, d, l) 68 #endif 69 70 /* FreeBSD equivalents of Solaris things */ 71 #ifndef _NOTE 72 #define _NOTE(s) 73 #endif 74 75 #ifndef B_FALSE 76 #define B_FALSE FALSE 77 #endif 78 #ifndef B_TRUE 79 #define B_TRUE TRUE 80 #endif 81 82 #ifndef IS_P2ALIGNED 83 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 84 #endif 85 86 #ifndef P2ROUNDUP 87 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 88 #endif 89 90 #ifndef IS2P 91 #define ISP2(x) (((x) & ((x) - 1)) == 0) 92 #endif 93 94 #define ENOTACTIVE EINVAL 95 96 /* Memory type to use on FreeBSD */ 97 MALLOC_DECLARE(M_SFXGE); 98 99 /* Machine dependend prefetch wrappers */ 100 #if defined(__i386__) || defined(__amd64__) 101 static __inline void 102 prefetch_read_many(void *addr) 103 { 104 105 __asm__( 106 "prefetcht0 (%0)" 107 : 108 : "r" (addr)); 109 } 110 111 static __inline void 112 prefetch_read_once(void *addr) 113 { 114 115 __asm__( 116 "prefetchnta (%0)" 117 : 118 : "r" (addr)); 119 } 120 #elif defined(__sparc64__) 121 static __inline void 122 prefetch_read_many(void *addr) 123 { 124 125 __asm__( 126 "prefetch [%0], 0" 127 : 128 : "r" (addr)); 129 } 130 131 static __inline void 132 prefetch_read_once(void *addr) 133 { 134 135 __asm__( 136 "prefetch [%0], 1" 137 : 138 : "r" (addr)); 139 } 140 #else 141 static __inline void 142 prefetch_read_many(void *addr) 143 { 144 145 } 146 147 static __inline void 148 prefetch_read_once(void *addr) 149 { 150 151 } 152 #endif 153 154 #if defined(__i386__) || defined(__amd64__) 155 #include <vm/vm.h> 156 #include <vm/pmap.h> 157 #endif 158 static __inline void 159 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 160 struct mbuf *m, bus_dma_segment_t *seg) 161 { 162 #if defined(__i386__) || defined(__amd64__) 163 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 164 seg->ds_len = m->m_len; 165 #else 166 int nsegstmp; 167 168 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 169 #endif 170 } 171 172 /* Modifiers used for DOS builds */ 173 #define __cs 174 #define __far 175 176 /* Modifiers used for Windows builds */ 177 #define __in 178 #define __in_opt 179 #define __in_ecount(_n) 180 #define __in_ecount_opt(_n) 181 #define __in_bcount(_n) 182 #define __in_bcount_opt(_n) 183 184 #define __out 185 #define __out_opt 186 #define __out_ecount(_n) 187 #define __out_ecount_opt(_n) 188 #define __out_bcount(_n) 189 #define __out_bcount_opt(_n) 190 191 #define __deref_out 192 193 #define __inout 194 #define __inout_opt 195 #define __inout_ecount(_n) 196 #define __inout_ecount_opt(_n) 197 #define __inout_bcount(_n) 198 #define __inout_bcount_opt(_n) 199 #define __inout_bcount_full_opt(_n) 200 201 #define __deref_out_bcount_opt(n) 202 203 #define __checkReturn 204 205 #define __drv_when(_p, _c) 206 207 /* Code inclusion options */ 208 209 210 #define EFSYS_OPT_NAMES 1 211 212 #define EFSYS_OPT_FALCON 0 213 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 214 #define EFSYS_OPT_SIENA 1 215 #ifdef DEBUG 216 #define EFSYS_OPT_CHECK_REG 1 217 #else 218 #define EFSYS_OPT_CHECK_REG 0 219 #endif 220 221 #define EFSYS_OPT_MCDI 1 222 223 #define EFSYS_OPT_MAC_FALCON_GMAC 0 224 #define EFSYS_OPT_MAC_FALCON_XMAC 0 225 #define EFSYS_OPT_MAC_STATS 1 226 227 #define EFSYS_OPT_LOOPBACK 0 228 229 #define EFSYS_OPT_MON_NULL 0 230 #define EFSYS_OPT_MON_LM87 0 231 #define EFSYS_OPT_MON_MAX6647 0 232 #define EFSYS_OPT_MON_SIENA 0 233 #define EFSYS_OPT_MON_STATS 0 234 235 #define EFSYS_OPT_PHY_NULL 0 236 #define EFSYS_OPT_PHY_QT2022C2 0 237 #define EFSYS_OPT_PHY_SFX7101 0 238 #define EFSYS_OPT_PHY_TXC43128 0 239 #define EFSYS_OPT_PHY_PM8358 0 240 #define EFSYS_OPT_PHY_SFT9001 0 241 #define EFSYS_OPT_PHY_QT2025C 0 242 #define EFSYS_OPT_PHY_STATS 1 243 #define EFSYS_OPT_PHY_PROPS 0 244 #define EFSYS_OPT_PHY_BIST 1 245 #define EFSYS_OPT_PHY_LED_CONTROL 1 246 #define EFSYS_OPT_PHY_FLAGS 0 247 248 #define EFSYS_OPT_VPD 1 249 #define EFSYS_OPT_NVRAM 1 250 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 251 #define EFSYS_OPT_NVRAM_SFT9001 0 252 #define EFSYS_OPT_NVRAM_SFX7101 0 253 #define EFSYS_OPT_BOOTCFG 0 254 255 #define EFSYS_OPT_PCIE_TUNE 0 256 #define EFSYS_OPT_DIAG 0 257 #define EFSYS_OPT_WOL 1 258 #define EFSYS_OPT_RX_SCALE 1 259 #define EFSYS_OPT_QSTATS 1 260 #define EFSYS_OPT_FILTER 0 261 #define EFSYS_OPT_RX_SCATTER 0 262 #define EFSYS_OPT_RX_HDR_SPLIT 0 263 264 #define EFSYS_OPT_EV_PREFETCH 0 265 266 #define EFSYS_OPT_DECODE_INTR_FATAL 1 267 268 /* ID */ 269 270 typedef struct __efsys_identifier_s efsys_identifier_t; 271 272 /* PROBE */ 273 274 #ifndef KDTRACE_HOOKS 275 276 #define EFSYS_PROBE(_name) 277 278 #define EFSYS_PROBE1(_name, _type1, _arg1) 279 280 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 281 282 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 283 _type3, _arg3) 284 285 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 286 _type3, _arg3, _type4, _arg4) 287 288 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 289 _type3, _arg3, _type4, _arg4, _type5, _arg5) 290 291 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 292 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 293 _type6, _arg6) 294 295 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 296 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 297 _type6, _arg6, _type7, _arg7) 298 299 #else /* KDTRACE_HOOKS */ 300 301 #define EFSYS_PROBE(_name) \ 302 DTRACE_PROBE(_name) 303 304 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 305 DTRACE_PROBE1(_name, _type1, _arg1) 306 307 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 308 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 309 310 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 311 _type3, _arg3) \ 312 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 313 _type3, _arg3) 314 315 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 316 _type3, _arg3, _type4, _arg4) \ 317 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 318 _type3, _arg3, _type4, _arg4) 319 320 #ifdef DTRACE_PROBE5 321 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 322 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 323 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 324 _type3, _arg3, _type4, _arg4, _type5, _arg5) 325 #else 326 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 327 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 328 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 329 _type3, _arg3, _type4, _arg4) 330 #endif 331 332 #ifdef DTRACE_PROBE6 333 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 334 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 335 _type6, _arg6) \ 336 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 337 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 338 _type6, _arg6) 339 #else 340 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 341 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 342 _type6, _arg6) \ 343 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 344 _type3, _arg3, _type4, _arg4, _type5, _arg5) 345 #endif 346 347 #ifdef DTRACE_PROBE7 348 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 349 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 350 _type6, _arg6, _type7, _arg7) \ 351 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 352 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 353 _type6, _arg6, _type7, _arg7) 354 #else 355 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 356 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 357 _type6, _arg6, _type7, _arg7) \ 358 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 359 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 360 _type6, _arg6) 361 #endif 362 363 #endif /* KDTRACE_HOOKS */ 364 365 /* DMA */ 366 367 typedef uint64_t efsys_dma_addr_t; 368 369 typedef struct efsys_mem_s { 370 bus_dma_tag_t esm_tag; 371 bus_dmamap_t esm_map; 372 caddr_t esm_base; 373 efsys_dma_addr_t esm_addr; 374 size_t esm_size; 375 } efsys_mem_t; 376 377 378 #define EFSYS_MEM_ZERO(_esmp, _size) \ 379 do { \ 380 (void) memset((_esmp)->esm_base, 0, (_size)); \ 381 \ 382 _NOTE(CONSTANTCONDITION) \ 383 } while (B_FALSE) 384 385 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 386 do { \ 387 uint32_t *addr; \ 388 \ 389 _NOTE(CONSTANTCONDITION) \ 390 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 391 ("not power of 2 aligned")); \ 392 \ 393 addr = (void *)((_esmp)->esm_base + (_offset)); \ 394 \ 395 (_edp)->ed_u32[0] = *addr; \ 396 \ 397 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 398 uint32_t, (_edp)->ed_u32[0]); \ 399 \ 400 _NOTE(CONSTANTCONDITION) \ 401 } while (B_FALSE) 402 403 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 404 do { \ 405 uint32_t *addr; \ 406 \ 407 _NOTE(CONSTANTCONDITION) \ 408 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 409 ("not power of 2 aligned")); \ 410 \ 411 addr = (void *)((_esmp)->esm_base + (_offset)); \ 412 \ 413 (_eqp)->eq_u32[0] = *addr++; \ 414 (_eqp)->eq_u32[1] = *addr; \ 415 \ 416 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 417 uint32_t, (_eqp)->eq_u32[1], \ 418 uint32_t, (_eqp)->eq_u32[0]); \ 419 \ 420 _NOTE(CONSTANTCONDITION) \ 421 } while (B_FALSE) 422 423 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 424 do { \ 425 uint32_t *addr; \ 426 \ 427 _NOTE(CONSTANTCONDITION) \ 428 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 429 ("not power of 2 aligned")); \ 430 \ 431 addr = (void *)((_esmp)->esm_base + (_offset)); \ 432 \ 433 (_eop)->eo_u32[0] = *addr++; \ 434 (_eop)->eo_u32[1] = *addr++; \ 435 (_eop)->eo_u32[2] = *addr++; \ 436 (_eop)->eo_u32[3] = *addr; \ 437 \ 438 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 439 uint32_t, (_eop)->eo_u32[3], \ 440 uint32_t, (_eop)->eo_u32[2], \ 441 uint32_t, (_eop)->eo_u32[1], \ 442 uint32_t, (_eop)->eo_u32[0]); \ 443 \ 444 _NOTE(CONSTANTCONDITION) \ 445 } while (B_FALSE) 446 447 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 448 do { \ 449 uint32_t *addr; \ 450 \ 451 _NOTE(CONSTANTCONDITION) \ 452 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 453 ("not power of 2 aligned")); \ 454 \ 455 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 456 uint32_t, (_edp)->ed_u32[0]); \ 457 \ 458 addr = (void *)((_esmp)->esm_base + (_offset)); \ 459 \ 460 *addr = (_edp)->ed_u32[0]; \ 461 \ 462 _NOTE(CONSTANTCONDITION) \ 463 } while (B_FALSE) 464 465 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 466 do { \ 467 uint32_t *addr; \ 468 \ 469 _NOTE(CONSTANTCONDITION) \ 470 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 471 ("not power of 2 aligned")); \ 472 \ 473 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 474 uint32_t, (_eqp)->eq_u32[1], \ 475 uint32_t, (_eqp)->eq_u32[0]); \ 476 \ 477 addr = (void *)((_esmp)->esm_base + (_offset)); \ 478 \ 479 *addr++ = (_eqp)->eq_u32[0]; \ 480 *addr = (_eqp)->eq_u32[1]; \ 481 \ 482 _NOTE(CONSTANTCONDITION) \ 483 } while (B_FALSE) 484 485 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 486 do { \ 487 uint32_t *addr; \ 488 \ 489 _NOTE(CONSTANTCONDITION) \ 490 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 491 ("not power of 2 aligned")); \ 492 \ 493 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 494 uint32_t, (_eop)->eo_u32[3], \ 495 uint32_t, (_eop)->eo_u32[2], \ 496 uint32_t, (_eop)->eo_u32[1], \ 497 uint32_t, (_eop)->eo_u32[0]); \ 498 \ 499 addr = (void *)((_esmp)->esm_base + (_offset)); \ 500 \ 501 *addr++ = (_eop)->eo_u32[0]; \ 502 *addr++ = (_eop)->eo_u32[1]; \ 503 *addr++ = (_eop)->eo_u32[2]; \ 504 *addr = (_eop)->eo_u32[3]; \ 505 \ 506 _NOTE(CONSTANTCONDITION) \ 507 } while (B_FALSE) 508 509 #define EFSYS_MEM_ADDR(_esmp) \ 510 ((_esmp)->esm_addr) 511 512 /* BAR */ 513 514 typedef struct efsys_bar_s { 515 struct mtx esb_lock; 516 bus_space_tag_t esb_tag; 517 bus_space_handle_t esb_handle; 518 int esb_rid; 519 struct resource *esb_res; 520 } efsys_bar_t; 521 522 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 523 do { \ 524 _NOTE(CONSTANTCONDITION) \ 525 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 526 ("not power of 2 aligned")); \ 527 \ 528 _NOTE(CONSTANTCONDITION) \ 529 if (_lock) \ 530 mtx_lock(&((_esbp)->esb_lock)); \ 531 \ 532 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 533 (_esbp)->esb_handle, (_offset)); \ 534 \ 535 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 536 uint32_t, (_edp)->ed_u32[0]); \ 537 \ 538 _NOTE(CONSTANTCONDITION) \ 539 if (_lock) \ 540 mtx_unlock(&((_esbp)->esb_lock)); \ 541 _NOTE(CONSTANTCONDITION) \ 542 } while (B_FALSE) 543 544 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 545 do { \ 546 _NOTE(CONSTANTCONDITION) \ 547 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 548 ("not power of 2 aligned")); \ 549 \ 550 mtx_lock(&((_esbp)->esb_lock)); \ 551 \ 552 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 553 (_esbp)->esb_handle, (_offset)); \ 554 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 555 (_esbp)->esb_handle, (_offset+4)); \ 556 \ 557 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 558 uint32_t, (_eqp)->eq_u32[1], \ 559 uint32_t, (_eqp)->eq_u32[0]); \ 560 \ 561 mtx_unlock(&((_esbp)->esb_lock)); \ 562 _NOTE(CONSTANTCONDITION) \ 563 } while (B_FALSE) 564 565 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 566 do { \ 567 _NOTE(CONSTANTCONDITION) \ 568 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 569 ("not power of 2 aligned")); \ 570 \ 571 _NOTE(CONSTANTCONDITION) \ 572 if (_lock) \ 573 mtx_lock(&((_esbp)->esb_lock)); \ 574 \ 575 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 576 (_esbp)->esb_handle, (_offset)); \ 577 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 578 (_esbp)->esb_handle, (_offset+4)); \ 579 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \ 580 (_esbp)->esb_handle, (_offset+8)); \ 581 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \ 582 (_esbp)->esb_handle, (_offset+12)); \ 583 \ 584 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 585 uint32_t, (_eop)->eo_u32[3], \ 586 uint32_t, (_eop)->eo_u32[2], \ 587 uint32_t, (_eop)->eo_u32[1], \ 588 uint32_t, (_eop)->eo_u32[0]); \ 589 \ 590 _NOTE(CONSTANTCONDITION) \ 591 if (_lock) \ 592 mtx_unlock(&((_esbp)->esb_lock)); \ 593 _NOTE(CONSTANTCONDITION) \ 594 } while (B_FALSE) 595 596 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 597 do { \ 598 _NOTE(CONSTANTCONDITION) \ 599 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 600 ("not power of 2 aligned")); \ 601 \ 602 _NOTE(CONSTANTCONDITION) \ 603 if (_lock) \ 604 mtx_lock(&((_esbp)->esb_lock)); \ 605 \ 606 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 607 uint32_t, (_edp)->ed_u32[0]); \ 608 \ 609 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 610 (_offset), (_edp)->ed_u32[0]); \ 611 \ 612 _NOTE(CONSTANTCONDITION) \ 613 if (_lock) \ 614 mtx_unlock(&((_esbp)->esb_lock)); \ 615 _NOTE(CONSTANTCONDITION) \ 616 } while (B_FALSE) 617 618 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 619 do { \ 620 _NOTE(CONSTANTCONDITION) \ 621 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 622 ("not power of 2 aligned")); \ 623 \ 624 mtx_lock(&((_esbp)->esb_lock)); \ 625 \ 626 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 627 uint32_t, (_eqp)->eq_u32[1], \ 628 uint32_t, (_eqp)->eq_u32[0]); \ 629 \ 630 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 631 (_offset), (_eqp)->eq_u32[0]); \ 632 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 633 (_offset+4), (_eqp)->eq_u32[1]); \ 634 \ 635 mtx_unlock(&((_esbp)->esb_lock)); \ 636 _NOTE(CONSTANTCONDITION) \ 637 } while (B_FALSE) 638 639 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 640 do { \ 641 _NOTE(CONSTANTCONDITION) \ 642 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 643 ("not power of 2 aligned")); \ 644 \ 645 _NOTE(CONSTANTCONDITION) \ 646 if (_lock) \ 647 mtx_lock(&((_esbp)->esb_lock)); \ 648 \ 649 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 650 uint32_t, (_eop)->eo_u32[3], \ 651 uint32_t, (_eop)->eo_u32[2], \ 652 uint32_t, (_eop)->eo_u32[1], \ 653 uint32_t, (_eop)->eo_u32[0]); \ 654 \ 655 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 656 (_offset), (_eop)->eo_u32[0]); \ 657 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 658 (_offset+4), (_eop)->eo_u32[1]); \ 659 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 660 (_offset+8), (_eop)->eo_u32[2]); \ 661 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 662 (_offset+12), (_eop)->eo_u32[3]); \ 663 \ 664 _NOTE(CONSTANTCONDITION) \ 665 if (_lock) \ 666 mtx_unlock(&((_esbp)->esb_lock)); \ 667 _NOTE(CONSTANTCONDITION) \ 668 } while (B_FALSE) 669 670 /* SPIN */ 671 672 #define EFSYS_SPIN(_us) \ 673 do { \ 674 DELAY(_us); \ 675 _NOTE(CONSTANTCONDITION) \ 676 } while (B_FALSE) 677 678 #define EFSYS_SLEEP EFSYS_SPIN 679 680 /* BARRIERS */ 681 682 /* Strict ordering guaranteed by devacc.devacc_attr_dataorder */ 683 #define EFSYS_MEM_READ_BARRIER() 684 #define EFSYS_PIO_WRITE_BARRIER() 685 686 /* TIMESTAMP */ 687 688 typedef clock_t efsys_timestamp_t; 689 690 #define EFSYS_TIMESTAMP(_usp) \ 691 do { \ 692 clock_t now; \ 693 \ 694 now = ticks; \ 695 *(_usp) = now * hz / 1000000; \ 696 _NOTE(CONSTANTCONDITION) \ 697 } while (B_FALSE) 698 699 /* KMEM */ 700 701 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 702 do { \ 703 (_esip) = (_esip); \ 704 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \ 705 _NOTE(CONSTANTCONDITION) \ 706 } while (B_FALSE) 707 708 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 709 do { \ 710 (void) (_esip); \ 711 (void) (_size); \ 712 free((_p), M_SFXGE); \ 713 _NOTE(CONSTANTCONDITION) \ 714 } while (B_FALSE) 715 716 /* LOCK */ 717 718 typedef struct mtx efsys_lock_t; 719 720 #define EFSYS_LOCK_MAGIC 0x000010c4 721 722 #define EFSYS_LOCK(_lockp, _state) \ 723 do { \ 724 mtx_lock(_lockp); \ 725 (_state) = EFSYS_LOCK_MAGIC; \ 726 _NOTE(CONSTANTCONDITION) \ 727 } while (B_FALSE) 728 729 #define EFSYS_UNLOCK(_lockp, _state) \ 730 do { \ 731 if ((_state) != EFSYS_LOCK_MAGIC) \ 732 KASSERT(B_FALSE, ("not locked")); \ 733 mtx_unlock(_lockp); \ 734 _NOTE(CONSTANTCONDITION) \ 735 } while (B_FALSE) 736 737 /* PREEMPT */ 738 739 #define EFSYS_PREEMPT_DISABLE(_state) \ 740 do { \ 741 (_state) = (_state); \ 742 critical_enter(); \ 743 _NOTE(CONSTANTCONDITION) \ 744 } while (B_FALSE) 745 746 #define EFSYS_PREEMPT_ENABLE(_state) \ 747 do { \ 748 (_state) = (_state); \ 749 critical_exit(_state); \ 750 _NOTE(CONSTANTCONDITION) \ 751 } while (B_FALSE) 752 753 /* STAT */ 754 755 typedef uint64_t efsys_stat_t; 756 757 #define EFSYS_STAT_INCR(_knp, _delta) \ 758 do { \ 759 *(_knp) += (_delta); \ 760 _NOTE(CONSTANTCONDITION) \ 761 } while (B_FALSE) 762 763 #define EFSYS_STAT_DECR(_knp, _delta) \ 764 do { \ 765 *(_knp) -= (_delta); \ 766 _NOTE(CONSTANTCONDITION) \ 767 } while (B_FALSE) 768 769 #define EFSYS_STAT_SET(_knp, _val) \ 770 do { \ 771 *(_knp) = (_val); \ 772 _NOTE(CONSTANTCONDITION) \ 773 } while (B_FALSE) 774 775 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 776 do { \ 777 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 778 _NOTE(CONSTANTCONDITION) \ 779 } while (B_FALSE) 780 781 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 782 do { \ 783 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 784 _NOTE(CONSTANTCONDITION) \ 785 } while (B_FALSE) 786 787 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 788 do { \ 789 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 790 _NOTE(CONSTANTCONDITION) \ 791 } while (B_FALSE) 792 793 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 794 do { \ 795 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 796 _NOTE(CONSTANTCONDITION) \ 797 } while (B_FALSE) 798 799 /* ERR */ 800 801 extern void sfxge_err(efsys_identifier_t *, unsigned int, 802 uint32_t, uint32_t); 803 804 #if EFSYS_OPT_DECODE_INTR_FATAL 805 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 806 do { \ 807 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 808 _NOTE(CONSTANTCONDITION) \ 809 } while (B_FALSE) 810 #endif 811 812 /* ASSERT */ 813 814 #define EFSYS_ASSERT(_exp) do { \ 815 if (!(_exp)) \ 816 panic(#_exp); \ 817 } while (0) 818 819 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 820 const _t __x = (_t)(_x); \ 821 const _t __y = (_t)(_y); \ 822 if (!(__x _op __y)) \ 823 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 824 } while(0) 825 826 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 827 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 828 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 829 830 #ifdef __cplusplus 831 } 832 #endif 833 834 #endif /* _SYS_EFSYS_H */ 835