1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 * 33 * $FreeBSD$ 34 */ 35 36 #ifndef _SYS_EFSYS_H 37 #define _SYS_EFSYS_H 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/bus.h> 45 #include <sys/endian.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/mutex.h> 50 #include <sys/rwlock.h> 51 #include <sys/sdt.h> 52 #include <sys/systm.h> 53 54 #include <machine/bus.h> 55 #include <machine/endian.h> 56 57 #define EFSYS_HAS_UINT64 1 58 #if defined(__x86_64__) 59 #define EFSYS_USE_UINT64 1 60 #else 61 #define EFSYS_USE_UINT64 0 62 #endif 63 #define EFSYS_HAS_SSE2_M128 0 64 #if _BYTE_ORDER == _BIG_ENDIAN 65 #define EFSYS_IS_BIG_ENDIAN 1 66 #define EFSYS_IS_LITTLE_ENDIAN 0 67 #elif _BYTE_ORDER == _LITTLE_ENDIAN 68 #define EFSYS_IS_BIG_ENDIAN 0 69 #define EFSYS_IS_LITTLE_ENDIAN 1 70 #endif 71 #include "efx_types.h" 72 73 /* Common code requires this */ 74 #if __FreeBSD_version < 800068 75 #define memmove(d, s, l) bcopy(s, d, l) 76 #endif 77 78 /* FreeBSD equivalents of Solaris things */ 79 #ifndef _NOTE 80 #define _NOTE(s) 81 #endif 82 83 #ifndef B_FALSE 84 #define B_FALSE FALSE 85 #endif 86 #ifndef B_TRUE 87 #define B_TRUE TRUE 88 #endif 89 90 #ifndef IS_P2ALIGNED 91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92 #endif 93 94 #ifndef P2ROUNDUP 95 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96 #endif 97 98 #ifndef P2ALIGN 99 #define P2ALIGN(_x, _a) ((_x) & -(_a)) 100 #endif 101 102 #ifndef IS2P 103 #define ISP2(x) (((x) & ((x) - 1)) == 0) 104 #endif 105 106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 107 108 #define SFXGE_USE_BUS_SPACE_8 1 109 110 #if !defined(bus_space_read_stream_8) 111 112 #define bus_space_read_stream_8(t, h, o) \ 113 bus_space_read_8((t), (h), (o)) 114 115 #define bus_space_write_stream_8(t, h, o, v) \ 116 bus_space_write_8((t), (h), (o), (v)) 117 118 #endif 119 120 #endif 121 122 #define ENOTACTIVE EINVAL 123 124 /* Memory type to use on FreeBSD */ 125 MALLOC_DECLARE(M_SFXGE); 126 127 /* Machine dependend prefetch wrappers */ 128 #if defined(__i386__) || defined(__amd64__) 129 static __inline void 130 prefetch_read_many(void *addr) 131 { 132 133 __asm__( 134 "prefetcht0 (%0)" 135 : 136 : "r" (addr)); 137 } 138 139 static __inline void 140 prefetch_read_once(void *addr) 141 { 142 143 __asm__( 144 "prefetchnta (%0)" 145 : 146 : "r" (addr)); 147 } 148 #elif defined(__sparc64__) 149 static __inline void 150 prefetch_read_many(void *addr) 151 { 152 153 __asm__( 154 "prefetch [%0], 0" 155 : 156 : "r" (addr)); 157 } 158 159 static __inline void 160 prefetch_read_once(void *addr) 161 { 162 163 __asm__( 164 "prefetch [%0], 1" 165 : 166 : "r" (addr)); 167 } 168 #else 169 static __inline void 170 prefetch_read_many(void *addr) 171 { 172 173 } 174 175 static __inline void 176 prefetch_read_once(void *addr) 177 { 178 179 } 180 #endif 181 182 #if defined(__i386__) || defined(__amd64__) 183 #include <vm/vm.h> 184 #include <vm/pmap.h> 185 #endif 186 static __inline void 187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188 struct mbuf *m, bus_dma_segment_t *seg) 189 { 190 #if defined(__i386__) || defined(__amd64__) 191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192 seg->ds_len = m->m_len; 193 #else 194 int nsegstmp; 195 196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197 #endif 198 } 199 200 /* Modifiers used for Windows builds */ 201 #define __in 202 #define __in_opt 203 #define __in_ecount(_n) 204 #define __in_ecount_opt(_n) 205 #define __in_bcount(_n) 206 #define __in_bcount_opt(_n) 207 208 #define __out 209 #define __out_opt 210 #define __out_ecount(_n) 211 #define __out_ecount_opt(_n) 212 #define __out_bcount(_n) 213 #define __out_bcount_opt(_n) 214 #define __out_bcount_part(_n, _l) 215 #define __out_bcount_part_opt(_n, _l) 216 217 #define __deref_out 218 219 #define __inout 220 #define __inout_opt 221 #define __inout_ecount(_n) 222 #define __inout_ecount_opt(_n) 223 #define __inout_bcount(_n) 224 #define __inout_bcount_opt(_n) 225 #define __inout_bcount_full_opt(_n) 226 227 #define __deref_out_bcount_opt(n) 228 229 #define __checkReturn 230 #define __success(_x) 231 232 #define __drv_when(_p, _c) 233 234 /* Code inclusion options */ 235 236 237 #define EFSYS_OPT_NAMES 1 238 239 #define EFSYS_OPT_FALCON 0 240 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 241 #define EFSYS_OPT_SIENA 1 242 #define EFSYS_OPT_HUNTINGTON 1 243 #define EFSYS_OPT_MEDFORD 0 244 #ifdef DEBUG 245 #define EFSYS_OPT_CHECK_REG 1 246 #else 247 #define EFSYS_OPT_CHECK_REG 0 248 #endif 249 250 #define EFSYS_OPT_MCDI 1 251 #define EFSYS_OPT_MCDI_LOGGING 0 252 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 253 254 #define EFSYS_OPT_MAC_FALCON_GMAC 0 255 #define EFSYS_OPT_MAC_FALCON_XMAC 0 256 #define EFSYS_OPT_MAC_STATS 1 257 258 #define EFSYS_OPT_LOOPBACK 0 259 260 #define EFSYS_OPT_MON_NULL 0 261 #define EFSYS_OPT_MON_LM87 0 262 #define EFSYS_OPT_MON_MAX6647 0 263 #define EFSYS_OPT_MON_MCDI 0 264 #define EFSYS_OPT_MON_STATS 0 265 266 #define EFSYS_OPT_PHY_NULL 0 267 #define EFSYS_OPT_PHY_QT2022C2 0 268 #define EFSYS_OPT_PHY_SFX7101 0 269 #define EFSYS_OPT_PHY_TXC43128 0 270 #define EFSYS_OPT_PHY_SFT9001 0 271 #define EFSYS_OPT_PHY_QT2025C 0 272 #define EFSYS_OPT_PHY_STATS 1 273 #define EFSYS_OPT_PHY_PROPS 0 274 #define EFSYS_OPT_PHY_BIST 0 275 #define EFSYS_OPT_BIST 1 276 #define EFSYS_OPT_PHY_LED_CONTROL 1 277 #define EFSYS_OPT_PHY_FLAGS 0 278 279 #define EFSYS_OPT_VPD 1 280 #define EFSYS_OPT_NVRAM 1 281 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 282 #define EFSYS_OPT_NVRAM_SFT9001 0 283 #define EFSYS_OPT_NVRAM_SFX7101 0 284 #define EFSYS_OPT_BOOTCFG 0 285 286 #define EFSYS_OPT_PCIE_TUNE 0 287 #define EFSYS_OPT_DIAG 0 288 #define EFSYS_OPT_WOL 1 289 #define EFSYS_OPT_RX_SCALE 1 290 #define EFSYS_OPT_QSTATS 1 291 #define EFSYS_OPT_FILTER 1 292 #define EFSYS_OPT_RX_SCATTER 0 293 294 #define EFSYS_OPT_EV_PREFETCH 0 295 296 #define EFSYS_OPT_DECODE_INTR_FATAL 1 297 298 #define EFSYS_OPT_LICENSING 0 299 300 /* ID */ 301 302 typedef struct __efsys_identifier_s efsys_identifier_t; 303 304 /* PROBE */ 305 306 #ifndef DTRACE_PROBE 307 308 #define EFSYS_PROBE(_name) 309 310 #define EFSYS_PROBE1(_name, _type1, _arg1) 311 312 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 313 314 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 315 _type3, _arg3) 316 317 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 318 _type3, _arg3, _type4, _arg4) 319 320 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 321 _type3, _arg3, _type4, _arg4, _type5, _arg5) 322 323 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 324 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 325 _type6, _arg6) 326 327 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 328 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 329 _type6, _arg6, _type7, _arg7) 330 331 #else /* DTRACE_PROBE */ 332 333 #define EFSYS_PROBE(_name) \ 334 DTRACE_PROBE(_name) 335 336 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 337 DTRACE_PROBE1(_name, _type1, _arg1) 338 339 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 340 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 341 342 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 343 _type3, _arg3) \ 344 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 345 _type3, _arg3) 346 347 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 348 _type3, _arg3, _type4, _arg4) \ 349 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 350 _type3, _arg3, _type4, _arg4) 351 352 #ifdef DTRACE_PROBE5 353 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 354 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 355 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 356 _type3, _arg3, _type4, _arg4, _type5, _arg5) 357 #else 358 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 359 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 360 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 361 _type3, _arg3, _type4, _arg4) 362 #endif 363 364 #ifdef DTRACE_PROBE6 365 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 366 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 367 _type6, _arg6) \ 368 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 369 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 370 _type6, _arg6) 371 #else 372 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 373 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 374 _type6, _arg6) \ 375 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 376 _type3, _arg3, _type4, _arg4, _type5, _arg5) 377 #endif 378 379 #ifdef DTRACE_PROBE7 380 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 381 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 382 _type6, _arg6, _type7, _arg7) \ 383 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 384 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 385 _type6, _arg6, _type7, _arg7) 386 #else 387 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 388 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 389 _type6, _arg6, _type7, _arg7) \ 390 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 391 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 392 _type6, _arg6) 393 #endif 394 395 #endif /* DTRACE_PROBE */ 396 397 /* DMA */ 398 399 typedef uint64_t efsys_dma_addr_t; 400 401 typedef struct efsys_mem_s { 402 bus_dma_tag_t esm_tag; 403 bus_dmamap_t esm_map; 404 caddr_t esm_base; 405 efsys_dma_addr_t esm_addr; 406 } efsys_mem_t; 407 408 409 #define EFSYS_MEM_ZERO(_esmp, _size) \ 410 do { \ 411 (void) memset((_esmp)->esm_base, 0, (_size)); \ 412 \ 413 _NOTE(CONSTANTCONDITION) \ 414 } while (B_FALSE) 415 416 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 417 do { \ 418 uint32_t *addr; \ 419 \ 420 _NOTE(CONSTANTCONDITION) \ 421 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 422 ("not power of 2 aligned")); \ 423 \ 424 addr = (void *)((_esmp)->esm_base + (_offset)); \ 425 \ 426 (_edp)->ed_u32[0] = *addr; \ 427 \ 428 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 429 uint32_t, (_edp)->ed_u32[0]); \ 430 \ 431 _NOTE(CONSTANTCONDITION) \ 432 } while (B_FALSE) 433 434 #if defined(__x86_64__) 435 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 436 do { \ 437 uint64_t *addr; \ 438 \ 439 _NOTE(CONSTANTCONDITION) \ 440 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 441 ("not power of 2 aligned")); \ 442 \ 443 addr = (void *)((_esmp)->esm_base + (_offset)); \ 444 \ 445 (_eqp)->eq_u64[0] = *addr; \ 446 \ 447 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 448 uint32_t, (_eqp)->eq_u32[1], \ 449 uint32_t, (_eqp)->eq_u32[0]); \ 450 \ 451 _NOTE(CONSTANTCONDITION) \ 452 } while (B_FALSE) 453 #else 454 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 455 do { \ 456 uint32_t *addr; \ 457 \ 458 _NOTE(CONSTANTCONDITION) \ 459 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 460 ("not power of 2 aligned")); \ 461 \ 462 addr = (void *)((_esmp)->esm_base + (_offset)); \ 463 \ 464 (_eqp)->eq_u32[0] = *addr++; \ 465 (_eqp)->eq_u32[1] = *addr; \ 466 \ 467 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 468 uint32_t, (_eqp)->eq_u32[1], \ 469 uint32_t, (_eqp)->eq_u32[0]); \ 470 \ 471 _NOTE(CONSTANTCONDITION) \ 472 } while (B_FALSE) 473 #endif 474 475 #if defined(__x86_64__) 476 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 477 do { \ 478 uint64_t *addr; \ 479 \ 480 _NOTE(CONSTANTCONDITION) \ 481 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 482 ("not power of 2 aligned")); \ 483 \ 484 addr = (void *)((_esmp)->esm_base + (_offset)); \ 485 \ 486 (_eop)->eo_u64[0] = *addr++; \ 487 (_eop)->eo_u64[1] = *addr; \ 488 \ 489 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 490 uint32_t, (_eop)->eo_u32[3], \ 491 uint32_t, (_eop)->eo_u32[2], \ 492 uint32_t, (_eop)->eo_u32[1], \ 493 uint32_t, (_eop)->eo_u32[0]); \ 494 \ 495 _NOTE(CONSTANTCONDITION) \ 496 } while (B_FALSE) 497 #else 498 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 499 do { \ 500 uint32_t *addr; \ 501 \ 502 _NOTE(CONSTANTCONDITION) \ 503 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 504 ("not power of 2 aligned")); \ 505 \ 506 addr = (void *)((_esmp)->esm_base + (_offset)); \ 507 \ 508 (_eop)->eo_u32[0] = *addr++; \ 509 (_eop)->eo_u32[1] = *addr++; \ 510 (_eop)->eo_u32[2] = *addr++; \ 511 (_eop)->eo_u32[3] = *addr; \ 512 \ 513 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 514 uint32_t, (_eop)->eo_u32[3], \ 515 uint32_t, (_eop)->eo_u32[2], \ 516 uint32_t, (_eop)->eo_u32[1], \ 517 uint32_t, (_eop)->eo_u32[0]); \ 518 \ 519 _NOTE(CONSTANTCONDITION) \ 520 } while (B_FALSE) 521 #endif 522 523 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 524 do { \ 525 uint32_t *addr; \ 526 \ 527 _NOTE(CONSTANTCONDITION) \ 528 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 529 ("not power of 2 aligned")); \ 530 \ 531 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 532 uint32_t, (_edp)->ed_u32[0]); \ 533 \ 534 addr = (void *)((_esmp)->esm_base + (_offset)); \ 535 \ 536 *addr = (_edp)->ed_u32[0]; \ 537 \ 538 _NOTE(CONSTANTCONDITION) \ 539 } while (B_FALSE) 540 541 #if defined(__x86_64__) 542 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 543 do { \ 544 uint64_t *addr; \ 545 \ 546 _NOTE(CONSTANTCONDITION) \ 547 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 548 ("not power of 2 aligned")); \ 549 \ 550 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 551 uint32_t, (_eqp)->eq_u32[1], \ 552 uint32_t, (_eqp)->eq_u32[0]); \ 553 \ 554 addr = (void *)((_esmp)->esm_base + (_offset)); \ 555 \ 556 *addr = (_eqp)->eq_u64[0]; \ 557 \ 558 _NOTE(CONSTANTCONDITION) \ 559 } while (B_FALSE) 560 561 #else 562 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 563 do { \ 564 uint32_t *addr; \ 565 \ 566 _NOTE(CONSTANTCONDITION) \ 567 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 568 ("not power of 2 aligned")); \ 569 \ 570 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 571 uint32_t, (_eqp)->eq_u32[1], \ 572 uint32_t, (_eqp)->eq_u32[0]); \ 573 \ 574 addr = (void *)((_esmp)->esm_base + (_offset)); \ 575 \ 576 *addr++ = (_eqp)->eq_u32[0]; \ 577 *addr = (_eqp)->eq_u32[1]; \ 578 \ 579 _NOTE(CONSTANTCONDITION) \ 580 } while (B_FALSE) 581 #endif 582 583 #if defined(__x86_64__) 584 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 585 do { \ 586 uint64_t *addr; \ 587 \ 588 _NOTE(CONSTANTCONDITION) \ 589 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 590 ("not power of 2 aligned")); \ 591 \ 592 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 593 uint32_t, (_eop)->eo_u32[3], \ 594 uint32_t, (_eop)->eo_u32[2], \ 595 uint32_t, (_eop)->eo_u32[1], \ 596 uint32_t, (_eop)->eo_u32[0]); \ 597 \ 598 addr = (void *)((_esmp)->esm_base + (_offset)); \ 599 \ 600 *addr++ = (_eop)->eo_u64[0]; \ 601 *addr = (_eop)->eo_u64[1]; \ 602 \ 603 _NOTE(CONSTANTCONDITION) \ 604 } while (B_FALSE) 605 #else 606 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 607 do { \ 608 uint32_t *addr; \ 609 \ 610 _NOTE(CONSTANTCONDITION) \ 611 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 612 ("not power of 2 aligned")); \ 613 \ 614 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 615 uint32_t, (_eop)->eo_u32[3], \ 616 uint32_t, (_eop)->eo_u32[2], \ 617 uint32_t, (_eop)->eo_u32[1], \ 618 uint32_t, (_eop)->eo_u32[0]); \ 619 \ 620 addr = (void *)((_esmp)->esm_base + (_offset)); \ 621 \ 622 *addr++ = (_eop)->eo_u32[0]; \ 623 *addr++ = (_eop)->eo_u32[1]; \ 624 *addr++ = (_eop)->eo_u32[2]; \ 625 *addr = (_eop)->eo_u32[3]; \ 626 \ 627 _NOTE(CONSTANTCONDITION) \ 628 } while (B_FALSE) 629 #endif 630 631 #define EFSYS_MEM_ADDR(_esmp) \ 632 ((_esmp)->esm_addr) 633 634 #define EFSYS_MEM_IS_NULL(_esmp) \ 635 ((_esmp)->esm_base == NULL) 636 637 /* BAR */ 638 639 #define SFXGE_LOCK_NAME_MAX 16 640 641 typedef struct efsys_bar_s { 642 struct mtx esb_lock; 643 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 644 bus_space_tag_t esb_tag; 645 bus_space_handle_t esb_handle; 646 int esb_rid; 647 struct resource *esb_res; 648 } efsys_bar_t; 649 650 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 651 do { \ 652 snprintf((_esbp)->esb_lock_name, \ 653 sizeof((_esbp)->esb_lock_name), \ 654 "%s:bar", (_ifname)); \ 655 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 656 NULL, MTX_DEF); \ 657 _NOTE(CONSTANTCONDITION) \ 658 } while (B_FALSE) 659 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 660 mtx_destroy(&(_esbp)->esb_lock) 661 #define SFXGE_BAR_LOCK(_esbp) \ 662 mtx_lock(&(_esbp)->esb_lock) 663 #define SFXGE_BAR_UNLOCK(_esbp) \ 664 mtx_unlock(&(_esbp)->esb_lock) 665 666 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 667 do { \ 668 _NOTE(CONSTANTCONDITION) \ 669 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 670 ("not power of 2 aligned")); \ 671 \ 672 _NOTE(CONSTANTCONDITION) \ 673 if (_lock) \ 674 SFXGE_BAR_LOCK(_esbp); \ 675 \ 676 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 677 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 678 (_offset)); \ 679 \ 680 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 681 uint32_t, (_edp)->ed_u32[0]); \ 682 \ 683 _NOTE(CONSTANTCONDITION) \ 684 if (_lock) \ 685 SFXGE_BAR_UNLOCK(_esbp); \ 686 _NOTE(CONSTANTCONDITION) \ 687 } while (B_FALSE) 688 689 #if defined(SFXGE_USE_BUS_SPACE_8) 690 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 691 do { \ 692 _NOTE(CONSTANTCONDITION) \ 693 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 694 ("not power of 2 aligned")); \ 695 \ 696 SFXGE_BAR_LOCK(_esbp); \ 697 \ 698 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 699 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 700 (_offset)); \ 701 \ 702 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 703 uint32_t, (_eqp)->eq_u32[1], \ 704 uint32_t, (_eqp)->eq_u32[0]); \ 705 \ 706 SFXGE_BAR_UNLOCK(_esbp); \ 707 _NOTE(CONSTANTCONDITION) \ 708 } while (B_FALSE) 709 710 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 711 do { \ 712 _NOTE(CONSTANTCONDITION) \ 713 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 714 ("not power of 2 aligned")); \ 715 \ 716 _NOTE(CONSTANTCONDITION) \ 717 if (_lock) \ 718 SFXGE_BAR_LOCK(_esbp); \ 719 \ 720 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 721 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 722 (_offset)); \ 723 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 724 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 725 (_offset) + 8); \ 726 \ 727 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 728 uint32_t, (_eop)->eo_u32[3], \ 729 uint32_t, (_eop)->eo_u32[2], \ 730 uint32_t, (_eop)->eo_u32[1], \ 731 uint32_t, (_eop)->eo_u32[0]); \ 732 \ 733 _NOTE(CONSTANTCONDITION) \ 734 if (_lock) \ 735 SFXGE_BAR_UNLOCK(_esbp); \ 736 _NOTE(CONSTANTCONDITION) \ 737 } while (B_FALSE) 738 739 #else 740 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 741 do { \ 742 _NOTE(CONSTANTCONDITION) \ 743 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 744 ("not power of 2 aligned")); \ 745 \ 746 SFXGE_BAR_LOCK(_esbp); \ 747 \ 748 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 749 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 750 (_offset)); \ 751 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 752 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 753 (_offset) + 4); \ 754 \ 755 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 756 uint32_t, (_eqp)->eq_u32[1], \ 757 uint32_t, (_eqp)->eq_u32[0]); \ 758 \ 759 SFXGE_BAR_UNLOCK(_esbp); \ 760 _NOTE(CONSTANTCONDITION) \ 761 } while (B_FALSE) 762 763 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 764 do { \ 765 _NOTE(CONSTANTCONDITION) \ 766 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 767 ("not power of 2 aligned")); \ 768 \ 769 _NOTE(CONSTANTCONDITION) \ 770 if (_lock) \ 771 SFXGE_BAR_LOCK(_esbp); \ 772 \ 773 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 774 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 775 (_offset)); \ 776 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 777 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 778 (_offset) + 4); \ 779 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 780 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 781 (_offset) + 8); \ 782 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 783 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 784 (_offset) + 12); \ 785 \ 786 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 787 uint32_t, (_eop)->eo_u32[3], \ 788 uint32_t, (_eop)->eo_u32[2], \ 789 uint32_t, (_eop)->eo_u32[1], \ 790 uint32_t, (_eop)->eo_u32[0]); \ 791 \ 792 _NOTE(CONSTANTCONDITION) \ 793 if (_lock) \ 794 SFXGE_BAR_UNLOCK(_esbp); \ 795 _NOTE(CONSTANTCONDITION) \ 796 } while (B_FALSE) 797 #endif 798 799 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 800 do { \ 801 _NOTE(CONSTANTCONDITION) \ 802 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 803 ("not power of 2 aligned")); \ 804 \ 805 _NOTE(CONSTANTCONDITION) \ 806 if (_lock) \ 807 SFXGE_BAR_LOCK(_esbp); \ 808 \ 809 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 810 uint32_t, (_edp)->ed_u32[0]); \ 811 \ 812 /* \ 813 * Make sure that previous writes to the dword have \ 814 * been done. It should be cheaper than barrier just \ 815 * after the write below. \ 816 */ \ 817 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 818 (_offset), sizeof (efx_dword_t), \ 819 BUS_SPACE_BARRIER_WRITE); \ 820 bus_space_write_stream_4((_esbp)->esb_tag, \ 821 (_esbp)->esb_handle, \ 822 (_offset), (_edp)->ed_u32[0]); \ 823 \ 824 _NOTE(CONSTANTCONDITION) \ 825 if (_lock) \ 826 SFXGE_BAR_UNLOCK(_esbp); \ 827 _NOTE(CONSTANTCONDITION) \ 828 } while (B_FALSE) 829 830 #if defined(SFXGE_USE_BUS_SPACE_8) 831 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 832 do { \ 833 _NOTE(CONSTANTCONDITION) \ 834 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 835 ("not power of 2 aligned")); \ 836 \ 837 SFXGE_BAR_LOCK(_esbp); \ 838 \ 839 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 840 uint32_t, (_eqp)->eq_u32[1], \ 841 uint32_t, (_eqp)->eq_u32[0]); \ 842 \ 843 /* \ 844 * Make sure that previous writes to the qword have \ 845 * been done. It should be cheaper than barrier just \ 846 * after the write below. \ 847 */ \ 848 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 849 (_offset), sizeof (efx_qword_t), \ 850 BUS_SPACE_BARRIER_WRITE); \ 851 bus_space_write_stream_8((_esbp)->esb_tag, \ 852 (_esbp)->esb_handle, \ 853 (_offset), (_eqp)->eq_u64[0]); \ 854 \ 855 SFXGE_BAR_UNLOCK(_esbp); \ 856 _NOTE(CONSTANTCONDITION) \ 857 } while (B_FALSE) 858 #else 859 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 860 do { \ 861 _NOTE(CONSTANTCONDITION) \ 862 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 863 ("not power of 2 aligned")); \ 864 \ 865 SFXGE_BAR_LOCK(_esbp); \ 866 \ 867 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 868 uint32_t, (_eqp)->eq_u32[1], \ 869 uint32_t, (_eqp)->eq_u32[0]); \ 870 \ 871 /* \ 872 * Make sure that previous writes to the qword have \ 873 * been done. It should be cheaper than barrier just \ 874 * after the last write below. \ 875 */ \ 876 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 877 (_offset), sizeof (efx_qword_t), \ 878 BUS_SPACE_BARRIER_WRITE); \ 879 bus_space_write_stream_4((_esbp)->esb_tag, \ 880 (_esbp)->esb_handle, \ 881 (_offset), (_eqp)->eq_u32[0]); \ 882 /* \ 883 * It should be guaranteed that the last dword comes \ 884 * the last, so barrier entire qword to be sure that \ 885 * neither above nor below writes are reordered. \ 886 */ \ 887 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 888 (_offset), sizeof (efx_qword_t), \ 889 BUS_SPACE_BARRIER_WRITE); \ 890 bus_space_write_stream_4((_esbp)->esb_tag, \ 891 (_esbp)->esb_handle, \ 892 (_offset) + 4, (_eqp)->eq_u32[1]); \ 893 \ 894 SFXGE_BAR_UNLOCK(_esbp); \ 895 _NOTE(CONSTANTCONDITION) \ 896 } while (B_FALSE) 897 #endif 898 899 /* 900 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 901 * (required by PIO hardware) 902 */ 903 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 904 do { \ 905 _NOTE(CONSTANTCONDITION) \ 906 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 907 ("not power of 2 aligned")); \ 908 \ 909 (void) (_esbp); \ 910 \ 911 /* FIXME: Perform a 64-bit write */ \ 912 KASSERT(0, ("not implemented")); \ 913 \ 914 _NOTE(CONSTANTCONDITION) \ 915 } while (B_FALSE) 916 917 #if defined(SFXGE_USE_BUS_SPACE_8) 918 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 919 do { \ 920 _NOTE(CONSTANTCONDITION) \ 921 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 922 ("not power of 2 aligned")); \ 923 \ 924 _NOTE(CONSTANTCONDITION) \ 925 if (_lock) \ 926 SFXGE_BAR_LOCK(_esbp); \ 927 \ 928 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 929 uint32_t, (_eop)->eo_u32[3], \ 930 uint32_t, (_eop)->eo_u32[2], \ 931 uint32_t, (_eop)->eo_u32[1], \ 932 uint32_t, (_eop)->eo_u32[0]); \ 933 \ 934 /* \ 935 * Make sure that previous writes to the oword have \ 936 * been done. It should be cheaper than barrier just \ 937 * after the last write below. \ 938 */ \ 939 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 940 (_offset), sizeof (efx_oword_t), \ 941 BUS_SPACE_BARRIER_WRITE); \ 942 bus_space_write_stream_8((_esbp)->esb_tag, \ 943 (_esbp)->esb_handle, \ 944 (_offset), (_eop)->eo_u64[0]); \ 945 /* \ 946 * It should be guaranteed that the last qword comes \ 947 * the last, so barrier entire oword to be sure that \ 948 * neither above nor below writes are reordered. \ 949 */ \ 950 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 951 (_offset), sizeof (efx_oword_t), \ 952 BUS_SPACE_BARRIER_WRITE); \ 953 bus_space_write_stream_8((_esbp)->esb_tag, \ 954 (_esbp)->esb_handle, \ 955 (_offset) + 8, (_eop)->eo_u64[1]); \ 956 \ 957 _NOTE(CONSTANTCONDITION) \ 958 if (_lock) \ 959 SFXGE_BAR_UNLOCK(_esbp); \ 960 _NOTE(CONSTANTCONDITION) \ 961 } while (B_FALSE) 962 963 #else 964 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 965 do { \ 966 _NOTE(CONSTANTCONDITION) \ 967 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 968 ("not power of 2 aligned")); \ 969 \ 970 _NOTE(CONSTANTCONDITION) \ 971 if (_lock) \ 972 SFXGE_BAR_LOCK(_esbp); \ 973 \ 974 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 975 uint32_t, (_eop)->eo_u32[3], \ 976 uint32_t, (_eop)->eo_u32[2], \ 977 uint32_t, (_eop)->eo_u32[1], \ 978 uint32_t, (_eop)->eo_u32[0]); \ 979 \ 980 /* \ 981 * Make sure that previous writes to the oword have \ 982 * been done. It should be cheaper than barrier just \ 983 * after the last write below. \ 984 */ \ 985 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 986 (_offset), sizeof (efx_oword_t), \ 987 BUS_SPACE_BARRIER_WRITE); \ 988 bus_space_write_stream_4((_esbp)->esb_tag, \ 989 (_esbp)->esb_handle, \ 990 (_offset), (_eop)->eo_u32[0]); \ 991 bus_space_write_stream_4((_esbp)->esb_tag, \ 992 (_esbp)->esb_handle, \ 993 (_offset) + 4, (_eop)->eo_u32[1]); \ 994 bus_space_write_stream_4((_esbp)->esb_tag, \ 995 (_esbp)->esb_handle, \ 996 (_offset) + 8, (_eop)->eo_u32[2]); \ 997 /* \ 998 * It should be guaranteed that the last dword comes \ 999 * the last, so barrier entire oword to be sure that \ 1000 * neither above nor below writes are reordered. \ 1001 */ \ 1002 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 1003 (_offset), sizeof (efx_oword_t), \ 1004 BUS_SPACE_BARRIER_WRITE); \ 1005 bus_space_write_stream_4((_esbp)->esb_tag, \ 1006 (_esbp)->esb_handle, \ 1007 (_offset) + 12, (_eop)->eo_u32[3]); \ 1008 \ 1009 _NOTE(CONSTANTCONDITION) \ 1010 if (_lock) \ 1011 SFXGE_BAR_UNLOCK(_esbp); \ 1012 _NOTE(CONSTANTCONDITION) \ 1013 } while (B_FALSE) 1014 #endif 1015 1016 /* Use the standard octo-word write for doorbell writes */ 1017 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1018 do { \ 1019 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1020 _NOTE(CONSTANTCONDITION) \ 1021 } while (B_FALSE) 1022 1023 /* SPIN */ 1024 1025 #define EFSYS_SPIN(_us) \ 1026 do { \ 1027 DELAY(_us); \ 1028 _NOTE(CONSTANTCONDITION) \ 1029 } while (B_FALSE) 1030 1031 #define EFSYS_SLEEP EFSYS_SPIN 1032 1033 /* BARRIERS */ 1034 1035 #define EFSYS_MEM_READ_BARRIER() rmb() 1036 #define EFSYS_PIO_WRITE_BARRIER() 1037 1038 /* DMA SYNC */ 1039 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1040 do { \ 1041 bus_dmamap_sync((_esmp)->esm_tag, \ 1042 (_esmp)->esm_map, \ 1043 BUS_DMASYNC_POSTREAD); \ 1044 _NOTE(CONSTANTCONDITION) \ 1045 } while (B_FALSE) 1046 1047 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1048 do { \ 1049 bus_dmamap_sync((_esmp)->esm_tag, \ 1050 (_esmp)->esm_map, \ 1051 BUS_DMASYNC_PREWRITE); \ 1052 _NOTE(CONSTANTCONDITION) \ 1053 } while (B_FALSE) 1054 1055 /* TIMESTAMP */ 1056 1057 typedef clock_t efsys_timestamp_t; 1058 1059 #define EFSYS_TIMESTAMP(_usp) \ 1060 do { \ 1061 clock_t now; \ 1062 \ 1063 now = ticks; \ 1064 *(_usp) = now * hz / 1000000; \ 1065 _NOTE(CONSTANTCONDITION) \ 1066 } while (B_FALSE) 1067 1068 /* KMEM */ 1069 1070 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1071 do { \ 1072 (_esip) = (_esip); \ 1073 /* \ 1074 * The macro is used in non-sleepable contexts, for \ 1075 * example, holding a mutex. \ 1076 */ \ 1077 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1078 _NOTE(CONSTANTCONDITION) \ 1079 } while (B_FALSE) 1080 1081 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1082 do { \ 1083 (void) (_esip); \ 1084 (void) (_size); \ 1085 free((_p), M_SFXGE); \ 1086 _NOTE(CONSTANTCONDITION) \ 1087 } while (B_FALSE) 1088 1089 /* LOCK */ 1090 1091 typedef struct efsys_lock_s { 1092 struct mtx lock; 1093 char lock_name[SFXGE_LOCK_NAME_MAX]; 1094 } efsys_lock_t; 1095 1096 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1097 do { \ 1098 efsys_lock_t *__eslp = (_eslp); \ 1099 \ 1100 snprintf((__eslp)->lock_name, \ 1101 sizeof((__eslp)->lock_name), \ 1102 "%s:%s", (_ifname), (_label)); \ 1103 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1104 NULL, MTX_DEF); \ 1105 } while (B_FALSE) 1106 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1107 mtx_destroy(&(_eslp)->lock) 1108 #define SFXGE_EFSYS_LOCK(_eslp) \ 1109 mtx_lock(&(_eslp)->lock) 1110 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1111 mtx_unlock(&(_eslp)->lock) 1112 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1113 mtx_assert(&(_eslp)->lock, MA_OWNED) 1114 1115 #define EFSYS_LOCK_MAGIC 0x000010c4 1116 1117 #define EFSYS_LOCK(_lockp, _state) \ 1118 do { \ 1119 SFXGE_EFSYS_LOCK(_lockp); \ 1120 (_state) = EFSYS_LOCK_MAGIC; \ 1121 _NOTE(CONSTANTCONDITION) \ 1122 } while (B_FALSE) 1123 1124 #define EFSYS_UNLOCK(_lockp, _state) \ 1125 do { \ 1126 if ((_state) != EFSYS_LOCK_MAGIC) \ 1127 KASSERT(B_FALSE, ("not locked")); \ 1128 SFXGE_EFSYS_UNLOCK(_lockp); \ 1129 _NOTE(CONSTANTCONDITION) \ 1130 } while (B_FALSE) 1131 1132 /* PREEMPT */ 1133 1134 #define EFSYS_PREEMPT_DISABLE(_state) \ 1135 do { \ 1136 (_state) = (_state); \ 1137 critical_enter(); \ 1138 _NOTE(CONSTANTCONDITION) \ 1139 } while (B_FALSE) 1140 1141 #define EFSYS_PREEMPT_ENABLE(_state) \ 1142 do { \ 1143 (_state) = (_state); \ 1144 critical_exit(_state); \ 1145 _NOTE(CONSTANTCONDITION) \ 1146 } while (B_FALSE) 1147 1148 /* STAT */ 1149 1150 typedef uint64_t efsys_stat_t; 1151 1152 #define EFSYS_STAT_INCR(_knp, _delta) \ 1153 do { \ 1154 *(_knp) += (_delta); \ 1155 _NOTE(CONSTANTCONDITION) \ 1156 } while (B_FALSE) 1157 1158 #define EFSYS_STAT_DECR(_knp, _delta) \ 1159 do { \ 1160 *(_knp) -= (_delta); \ 1161 _NOTE(CONSTANTCONDITION) \ 1162 } while (B_FALSE) 1163 1164 #define EFSYS_STAT_SET(_knp, _val) \ 1165 do { \ 1166 *(_knp) = (_val); \ 1167 _NOTE(CONSTANTCONDITION) \ 1168 } while (B_FALSE) 1169 1170 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1171 do { \ 1172 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1173 _NOTE(CONSTANTCONDITION) \ 1174 } while (B_FALSE) 1175 1176 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1177 do { \ 1178 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1179 _NOTE(CONSTANTCONDITION) \ 1180 } while (B_FALSE) 1181 1182 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1183 do { \ 1184 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1185 _NOTE(CONSTANTCONDITION) \ 1186 } while (B_FALSE) 1187 1188 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1189 do { \ 1190 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1191 _NOTE(CONSTANTCONDITION) \ 1192 } while (B_FALSE) 1193 1194 /* ERR */ 1195 1196 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1197 uint32_t, uint32_t); 1198 1199 #if EFSYS_OPT_DECODE_INTR_FATAL 1200 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1201 do { \ 1202 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1203 _NOTE(CONSTANTCONDITION) \ 1204 } while (B_FALSE) 1205 #endif 1206 1207 /* ASSERT */ 1208 1209 #define EFSYS_ASSERT(_exp) do { \ 1210 if (!(_exp)) \ 1211 panic("%s", #_exp); \ 1212 } while (0) 1213 1214 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1215 const _t __x = (_t)(_x); \ 1216 const _t __y = (_t)(_y); \ 1217 if (!(__x _op __y)) \ 1218 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1219 } while(0) 1220 1221 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1222 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1223 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1224 1225 /* ROTATE */ 1226 1227 #define EFSYS_HAS_ROTL_DWORD 0 1228 1229 #ifdef __cplusplus 1230 } 1231 #endif 1232 1233 #endif /* _SYS_EFSYS_H */ 1234