1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 * 33 * $FreeBSD$ 34 */ 35 36 #ifndef _SYS_EFSYS_H 37 #define _SYS_EFSYS_H 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/bus.h> 45 #include <sys/endian.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/mutex.h> 50 #include <sys/rwlock.h> 51 #include <sys/sdt.h> 52 #include <sys/systm.h> 53 54 #include <machine/bus.h> 55 #include <machine/endian.h> 56 57 #define EFSYS_HAS_UINT64 1 58 #if defined(__x86_64__) 59 #define EFSYS_USE_UINT64 1 60 #else 61 #define EFSYS_USE_UINT64 0 62 #endif 63 #define EFSYS_HAS_SSE2_M128 0 64 #if _BYTE_ORDER == _BIG_ENDIAN 65 #define EFSYS_IS_BIG_ENDIAN 1 66 #define EFSYS_IS_LITTLE_ENDIAN 0 67 #elif _BYTE_ORDER == _LITTLE_ENDIAN 68 #define EFSYS_IS_BIG_ENDIAN 0 69 #define EFSYS_IS_LITTLE_ENDIAN 1 70 #endif 71 #include "efx_types.h" 72 73 /* Common code requires this */ 74 #if __FreeBSD_version < 800068 75 #define memmove(d, s, l) bcopy(s, d, l) 76 #endif 77 78 /* FreeBSD equivalents of Solaris things */ 79 #ifndef _NOTE 80 #define _NOTE(s) 81 #endif 82 83 #ifndef B_FALSE 84 #define B_FALSE FALSE 85 #endif 86 #ifndef B_TRUE 87 #define B_TRUE TRUE 88 #endif 89 90 #ifndef IS_P2ALIGNED 91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92 #endif 93 94 #ifndef P2ROUNDUP 95 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96 #endif 97 98 #ifndef P2ALIGN 99 #define P2ALIGN(_x, _a) ((_x) & -(_a)) 100 #endif 101 102 #ifndef IS2P 103 #define ISP2(x) (((x) & ((x) - 1)) == 0) 104 #endif 105 106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 107 108 #define SFXGE_USE_BUS_SPACE_8 1 109 110 #if !defined(bus_space_read_stream_8) 111 112 #define bus_space_read_stream_8(t, h, o) \ 113 bus_space_read_8((t), (h), (o)) 114 115 #define bus_space_write_stream_8(t, h, o, v) \ 116 bus_space_write_8((t), (h), (o), (v)) 117 118 #endif 119 120 #endif 121 122 #define ENOTACTIVE EINVAL 123 124 /* Memory type to use on FreeBSD */ 125 MALLOC_DECLARE(M_SFXGE); 126 127 /* Machine dependend prefetch wrappers */ 128 #if defined(__i386__) || defined(__amd64__) 129 static __inline void 130 prefetch_read_many(void *addr) 131 { 132 133 __asm__( 134 "prefetcht0 (%0)" 135 : 136 : "r" (addr)); 137 } 138 139 static __inline void 140 prefetch_read_once(void *addr) 141 { 142 143 __asm__( 144 "prefetchnta (%0)" 145 : 146 : "r" (addr)); 147 } 148 #elif defined(__sparc64__) 149 static __inline void 150 prefetch_read_many(void *addr) 151 { 152 153 __asm__( 154 "prefetch [%0], 0" 155 : 156 : "r" (addr)); 157 } 158 159 static __inline void 160 prefetch_read_once(void *addr) 161 { 162 163 __asm__( 164 "prefetch [%0], 1" 165 : 166 : "r" (addr)); 167 } 168 #else 169 static __inline void 170 prefetch_read_many(void *addr) 171 { 172 173 } 174 175 static __inline void 176 prefetch_read_once(void *addr) 177 { 178 179 } 180 #endif 181 182 #if defined(__i386__) || defined(__amd64__) 183 #include <vm/vm.h> 184 #include <vm/pmap.h> 185 #endif 186 static __inline void 187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188 struct mbuf *m, bus_dma_segment_t *seg) 189 { 190 #if defined(__i386__) || defined(__amd64__) 191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192 seg->ds_len = m->m_len; 193 #else 194 int nsegstmp; 195 196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197 #endif 198 } 199 200 /* Modifiers used for Windows builds */ 201 #define __in 202 #define __in_opt 203 #define __in_ecount(_n) 204 #define __in_ecount_opt(_n) 205 #define __in_bcount(_n) 206 #define __in_bcount_opt(_n) 207 208 #define __out 209 #define __out_opt 210 #define __out_ecount(_n) 211 #define __out_ecount_opt(_n) 212 #define __out_bcount(_n) 213 #define __out_bcount_opt(_n) 214 215 #define __deref_out 216 217 #define __inout 218 #define __inout_opt 219 #define __inout_ecount(_n) 220 #define __inout_ecount_opt(_n) 221 #define __inout_bcount(_n) 222 #define __inout_bcount_opt(_n) 223 #define __inout_bcount_full_opt(_n) 224 225 #define __deref_out_bcount_opt(n) 226 227 #define __checkReturn 228 #define __success(_x) 229 230 #define __drv_when(_p, _c) 231 232 /* Code inclusion options */ 233 234 235 #define EFSYS_OPT_NAMES 1 236 237 #define EFSYS_OPT_FALCON 0 238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 239 #define EFSYS_OPT_SIENA 1 240 #define EFSYS_OPT_HUNTINGTON 1 241 #define EFSYS_OPT_MEDFORD 0 242 #ifdef DEBUG 243 #define EFSYS_OPT_CHECK_REG 1 244 #else 245 #define EFSYS_OPT_CHECK_REG 0 246 #endif 247 248 #define EFSYS_OPT_MCDI 1 249 #define EFSYS_OPT_MCDI_LOGGING 0 250 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 251 252 #define EFSYS_OPT_MAC_FALCON_GMAC 0 253 #define EFSYS_OPT_MAC_FALCON_XMAC 0 254 #define EFSYS_OPT_MAC_STATS 1 255 256 #define EFSYS_OPT_LOOPBACK 0 257 258 #define EFSYS_OPT_MON_NULL 0 259 #define EFSYS_OPT_MON_LM87 0 260 #define EFSYS_OPT_MON_MAX6647 0 261 #define EFSYS_OPT_MON_MCDI 0 262 #define EFSYS_OPT_MON_STATS 0 263 264 #define EFSYS_OPT_PHY_NULL 0 265 #define EFSYS_OPT_PHY_QT2022C2 0 266 #define EFSYS_OPT_PHY_SFX7101 0 267 #define EFSYS_OPT_PHY_TXC43128 0 268 #define EFSYS_OPT_PHY_SFT9001 0 269 #define EFSYS_OPT_PHY_QT2025C 0 270 #define EFSYS_OPT_PHY_STATS 1 271 #define EFSYS_OPT_PHY_PROPS 0 272 #define EFSYS_OPT_PHY_BIST 0 273 #define EFSYS_OPT_BIST 1 274 #define EFSYS_OPT_PHY_LED_CONTROL 1 275 #define EFSYS_OPT_PHY_FLAGS 0 276 277 #define EFSYS_OPT_VPD 1 278 #define EFSYS_OPT_NVRAM 1 279 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 280 #define EFSYS_OPT_NVRAM_SFT9001 0 281 #define EFSYS_OPT_NVRAM_SFX7101 0 282 #define EFSYS_OPT_BOOTCFG 0 283 284 #define EFSYS_OPT_PCIE_TUNE 0 285 #define EFSYS_OPT_DIAG 0 286 #define EFSYS_OPT_WOL 1 287 #define EFSYS_OPT_RX_SCALE 1 288 #define EFSYS_OPT_QSTATS 1 289 #define EFSYS_OPT_FILTER 1 290 #define EFSYS_OPT_RX_SCATTER 0 291 292 #define EFSYS_OPT_EV_PREFETCH 0 293 294 #define EFSYS_OPT_DECODE_INTR_FATAL 1 295 296 /* ID */ 297 298 typedef struct __efsys_identifier_s efsys_identifier_t; 299 300 /* PROBE */ 301 302 #ifndef DTRACE_PROBE 303 304 #define EFSYS_PROBE(_name) 305 306 #define EFSYS_PROBE1(_name, _type1, _arg1) 307 308 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 309 310 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 311 _type3, _arg3) 312 313 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 314 _type3, _arg3, _type4, _arg4) 315 316 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 317 _type3, _arg3, _type4, _arg4, _type5, _arg5) 318 319 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 320 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 321 _type6, _arg6) 322 323 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 324 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 325 _type6, _arg6, _type7, _arg7) 326 327 #else /* DTRACE_PROBE */ 328 329 #define EFSYS_PROBE(_name) \ 330 DTRACE_PROBE(_name) 331 332 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 333 DTRACE_PROBE1(_name, _type1, _arg1) 334 335 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 336 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 337 338 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 339 _type3, _arg3) \ 340 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 341 _type3, _arg3) 342 343 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 344 _type3, _arg3, _type4, _arg4) \ 345 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 346 _type3, _arg3, _type4, _arg4) 347 348 #ifdef DTRACE_PROBE5 349 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 350 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 351 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 352 _type3, _arg3, _type4, _arg4, _type5, _arg5) 353 #else 354 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 355 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 356 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 357 _type3, _arg3, _type4, _arg4) 358 #endif 359 360 #ifdef DTRACE_PROBE6 361 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 362 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 363 _type6, _arg6) \ 364 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 365 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 366 _type6, _arg6) 367 #else 368 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 369 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 370 _type6, _arg6) \ 371 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 372 _type3, _arg3, _type4, _arg4, _type5, _arg5) 373 #endif 374 375 #ifdef DTRACE_PROBE7 376 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 377 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 378 _type6, _arg6, _type7, _arg7) \ 379 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 380 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 381 _type6, _arg6, _type7, _arg7) 382 #else 383 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 384 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 385 _type6, _arg6, _type7, _arg7) \ 386 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 387 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 388 _type6, _arg6) 389 #endif 390 391 #endif /* DTRACE_PROBE */ 392 393 /* DMA */ 394 395 typedef uint64_t efsys_dma_addr_t; 396 397 typedef struct efsys_mem_s { 398 bus_dma_tag_t esm_tag; 399 bus_dmamap_t esm_map; 400 caddr_t esm_base; 401 efsys_dma_addr_t esm_addr; 402 } efsys_mem_t; 403 404 405 #define EFSYS_MEM_ZERO(_esmp, _size) \ 406 do { \ 407 (void) memset((_esmp)->esm_base, 0, (_size)); \ 408 \ 409 _NOTE(CONSTANTCONDITION) \ 410 } while (B_FALSE) 411 412 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 413 do { \ 414 uint32_t *addr; \ 415 \ 416 _NOTE(CONSTANTCONDITION) \ 417 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 418 ("not power of 2 aligned")); \ 419 \ 420 addr = (void *)((_esmp)->esm_base + (_offset)); \ 421 \ 422 (_edp)->ed_u32[0] = *addr; \ 423 \ 424 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 425 uint32_t, (_edp)->ed_u32[0]); \ 426 \ 427 _NOTE(CONSTANTCONDITION) \ 428 } while (B_FALSE) 429 430 #if defined(__x86_64__) 431 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 432 do { \ 433 uint64_t *addr; \ 434 \ 435 _NOTE(CONSTANTCONDITION) \ 436 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 437 ("not power of 2 aligned")); \ 438 \ 439 addr = (void *)((_esmp)->esm_base + (_offset)); \ 440 \ 441 (_eqp)->eq_u64[0] = *addr; \ 442 \ 443 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 444 uint32_t, (_eqp)->eq_u32[1], \ 445 uint32_t, (_eqp)->eq_u32[0]); \ 446 \ 447 _NOTE(CONSTANTCONDITION) \ 448 } while (B_FALSE) 449 #else 450 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 451 do { \ 452 uint32_t *addr; \ 453 \ 454 _NOTE(CONSTANTCONDITION) \ 455 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 456 ("not power of 2 aligned")); \ 457 \ 458 addr = (void *)((_esmp)->esm_base + (_offset)); \ 459 \ 460 (_eqp)->eq_u32[0] = *addr++; \ 461 (_eqp)->eq_u32[1] = *addr; \ 462 \ 463 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 464 uint32_t, (_eqp)->eq_u32[1], \ 465 uint32_t, (_eqp)->eq_u32[0]); \ 466 \ 467 _NOTE(CONSTANTCONDITION) \ 468 } while (B_FALSE) 469 #endif 470 471 #if defined(__x86_64__) 472 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 473 do { \ 474 uint64_t *addr; \ 475 \ 476 _NOTE(CONSTANTCONDITION) \ 477 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 478 ("not power of 2 aligned")); \ 479 \ 480 addr = (void *)((_esmp)->esm_base + (_offset)); \ 481 \ 482 (_eop)->eo_u64[0] = *addr++; \ 483 (_eop)->eo_u64[1] = *addr; \ 484 \ 485 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 486 uint32_t, (_eop)->eo_u32[3], \ 487 uint32_t, (_eop)->eo_u32[2], \ 488 uint32_t, (_eop)->eo_u32[1], \ 489 uint32_t, (_eop)->eo_u32[0]); \ 490 \ 491 _NOTE(CONSTANTCONDITION) \ 492 } while (B_FALSE) 493 #else 494 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 495 do { \ 496 uint32_t *addr; \ 497 \ 498 _NOTE(CONSTANTCONDITION) \ 499 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 500 ("not power of 2 aligned")); \ 501 \ 502 addr = (void *)((_esmp)->esm_base + (_offset)); \ 503 \ 504 (_eop)->eo_u32[0] = *addr++; \ 505 (_eop)->eo_u32[1] = *addr++; \ 506 (_eop)->eo_u32[2] = *addr++; \ 507 (_eop)->eo_u32[3] = *addr; \ 508 \ 509 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 510 uint32_t, (_eop)->eo_u32[3], \ 511 uint32_t, (_eop)->eo_u32[2], \ 512 uint32_t, (_eop)->eo_u32[1], \ 513 uint32_t, (_eop)->eo_u32[0]); \ 514 \ 515 _NOTE(CONSTANTCONDITION) \ 516 } while (B_FALSE) 517 #endif 518 519 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 520 do { \ 521 uint32_t *addr; \ 522 \ 523 _NOTE(CONSTANTCONDITION) \ 524 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 525 ("not power of 2 aligned")); \ 526 \ 527 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 528 uint32_t, (_edp)->ed_u32[0]); \ 529 \ 530 addr = (void *)((_esmp)->esm_base + (_offset)); \ 531 \ 532 *addr = (_edp)->ed_u32[0]; \ 533 \ 534 _NOTE(CONSTANTCONDITION) \ 535 } while (B_FALSE) 536 537 #if defined(__x86_64__) 538 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 539 do { \ 540 uint64_t *addr; \ 541 \ 542 _NOTE(CONSTANTCONDITION) \ 543 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 544 ("not power of 2 aligned")); \ 545 \ 546 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 547 uint32_t, (_eqp)->eq_u32[1], \ 548 uint32_t, (_eqp)->eq_u32[0]); \ 549 \ 550 addr = (void *)((_esmp)->esm_base + (_offset)); \ 551 \ 552 *addr = (_eqp)->eq_u64[0]; \ 553 \ 554 _NOTE(CONSTANTCONDITION) \ 555 } while (B_FALSE) 556 557 #else 558 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 559 do { \ 560 uint32_t *addr; \ 561 \ 562 _NOTE(CONSTANTCONDITION) \ 563 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 564 ("not power of 2 aligned")); \ 565 \ 566 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 567 uint32_t, (_eqp)->eq_u32[1], \ 568 uint32_t, (_eqp)->eq_u32[0]); \ 569 \ 570 addr = (void *)((_esmp)->esm_base + (_offset)); \ 571 \ 572 *addr++ = (_eqp)->eq_u32[0]; \ 573 *addr = (_eqp)->eq_u32[1]; \ 574 \ 575 _NOTE(CONSTANTCONDITION) \ 576 } while (B_FALSE) 577 #endif 578 579 #if defined(__x86_64__) 580 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 581 do { \ 582 uint64_t *addr; \ 583 \ 584 _NOTE(CONSTANTCONDITION) \ 585 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 586 ("not power of 2 aligned")); \ 587 \ 588 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 589 uint32_t, (_eop)->eo_u32[3], \ 590 uint32_t, (_eop)->eo_u32[2], \ 591 uint32_t, (_eop)->eo_u32[1], \ 592 uint32_t, (_eop)->eo_u32[0]); \ 593 \ 594 addr = (void *)((_esmp)->esm_base + (_offset)); \ 595 \ 596 *addr++ = (_eop)->eo_u64[0]; \ 597 *addr = (_eop)->eo_u64[1]; \ 598 \ 599 _NOTE(CONSTANTCONDITION) \ 600 } while (B_FALSE) 601 #else 602 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 603 do { \ 604 uint32_t *addr; \ 605 \ 606 _NOTE(CONSTANTCONDITION) \ 607 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 608 ("not power of 2 aligned")); \ 609 \ 610 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 611 uint32_t, (_eop)->eo_u32[3], \ 612 uint32_t, (_eop)->eo_u32[2], \ 613 uint32_t, (_eop)->eo_u32[1], \ 614 uint32_t, (_eop)->eo_u32[0]); \ 615 \ 616 addr = (void *)((_esmp)->esm_base + (_offset)); \ 617 \ 618 *addr++ = (_eop)->eo_u32[0]; \ 619 *addr++ = (_eop)->eo_u32[1]; \ 620 *addr++ = (_eop)->eo_u32[2]; \ 621 *addr = (_eop)->eo_u32[3]; \ 622 \ 623 _NOTE(CONSTANTCONDITION) \ 624 } while (B_FALSE) 625 #endif 626 627 #define EFSYS_MEM_ADDR(_esmp) \ 628 ((_esmp)->esm_addr) 629 630 #define EFSYS_MEM_IS_NULL(_esmp) \ 631 ((_esmp)->esm_base == NULL) 632 633 /* BAR */ 634 635 #define SFXGE_LOCK_NAME_MAX 16 636 637 typedef struct efsys_bar_s { 638 struct mtx esb_lock; 639 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 640 bus_space_tag_t esb_tag; 641 bus_space_handle_t esb_handle; 642 int esb_rid; 643 struct resource *esb_res; 644 } efsys_bar_t; 645 646 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 647 do { \ 648 snprintf((_esbp)->esb_lock_name, \ 649 sizeof((_esbp)->esb_lock_name), \ 650 "%s:bar", (_ifname)); \ 651 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 652 NULL, MTX_DEF); \ 653 _NOTE(CONSTANTCONDITION) \ 654 } while (B_FALSE) 655 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 656 mtx_destroy(&(_esbp)->esb_lock) 657 #define SFXGE_BAR_LOCK(_esbp) \ 658 mtx_lock(&(_esbp)->esb_lock) 659 #define SFXGE_BAR_UNLOCK(_esbp) \ 660 mtx_unlock(&(_esbp)->esb_lock) 661 662 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 663 do { \ 664 _NOTE(CONSTANTCONDITION) \ 665 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 666 ("not power of 2 aligned")); \ 667 \ 668 _NOTE(CONSTANTCONDITION) \ 669 if (_lock) \ 670 SFXGE_BAR_LOCK(_esbp); \ 671 \ 672 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 673 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 674 (_offset)); \ 675 \ 676 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 677 uint32_t, (_edp)->ed_u32[0]); \ 678 \ 679 _NOTE(CONSTANTCONDITION) \ 680 if (_lock) \ 681 SFXGE_BAR_UNLOCK(_esbp); \ 682 _NOTE(CONSTANTCONDITION) \ 683 } while (B_FALSE) 684 685 #if defined(SFXGE_USE_BUS_SPACE_8) 686 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 687 do { \ 688 _NOTE(CONSTANTCONDITION) \ 689 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 690 ("not power of 2 aligned")); \ 691 \ 692 SFXGE_BAR_LOCK(_esbp); \ 693 \ 694 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 695 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 696 (_offset)); \ 697 \ 698 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 699 uint32_t, (_eqp)->eq_u32[1], \ 700 uint32_t, (_eqp)->eq_u32[0]); \ 701 \ 702 SFXGE_BAR_UNLOCK(_esbp); \ 703 _NOTE(CONSTANTCONDITION) \ 704 } while (B_FALSE) 705 706 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 707 do { \ 708 _NOTE(CONSTANTCONDITION) \ 709 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 710 ("not power of 2 aligned")); \ 711 \ 712 _NOTE(CONSTANTCONDITION) \ 713 if (_lock) \ 714 SFXGE_BAR_LOCK(_esbp); \ 715 \ 716 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 717 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 718 (_offset)); \ 719 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 720 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 721 (_offset) + 8); \ 722 \ 723 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 724 uint32_t, (_eop)->eo_u32[3], \ 725 uint32_t, (_eop)->eo_u32[2], \ 726 uint32_t, (_eop)->eo_u32[1], \ 727 uint32_t, (_eop)->eo_u32[0]); \ 728 \ 729 _NOTE(CONSTANTCONDITION) \ 730 if (_lock) \ 731 SFXGE_BAR_UNLOCK(_esbp); \ 732 _NOTE(CONSTANTCONDITION) \ 733 } while (B_FALSE) 734 735 #else 736 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 737 do { \ 738 _NOTE(CONSTANTCONDITION) \ 739 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 740 ("not power of 2 aligned")); \ 741 \ 742 SFXGE_BAR_LOCK(_esbp); \ 743 \ 744 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 745 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 746 (_offset)); \ 747 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 748 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 749 (_offset) + 4); \ 750 \ 751 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 752 uint32_t, (_eqp)->eq_u32[1], \ 753 uint32_t, (_eqp)->eq_u32[0]); \ 754 \ 755 SFXGE_BAR_UNLOCK(_esbp); \ 756 _NOTE(CONSTANTCONDITION) \ 757 } while (B_FALSE) 758 759 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 760 do { \ 761 _NOTE(CONSTANTCONDITION) \ 762 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 763 ("not power of 2 aligned")); \ 764 \ 765 _NOTE(CONSTANTCONDITION) \ 766 if (_lock) \ 767 SFXGE_BAR_LOCK(_esbp); \ 768 \ 769 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 770 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 771 (_offset)); \ 772 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 773 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 774 (_offset) + 4); \ 775 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 776 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 777 (_offset) + 8); \ 778 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 779 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 780 (_offset) + 12); \ 781 \ 782 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 783 uint32_t, (_eop)->eo_u32[3], \ 784 uint32_t, (_eop)->eo_u32[2], \ 785 uint32_t, (_eop)->eo_u32[1], \ 786 uint32_t, (_eop)->eo_u32[0]); \ 787 \ 788 _NOTE(CONSTANTCONDITION) \ 789 if (_lock) \ 790 SFXGE_BAR_UNLOCK(_esbp); \ 791 _NOTE(CONSTANTCONDITION) \ 792 } while (B_FALSE) 793 #endif 794 795 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 796 do { \ 797 _NOTE(CONSTANTCONDITION) \ 798 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 799 ("not power of 2 aligned")); \ 800 \ 801 _NOTE(CONSTANTCONDITION) \ 802 if (_lock) \ 803 SFXGE_BAR_LOCK(_esbp); \ 804 \ 805 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 806 uint32_t, (_edp)->ed_u32[0]); \ 807 \ 808 /* \ 809 * Make sure that previous writes to the dword have \ 810 * been done. It should be cheaper than barrier just \ 811 * after the write below. \ 812 */ \ 813 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 814 (_offset), sizeof (efx_dword_t), \ 815 BUS_SPACE_BARRIER_WRITE); \ 816 bus_space_write_stream_4((_esbp)->esb_tag, \ 817 (_esbp)->esb_handle, \ 818 (_offset), (_edp)->ed_u32[0]); \ 819 \ 820 _NOTE(CONSTANTCONDITION) \ 821 if (_lock) \ 822 SFXGE_BAR_UNLOCK(_esbp); \ 823 _NOTE(CONSTANTCONDITION) \ 824 } while (B_FALSE) 825 826 #if defined(SFXGE_USE_BUS_SPACE_8) 827 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 828 do { \ 829 _NOTE(CONSTANTCONDITION) \ 830 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 831 ("not power of 2 aligned")); \ 832 \ 833 SFXGE_BAR_LOCK(_esbp); \ 834 \ 835 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 836 uint32_t, (_eqp)->eq_u32[1], \ 837 uint32_t, (_eqp)->eq_u32[0]); \ 838 \ 839 /* \ 840 * Make sure that previous writes to the qword have \ 841 * been done. It should be cheaper than barrier just \ 842 * after the write below. \ 843 */ \ 844 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 845 (_offset), sizeof (efx_qword_t), \ 846 BUS_SPACE_BARRIER_WRITE); \ 847 bus_space_write_stream_8((_esbp)->esb_tag, \ 848 (_esbp)->esb_handle, \ 849 (_offset), (_eqp)->eq_u64[0]); \ 850 \ 851 SFXGE_BAR_UNLOCK(_esbp); \ 852 _NOTE(CONSTANTCONDITION) \ 853 } while (B_FALSE) 854 #else 855 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 856 do { \ 857 _NOTE(CONSTANTCONDITION) \ 858 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 859 ("not power of 2 aligned")); \ 860 \ 861 SFXGE_BAR_LOCK(_esbp); \ 862 \ 863 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 864 uint32_t, (_eqp)->eq_u32[1], \ 865 uint32_t, (_eqp)->eq_u32[0]); \ 866 \ 867 /* \ 868 * Make sure that previous writes to the qword have \ 869 * been done. It should be cheaper than barrier just \ 870 * after the last write below. \ 871 */ \ 872 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 873 (_offset), sizeof (efx_qword_t), \ 874 BUS_SPACE_BARRIER_WRITE); \ 875 bus_space_write_stream_4((_esbp)->esb_tag, \ 876 (_esbp)->esb_handle, \ 877 (_offset), (_eqp)->eq_u32[0]); \ 878 /* \ 879 * It should be guaranteed that the last dword comes \ 880 * the last, so barrier entire qword to be sure that \ 881 * neither above nor below writes are reordered. \ 882 */ \ 883 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 884 (_offset), sizeof (efx_qword_t), \ 885 BUS_SPACE_BARRIER_WRITE); \ 886 bus_space_write_stream_4((_esbp)->esb_tag, \ 887 (_esbp)->esb_handle, \ 888 (_offset) + 4, (_eqp)->eq_u32[1]); \ 889 \ 890 SFXGE_BAR_UNLOCK(_esbp); \ 891 _NOTE(CONSTANTCONDITION) \ 892 } while (B_FALSE) 893 #endif 894 895 /* 896 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 897 * (required by PIO hardware) 898 */ 899 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 900 do { \ 901 _NOTE(CONSTANTCONDITION) \ 902 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 903 ("not power of 2 aligned")); \ 904 \ 905 (void) (_esbp); \ 906 \ 907 /* FIXME: Perform a 64-bit write */ \ 908 KASSERT(0, ("not implemented")); \ 909 \ 910 _NOTE(CONSTANTCONDITION) \ 911 } while (B_FALSE) 912 913 #if defined(SFXGE_USE_BUS_SPACE_8) 914 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 915 do { \ 916 _NOTE(CONSTANTCONDITION) \ 917 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 918 ("not power of 2 aligned")); \ 919 \ 920 _NOTE(CONSTANTCONDITION) \ 921 if (_lock) \ 922 SFXGE_BAR_LOCK(_esbp); \ 923 \ 924 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 925 uint32_t, (_eop)->eo_u32[3], \ 926 uint32_t, (_eop)->eo_u32[2], \ 927 uint32_t, (_eop)->eo_u32[1], \ 928 uint32_t, (_eop)->eo_u32[0]); \ 929 \ 930 /* \ 931 * Make sure that previous writes to the oword have \ 932 * been done. It should be cheaper than barrier just \ 933 * after the last write below. \ 934 */ \ 935 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 936 (_offset), sizeof (efx_oword_t), \ 937 BUS_SPACE_BARRIER_WRITE); \ 938 bus_space_write_stream_8((_esbp)->esb_tag, \ 939 (_esbp)->esb_handle, \ 940 (_offset), (_eop)->eo_u64[0]); \ 941 /* \ 942 * It should be guaranteed that the last qword comes \ 943 * the last, so barrier entire oword to be sure that \ 944 * neither above nor below writes are reordered. \ 945 */ \ 946 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 947 (_offset), sizeof (efx_oword_t), \ 948 BUS_SPACE_BARRIER_WRITE); \ 949 bus_space_write_stream_8((_esbp)->esb_tag, \ 950 (_esbp)->esb_handle, \ 951 (_offset) + 8, (_eop)->eo_u64[1]); \ 952 \ 953 _NOTE(CONSTANTCONDITION) \ 954 if (_lock) \ 955 SFXGE_BAR_UNLOCK(_esbp); \ 956 _NOTE(CONSTANTCONDITION) \ 957 } while (B_FALSE) 958 959 #else 960 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 961 do { \ 962 _NOTE(CONSTANTCONDITION) \ 963 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 964 ("not power of 2 aligned")); \ 965 \ 966 _NOTE(CONSTANTCONDITION) \ 967 if (_lock) \ 968 SFXGE_BAR_LOCK(_esbp); \ 969 \ 970 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 971 uint32_t, (_eop)->eo_u32[3], \ 972 uint32_t, (_eop)->eo_u32[2], \ 973 uint32_t, (_eop)->eo_u32[1], \ 974 uint32_t, (_eop)->eo_u32[0]); \ 975 \ 976 /* \ 977 * Make sure that previous writes to the oword have \ 978 * been done. It should be cheaper than barrier just \ 979 * after the last write below. \ 980 */ \ 981 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 982 (_offset), sizeof (efx_oword_t), \ 983 BUS_SPACE_BARRIER_WRITE); \ 984 bus_space_write_stream_4((_esbp)->esb_tag, \ 985 (_esbp)->esb_handle, \ 986 (_offset), (_eop)->eo_u32[0]); \ 987 bus_space_write_stream_4((_esbp)->esb_tag, \ 988 (_esbp)->esb_handle, \ 989 (_offset) + 4, (_eop)->eo_u32[1]); \ 990 bus_space_write_stream_4((_esbp)->esb_tag, \ 991 (_esbp)->esb_handle, \ 992 (_offset) + 8, (_eop)->eo_u32[2]); \ 993 /* \ 994 * It should be guaranteed that the last dword comes \ 995 * the last, so barrier entire oword to be sure that \ 996 * neither above nor below writes are reordered. \ 997 */ \ 998 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 999 (_offset), sizeof (efx_oword_t), \ 1000 BUS_SPACE_BARRIER_WRITE); \ 1001 bus_space_write_stream_4((_esbp)->esb_tag, \ 1002 (_esbp)->esb_handle, \ 1003 (_offset) + 12, (_eop)->eo_u32[3]); \ 1004 \ 1005 _NOTE(CONSTANTCONDITION) \ 1006 if (_lock) \ 1007 SFXGE_BAR_UNLOCK(_esbp); \ 1008 _NOTE(CONSTANTCONDITION) \ 1009 } while (B_FALSE) 1010 #endif 1011 1012 /* Use the standard octo-word write for doorbell writes */ 1013 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1014 do { \ 1015 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1016 _NOTE(CONSTANTCONDITION) \ 1017 } while (B_FALSE) 1018 1019 /* SPIN */ 1020 1021 #define EFSYS_SPIN(_us) \ 1022 do { \ 1023 DELAY(_us); \ 1024 _NOTE(CONSTANTCONDITION) \ 1025 } while (B_FALSE) 1026 1027 #define EFSYS_SLEEP EFSYS_SPIN 1028 1029 /* BARRIERS */ 1030 1031 #define EFSYS_MEM_READ_BARRIER() rmb() 1032 #define EFSYS_PIO_WRITE_BARRIER() 1033 1034 /* DMA SYNC */ 1035 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1036 do { \ 1037 bus_dmamap_sync((_esmp)->esm_tag, \ 1038 (_esmp)->esm_map, \ 1039 BUS_DMASYNC_POSTREAD); \ 1040 _NOTE(CONSTANTCONDITION) \ 1041 } while (B_FALSE) 1042 1043 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1044 do { \ 1045 bus_dmamap_sync((_esmp)->esm_tag, \ 1046 (_esmp)->esm_map, \ 1047 BUS_DMASYNC_PREWRITE); \ 1048 _NOTE(CONSTANTCONDITION) \ 1049 } while (B_FALSE) 1050 1051 /* TIMESTAMP */ 1052 1053 typedef clock_t efsys_timestamp_t; 1054 1055 #define EFSYS_TIMESTAMP(_usp) \ 1056 do { \ 1057 clock_t now; \ 1058 \ 1059 now = ticks; \ 1060 *(_usp) = now * hz / 1000000; \ 1061 _NOTE(CONSTANTCONDITION) \ 1062 } while (B_FALSE) 1063 1064 /* KMEM */ 1065 1066 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1067 do { \ 1068 (_esip) = (_esip); \ 1069 /* \ 1070 * The macro is used in non-sleepable contexts, for \ 1071 * example, holding a mutex. \ 1072 */ \ 1073 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1074 _NOTE(CONSTANTCONDITION) \ 1075 } while (B_FALSE) 1076 1077 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1078 do { \ 1079 (void) (_esip); \ 1080 (void) (_size); \ 1081 free((_p), M_SFXGE); \ 1082 _NOTE(CONSTANTCONDITION) \ 1083 } while (B_FALSE) 1084 1085 /* LOCK */ 1086 1087 typedef struct efsys_lock_s { 1088 struct mtx lock; 1089 char lock_name[SFXGE_LOCK_NAME_MAX]; 1090 } efsys_lock_t; 1091 1092 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1093 do { \ 1094 efsys_lock_t *__eslp = (_eslp); \ 1095 \ 1096 snprintf((__eslp)->lock_name, \ 1097 sizeof((__eslp)->lock_name), \ 1098 "%s:%s", (_ifname), (_label)); \ 1099 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1100 NULL, MTX_DEF); \ 1101 } while (B_FALSE) 1102 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1103 mtx_destroy(&(_eslp)->lock) 1104 #define SFXGE_EFSYS_LOCK(_eslp) \ 1105 mtx_lock(&(_eslp)->lock) 1106 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1107 mtx_unlock(&(_eslp)->lock) 1108 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1109 mtx_assert(&(_eslp)->lock, MA_OWNED) 1110 1111 #define EFSYS_LOCK_MAGIC 0x000010c4 1112 1113 #define EFSYS_LOCK(_lockp, _state) \ 1114 do { \ 1115 SFXGE_EFSYS_LOCK(_lockp); \ 1116 (_state) = EFSYS_LOCK_MAGIC; \ 1117 _NOTE(CONSTANTCONDITION) \ 1118 } while (B_FALSE) 1119 1120 #define EFSYS_UNLOCK(_lockp, _state) \ 1121 do { \ 1122 if ((_state) != EFSYS_LOCK_MAGIC) \ 1123 KASSERT(B_FALSE, ("not locked")); \ 1124 SFXGE_EFSYS_UNLOCK(_lockp); \ 1125 _NOTE(CONSTANTCONDITION) \ 1126 } while (B_FALSE) 1127 1128 /* PREEMPT */ 1129 1130 #define EFSYS_PREEMPT_DISABLE(_state) \ 1131 do { \ 1132 (_state) = (_state); \ 1133 critical_enter(); \ 1134 _NOTE(CONSTANTCONDITION) \ 1135 } while (B_FALSE) 1136 1137 #define EFSYS_PREEMPT_ENABLE(_state) \ 1138 do { \ 1139 (_state) = (_state); \ 1140 critical_exit(_state); \ 1141 _NOTE(CONSTANTCONDITION) \ 1142 } while (B_FALSE) 1143 1144 /* STAT */ 1145 1146 typedef uint64_t efsys_stat_t; 1147 1148 #define EFSYS_STAT_INCR(_knp, _delta) \ 1149 do { \ 1150 *(_knp) += (_delta); \ 1151 _NOTE(CONSTANTCONDITION) \ 1152 } while (B_FALSE) 1153 1154 #define EFSYS_STAT_DECR(_knp, _delta) \ 1155 do { \ 1156 *(_knp) -= (_delta); \ 1157 _NOTE(CONSTANTCONDITION) \ 1158 } while (B_FALSE) 1159 1160 #define EFSYS_STAT_SET(_knp, _val) \ 1161 do { \ 1162 *(_knp) = (_val); \ 1163 _NOTE(CONSTANTCONDITION) \ 1164 } while (B_FALSE) 1165 1166 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1167 do { \ 1168 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1169 _NOTE(CONSTANTCONDITION) \ 1170 } while (B_FALSE) 1171 1172 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1173 do { \ 1174 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1175 _NOTE(CONSTANTCONDITION) \ 1176 } while (B_FALSE) 1177 1178 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1179 do { \ 1180 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1181 _NOTE(CONSTANTCONDITION) \ 1182 } while (B_FALSE) 1183 1184 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1185 do { \ 1186 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1187 _NOTE(CONSTANTCONDITION) \ 1188 } while (B_FALSE) 1189 1190 /* ERR */ 1191 1192 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1193 uint32_t, uint32_t); 1194 1195 #if EFSYS_OPT_DECODE_INTR_FATAL 1196 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1197 do { \ 1198 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1199 _NOTE(CONSTANTCONDITION) \ 1200 } while (B_FALSE) 1201 #endif 1202 1203 /* ASSERT */ 1204 1205 #define EFSYS_ASSERT(_exp) do { \ 1206 if (!(_exp)) \ 1207 panic("%s", #_exp); \ 1208 } while (0) 1209 1210 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1211 const _t __x = (_t)(_x); \ 1212 const _t __y = (_t)(_y); \ 1213 if (!(__x _op __y)) \ 1214 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1215 } while(0) 1216 1217 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1218 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1219 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1220 1221 /* ROTATE */ 1222 1223 #define EFSYS_HAS_ROTL_DWORD 0 1224 1225 #ifdef __cplusplus 1226 } 1227 #endif 1228 1229 #endif /* _SYS_EFSYS_H */ 1230