1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 * 33 * $FreeBSD$ 34 */ 35 36 #ifndef _SYS_EFSYS_H 37 #define _SYS_EFSYS_H 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/bus.h> 45 #include <sys/endian.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/mutex.h> 50 #include <sys/rwlock.h> 51 #include <sys/sdt.h> 52 #include <sys/systm.h> 53 54 #include <machine/bus.h> 55 #include <machine/endian.h> 56 57 #define EFSYS_HAS_UINT64 1 58 #if defined(__x86_64__) 59 #define EFSYS_USE_UINT64 1 60 #else 61 #define EFSYS_USE_UINT64 0 62 #endif 63 #define EFSYS_HAS_SSE2_M128 0 64 #if _BYTE_ORDER == _BIG_ENDIAN 65 #define EFSYS_IS_BIG_ENDIAN 1 66 #define EFSYS_IS_LITTLE_ENDIAN 0 67 #elif _BYTE_ORDER == _LITTLE_ENDIAN 68 #define EFSYS_IS_BIG_ENDIAN 0 69 #define EFSYS_IS_LITTLE_ENDIAN 1 70 #endif 71 #include "efx_types.h" 72 73 /* Common code requires this */ 74 #if __FreeBSD_version < 800068 75 #define memmove(d, s, l) bcopy(s, d, l) 76 #endif 77 78 /* FreeBSD equivalents of Solaris things */ 79 #ifndef _NOTE 80 #define _NOTE(s) 81 #endif 82 83 #ifndef B_FALSE 84 #define B_FALSE FALSE 85 #endif 86 #ifndef B_TRUE 87 #define B_TRUE TRUE 88 #endif 89 90 #ifndef IS_P2ALIGNED 91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92 #endif 93 94 #ifndef P2ROUNDUP 95 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96 #endif 97 98 #ifndef P2ALIGN 99 #define P2ALIGN(_x, _a) ((_x) & -(_a)) 100 #endif 101 102 #ifndef IS2P 103 #define ISP2(x) (((x) & ((x) - 1)) == 0) 104 #endif 105 106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 107 108 #define SFXGE_USE_BUS_SPACE_8 1 109 110 #if !defined(bus_space_read_stream_8) 111 112 #define bus_space_read_stream_8(t, h, o) \ 113 bus_space_read_8((t), (h), (o)) 114 115 #define bus_space_write_stream_8(t, h, o, v) \ 116 bus_space_write_8((t), (h), (o), (v)) 117 118 #endif 119 120 #endif 121 122 #define ENOTACTIVE EINVAL 123 124 /* Memory type to use on FreeBSD */ 125 MALLOC_DECLARE(M_SFXGE); 126 127 /* Machine dependend prefetch wrappers */ 128 #if defined(__i386__) || defined(__amd64__) 129 static __inline void 130 prefetch_read_many(void *addr) 131 { 132 133 __asm__( 134 "prefetcht0 (%0)" 135 : 136 : "r" (addr)); 137 } 138 139 static __inline void 140 prefetch_read_once(void *addr) 141 { 142 143 __asm__( 144 "prefetchnta (%0)" 145 : 146 : "r" (addr)); 147 } 148 #elif defined(__sparc64__) 149 static __inline void 150 prefetch_read_many(void *addr) 151 { 152 153 __asm__( 154 "prefetch [%0], 0" 155 : 156 : "r" (addr)); 157 } 158 159 static __inline void 160 prefetch_read_once(void *addr) 161 { 162 163 __asm__( 164 "prefetch [%0], 1" 165 : 166 : "r" (addr)); 167 } 168 #else 169 static __inline void 170 prefetch_read_many(void *addr) 171 { 172 173 } 174 175 static __inline void 176 prefetch_read_once(void *addr) 177 { 178 179 } 180 #endif 181 182 #if defined(__i386__) || defined(__amd64__) 183 #include <vm/vm.h> 184 #include <vm/pmap.h> 185 #endif 186 static __inline void 187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188 struct mbuf *m, bus_dma_segment_t *seg) 189 { 190 #if defined(__i386__) || defined(__amd64__) 191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192 seg->ds_len = m->m_len; 193 #else 194 int nsegstmp; 195 196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197 #endif 198 } 199 200 /* Modifiers used for Windows builds */ 201 #define __in 202 #define __in_opt 203 #define __in_ecount(_n) 204 #define __in_ecount_opt(_n) 205 #define __in_bcount(_n) 206 #define __in_bcount_opt(_n) 207 208 #define __out 209 #define __out_opt 210 #define __out_ecount(_n) 211 #define __out_ecount_opt(_n) 212 #define __out_bcount(_n) 213 #define __out_bcount_opt(_n) 214 215 #define __deref_out 216 217 #define __inout 218 #define __inout_opt 219 #define __inout_ecount(_n) 220 #define __inout_ecount_opt(_n) 221 #define __inout_bcount(_n) 222 #define __inout_bcount_opt(_n) 223 #define __inout_bcount_full_opt(_n) 224 225 #define __deref_out_bcount_opt(n) 226 227 #define __checkReturn 228 #define __success(_x) 229 230 #define __drv_when(_p, _c) 231 232 /* Code inclusion options */ 233 234 235 #define EFSYS_OPT_NAMES 1 236 237 #define EFSYS_OPT_FALCON 0 238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 239 #define EFSYS_OPT_SIENA 1 240 #define EFSYS_OPT_HUNTINGTON 1 241 #ifdef DEBUG 242 #define EFSYS_OPT_CHECK_REG 1 243 #else 244 #define EFSYS_OPT_CHECK_REG 0 245 #endif 246 247 #define EFSYS_OPT_MCDI 1 248 #define EFSYS_OPT_MCDI_LOGGING 0 249 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 250 251 #define EFSYS_OPT_MAC_FALCON_GMAC 0 252 #define EFSYS_OPT_MAC_FALCON_XMAC 0 253 #define EFSYS_OPT_MAC_STATS 1 254 255 #define EFSYS_OPT_LOOPBACK 0 256 257 #define EFSYS_OPT_MON_NULL 0 258 #define EFSYS_OPT_MON_LM87 0 259 #define EFSYS_OPT_MON_MAX6647 0 260 #define EFSYS_OPT_MON_MCDI 0 261 #define EFSYS_OPT_MON_STATS 0 262 263 #define EFSYS_OPT_PHY_NULL 0 264 #define EFSYS_OPT_PHY_QT2022C2 0 265 #define EFSYS_OPT_PHY_SFX7101 0 266 #define EFSYS_OPT_PHY_TXC43128 0 267 #define EFSYS_OPT_PHY_SFT9001 0 268 #define EFSYS_OPT_PHY_QT2025C 0 269 #define EFSYS_OPT_PHY_STATS 1 270 #define EFSYS_OPT_PHY_PROPS 0 271 #define EFSYS_OPT_PHY_BIST 0 272 #define EFSYS_OPT_BIST 1 273 #define EFSYS_OPT_PHY_LED_CONTROL 1 274 #define EFSYS_OPT_PHY_FLAGS 0 275 276 #define EFSYS_OPT_VPD 1 277 #define EFSYS_OPT_NVRAM 1 278 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 279 #define EFSYS_OPT_NVRAM_SFT9001 0 280 #define EFSYS_OPT_NVRAM_SFX7101 0 281 #define EFSYS_OPT_BOOTCFG 0 282 283 #define EFSYS_OPT_PCIE_TUNE 0 284 #define EFSYS_OPT_DIAG 0 285 #define EFSYS_OPT_WOL 1 286 #define EFSYS_OPT_RX_SCALE 1 287 #define EFSYS_OPT_QSTATS 1 288 #define EFSYS_OPT_FILTER 1 289 #define EFSYS_OPT_MCAST_FILTER_LIST 1 290 #define EFSYS_OPT_RX_SCATTER 0 291 #define EFSYS_OPT_RX_HDR_SPLIT 0 292 293 #define EFSYS_OPT_EV_PREFETCH 0 294 295 #define EFSYS_OPT_DECODE_INTR_FATAL 1 296 297 /* ID */ 298 299 typedef struct __efsys_identifier_s efsys_identifier_t; 300 301 /* PROBE */ 302 303 #ifndef DTRACE_PROBE 304 305 #define EFSYS_PROBE(_name) 306 307 #define EFSYS_PROBE1(_name, _type1, _arg1) 308 309 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 310 311 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 312 _type3, _arg3) 313 314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 315 _type3, _arg3, _type4, _arg4) 316 317 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 318 _type3, _arg3, _type4, _arg4, _type5, _arg5) 319 320 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 321 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 322 _type6, _arg6) 323 324 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 325 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 326 _type6, _arg6, _type7, _arg7) 327 328 #else /* DTRACE_PROBE */ 329 330 #define EFSYS_PROBE(_name) \ 331 DTRACE_PROBE(_name) 332 333 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 334 DTRACE_PROBE1(_name, _type1, _arg1) 335 336 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 337 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 338 339 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 340 _type3, _arg3) \ 341 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 342 _type3, _arg3) 343 344 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 345 _type3, _arg3, _type4, _arg4) \ 346 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 347 _type3, _arg3, _type4, _arg4) 348 349 #ifdef DTRACE_PROBE5 350 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 351 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 352 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 353 _type3, _arg3, _type4, _arg4, _type5, _arg5) 354 #else 355 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 356 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 357 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 358 _type3, _arg3, _type4, _arg4) 359 #endif 360 361 #ifdef DTRACE_PROBE6 362 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 363 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 364 _type6, _arg6) \ 365 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 366 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 367 _type6, _arg6) 368 #else 369 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 370 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 371 _type6, _arg6) \ 372 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 373 _type3, _arg3, _type4, _arg4, _type5, _arg5) 374 #endif 375 376 #ifdef DTRACE_PROBE7 377 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 378 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 379 _type6, _arg6, _type7, _arg7) \ 380 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 381 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 382 _type6, _arg6, _type7, _arg7) 383 #else 384 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 385 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 386 _type6, _arg6, _type7, _arg7) \ 387 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 388 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 389 _type6, _arg6) 390 #endif 391 392 #endif /* DTRACE_PROBE */ 393 394 /* DMA */ 395 396 typedef uint64_t efsys_dma_addr_t; 397 398 typedef struct efsys_mem_s { 399 bus_dma_tag_t esm_tag; 400 bus_dmamap_t esm_map; 401 caddr_t esm_base; 402 efsys_dma_addr_t esm_addr; 403 } efsys_mem_t; 404 405 406 #define EFSYS_MEM_ZERO(_esmp, _size) \ 407 do { \ 408 (void) memset((_esmp)->esm_base, 0, (_size)); \ 409 \ 410 _NOTE(CONSTANTCONDITION) \ 411 } while (B_FALSE) 412 413 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 414 do { \ 415 uint32_t *addr; \ 416 \ 417 _NOTE(CONSTANTCONDITION) \ 418 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 419 ("not power of 2 aligned")); \ 420 \ 421 addr = (void *)((_esmp)->esm_base + (_offset)); \ 422 \ 423 (_edp)->ed_u32[0] = *addr; \ 424 \ 425 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 426 uint32_t, (_edp)->ed_u32[0]); \ 427 \ 428 _NOTE(CONSTANTCONDITION) \ 429 } while (B_FALSE) 430 431 #if defined(__x86_64__) 432 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 433 do { \ 434 uint64_t *addr; \ 435 \ 436 _NOTE(CONSTANTCONDITION) \ 437 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 438 ("not power of 2 aligned")); \ 439 \ 440 addr = (void *)((_esmp)->esm_base + (_offset)); \ 441 \ 442 (_eqp)->eq_u64[0] = *addr; \ 443 \ 444 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 445 uint32_t, (_eqp)->eq_u32[1], \ 446 uint32_t, (_eqp)->eq_u32[0]); \ 447 \ 448 _NOTE(CONSTANTCONDITION) \ 449 } while (B_FALSE) 450 #else 451 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 452 do { \ 453 uint32_t *addr; \ 454 \ 455 _NOTE(CONSTANTCONDITION) \ 456 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 457 ("not power of 2 aligned")); \ 458 \ 459 addr = (void *)((_esmp)->esm_base + (_offset)); \ 460 \ 461 (_eqp)->eq_u32[0] = *addr++; \ 462 (_eqp)->eq_u32[1] = *addr; \ 463 \ 464 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 465 uint32_t, (_eqp)->eq_u32[1], \ 466 uint32_t, (_eqp)->eq_u32[0]); \ 467 \ 468 _NOTE(CONSTANTCONDITION) \ 469 } while (B_FALSE) 470 #endif 471 472 #if defined(__x86_64__) 473 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 474 do { \ 475 uint64_t *addr; \ 476 \ 477 _NOTE(CONSTANTCONDITION) \ 478 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 479 ("not power of 2 aligned")); \ 480 \ 481 addr = (void *)((_esmp)->esm_base + (_offset)); \ 482 \ 483 (_eop)->eo_u64[0] = *addr++; \ 484 (_eop)->eo_u64[1] = *addr; \ 485 \ 486 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 487 uint32_t, (_eop)->eo_u32[3], \ 488 uint32_t, (_eop)->eo_u32[2], \ 489 uint32_t, (_eop)->eo_u32[1], \ 490 uint32_t, (_eop)->eo_u32[0]); \ 491 \ 492 _NOTE(CONSTANTCONDITION) \ 493 } while (B_FALSE) 494 #else 495 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 496 do { \ 497 uint32_t *addr; \ 498 \ 499 _NOTE(CONSTANTCONDITION) \ 500 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 501 ("not power of 2 aligned")); \ 502 \ 503 addr = (void *)((_esmp)->esm_base + (_offset)); \ 504 \ 505 (_eop)->eo_u32[0] = *addr++; \ 506 (_eop)->eo_u32[1] = *addr++; \ 507 (_eop)->eo_u32[2] = *addr++; \ 508 (_eop)->eo_u32[3] = *addr; \ 509 \ 510 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 511 uint32_t, (_eop)->eo_u32[3], \ 512 uint32_t, (_eop)->eo_u32[2], \ 513 uint32_t, (_eop)->eo_u32[1], \ 514 uint32_t, (_eop)->eo_u32[0]); \ 515 \ 516 _NOTE(CONSTANTCONDITION) \ 517 } while (B_FALSE) 518 #endif 519 520 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 521 do { \ 522 uint32_t *addr; \ 523 \ 524 _NOTE(CONSTANTCONDITION) \ 525 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 526 ("not power of 2 aligned")); \ 527 \ 528 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 529 uint32_t, (_edp)->ed_u32[0]); \ 530 \ 531 addr = (void *)((_esmp)->esm_base + (_offset)); \ 532 \ 533 *addr = (_edp)->ed_u32[0]; \ 534 \ 535 _NOTE(CONSTANTCONDITION) \ 536 } while (B_FALSE) 537 538 #if defined(__x86_64__) 539 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 540 do { \ 541 uint64_t *addr; \ 542 \ 543 _NOTE(CONSTANTCONDITION) \ 544 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 545 ("not power of 2 aligned")); \ 546 \ 547 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 548 uint32_t, (_eqp)->eq_u32[1], \ 549 uint32_t, (_eqp)->eq_u32[0]); \ 550 \ 551 addr = (void *)((_esmp)->esm_base + (_offset)); \ 552 \ 553 *addr = (_eqp)->eq_u64[0]; \ 554 \ 555 _NOTE(CONSTANTCONDITION) \ 556 } while (B_FALSE) 557 558 #else 559 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 560 do { \ 561 uint32_t *addr; \ 562 \ 563 _NOTE(CONSTANTCONDITION) \ 564 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 565 ("not power of 2 aligned")); \ 566 \ 567 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 568 uint32_t, (_eqp)->eq_u32[1], \ 569 uint32_t, (_eqp)->eq_u32[0]); \ 570 \ 571 addr = (void *)((_esmp)->esm_base + (_offset)); \ 572 \ 573 *addr++ = (_eqp)->eq_u32[0]; \ 574 *addr = (_eqp)->eq_u32[1]; \ 575 \ 576 _NOTE(CONSTANTCONDITION) \ 577 } while (B_FALSE) 578 #endif 579 580 #if defined(__x86_64__) 581 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 582 do { \ 583 uint64_t *addr; \ 584 \ 585 _NOTE(CONSTANTCONDITION) \ 586 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 587 ("not power of 2 aligned")); \ 588 \ 589 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 590 uint32_t, (_eop)->eo_u32[3], \ 591 uint32_t, (_eop)->eo_u32[2], \ 592 uint32_t, (_eop)->eo_u32[1], \ 593 uint32_t, (_eop)->eo_u32[0]); \ 594 \ 595 addr = (void *)((_esmp)->esm_base + (_offset)); \ 596 \ 597 *addr++ = (_eop)->eo_u64[0]; \ 598 *addr = (_eop)->eo_u64[1]; \ 599 \ 600 _NOTE(CONSTANTCONDITION) \ 601 } while (B_FALSE) 602 #else 603 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 604 do { \ 605 uint32_t *addr; \ 606 \ 607 _NOTE(CONSTANTCONDITION) \ 608 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 609 ("not power of 2 aligned")); \ 610 \ 611 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 612 uint32_t, (_eop)->eo_u32[3], \ 613 uint32_t, (_eop)->eo_u32[2], \ 614 uint32_t, (_eop)->eo_u32[1], \ 615 uint32_t, (_eop)->eo_u32[0]); \ 616 \ 617 addr = (void *)((_esmp)->esm_base + (_offset)); \ 618 \ 619 *addr++ = (_eop)->eo_u32[0]; \ 620 *addr++ = (_eop)->eo_u32[1]; \ 621 *addr++ = (_eop)->eo_u32[2]; \ 622 *addr = (_eop)->eo_u32[3]; \ 623 \ 624 _NOTE(CONSTANTCONDITION) \ 625 } while (B_FALSE) 626 #endif 627 628 #define EFSYS_MEM_ADDR(_esmp) \ 629 ((_esmp)->esm_addr) 630 631 #define EFSYS_MEM_IS_NULL(_esmp) \ 632 ((_esmp)->esm_base == NULL) 633 634 /* BAR */ 635 636 #define SFXGE_LOCK_NAME_MAX 16 637 638 typedef struct efsys_bar_s { 639 struct mtx esb_lock; 640 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 641 bus_space_tag_t esb_tag; 642 bus_space_handle_t esb_handle; 643 int esb_rid; 644 struct resource *esb_res; 645 } efsys_bar_t; 646 647 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 648 do { \ 649 snprintf((_esbp)->esb_lock_name, \ 650 sizeof((_esbp)->esb_lock_name), \ 651 "%s:bar", (_ifname)); \ 652 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 653 NULL, MTX_DEF); \ 654 _NOTE(CONSTANTCONDITION) \ 655 } while (B_FALSE) 656 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 657 mtx_destroy(&(_esbp)->esb_lock) 658 #define SFXGE_BAR_LOCK(_esbp) \ 659 mtx_lock(&(_esbp)->esb_lock) 660 #define SFXGE_BAR_UNLOCK(_esbp) \ 661 mtx_unlock(&(_esbp)->esb_lock) 662 663 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 664 do { \ 665 _NOTE(CONSTANTCONDITION) \ 666 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 667 ("not power of 2 aligned")); \ 668 \ 669 _NOTE(CONSTANTCONDITION) \ 670 if (_lock) \ 671 SFXGE_BAR_LOCK(_esbp); \ 672 \ 673 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 674 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 675 (_offset)); \ 676 \ 677 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 678 uint32_t, (_edp)->ed_u32[0]); \ 679 \ 680 _NOTE(CONSTANTCONDITION) \ 681 if (_lock) \ 682 SFXGE_BAR_UNLOCK(_esbp); \ 683 _NOTE(CONSTANTCONDITION) \ 684 } while (B_FALSE) 685 686 #if defined(SFXGE_USE_BUS_SPACE_8) 687 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 688 do { \ 689 _NOTE(CONSTANTCONDITION) \ 690 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 691 ("not power of 2 aligned")); \ 692 \ 693 SFXGE_BAR_LOCK(_esbp); \ 694 \ 695 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 696 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 697 (_offset)); \ 698 \ 699 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 700 uint32_t, (_eqp)->eq_u32[1], \ 701 uint32_t, (_eqp)->eq_u32[0]); \ 702 \ 703 SFXGE_BAR_UNLOCK(_esbp); \ 704 _NOTE(CONSTANTCONDITION) \ 705 } while (B_FALSE) 706 707 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 708 do { \ 709 _NOTE(CONSTANTCONDITION) \ 710 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 711 ("not power of 2 aligned")); \ 712 \ 713 _NOTE(CONSTANTCONDITION) \ 714 if (_lock) \ 715 SFXGE_BAR_LOCK(_esbp); \ 716 \ 717 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 718 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 719 (_offset)); \ 720 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 721 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 722 (_offset) + 8); \ 723 \ 724 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 725 uint32_t, (_eop)->eo_u32[3], \ 726 uint32_t, (_eop)->eo_u32[2], \ 727 uint32_t, (_eop)->eo_u32[1], \ 728 uint32_t, (_eop)->eo_u32[0]); \ 729 \ 730 _NOTE(CONSTANTCONDITION) \ 731 if (_lock) \ 732 SFXGE_BAR_UNLOCK(_esbp); \ 733 _NOTE(CONSTANTCONDITION) \ 734 } while (B_FALSE) 735 736 #else 737 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 738 do { \ 739 _NOTE(CONSTANTCONDITION) \ 740 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 741 ("not power of 2 aligned")); \ 742 \ 743 SFXGE_BAR_LOCK(_esbp); \ 744 \ 745 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 746 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 747 (_offset)); \ 748 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 749 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 750 (_offset) + 4); \ 751 \ 752 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 753 uint32_t, (_eqp)->eq_u32[1], \ 754 uint32_t, (_eqp)->eq_u32[0]); \ 755 \ 756 SFXGE_BAR_UNLOCK(_esbp); \ 757 _NOTE(CONSTANTCONDITION) \ 758 } while (B_FALSE) 759 760 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 761 do { \ 762 _NOTE(CONSTANTCONDITION) \ 763 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 764 ("not power of 2 aligned")); \ 765 \ 766 _NOTE(CONSTANTCONDITION) \ 767 if (_lock) \ 768 SFXGE_BAR_LOCK(_esbp); \ 769 \ 770 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 771 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 772 (_offset)); \ 773 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 774 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 775 (_offset) + 4); \ 776 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 777 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 778 (_offset) + 8); \ 779 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 780 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 781 (_offset) + 12); \ 782 \ 783 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 784 uint32_t, (_eop)->eo_u32[3], \ 785 uint32_t, (_eop)->eo_u32[2], \ 786 uint32_t, (_eop)->eo_u32[1], \ 787 uint32_t, (_eop)->eo_u32[0]); \ 788 \ 789 _NOTE(CONSTANTCONDITION) \ 790 if (_lock) \ 791 SFXGE_BAR_UNLOCK(_esbp); \ 792 _NOTE(CONSTANTCONDITION) \ 793 } while (B_FALSE) 794 #endif 795 796 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 797 do { \ 798 _NOTE(CONSTANTCONDITION) \ 799 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 800 ("not power of 2 aligned")); \ 801 \ 802 _NOTE(CONSTANTCONDITION) \ 803 if (_lock) \ 804 SFXGE_BAR_LOCK(_esbp); \ 805 \ 806 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 807 uint32_t, (_edp)->ed_u32[0]); \ 808 \ 809 /* \ 810 * Make sure that previous writes to the dword have \ 811 * been done. It should be cheaper than barrier just \ 812 * after the write below. \ 813 */ \ 814 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 815 (_offset), sizeof (efx_dword_t), \ 816 BUS_SPACE_BARRIER_WRITE); \ 817 bus_space_write_stream_4((_esbp)->esb_tag, \ 818 (_esbp)->esb_handle, \ 819 (_offset), (_edp)->ed_u32[0]); \ 820 \ 821 _NOTE(CONSTANTCONDITION) \ 822 if (_lock) \ 823 SFXGE_BAR_UNLOCK(_esbp); \ 824 _NOTE(CONSTANTCONDITION) \ 825 } while (B_FALSE) 826 827 #if defined(SFXGE_USE_BUS_SPACE_8) 828 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 829 do { \ 830 _NOTE(CONSTANTCONDITION) \ 831 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 832 ("not power of 2 aligned")); \ 833 \ 834 SFXGE_BAR_LOCK(_esbp); \ 835 \ 836 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 837 uint32_t, (_eqp)->eq_u32[1], \ 838 uint32_t, (_eqp)->eq_u32[0]); \ 839 \ 840 /* \ 841 * Make sure that previous writes to the qword have \ 842 * been done. It should be cheaper than barrier just \ 843 * after the write below. \ 844 */ \ 845 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 846 (_offset), sizeof (efx_qword_t), \ 847 BUS_SPACE_BARRIER_WRITE); \ 848 bus_space_write_stream_8((_esbp)->esb_tag, \ 849 (_esbp)->esb_handle, \ 850 (_offset), (_eqp)->eq_u64[0]); \ 851 \ 852 SFXGE_BAR_UNLOCK(_esbp); \ 853 _NOTE(CONSTANTCONDITION) \ 854 } while (B_FALSE) 855 #else 856 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 857 do { \ 858 _NOTE(CONSTANTCONDITION) \ 859 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 860 ("not power of 2 aligned")); \ 861 \ 862 SFXGE_BAR_LOCK(_esbp); \ 863 \ 864 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 865 uint32_t, (_eqp)->eq_u32[1], \ 866 uint32_t, (_eqp)->eq_u32[0]); \ 867 \ 868 /* \ 869 * Make sure that previous writes to the qword have \ 870 * been done. It should be cheaper than barrier just \ 871 * after the last write below. \ 872 */ \ 873 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 874 (_offset), sizeof (efx_qword_t), \ 875 BUS_SPACE_BARRIER_WRITE); \ 876 bus_space_write_stream_4((_esbp)->esb_tag, \ 877 (_esbp)->esb_handle, \ 878 (_offset), (_eqp)->eq_u32[0]); \ 879 /* \ 880 * It should be guaranteed that the last dword comes \ 881 * the last, so barrier entire qword to be sure that \ 882 * neither above nor below writes are reordered. \ 883 */ \ 884 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 885 (_offset), sizeof (efx_qword_t), \ 886 BUS_SPACE_BARRIER_WRITE); \ 887 bus_space_write_stream_4((_esbp)->esb_tag, \ 888 (_esbp)->esb_handle, \ 889 (_offset) + 4, (_eqp)->eq_u32[1]); \ 890 \ 891 SFXGE_BAR_UNLOCK(_esbp); \ 892 _NOTE(CONSTANTCONDITION) \ 893 } while (B_FALSE) 894 #endif 895 896 /* 897 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 898 * (required by PIO hardware) 899 */ 900 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 901 do { \ 902 _NOTE(CONSTANTCONDITION) \ 903 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 904 ("not power of 2 aligned")); \ 905 \ 906 (void) (_esbp); \ 907 \ 908 /* FIXME: Perform a 64-bit write */ \ 909 KASSERT(0, ("not implemented")); \ 910 \ 911 _NOTE(CONSTANTCONDITION) \ 912 } while (B_FALSE) 913 914 #if defined(SFXGE_USE_BUS_SPACE_8) 915 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 916 do { \ 917 _NOTE(CONSTANTCONDITION) \ 918 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 919 ("not power of 2 aligned")); \ 920 \ 921 _NOTE(CONSTANTCONDITION) \ 922 if (_lock) \ 923 SFXGE_BAR_LOCK(_esbp); \ 924 \ 925 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 926 uint32_t, (_eop)->eo_u32[3], \ 927 uint32_t, (_eop)->eo_u32[2], \ 928 uint32_t, (_eop)->eo_u32[1], \ 929 uint32_t, (_eop)->eo_u32[0]); \ 930 \ 931 /* \ 932 * Make sure that previous writes to the oword have \ 933 * been done. It should be cheaper than barrier just \ 934 * after the last write below. \ 935 */ \ 936 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 937 (_offset), sizeof (efx_oword_t), \ 938 BUS_SPACE_BARRIER_WRITE); \ 939 bus_space_write_stream_8((_esbp)->esb_tag, \ 940 (_esbp)->esb_handle, \ 941 (_offset), (_eop)->eo_u64[0]); \ 942 /* \ 943 * It should be guaranteed that the last qword comes \ 944 * the last, so barrier entire oword to be sure that \ 945 * neither above nor below writes are reordered. \ 946 */ \ 947 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 948 (_offset), sizeof (efx_oword_t), \ 949 BUS_SPACE_BARRIER_WRITE); \ 950 bus_space_write_stream_8((_esbp)->esb_tag, \ 951 (_esbp)->esb_handle, \ 952 (_offset) + 8, (_eop)->eo_u64[1]); \ 953 \ 954 _NOTE(CONSTANTCONDITION) \ 955 if (_lock) \ 956 SFXGE_BAR_UNLOCK(_esbp); \ 957 _NOTE(CONSTANTCONDITION) \ 958 } while (B_FALSE) 959 960 #else 961 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 962 do { \ 963 _NOTE(CONSTANTCONDITION) \ 964 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 965 ("not power of 2 aligned")); \ 966 \ 967 _NOTE(CONSTANTCONDITION) \ 968 if (_lock) \ 969 SFXGE_BAR_LOCK(_esbp); \ 970 \ 971 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 972 uint32_t, (_eop)->eo_u32[3], \ 973 uint32_t, (_eop)->eo_u32[2], \ 974 uint32_t, (_eop)->eo_u32[1], \ 975 uint32_t, (_eop)->eo_u32[0]); \ 976 \ 977 /* \ 978 * Make sure that previous writes to the oword have \ 979 * been done. It should be cheaper than barrier just \ 980 * after the last write below. \ 981 */ \ 982 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 983 (_offset), sizeof (efx_oword_t), \ 984 BUS_SPACE_BARRIER_WRITE); \ 985 bus_space_write_stream_4((_esbp)->esb_tag, \ 986 (_esbp)->esb_handle, \ 987 (_offset), (_eop)->eo_u32[0]); \ 988 bus_space_write_stream_4((_esbp)->esb_tag, \ 989 (_esbp)->esb_handle, \ 990 (_offset) + 4, (_eop)->eo_u32[1]); \ 991 bus_space_write_stream_4((_esbp)->esb_tag, \ 992 (_esbp)->esb_handle, \ 993 (_offset) + 8, (_eop)->eo_u32[2]); \ 994 /* \ 995 * It should be guaranteed that the last dword comes \ 996 * the last, so barrier entire oword to be sure that \ 997 * neither above nor below writes are reordered. \ 998 */ \ 999 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 1000 (_offset), sizeof (efx_oword_t), \ 1001 BUS_SPACE_BARRIER_WRITE); \ 1002 bus_space_write_stream_4((_esbp)->esb_tag, \ 1003 (_esbp)->esb_handle, \ 1004 (_offset) + 12, (_eop)->eo_u32[3]); \ 1005 \ 1006 _NOTE(CONSTANTCONDITION) \ 1007 if (_lock) \ 1008 SFXGE_BAR_UNLOCK(_esbp); \ 1009 _NOTE(CONSTANTCONDITION) \ 1010 } while (B_FALSE) 1011 #endif 1012 1013 /* Use the standard octo-word write for doorbell writes */ 1014 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1015 do { \ 1016 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1017 _NOTE(CONSTANTCONDITION) \ 1018 } while (B_FALSE) 1019 1020 /* SPIN */ 1021 1022 #define EFSYS_SPIN(_us) \ 1023 do { \ 1024 DELAY(_us); \ 1025 _NOTE(CONSTANTCONDITION) \ 1026 } while (B_FALSE) 1027 1028 #define EFSYS_SLEEP EFSYS_SPIN 1029 1030 /* BARRIERS */ 1031 1032 #define EFSYS_MEM_READ_BARRIER() rmb() 1033 #define EFSYS_PIO_WRITE_BARRIER() 1034 1035 /* DMA SYNC */ 1036 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1037 do { \ 1038 bus_dmamap_sync((_esmp)->esm_tag, \ 1039 (_esmp)->esm_map, \ 1040 BUS_DMASYNC_POSTREAD); \ 1041 _NOTE(CONSTANTCONDITION) \ 1042 } while (B_FALSE) 1043 1044 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1045 do { \ 1046 bus_dmamap_sync((_esmp)->esm_tag, \ 1047 (_esmp)->esm_map, \ 1048 BUS_DMASYNC_PREWRITE); \ 1049 _NOTE(CONSTANTCONDITION) \ 1050 } while (B_FALSE) 1051 1052 /* TIMESTAMP */ 1053 1054 typedef clock_t efsys_timestamp_t; 1055 1056 #define EFSYS_TIMESTAMP(_usp) \ 1057 do { \ 1058 clock_t now; \ 1059 \ 1060 now = ticks; \ 1061 *(_usp) = now * hz / 1000000; \ 1062 _NOTE(CONSTANTCONDITION) \ 1063 } while (B_FALSE) 1064 1065 /* KMEM */ 1066 1067 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1068 do { \ 1069 (_esip) = (_esip); \ 1070 /* \ 1071 * The macro is used in non-sleepable contexts, for \ 1072 * example, holding a mutex. \ 1073 */ \ 1074 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1075 _NOTE(CONSTANTCONDITION) \ 1076 } while (B_FALSE) 1077 1078 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1079 do { \ 1080 (void) (_esip); \ 1081 (void) (_size); \ 1082 free((_p), M_SFXGE); \ 1083 _NOTE(CONSTANTCONDITION) \ 1084 } while (B_FALSE) 1085 1086 /* LOCK */ 1087 1088 typedef struct efsys_lock_s { 1089 struct mtx lock; 1090 char lock_name[SFXGE_LOCK_NAME_MAX]; 1091 } efsys_lock_t; 1092 1093 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1094 do { \ 1095 efsys_lock_t *__eslp = (_eslp); \ 1096 \ 1097 snprintf((__eslp)->lock_name, \ 1098 sizeof((__eslp)->lock_name), \ 1099 "%s:%s", (_ifname), (_label)); \ 1100 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1101 NULL, MTX_DEF); \ 1102 } while (B_FALSE) 1103 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1104 mtx_destroy(&(_eslp)->lock) 1105 #define SFXGE_EFSYS_LOCK(_eslp) \ 1106 mtx_lock(&(_eslp)->lock) 1107 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1108 mtx_unlock(&(_eslp)->lock) 1109 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1110 mtx_assert(&(_eslp)->lock, MA_OWNED) 1111 1112 #define EFSYS_LOCK_MAGIC 0x000010c4 1113 1114 #define EFSYS_LOCK(_lockp, _state) \ 1115 do { \ 1116 SFXGE_EFSYS_LOCK(_lockp); \ 1117 (_state) = EFSYS_LOCK_MAGIC; \ 1118 _NOTE(CONSTANTCONDITION) \ 1119 } while (B_FALSE) 1120 1121 #define EFSYS_UNLOCK(_lockp, _state) \ 1122 do { \ 1123 if ((_state) != EFSYS_LOCK_MAGIC) \ 1124 KASSERT(B_FALSE, ("not locked")); \ 1125 SFXGE_EFSYS_UNLOCK(_lockp); \ 1126 _NOTE(CONSTANTCONDITION) \ 1127 } while (B_FALSE) 1128 1129 /* PREEMPT */ 1130 1131 #define EFSYS_PREEMPT_DISABLE(_state) \ 1132 do { \ 1133 (_state) = (_state); \ 1134 critical_enter(); \ 1135 _NOTE(CONSTANTCONDITION) \ 1136 } while (B_FALSE) 1137 1138 #define EFSYS_PREEMPT_ENABLE(_state) \ 1139 do { \ 1140 (_state) = (_state); \ 1141 critical_exit(_state); \ 1142 _NOTE(CONSTANTCONDITION) \ 1143 } while (B_FALSE) 1144 1145 /* STAT */ 1146 1147 typedef uint64_t efsys_stat_t; 1148 1149 #define EFSYS_STAT_INCR(_knp, _delta) \ 1150 do { \ 1151 *(_knp) += (_delta); \ 1152 _NOTE(CONSTANTCONDITION) \ 1153 } while (B_FALSE) 1154 1155 #define EFSYS_STAT_DECR(_knp, _delta) \ 1156 do { \ 1157 *(_knp) -= (_delta); \ 1158 _NOTE(CONSTANTCONDITION) \ 1159 } while (B_FALSE) 1160 1161 #define EFSYS_STAT_SET(_knp, _val) \ 1162 do { \ 1163 *(_knp) = (_val); \ 1164 _NOTE(CONSTANTCONDITION) \ 1165 } while (B_FALSE) 1166 1167 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1168 do { \ 1169 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1170 _NOTE(CONSTANTCONDITION) \ 1171 } while (B_FALSE) 1172 1173 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1174 do { \ 1175 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1176 _NOTE(CONSTANTCONDITION) \ 1177 } while (B_FALSE) 1178 1179 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1180 do { \ 1181 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1182 _NOTE(CONSTANTCONDITION) \ 1183 } while (B_FALSE) 1184 1185 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1186 do { \ 1187 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1188 _NOTE(CONSTANTCONDITION) \ 1189 } while (B_FALSE) 1190 1191 /* ERR */ 1192 1193 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1194 uint32_t, uint32_t); 1195 1196 #if EFSYS_OPT_DECODE_INTR_FATAL 1197 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1198 do { \ 1199 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1200 _NOTE(CONSTANTCONDITION) \ 1201 } while (B_FALSE) 1202 #endif 1203 1204 /* ASSERT */ 1205 1206 #define EFSYS_ASSERT(_exp) do { \ 1207 if (!(_exp)) \ 1208 panic("%s", #_exp); \ 1209 } while (0) 1210 1211 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1212 const _t __x = (_t)(_x); \ 1213 const _t __y = (_t)(_y); \ 1214 if (!(__x _op __y)) \ 1215 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1216 } while(0) 1217 1218 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1219 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1220 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1221 1222 /* ROTATE */ 1223 1224 #define EFSYS_HAS_ROTL_DWORD 0 1225 1226 #ifdef __cplusplus 1227 } 1228 #endif 1229 1230 #endif /* _SYS_EFSYS_H */ 1231