1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 * 35 * $FreeBSD$ 36 */ 37 38 #ifndef _SYS_EFSYS_H 39 #define _SYS_EFSYS_H 40 41 #ifdef __cplusplus 42 extern "C" { 43 #endif 44 45 #include <sys/param.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/mutex.h> 52 #include <sys/rwlock.h> 53 #include <sys/sdt.h> 54 #include <sys/systm.h> 55 56 #include <machine/bus.h> 57 #include <machine/endian.h> 58 59 #define EFSYS_HAS_UINT64 1 60 #if defined(__x86_64__) 61 #define EFSYS_USE_UINT64 1 62 #else 63 #define EFSYS_USE_UINT64 0 64 #endif 65 #define EFSYS_HAS_SSE2_M128 0 66 #if _BYTE_ORDER == _BIG_ENDIAN 67 #define EFSYS_IS_BIG_ENDIAN 1 68 #define EFSYS_IS_LITTLE_ENDIAN 0 69 #elif _BYTE_ORDER == _LITTLE_ENDIAN 70 #define EFSYS_IS_BIG_ENDIAN 0 71 #define EFSYS_IS_LITTLE_ENDIAN 1 72 #endif 73 #include "efx_types.h" 74 75 /* Common code requires this */ 76 #if __FreeBSD_version < 800068 77 #define memmove(d, s, l) bcopy(s, d, l) 78 #endif 79 80 #ifndef B_FALSE 81 #define B_FALSE FALSE 82 #endif 83 #ifndef B_TRUE 84 #define B_TRUE TRUE 85 #endif 86 87 #ifndef IS2P 88 #define ISP2(x) (((x) & ((x) - 1)) == 0) 89 #endif 90 91 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 92 93 #define SFXGE_USE_BUS_SPACE_8 1 94 95 #if !defined(bus_space_read_stream_8) 96 97 #define bus_space_read_stream_8(t, h, o) \ 98 bus_space_read_8((t), (h), (o)) 99 100 #define bus_space_write_stream_8(t, h, o, v) \ 101 bus_space_write_8((t), (h), (o), (v)) 102 103 #endif 104 105 #endif 106 107 #define ENOTACTIVE EINVAL 108 109 /* Memory type to use on FreeBSD */ 110 MALLOC_DECLARE(M_SFXGE); 111 112 /* Machine dependend prefetch wrappers */ 113 #if defined(__i386__) || defined(__amd64__) 114 static __inline void 115 prefetch_read_many(void *addr) 116 { 117 118 __asm__( 119 "prefetcht0 (%0)" 120 : 121 : "r" (addr)); 122 } 123 124 static __inline void 125 prefetch_read_once(void *addr) 126 { 127 128 __asm__( 129 "prefetchnta (%0)" 130 : 131 : "r" (addr)); 132 } 133 #else 134 static __inline void 135 prefetch_read_many(void *addr) 136 { 137 138 } 139 140 static __inline void 141 prefetch_read_once(void *addr) 142 { 143 144 } 145 #endif 146 147 #if defined(__i386__) || defined(__amd64__) 148 #include <vm/vm.h> 149 #include <vm/pmap.h> 150 #endif 151 static __inline void 152 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 153 struct mbuf *m, bus_dma_segment_t *seg) 154 { 155 #if defined(__i386__) || defined(__amd64__) 156 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 157 seg->ds_len = m->m_len; 158 #else 159 int nsegstmp; 160 161 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 162 #endif 163 } 164 165 /* Code inclusion options */ 166 167 168 #define EFSYS_OPT_NAMES 1 169 170 #define EFSYS_OPT_SIENA 1 171 #define EFSYS_OPT_HUNTINGTON 1 172 #define EFSYS_OPT_MEDFORD 1 173 #define EFSYS_OPT_MEDFORD2 1 174 #ifdef DEBUG 175 #define EFSYS_OPT_CHECK_REG 1 176 #else 177 #define EFSYS_OPT_CHECK_REG 0 178 #endif 179 180 #define EFSYS_OPT_MCDI 1 181 #define EFSYS_OPT_MCDI_LOGGING 0 182 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 183 184 #define EFSYS_OPT_MAC_STATS 1 185 186 #define EFSYS_OPT_LOOPBACK 0 187 188 #define EFSYS_OPT_MON_MCDI 0 189 #define EFSYS_OPT_MON_STATS 0 190 191 #define EFSYS_OPT_PHY_STATS 1 192 #define EFSYS_OPT_BIST 1 193 #define EFSYS_OPT_PHY_LED_CONTROL 1 194 #define EFSYS_OPT_PHY_FLAGS 0 195 196 #define EFSYS_OPT_VPD 1 197 #define EFSYS_OPT_NVRAM 1 198 #define EFSYS_OPT_BOOTCFG 0 199 #define EFSYS_OPT_IMAGE_LAYOUT 0 200 201 #define EFSYS_OPT_DIAG 0 202 #define EFSYS_OPT_RX_SCALE 1 203 #define EFSYS_OPT_QSTATS 1 204 #define EFSYS_OPT_FILTER 1 205 #define EFSYS_OPT_RX_SCATTER 0 206 207 #define EFSYS_OPT_EV_PREFETCH 0 208 209 #define EFSYS_OPT_DECODE_INTR_FATAL 1 210 211 #define EFSYS_OPT_LICENSING 0 212 213 #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 214 215 #define EFSYS_OPT_RX_PACKED_STREAM 0 216 217 #define EFSYS_OPT_RX_ES_SUPER_BUFFER 0 218 219 #define EFSYS_OPT_TUNNEL 0 220 221 #define EFSYS_OPT_FW_SUBVARIANT_AWARE 0 222 223 /* ID */ 224 225 typedef struct __efsys_identifier_s efsys_identifier_t; 226 227 /* PROBE */ 228 229 #ifndef DTRACE_PROBE 230 231 #define EFSYS_PROBE(_name) 232 233 #define EFSYS_PROBE1(_name, _type1, _arg1) 234 235 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 236 237 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 238 _type3, _arg3) 239 240 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 241 _type3, _arg3, _type4, _arg4) 242 243 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 244 _type3, _arg3, _type4, _arg4, _type5, _arg5) 245 246 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 247 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 248 _type6, _arg6) 249 250 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 251 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 252 _type6, _arg6, _type7, _arg7) 253 254 #else /* DTRACE_PROBE */ 255 256 #define EFSYS_PROBE(_name) \ 257 DTRACE_PROBE(_name) 258 259 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 260 DTRACE_PROBE1(_name, _type1, _arg1) 261 262 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 263 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 264 265 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 266 _type3, _arg3) \ 267 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 268 _type3, _arg3) 269 270 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 271 _type3, _arg3, _type4, _arg4) \ 272 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 273 _type3, _arg3, _type4, _arg4) 274 275 #ifdef DTRACE_PROBE5 276 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 277 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 278 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 279 _type3, _arg3, _type4, _arg4, _type5, _arg5) 280 #else 281 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 282 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 283 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 284 _type3, _arg3, _type4, _arg4) 285 #endif 286 287 #ifdef DTRACE_PROBE6 288 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 289 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 290 _type6, _arg6) \ 291 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 292 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 293 _type6, _arg6) 294 #else 295 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 296 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 297 _type6, _arg6) \ 298 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 299 _type3, _arg3, _type4, _arg4, _type5, _arg5) 300 #endif 301 302 #ifdef DTRACE_PROBE7 303 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 304 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 305 _type6, _arg6, _type7, _arg7) \ 306 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 307 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 308 _type6, _arg6, _type7, _arg7) 309 #else 310 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 311 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 312 _type6, _arg6, _type7, _arg7) \ 313 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 314 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 315 _type6, _arg6) 316 #endif 317 318 #endif /* DTRACE_PROBE */ 319 320 /* DMA */ 321 322 typedef uint64_t efsys_dma_addr_t; 323 324 typedef struct efsys_mem_s { 325 bus_dma_tag_t esm_tag; 326 bus_dmamap_t esm_map; 327 caddr_t esm_base; 328 efsys_dma_addr_t esm_addr; 329 size_t esm_size; 330 } efsys_mem_t; 331 332 #define EFSYS_MEM_SIZE(_esmp) \ 333 ((_esmp)->esm_size) 334 335 #define EFSYS_MEM_ADDR(_esmp) \ 336 ((_esmp)->esm_addr) 337 338 #define EFSYS_MEM_IS_NULL(_esmp) \ 339 ((_esmp)->esm_base == NULL) 340 341 342 #define EFSYS_MEM_ZERO(_esmp, _size) \ 343 do { \ 344 (void) memset((_esmp)->esm_base, 0, (_size)); \ 345 \ 346 _NOTE(CONSTANTCONDITION) \ 347 } while (B_FALSE) 348 349 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 350 do { \ 351 uint32_t *addr; \ 352 \ 353 _NOTE(CONSTANTCONDITION) \ 354 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 355 sizeof (efx_dword_t)), \ 356 ("not power of 2 aligned")); \ 357 \ 358 addr = (void *)((_esmp)->esm_base + (_offset)); \ 359 \ 360 (_edp)->ed_u32[0] = *addr; \ 361 \ 362 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 363 uint32_t, (_edp)->ed_u32[0]); \ 364 \ 365 _NOTE(CONSTANTCONDITION) \ 366 } while (B_FALSE) 367 368 #if defined(__x86_64__) 369 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 370 do { \ 371 uint64_t *addr; \ 372 \ 373 _NOTE(CONSTANTCONDITION) \ 374 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 375 sizeof (efx_qword_t)), \ 376 ("not power of 2 aligned")); \ 377 \ 378 addr = (void *)((_esmp)->esm_base + (_offset)); \ 379 \ 380 (_eqp)->eq_u64[0] = *addr; \ 381 \ 382 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 383 uint32_t, (_eqp)->eq_u32[1], \ 384 uint32_t, (_eqp)->eq_u32[0]); \ 385 \ 386 _NOTE(CONSTANTCONDITION) \ 387 } while (B_FALSE) 388 #else 389 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 390 do { \ 391 uint32_t *addr; \ 392 \ 393 _NOTE(CONSTANTCONDITION) \ 394 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 395 sizeof (efx_qword_t)), \ 396 ("not power of 2 aligned")); \ 397 \ 398 addr = (void *)((_esmp)->esm_base + (_offset)); \ 399 \ 400 (_eqp)->eq_u32[0] = *addr++; \ 401 (_eqp)->eq_u32[1] = *addr; \ 402 \ 403 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 404 uint32_t, (_eqp)->eq_u32[1], \ 405 uint32_t, (_eqp)->eq_u32[0]); \ 406 \ 407 _NOTE(CONSTANTCONDITION) \ 408 } while (B_FALSE) 409 #endif 410 411 #if defined(__x86_64__) 412 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 413 do { \ 414 uint64_t *addr; \ 415 \ 416 _NOTE(CONSTANTCONDITION) \ 417 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 418 sizeof (efx_oword_t)), \ 419 ("not power of 2 aligned")); \ 420 \ 421 addr = (void *)((_esmp)->esm_base + (_offset)); \ 422 \ 423 (_eop)->eo_u64[0] = *addr++; \ 424 (_eop)->eo_u64[1] = *addr; \ 425 \ 426 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 427 uint32_t, (_eop)->eo_u32[3], \ 428 uint32_t, (_eop)->eo_u32[2], \ 429 uint32_t, (_eop)->eo_u32[1], \ 430 uint32_t, (_eop)->eo_u32[0]); \ 431 \ 432 _NOTE(CONSTANTCONDITION) \ 433 } while (B_FALSE) 434 #else 435 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 436 do { \ 437 uint32_t *addr; \ 438 \ 439 _NOTE(CONSTANTCONDITION) \ 440 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 441 sizeof (efx_oword_t)), \ 442 ("not power of 2 aligned")); \ 443 \ 444 addr = (void *)((_esmp)->esm_base + (_offset)); \ 445 \ 446 (_eop)->eo_u32[0] = *addr++; \ 447 (_eop)->eo_u32[1] = *addr++; \ 448 (_eop)->eo_u32[2] = *addr++; \ 449 (_eop)->eo_u32[3] = *addr; \ 450 \ 451 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 452 uint32_t, (_eop)->eo_u32[3], \ 453 uint32_t, (_eop)->eo_u32[2], \ 454 uint32_t, (_eop)->eo_u32[1], \ 455 uint32_t, (_eop)->eo_u32[0]); \ 456 \ 457 _NOTE(CONSTANTCONDITION) \ 458 } while (B_FALSE) 459 #endif 460 461 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 462 do { \ 463 uint32_t *addr; \ 464 \ 465 _NOTE(CONSTANTCONDITION) \ 466 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 467 sizeof (efx_dword_t)), \ 468 ("not power of 2 aligned")); \ 469 \ 470 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 471 uint32_t, (_edp)->ed_u32[0]); \ 472 \ 473 addr = (void *)((_esmp)->esm_base + (_offset)); \ 474 \ 475 *addr = (_edp)->ed_u32[0]; \ 476 \ 477 _NOTE(CONSTANTCONDITION) \ 478 } while (B_FALSE) 479 480 #if defined(__x86_64__) 481 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 482 do { \ 483 uint64_t *addr; \ 484 \ 485 _NOTE(CONSTANTCONDITION) \ 486 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 487 sizeof (efx_qword_t)), \ 488 ("not power of 2 aligned")); \ 489 \ 490 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 491 uint32_t, (_eqp)->eq_u32[1], \ 492 uint32_t, (_eqp)->eq_u32[0]); \ 493 \ 494 addr = (void *)((_esmp)->esm_base + (_offset)); \ 495 \ 496 *addr = (_eqp)->eq_u64[0]; \ 497 \ 498 _NOTE(CONSTANTCONDITION) \ 499 } while (B_FALSE) 500 501 #else 502 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 503 do { \ 504 uint32_t *addr; \ 505 \ 506 _NOTE(CONSTANTCONDITION) \ 507 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 508 sizeof (efx_qword_t)), \ 509 ("not power of 2 aligned")); \ 510 \ 511 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 512 uint32_t, (_eqp)->eq_u32[1], \ 513 uint32_t, (_eqp)->eq_u32[0]); \ 514 \ 515 addr = (void *)((_esmp)->esm_base + (_offset)); \ 516 \ 517 *addr++ = (_eqp)->eq_u32[0]; \ 518 *addr = (_eqp)->eq_u32[1]; \ 519 \ 520 _NOTE(CONSTANTCONDITION) \ 521 } while (B_FALSE) 522 #endif 523 524 #if defined(__x86_64__) 525 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 526 do { \ 527 uint64_t *addr; \ 528 \ 529 _NOTE(CONSTANTCONDITION) \ 530 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 531 sizeof (efx_oword_t)), \ 532 ("not power of 2 aligned")); \ 533 \ 534 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 535 uint32_t, (_eop)->eo_u32[3], \ 536 uint32_t, (_eop)->eo_u32[2], \ 537 uint32_t, (_eop)->eo_u32[1], \ 538 uint32_t, (_eop)->eo_u32[0]); \ 539 \ 540 addr = (void *)((_esmp)->esm_base + (_offset)); \ 541 \ 542 *addr++ = (_eop)->eo_u64[0]; \ 543 *addr = (_eop)->eo_u64[1]; \ 544 \ 545 _NOTE(CONSTANTCONDITION) \ 546 } while (B_FALSE) 547 #else 548 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 549 do { \ 550 uint32_t *addr; \ 551 \ 552 _NOTE(CONSTANTCONDITION) \ 553 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 554 sizeof (efx_oword_t)), \ 555 ("not power of 2 aligned")); \ 556 \ 557 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 558 uint32_t, (_eop)->eo_u32[3], \ 559 uint32_t, (_eop)->eo_u32[2], \ 560 uint32_t, (_eop)->eo_u32[1], \ 561 uint32_t, (_eop)->eo_u32[0]); \ 562 \ 563 addr = (void *)((_esmp)->esm_base + (_offset)); \ 564 \ 565 *addr++ = (_eop)->eo_u32[0]; \ 566 *addr++ = (_eop)->eo_u32[1]; \ 567 *addr++ = (_eop)->eo_u32[2]; \ 568 *addr = (_eop)->eo_u32[3]; \ 569 \ 570 _NOTE(CONSTANTCONDITION) \ 571 } while (B_FALSE) 572 #endif 573 574 /* BAR */ 575 576 #define SFXGE_LOCK_NAME_MAX 16 577 578 typedef struct efsys_bar_s { 579 struct mtx esb_lock; 580 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 581 bus_space_tag_t esb_tag; 582 bus_space_handle_t esb_handle; 583 int esb_rid; 584 struct resource *esb_res; 585 } efsys_bar_t; 586 587 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 588 do { \ 589 snprintf((_esbp)->esb_lock_name, \ 590 sizeof((_esbp)->esb_lock_name), \ 591 "%s:bar", (_ifname)); \ 592 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 593 NULL, MTX_DEF); \ 594 _NOTE(CONSTANTCONDITION) \ 595 } while (B_FALSE) 596 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 597 mtx_destroy(&(_esbp)->esb_lock) 598 #define SFXGE_BAR_LOCK(_esbp) \ 599 mtx_lock(&(_esbp)->esb_lock) 600 #define SFXGE_BAR_UNLOCK(_esbp) \ 601 mtx_unlock(&(_esbp)->esb_lock) 602 603 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 604 do { \ 605 _NOTE(CONSTANTCONDITION) \ 606 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 607 sizeof (efx_dword_t)), \ 608 ("not power of 2 aligned")); \ 609 \ 610 _NOTE(CONSTANTCONDITION) \ 611 if (_lock) \ 612 SFXGE_BAR_LOCK(_esbp); \ 613 \ 614 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 615 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 616 (_offset)); \ 617 \ 618 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 619 uint32_t, (_edp)->ed_u32[0]); \ 620 \ 621 _NOTE(CONSTANTCONDITION) \ 622 if (_lock) \ 623 SFXGE_BAR_UNLOCK(_esbp); \ 624 _NOTE(CONSTANTCONDITION) \ 625 } while (B_FALSE) 626 627 #if defined(SFXGE_USE_BUS_SPACE_8) 628 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 629 do { \ 630 _NOTE(CONSTANTCONDITION) \ 631 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 632 sizeof (efx_qword_t)), \ 633 ("not power of 2 aligned")); \ 634 \ 635 SFXGE_BAR_LOCK(_esbp); \ 636 \ 637 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 638 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 639 (_offset)); \ 640 \ 641 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 642 uint32_t, (_eqp)->eq_u32[1], \ 643 uint32_t, (_eqp)->eq_u32[0]); \ 644 \ 645 SFXGE_BAR_UNLOCK(_esbp); \ 646 _NOTE(CONSTANTCONDITION) \ 647 } while (B_FALSE) 648 649 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 650 do { \ 651 _NOTE(CONSTANTCONDITION) \ 652 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 653 sizeof (efx_oword_t)), \ 654 ("not power of 2 aligned")); \ 655 \ 656 _NOTE(CONSTANTCONDITION) \ 657 if (_lock) \ 658 SFXGE_BAR_LOCK(_esbp); \ 659 \ 660 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 661 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 662 (_offset)); \ 663 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 664 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 665 (_offset) + 8); \ 666 \ 667 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 668 uint32_t, (_eop)->eo_u32[3], \ 669 uint32_t, (_eop)->eo_u32[2], \ 670 uint32_t, (_eop)->eo_u32[1], \ 671 uint32_t, (_eop)->eo_u32[0]); \ 672 \ 673 _NOTE(CONSTANTCONDITION) \ 674 if (_lock) \ 675 SFXGE_BAR_UNLOCK(_esbp); \ 676 _NOTE(CONSTANTCONDITION) \ 677 } while (B_FALSE) 678 679 #else 680 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 681 do { \ 682 _NOTE(CONSTANTCONDITION) \ 683 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 684 sizeof (efx_qword_t)), \ 685 ("not power of 2 aligned")); \ 686 \ 687 SFXGE_BAR_LOCK(_esbp); \ 688 \ 689 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 690 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 691 (_offset)); \ 692 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 693 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 694 (_offset) + 4); \ 695 \ 696 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 697 uint32_t, (_eqp)->eq_u32[1], \ 698 uint32_t, (_eqp)->eq_u32[0]); \ 699 \ 700 SFXGE_BAR_UNLOCK(_esbp); \ 701 _NOTE(CONSTANTCONDITION) \ 702 } while (B_FALSE) 703 704 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 705 do { \ 706 _NOTE(CONSTANTCONDITION) \ 707 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 708 sizeof (efx_oword_t)), \ 709 ("not power of 2 aligned")); \ 710 \ 711 _NOTE(CONSTANTCONDITION) \ 712 if (_lock) \ 713 SFXGE_BAR_LOCK(_esbp); \ 714 \ 715 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 716 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 717 (_offset)); \ 718 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 719 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 720 (_offset) + 4); \ 721 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 722 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 723 (_offset) + 8); \ 724 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 725 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 726 (_offset) + 12); \ 727 \ 728 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 729 uint32_t, (_eop)->eo_u32[3], \ 730 uint32_t, (_eop)->eo_u32[2], \ 731 uint32_t, (_eop)->eo_u32[1], \ 732 uint32_t, (_eop)->eo_u32[0]); \ 733 \ 734 _NOTE(CONSTANTCONDITION) \ 735 if (_lock) \ 736 SFXGE_BAR_UNLOCK(_esbp); \ 737 _NOTE(CONSTANTCONDITION) \ 738 } while (B_FALSE) 739 #endif 740 741 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 742 do { \ 743 _NOTE(CONSTANTCONDITION) \ 744 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 745 sizeof (efx_dword_t)), \ 746 ("not power of 2 aligned")); \ 747 \ 748 _NOTE(CONSTANTCONDITION) \ 749 if (_lock) \ 750 SFXGE_BAR_LOCK(_esbp); \ 751 \ 752 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 753 uint32_t, (_edp)->ed_u32[0]); \ 754 \ 755 /* \ 756 * Make sure that previous writes to the dword have \ 757 * been done. It should be cheaper than barrier just \ 758 * after the write below. \ 759 */ \ 760 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 761 (_offset), sizeof (efx_dword_t), \ 762 BUS_SPACE_BARRIER_WRITE); \ 763 bus_space_write_stream_4((_esbp)->esb_tag, \ 764 (_esbp)->esb_handle, \ 765 (_offset), (_edp)->ed_u32[0]); \ 766 \ 767 _NOTE(CONSTANTCONDITION) \ 768 if (_lock) \ 769 SFXGE_BAR_UNLOCK(_esbp); \ 770 _NOTE(CONSTANTCONDITION) \ 771 } while (B_FALSE) 772 773 #if defined(SFXGE_USE_BUS_SPACE_8) 774 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 775 do { \ 776 _NOTE(CONSTANTCONDITION) \ 777 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 778 sizeof (efx_qword_t)), \ 779 ("not power of 2 aligned")); \ 780 \ 781 SFXGE_BAR_LOCK(_esbp); \ 782 \ 783 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 784 uint32_t, (_eqp)->eq_u32[1], \ 785 uint32_t, (_eqp)->eq_u32[0]); \ 786 \ 787 /* \ 788 * Make sure that previous writes to the qword have \ 789 * been done. It should be cheaper than barrier just \ 790 * after the write below. \ 791 */ \ 792 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 793 (_offset), sizeof (efx_qword_t), \ 794 BUS_SPACE_BARRIER_WRITE); \ 795 bus_space_write_stream_8((_esbp)->esb_tag, \ 796 (_esbp)->esb_handle, \ 797 (_offset), (_eqp)->eq_u64[0]); \ 798 \ 799 SFXGE_BAR_UNLOCK(_esbp); \ 800 _NOTE(CONSTANTCONDITION) \ 801 } while (B_FALSE) 802 #else 803 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 804 do { \ 805 _NOTE(CONSTANTCONDITION) \ 806 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 807 sizeof (efx_qword_t)), \ 808 ("not power of 2 aligned")); \ 809 \ 810 SFXGE_BAR_LOCK(_esbp); \ 811 \ 812 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 813 uint32_t, (_eqp)->eq_u32[1], \ 814 uint32_t, (_eqp)->eq_u32[0]); \ 815 \ 816 /* \ 817 * Make sure that previous writes to the qword have \ 818 * been done. It should be cheaper than barrier just \ 819 * after the last write below. \ 820 */ \ 821 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 822 (_offset), sizeof (efx_qword_t), \ 823 BUS_SPACE_BARRIER_WRITE); \ 824 bus_space_write_stream_4((_esbp)->esb_tag, \ 825 (_esbp)->esb_handle, \ 826 (_offset), (_eqp)->eq_u32[0]); \ 827 /* \ 828 * It should be guaranteed that the last dword comes \ 829 * the last, so barrier entire qword to be sure that \ 830 * neither above nor below writes are reordered. \ 831 */ \ 832 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 833 (_offset), sizeof (efx_qword_t), \ 834 BUS_SPACE_BARRIER_WRITE); \ 835 bus_space_write_stream_4((_esbp)->esb_tag, \ 836 (_esbp)->esb_handle, \ 837 (_offset) + 4, (_eqp)->eq_u32[1]); \ 838 \ 839 SFXGE_BAR_UNLOCK(_esbp); \ 840 _NOTE(CONSTANTCONDITION) \ 841 } while (B_FALSE) 842 #endif 843 844 /* 845 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 846 * (required by PIO hardware) 847 */ 848 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 849 do { \ 850 _NOTE(CONSTANTCONDITION) \ 851 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 852 sizeof (efx_qword_t)), \ 853 ("not power of 2 aligned")); \ 854 \ 855 (void) (_esbp); \ 856 \ 857 /* FIXME: Perform a 64-bit write */ \ 858 KASSERT(0, ("not implemented")); \ 859 \ 860 _NOTE(CONSTANTCONDITION) \ 861 } while (B_FALSE) 862 863 #if defined(SFXGE_USE_BUS_SPACE_8) 864 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 865 do { \ 866 _NOTE(CONSTANTCONDITION) \ 867 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 868 sizeof (efx_oword_t)), \ 869 ("not power of 2 aligned")); \ 870 \ 871 _NOTE(CONSTANTCONDITION) \ 872 if (_lock) \ 873 SFXGE_BAR_LOCK(_esbp); \ 874 \ 875 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 876 uint32_t, (_eop)->eo_u32[3], \ 877 uint32_t, (_eop)->eo_u32[2], \ 878 uint32_t, (_eop)->eo_u32[1], \ 879 uint32_t, (_eop)->eo_u32[0]); \ 880 \ 881 /* \ 882 * Make sure that previous writes to the oword have \ 883 * been done. It should be cheaper than barrier just \ 884 * after the last write below. \ 885 */ \ 886 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 887 (_offset), sizeof (efx_oword_t), \ 888 BUS_SPACE_BARRIER_WRITE); \ 889 bus_space_write_stream_8((_esbp)->esb_tag, \ 890 (_esbp)->esb_handle, \ 891 (_offset), (_eop)->eo_u64[0]); \ 892 /* \ 893 * It should be guaranteed that the last qword comes \ 894 * the last, so barrier entire oword to be sure that \ 895 * neither above nor below writes are reordered. \ 896 */ \ 897 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 898 (_offset), sizeof (efx_oword_t), \ 899 BUS_SPACE_BARRIER_WRITE); \ 900 bus_space_write_stream_8((_esbp)->esb_tag, \ 901 (_esbp)->esb_handle, \ 902 (_offset) + 8, (_eop)->eo_u64[1]); \ 903 \ 904 _NOTE(CONSTANTCONDITION) \ 905 if (_lock) \ 906 SFXGE_BAR_UNLOCK(_esbp); \ 907 _NOTE(CONSTANTCONDITION) \ 908 } while (B_FALSE) 909 910 #else 911 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 912 do { \ 913 _NOTE(CONSTANTCONDITION) \ 914 KASSERT(EFX_IS_P2ALIGNED(size_t, _offset, \ 915 sizeof (efx_oword_t)), \ 916 ("not power of 2 aligned")); \ 917 \ 918 _NOTE(CONSTANTCONDITION) \ 919 if (_lock) \ 920 SFXGE_BAR_LOCK(_esbp); \ 921 \ 922 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 923 uint32_t, (_eop)->eo_u32[3], \ 924 uint32_t, (_eop)->eo_u32[2], \ 925 uint32_t, (_eop)->eo_u32[1], \ 926 uint32_t, (_eop)->eo_u32[0]); \ 927 \ 928 /* \ 929 * Make sure that previous writes to the oword have \ 930 * been done. It should be cheaper than barrier just \ 931 * after the last write below. \ 932 */ \ 933 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 934 (_offset), sizeof (efx_oword_t), \ 935 BUS_SPACE_BARRIER_WRITE); \ 936 bus_space_write_stream_4((_esbp)->esb_tag, \ 937 (_esbp)->esb_handle, \ 938 (_offset), (_eop)->eo_u32[0]); \ 939 bus_space_write_stream_4((_esbp)->esb_tag, \ 940 (_esbp)->esb_handle, \ 941 (_offset) + 4, (_eop)->eo_u32[1]); \ 942 bus_space_write_stream_4((_esbp)->esb_tag, \ 943 (_esbp)->esb_handle, \ 944 (_offset) + 8, (_eop)->eo_u32[2]); \ 945 /* \ 946 * It should be guaranteed that the last dword comes \ 947 * the last, so barrier entire oword to be sure that \ 948 * neither above nor below writes are reordered. \ 949 */ \ 950 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 951 (_offset), sizeof (efx_oword_t), \ 952 BUS_SPACE_BARRIER_WRITE); \ 953 bus_space_write_stream_4((_esbp)->esb_tag, \ 954 (_esbp)->esb_handle, \ 955 (_offset) + 12, (_eop)->eo_u32[3]); \ 956 \ 957 _NOTE(CONSTANTCONDITION) \ 958 if (_lock) \ 959 SFXGE_BAR_UNLOCK(_esbp); \ 960 _NOTE(CONSTANTCONDITION) \ 961 } while (B_FALSE) 962 #endif 963 964 /* Use the standard octo-word write for doorbell writes */ 965 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 966 do { \ 967 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 968 _NOTE(CONSTANTCONDITION) \ 969 } while (B_FALSE) 970 971 /* SPIN */ 972 973 #define EFSYS_SPIN(_us) \ 974 do { \ 975 DELAY(_us); \ 976 _NOTE(CONSTANTCONDITION) \ 977 } while (B_FALSE) 978 979 #define EFSYS_SLEEP EFSYS_SPIN 980 981 /* BARRIERS */ 982 983 #define EFSYS_MEM_READ_BARRIER() rmb() 984 #define EFSYS_PIO_WRITE_BARRIER() 985 986 /* DMA SYNC */ 987 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 988 do { \ 989 bus_dmamap_sync((_esmp)->esm_tag, \ 990 (_esmp)->esm_map, \ 991 BUS_DMASYNC_POSTREAD); \ 992 _NOTE(CONSTANTCONDITION) \ 993 } while (B_FALSE) 994 995 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 996 do { \ 997 bus_dmamap_sync((_esmp)->esm_tag, \ 998 (_esmp)->esm_map, \ 999 BUS_DMASYNC_PREWRITE); \ 1000 _NOTE(CONSTANTCONDITION) \ 1001 } while (B_FALSE) 1002 1003 /* TIMESTAMP */ 1004 1005 typedef clock_t efsys_timestamp_t; 1006 1007 #define EFSYS_TIMESTAMP(_usp) \ 1008 do { \ 1009 clock_t now; \ 1010 \ 1011 now = ticks; \ 1012 *(_usp) = now * hz / 1000000; \ 1013 _NOTE(CONSTANTCONDITION) \ 1014 } while (B_FALSE) 1015 1016 /* KMEM */ 1017 1018 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1019 do { \ 1020 (_esip) = (_esip); \ 1021 /* \ 1022 * The macro is used in non-sleepable contexts, for \ 1023 * example, holding a mutex. \ 1024 */ \ 1025 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1026 _NOTE(CONSTANTCONDITION) \ 1027 } while (B_FALSE) 1028 1029 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1030 do { \ 1031 (void) (_esip); \ 1032 (void) (_size); \ 1033 free((_p), M_SFXGE); \ 1034 _NOTE(CONSTANTCONDITION) \ 1035 } while (B_FALSE) 1036 1037 /* LOCK */ 1038 1039 typedef struct efsys_lock_s { 1040 struct mtx lock; 1041 char lock_name[SFXGE_LOCK_NAME_MAX]; 1042 } efsys_lock_t; 1043 1044 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1045 do { \ 1046 efsys_lock_t *__eslp = (_eslp); \ 1047 \ 1048 snprintf((__eslp)->lock_name, \ 1049 sizeof((__eslp)->lock_name), \ 1050 "%s:%s", (_ifname), (_label)); \ 1051 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1052 NULL, MTX_DEF); \ 1053 } while (B_FALSE) 1054 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1055 mtx_destroy(&(_eslp)->lock) 1056 #define SFXGE_EFSYS_LOCK(_eslp) \ 1057 mtx_lock(&(_eslp)->lock) 1058 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1059 mtx_unlock(&(_eslp)->lock) 1060 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1061 mtx_assert(&(_eslp)->lock, MA_OWNED) 1062 1063 typedef int efsys_lock_state_t; 1064 1065 #define EFSYS_LOCK_MAGIC 0x000010c4 1066 1067 #define EFSYS_LOCK(_lockp, _state) \ 1068 do { \ 1069 SFXGE_EFSYS_LOCK(_lockp); \ 1070 (_state) = EFSYS_LOCK_MAGIC; \ 1071 _NOTE(CONSTANTCONDITION) \ 1072 } while (B_FALSE) 1073 1074 #define EFSYS_UNLOCK(_lockp, _state) \ 1075 do { \ 1076 if ((_state) != EFSYS_LOCK_MAGIC) \ 1077 KASSERT(B_FALSE, ("not locked")); \ 1078 SFXGE_EFSYS_UNLOCK(_lockp); \ 1079 _NOTE(CONSTANTCONDITION) \ 1080 } while (B_FALSE) 1081 1082 /* STAT */ 1083 1084 typedef uint64_t efsys_stat_t; 1085 1086 #define EFSYS_STAT_INCR(_knp, _delta) \ 1087 do { \ 1088 *(_knp) += (_delta); \ 1089 _NOTE(CONSTANTCONDITION) \ 1090 } while (B_FALSE) 1091 1092 #define EFSYS_STAT_DECR(_knp, _delta) \ 1093 do { \ 1094 *(_knp) -= (_delta); \ 1095 _NOTE(CONSTANTCONDITION) \ 1096 } while (B_FALSE) 1097 1098 #define EFSYS_STAT_SET(_knp, _val) \ 1099 do { \ 1100 *(_knp) = (_val); \ 1101 _NOTE(CONSTANTCONDITION) \ 1102 } while (B_FALSE) 1103 1104 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1105 do { \ 1106 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1107 _NOTE(CONSTANTCONDITION) \ 1108 } while (B_FALSE) 1109 1110 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1111 do { \ 1112 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1113 _NOTE(CONSTANTCONDITION) \ 1114 } while (B_FALSE) 1115 1116 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1117 do { \ 1118 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1119 _NOTE(CONSTANTCONDITION) \ 1120 } while (B_FALSE) 1121 1122 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1123 do { \ 1124 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1125 _NOTE(CONSTANTCONDITION) \ 1126 } while (B_FALSE) 1127 1128 /* ERR */ 1129 1130 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1131 uint32_t, uint32_t); 1132 1133 #if EFSYS_OPT_DECODE_INTR_FATAL 1134 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1135 do { \ 1136 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1137 _NOTE(CONSTANTCONDITION) \ 1138 } while (B_FALSE) 1139 #endif 1140 1141 /* ASSERT */ 1142 1143 #define EFSYS_ASSERT(_exp) do { \ 1144 if (!(_exp)) \ 1145 panic("%s", #_exp); \ 1146 } while (0) 1147 1148 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1149 const _t __x = (_t)(_x); \ 1150 const _t __y = (_t)(_y); \ 1151 if (!(__x _op __y)) \ 1152 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1153 } while(0) 1154 1155 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1156 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1157 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1158 1159 /* ROTATE */ 1160 1161 #define EFSYS_HAS_ROTL_DWORD 0 1162 1163 #ifdef __cplusplus 1164 } 1165 #endif 1166 1167 #endif /* _SYS_EFSYS_H */ 1168