1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _SYS_EFSYS_H 33 #define _SYS_EFSYS_H 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/endian.h> 42 #include <sys/lock.h> 43 #include <sys/malloc.h> 44 #include <sys/mbuf.h> 45 #include <sys/mutex.h> 46 #include <sys/rwlock.h> 47 #include <sys/sdt.h> 48 #include <sys/systm.h> 49 50 #include <machine/bus.h> 51 #include <machine/endian.h> 52 53 #define EFSYS_HAS_UINT64 1 54 #if defined(__x86_64__) 55 #define EFSYS_USE_UINT64 1 56 #else 57 #define EFSYS_USE_UINT64 0 58 #endif 59 #if _BYTE_ORDER == _BIG_ENDIAN 60 #define EFSYS_IS_BIG_ENDIAN 1 61 #define EFSYS_IS_LITTLE_ENDIAN 0 62 #elif _BYTE_ORDER == _LITTLE_ENDIAN 63 #define EFSYS_IS_BIG_ENDIAN 0 64 #define EFSYS_IS_LITTLE_ENDIAN 1 65 #endif 66 #include "efx_types.h" 67 68 /* Common code requires this */ 69 #if __FreeBSD_version < 800068 70 #define memmove(d, s, l) bcopy(s, d, l) 71 #endif 72 73 /* FreeBSD equivalents of Solaris things */ 74 #ifndef _NOTE 75 #define _NOTE(s) 76 #endif 77 78 #ifndef B_FALSE 79 #define B_FALSE FALSE 80 #endif 81 #ifndef B_TRUE 82 #define B_TRUE TRUE 83 #endif 84 85 #ifndef IS_P2ALIGNED 86 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 87 #endif 88 89 #ifndef P2ROUNDUP 90 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 91 #endif 92 93 #ifndef IS2P 94 #define ISP2(x) (((x) & ((x) - 1)) == 0) 95 #endif 96 97 #define ENOTACTIVE EINVAL 98 99 /* Memory type to use on FreeBSD */ 100 MALLOC_DECLARE(M_SFXGE); 101 102 /* Machine dependend prefetch wrappers */ 103 #if defined(__i386__) || defined(__amd64__) 104 static __inline void 105 prefetch_read_many(void *addr) 106 { 107 108 __asm__( 109 "prefetcht0 (%0)" 110 : 111 : "r" (addr)); 112 } 113 114 static __inline void 115 prefetch_read_once(void *addr) 116 { 117 118 __asm__( 119 "prefetchnta (%0)" 120 : 121 : "r" (addr)); 122 } 123 #elif defined(__sparc64__) 124 static __inline void 125 prefetch_read_many(void *addr) 126 { 127 128 __asm__( 129 "prefetch [%0], 0" 130 : 131 : "r" (addr)); 132 } 133 134 static __inline void 135 prefetch_read_once(void *addr) 136 { 137 138 __asm__( 139 "prefetch [%0], 1" 140 : 141 : "r" (addr)); 142 } 143 #else 144 static __inline void 145 prefetch_read_many(void *addr) 146 { 147 148 } 149 150 static __inline void 151 prefetch_read_once(void *addr) 152 { 153 154 } 155 #endif 156 157 #if defined(__i386__) || defined(__amd64__) 158 #include <vm/vm.h> 159 #include <vm/pmap.h> 160 #endif 161 static __inline void 162 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 163 struct mbuf *m, bus_dma_segment_t *seg) 164 { 165 #if defined(__i386__) || defined(__amd64__) 166 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 167 seg->ds_len = m->m_len; 168 #else 169 int nsegstmp; 170 171 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 172 #endif 173 } 174 175 /* Modifiers used for DOS builds */ 176 #define __cs 177 #define __far 178 179 /* Modifiers used for Windows builds */ 180 #define __in 181 #define __in_opt 182 #define __in_ecount(_n) 183 #define __in_ecount_opt(_n) 184 #define __in_bcount(_n) 185 #define __in_bcount_opt(_n) 186 187 #define __out 188 #define __out_opt 189 #define __out_ecount(_n) 190 #define __out_ecount_opt(_n) 191 #define __out_bcount(_n) 192 #define __out_bcount_opt(_n) 193 194 #define __deref_out 195 196 #define __inout 197 #define __inout_opt 198 #define __inout_ecount(_n) 199 #define __inout_ecount_opt(_n) 200 #define __inout_bcount(_n) 201 #define __inout_bcount_opt(_n) 202 #define __inout_bcount_full_opt(_n) 203 204 #define __deref_out_bcount_opt(n) 205 206 #define __checkReturn 207 208 #define __drv_when(_p, _c) 209 210 /* Code inclusion options */ 211 212 213 #define EFSYS_OPT_NAMES 1 214 215 #define EFSYS_OPT_FALCON 0 216 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 217 #define EFSYS_OPT_SIENA 1 218 #ifdef DEBUG 219 #define EFSYS_OPT_CHECK_REG 1 220 #else 221 #define EFSYS_OPT_CHECK_REG 0 222 #endif 223 224 #define EFSYS_OPT_MCDI 1 225 226 #define EFSYS_OPT_MAC_FALCON_GMAC 0 227 #define EFSYS_OPT_MAC_FALCON_XMAC 0 228 #define EFSYS_OPT_MAC_STATS 1 229 230 #define EFSYS_OPT_LOOPBACK 0 231 232 #define EFSYS_OPT_MON_NULL 0 233 #define EFSYS_OPT_MON_LM87 0 234 #define EFSYS_OPT_MON_MAX6647 0 235 #define EFSYS_OPT_MON_SIENA 0 236 #define EFSYS_OPT_MON_STATS 0 237 238 #define EFSYS_OPT_PHY_NULL 0 239 #define EFSYS_OPT_PHY_QT2022C2 0 240 #define EFSYS_OPT_PHY_SFX7101 0 241 #define EFSYS_OPT_PHY_TXC43128 0 242 #define EFSYS_OPT_PHY_PM8358 0 243 #define EFSYS_OPT_PHY_SFT9001 0 244 #define EFSYS_OPT_PHY_QT2025C 0 245 #define EFSYS_OPT_PHY_STATS 1 246 #define EFSYS_OPT_PHY_PROPS 0 247 #define EFSYS_OPT_PHY_BIST 1 248 #define EFSYS_OPT_PHY_LED_CONTROL 1 249 #define EFSYS_OPT_PHY_FLAGS 0 250 251 #define EFSYS_OPT_VPD 1 252 #define EFSYS_OPT_NVRAM 1 253 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 254 #define EFSYS_OPT_NVRAM_SFT9001 0 255 #define EFSYS_OPT_NVRAM_SFX7101 0 256 #define EFSYS_OPT_BOOTCFG 0 257 258 #define EFSYS_OPT_PCIE_TUNE 0 259 #define EFSYS_OPT_DIAG 0 260 #define EFSYS_OPT_WOL 1 261 #define EFSYS_OPT_RX_SCALE 1 262 #define EFSYS_OPT_QSTATS 1 263 #define EFSYS_OPT_FILTER 0 264 #define EFSYS_OPT_RX_SCATTER 0 265 #define EFSYS_OPT_RX_HDR_SPLIT 0 266 267 #define EFSYS_OPT_EV_PREFETCH 0 268 269 #define EFSYS_OPT_DECODE_INTR_FATAL 1 270 271 /* ID */ 272 273 typedef struct __efsys_identifier_s efsys_identifier_t; 274 275 /* PROBE */ 276 277 #ifndef DTRACE_PROBE 278 279 #define EFSYS_PROBE(_name) 280 281 #define EFSYS_PROBE1(_name, _type1, _arg1) 282 283 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 284 285 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 286 _type3, _arg3) 287 288 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 289 _type3, _arg3, _type4, _arg4) 290 291 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 292 _type3, _arg3, _type4, _arg4, _type5, _arg5) 293 294 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 295 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 296 _type6, _arg6) 297 298 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 299 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 300 _type6, _arg6, _type7, _arg7) 301 302 #else /* DTRACE_PROBE */ 303 304 #define EFSYS_PROBE(_name) \ 305 DTRACE_PROBE(_name) 306 307 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 308 DTRACE_PROBE1(_name, _type1, _arg1) 309 310 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 311 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 312 313 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 314 _type3, _arg3) \ 315 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 316 _type3, _arg3) 317 318 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 319 _type3, _arg3, _type4, _arg4) \ 320 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 321 _type3, _arg3, _type4, _arg4) 322 323 #ifdef DTRACE_PROBE5 324 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 325 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 326 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 327 _type3, _arg3, _type4, _arg4, _type5, _arg5) 328 #else 329 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 330 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 331 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 332 _type3, _arg3, _type4, _arg4) 333 #endif 334 335 #ifdef DTRACE_PROBE6 336 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 337 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 338 _type6, _arg6) \ 339 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 340 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 341 _type6, _arg6) 342 #else 343 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 344 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 345 _type6, _arg6) \ 346 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 347 _type3, _arg3, _type4, _arg4, _type5, _arg5) 348 #endif 349 350 #ifdef DTRACE_PROBE7 351 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 352 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 353 _type6, _arg6, _type7, _arg7) \ 354 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 355 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 356 _type6, _arg6, _type7, _arg7) 357 #else 358 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 359 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 360 _type6, _arg6, _type7, _arg7) \ 361 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 362 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 363 _type6, _arg6) 364 #endif 365 366 #endif /* DTRACE_PROBE */ 367 368 /* DMA */ 369 370 typedef uint64_t efsys_dma_addr_t; 371 372 typedef struct efsys_mem_s { 373 bus_dma_tag_t esm_tag; 374 bus_dmamap_t esm_map; 375 caddr_t esm_base; 376 efsys_dma_addr_t esm_addr; 377 } efsys_mem_t; 378 379 380 #define EFSYS_MEM_ZERO(_esmp, _size) \ 381 do { \ 382 (void) memset((_esmp)->esm_base, 0, (_size)); \ 383 \ 384 _NOTE(CONSTANTCONDITION) \ 385 } while (B_FALSE) 386 387 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 388 do { \ 389 uint32_t *addr; \ 390 \ 391 _NOTE(CONSTANTCONDITION) \ 392 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 393 ("not power of 2 aligned")); \ 394 \ 395 addr = (void *)((_esmp)->esm_base + (_offset)); \ 396 \ 397 (_edp)->ed_u32[0] = *addr; \ 398 \ 399 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 400 uint32_t, (_edp)->ed_u32[0]); \ 401 \ 402 _NOTE(CONSTANTCONDITION) \ 403 } while (B_FALSE) 404 405 #if defined(__x86_64__) 406 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 407 do { \ 408 uint64_t *addr; \ 409 \ 410 _NOTE(CONSTANTCONDITION) \ 411 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 412 ("not power of 2 aligned")); \ 413 \ 414 addr = (void *)((_esmp)->esm_base + (_offset)); \ 415 \ 416 (_eqp)->eq_u64[0] = *addr; \ 417 \ 418 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 419 uint32_t, (_eqp)->eq_u32[1], \ 420 uint32_t, (_eqp)->eq_u32[0]); \ 421 \ 422 _NOTE(CONSTANTCONDITION) \ 423 } while (B_FALSE) 424 #else 425 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 426 do { \ 427 uint32_t *addr; \ 428 \ 429 _NOTE(CONSTANTCONDITION) \ 430 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 431 ("not power of 2 aligned")); \ 432 \ 433 addr = (void *)((_esmp)->esm_base + (_offset)); \ 434 \ 435 (_eqp)->eq_u32[0] = *addr++; \ 436 (_eqp)->eq_u32[1] = *addr; \ 437 \ 438 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 439 uint32_t, (_eqp)->eq_u32[1], \ 440 uint32_t, (_eqp)->eq_u32[0]); \ 441 \ 442 _NOTE(CONSTANTCONDITION) \ 443 } while (B_FALSE) 444 #endif 445 446 #if defined(__x86_64__) 447 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 448 do { \ 449 uint64_t *addr; \ 450 \ 451 _NOTE(CONSTANTCONDITION) \ 452 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 453 ("not power of 2 aligned")); \ 454 \ 455 addr = (void *)((_esmp)->esm_base + (_offset)); \ 456 \ 457 (_eop)->eo_u64[0] = *addr++; \ 458 (_eop)->eo_u64[1] = *addr; \ 459 \ 460 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 461 uint32_t, (_eop)->eo_u32[3], \ 462 uint32_t, (_eop)->eo_u32[2], \ 463 uint32_t, (_eop)->eo_u32[1], \ 464 uint32_t, (_eop)->eo_u32[0]); \ 465 \ 466 _NOTE(CONSTANTCONDITION) \ 467 } while (B_FALSE) 468 #else 469 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 470 do { \ 471 uint32_t *addr; \ 472 \ 473 _NOTE(CONSTANTCONDITION) \ 474 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 475 ("not power of 2 aligned")); \ 476 \ 477 addr = (void *)((_esmp)->esm_base + (_offset)); \ 478 \ 479 (_eop)->eo_u32[0] = *addr++; \ 480 (_eop)->eo_u32[1] = *addr++; \ 481 (_eop)->eo_u32[2] = *addr++; \ 482 (_eop)->eo_u32[3] = *addr; \ 483 \ 484 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 485 uint32_t, (_eop)->eo_u32[3], \ 486 uint32_t, (_eop)->eo_u32[2], \ 487 uint32_t, (_eop)->eo_u32[1], \ 488 uint32_t, (_eop)->eo_u32[0]); \ 489 \ 490 _NOTE(CONSTANTCONDITION) \ 491 } while (B_FALSE) 492 #endif 493 494 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 495 do { \ 496 uint32_t *addr; \ 497 \ 498 _NOTE(CONSTANTCONDITION) \ 499 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 500 ("not power of 2 aligned")); \ 501 \ 502 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 503 uint32_t, (_edp)->ed_u32[0]); \ 504 \ 505 addr = (void *)((_esmp)->esm_base + (_offset)); \ 506 \ 507 *addr = (_edp)->ed_u32[0]; \ 508 \ 509 _NOTE(CONSTANTCONDITION) \ 510 } while (B_FALSE) 511 512 #if defined(__x86_64__) 513 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 514 do { \ 515 uint64_t *addr; \ 516 \ 517 _NOTE(CONSTANTCONDITION) \ 518 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 519 ("not power of 2 aligned")); \ 520 \ 521 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 522 uint32_t, (_eqp)->eq_u32[1], \ 523 uint32_t, (_eqp)->eq_u32[0]); \ 524 \ 525 addr = (void *)((_esmp)->esm_base + (_offset)); \ 526 \ 527 *addr = (_eqp)->eq_u64[0]; \ 528 \ 529 _NOTE(CONSTANTCONDITION) \ 530 } while (B_FALSE) 531 532 #else 533 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 534 do { \ 535 uint32_t *addr; \ 536 \ 537 _NOTE(CONSTANTCONDITION) \ 538 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 539 ("not power of 2 aligned")); \ 540 \ 541 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 542 uint32_t, (_eqp)->eq_u32[1], \ 543 uint32_t, (_eqp)->eq_u32[0]); \ 544 \ 545 addr = (void *)((_esmp)->esm_base + (_offset)); \ 546 \ 547 *addr++ = (_eqp)->eq_u32[0]; \ 548 *addr = (_eqp)->eq_u32[1]; \ 549 \ 550 _NOTE(CONSTANTCONDITION) \ 551 } while (B_FALSE) 552 #endif 553 554 #if defined(__x86_64__) 555 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 556 do { \ 557 uint64_t *addr; \ 558 \ 559 _NOTE(CONSTANTCONDITION) \ 560 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 561 ("not power of 2 aligned")); \ 562 \ 563 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 564 uint32_t, (_eop)->eo_u32[3], \ 565 uint32_t, (_eop)->eo_u32[2], \ 566 uint32_t, (_eop)->eo_u32[1], \ 567 uint32_t, (_eop)->eo_u32[0]); \ 568 \ 569 addr = (void *)((_esmp)->esm_base + (_offset)); \ 570 \ 571 *addr++ = (_eop)->eo_u64[0]; \ 572 *addr = (_eop)->eo_u64[1]; \ 573 \ 574 _NOTE(CONSTANTCONDITION) \ 575 } while (B_FALSE) 576 #else 577 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 578 do { \ 579 uint32_t *addr; \ 580 \ 581 _NOTE(CONSTANTCONDITION) \ 582 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 583 ("not power of 2 aligned")); \ 584 \ 585 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 586 uint32_t, (_eop)->eo_u32[3], \ 587 uint32_t, (_eop)->eo_u32[2], \ 588 uint32_t, (_eop)->eo_u32[1], \ 589 uint32_t, (_eop)->eo_u32[0]); \ 590 \ 591 addr = (void *)((_esmp)->esm_base + (_offset)); \ 592 \ 593 *addr++ = (_eop)->eo_u32[0]; \ 594 *addr++ = (_eop)->eo_u32[1]; \ 595 *addr++ = (_eop)->eo_u32[2]; \ 596 *addr = (_eop)->eo_u32[3]; \ 597 \ 598 _NOTE(CONSTANTCONDITION) \ 599 } while (B_FALSE) 600 #endif 601 602 #define EFSYS_MEM_ADDR(_esmp) \ 603 ((_esmp)->esm_addr) 604 605 /* BAR */ 606 607 #define SFXGE_LOCK_NAME_MAX 16 608 609 typedef struct efsys_bar_s { 610 struct mtx esb_lock; 611 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 612 bus_space_tag_t esb_tag; 613 bus_space_handle_t esb_handle; 614 int esb_rid; 615 struct resource *esb_res; 616 } efsys_bar_t; 617 618 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 619 do { \ 620 snprintf((_esbp)->esb_lock_name, \ 621 sizeof((_esbp)->esb_lock_name), \ 622 "%s:bar", (_ifname)); \ 623 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 624 NULL, MTX_DEF); \ 625 _NOTE(CONSTANTCONDITION) \ 626 } while (B_FALSE) 627 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 628 mtx_destroy(&(_esbp)->esb_lock) 629 #define SFXGE_BAR_LOCK(_esbp) \ 630 mtx_lock(&(_esbp)->esb_lock) 631 #define SFXGE_BAR_UNLOCK(_esbp) \ 632 mtx_unlock(&(_esbp)->esb_lock) 633 634 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 635 do { \ 636 _NOTE(CONSTANTCONDITION) \ 637 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 638 ("not power of 2 aligned")); \ 639 \ 640 _NOTE(CONSTANTCONDITION) \ 641 if (_lock) \ 642 SFXGE_BAR_LOCK(_esbp); \ 643 \ 644 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 645 (_esbp)->esb_handle, (_offset)); \ 646 \ 647 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 648 uint32_t, (_edp)->ed_u32[0]); \ 649 \ 650 _NOTE(CONSTANTCONDITION) \ 651 if (_lock) \ 652 SFXGE_BAR_UNLOCK(_esbp); \ 653 _NOTE(CONSTANTCONDITION) \ 654 } while (B_FALSE) 655 656 #if defined(__x86_64__) 657 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 658 do { \ 659 _NOTE(CONSTANTCONDITION) \ 660 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 661 ("not power of 2 aligned")); \ 662 \ 663 SFXGE_BAR_LOCK(_esbp); \ 664 \ 665 (_eqp)->eq_u64[0] = bus_space_read_8((_esbp)->esb_tag, \ 666 (_esbp)->esb_handle, (_offset)); \ 667 \ 668 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 669 uint32_t, (_eqp)->eq_u32[1], \ 670 uint32_t, (_eqp)->eq_u32[0]); \ 671 \ 672 mtx_unlock(&((_esbp)->esb_lock)); \ 673 _NOTE(CONSTANTCONDITION) \ 674 } while (B_FALSE) 675 676 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 677 do { \ 678 _NOTE(CONSTANTCONDITION) \ 679 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 680 ("not power of 2 aligned")); \ 681 \ 682 _NOTE(CONSTANTCONDITION) \ 683 if (_lock) \ 684 mtx_lock(&((_esbp)->esb_lock)); \ 685 \ 686 (_eop)->eo_u64[0] = bus_space_read_8((_esbp)->esb_tag, \ 687 (_esbp)->esb_handle, (_offset)); \ 688 (_eop)->eo_u64[1] = bus_space_read_8((_esbp)->esb_tag, \ 689 (_esbp)->esb_handle, (_offset+8)); \ 690 \ 691 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 692 uint32_t, (_eop)->eo_u32[3], \ 693 uint32_t, (_eop)->eo_u32[2], \ 694 uint32_t, (_eop)->eo_u32[1], \ 695 uint32_t, (_eop)->eo_u32[0]); \ 696 \ 697 _NOTE(CONSTANTCONDITION) \ 698 if (_lock) \ 699 mtx_unlock(&((_esbp)->esb_lock)); \ 700 _NOTE(CONSTANTCONDITION) \ 701 } while (B_FALSE) 702 703 #else 704 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 705 do { \ 706 _NOTE(CONSTANTCONDITION) \ 707 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 708 ("not power of 2 aligned")); \ 709 \ 710 mtx_lock(&((_esbp)->esb_lock)); \ 711 \ 712 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 713 (_esbp)->esb_handle, (_offset)); \ 714 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 715 (_esbp)->esb_handle, (_offset+4)); \ 716 \ 717 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 718 uint32_t, (_eqp)->eq_u32[1], \ 719 uint32_t, (_eqp)->eq_u32[0]); \ 720 \ 721 SFXGE_BAR_UNLOCK(_esbp); \ 722 _NOTE(CONSTANTCONDITION) \ 723 } while (B_FALSE) 724 725 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 726 do { \ 727 _NOTE(CONSTANTCONDITION) \ 728 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 729 ("not power of 2 aligned")); \ 730 \ 731 _NOTE(CONSTANTCONDITION) \ 732 if (_lock) \ 733 SFXGE_BAR_LOCK(_esbp); \ 734 \ 735 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 736 (_esbp)->esb_handle, (_offset)); \ 737 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 738 (_esbp)->esb_handle, (_offset+4)); \ 739 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \ 740 (_esbp)->esb_handle, (_offset+8)); \ 741 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \ 742 (_esbp)->esb_handle, (_offset+12)); \ 743 \ 744 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 745 uint32_t, (_eop)->eo_u32[3], \ 746 uint32_t, (_eop)->eo_u32[2], \ 747 uint32_t, (_eop)->eo_u32[1], \ 748 uint32_t, (_eop)->eo_u32[0]); \ 749 \ 750 _NOTE(CONSTANTCONDITION) \ 751 if (_lock) \ 752 SFXGE_BAR_UNLOCK(_esbp); \ 753 _NOTE(CONSTANTCONDITION) \ 754 } while (B_FALSE) 755 #endif 756 757 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 758 do { \ 759 _NOTE(CONSTANTCONDITION) \ 760 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 761 ("not power of 2 aligned")); \ 762 \ 763 _NOTE(CONSTANTCONDITION) \ 764 if (_lock) \ 765 SFXGE_BAR_LOCK(_esbp); \ 766 \ 767 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 768 uint32_t, (_edp)->ed_u32[0]); \ 769 \ 770 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 771 (_offset), (_edp)->ed_u32[0]); \ 772 \ 773 _NOTE(CONSTANTCONDITION) \ 774 if (_lock) \ 775 SFXGE_BAR_UNLOCK(_esbp); \ 776 _NOTE(CONSTANTCONDITION) \ 777 } while (B_FALSE) 778 779 #if defined(__x86_64__) 780 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 781 do { \ 782 _NOTE(CONSTANTCONDITION) \ 783 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 784 ("not power of 2 aligned")); \ 785 \ 786 SFXGE_BAR_LOCK(_esbp); \ 787 \ 788 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 789 uint32_t, (_eqp)->eq_u32[1], \ 790 uint32_t, (_eqp)->eq_u32[0]); \ 791 \ 792 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\ 793 (_offset), (_eqp)->eq_u64[0]); \ 794 \ 795 mtx_unlock(&((_esbp)->esb_lock)); \ 796 _NOTE(CONSTANTCONDITION) \ 797 } while (B_FALSE) 798 #else 799 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 800 do { \ 801 _NOTE(CONSTANTCONDITION) \ 802 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 803 ("not power of 2 aligned")); \ 804 \ 805 mtx_lock(&((_esbp)->esb_lock)); \ 806 \ 807 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 808 uint32_t, (_eqp)->eq_u32[1], \ 809 uint32_t, (_eqp)->eq_u32[0]); \ 810 \ 811 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 812 (_offset), (_eqp)->eq_u32[0]); \ 813 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 814 (_offset+4), (_eqp)->eq_u32[1]); \ 815 \ 816 SFXGE_BAR_UNLOCK(_esbp); \ 817 _NOTE(CONSTANTCONDITION) \ 818 } while (B_FALSE) 819 #endif 820 821 #if defined(__x86_64__) 822 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 823 do { \ 824 _NOTE(CONSTANTCONDITION) \ 825 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 826 ("not power of 2 aligned")); \ 827 \ 828 _NOTE(CONSTANTCONDITION) \ 829 if (_lock) \ 830 SFXGE_BAR_LOCK(_esbp); \ 831 \ 832 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 833 uint32_t, (_eop)->eo_u32[3], \ 834 uint32_t, (_eop)->eo_u32[2], \ 835 uint32_t, (_eop)->eo_u32[1], \ 836 uint32_t, (_eop)->eo_u32[0]); \ 837 \ 838 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\ 839 (_offset), (_eop)->eo_u64[0]); \ 840 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\ 841 (_offset+8), (_eop)->eo_u64[1]); \ 842 \ 843 _NOTE(CONSTANTCONDITION) \ 844 if (_lock) \ 845 mtx_unlock(&((_esbp)->esb_lock)); \ 846 _NOTE(CONSTANTCONDITION) \ 847 } while (B_FALSE) 848 849 #else 850 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 851 do { \ 852 _NOTE(CONSTANTCONDITION) \ 853 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 854 ("not power of 2 aligned")); \ 855 \ 856 _NOTE(CONSTANTCONDITION) \ 857 if (_lock) \ 858 mtx_lock(&((_esbp)->esb_lock)); \ 859 \ 860 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 861 uint32_t, (_eop)->eo_u32[3], \ 862 uint32_t, (_eop)->eo_u32[2], \ 863 uint32_t, (_eop)->eo_u32[1], \ 864 uint32_t, (_eop)->eo_u32[0]); \ 865 \ 866 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 867 (_offset), (_eop)->eo_u32[0]); \ 868 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 869 (_offset+4), (_eop)->eo_u32[1]); \ 870 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 871 (_offset+8), (_eop)->eo_u32[2]); \ 872 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 873 (_offset+12), (_eop)->eo_u32[3]); \ 874 \ 875 _NOTE(CONSTANTCONDITION) \ 876 if (_lock) \ 877 SFXGE_BAR_UNLOCK(_esbp); \ 878 _NOTE(CONSTANTCONDITION) \ 879 } while (B_FALSE) 880 #endif 881 882 /* SPIN */ 883 884 #define EFSYS_SPIN(_us) \ 885 do { \ 886 DELAY(_us); \ 887 _NOTE(CONSTANTCONDITION) \ 888 } while (B_FALSE) 889 890 #define EFSYS_SLEEP EFSYS_SPIN 891 892 /* BARRIERS */ 893 894 #define EFSYS_MEM_READ_BARRIER() rmb() 895 #define EFSYS_PIO_WRITE_BARRIER() 896 897 /* TIMESTAMP */ 898 899 typedef clock_t efsys_timestamp_t; 900 901 #define EFSYS_TIMESTAMP(_usp) \ 902 do { \ 903 clock_t now; \ 904 \ 905 now = ticks; \ 906 *(_usp) = now * hz / 1000000; \ 907 _NOTE(CONSTANTCONDITION) \ 908 } while (B_FALSE) 909 910 /* KMEM */ 911 912 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 913 do { \ 914 (_esip) = (_esip); \ 915 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \ 916 _NOTE(CONSTANTCONDITION) \ 917 } while (B_FALSE) 918 919 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 920 do { \ 921 (void) (_esip); \ 922 (void) (_size); \ 923 free((_p), M_SFXGE); \ 924 _NOTE(CONSTANTCONDITION) \ 925 } while (B_FALSE) 926 927 /* LOCK */ 928 929 typedef struct efsys_lock_s { 930 struct mtx lock; 931 char lock_name[SFXGE_LOCK_NAME_MAX]; 932 } efsys_lock_t; 933 934 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 935 do { \ 936 efsys_lock_t *__eslp = (_eslp); \ 937 \ 938 snprintf((__eslp)->lock_name, \ 939 sizeof((__eslp)->lock_name), \ 940 "%s:%s", (_ifname), (_label)); \ 941 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 942 NULL, MTX_DEF); \ 943 } while (B_FALSE) 944 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 945 mtx_destroy(&(_eslp)->lock) 946 #define SFXGE_EFSYS_LOCK(_eslp) \ 947 mtx_lock(&(_eslp)->lock) 948 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 949 mtx_unlock(&(_eslp)->lock) 950 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 951 mtx_assert(&(_eslp)->lock, MA_OWNED) 952 953 #define EFSYS_LOCK_MAGIC 0x000010c4 954 955 #define EFSYS_LOCK(_lockp, _state) \ 956 do { \ 957 SFXGE_EFSYS_LOCK(_lockp); \ 958 (_state) = EFSYS_LOCK_MAGIC; \ 959 _NOTE(CONSTANTCONDITION) \ 960 } while (B_FALSE) 961 962 #define EFSYS_UNLOCK(_lockp, _state) \ 963 do { \ 964 if ((_state) != EFSYS_LOCK_MAGIC) \ 965 KASSERT(B_FALSE, ("not locked")); \ 966 SFXGE_EFSYS_UNLOCK(_lockp); \ 967 _NOTE(CONSTANTCONDITION) \ 968 } while (B_FALSE) 969 970 /* PREEMPT */ 971 972 #define EFSYS_PREEMPT_DISABLE(_state) \ 973 do { \ 974 (_state) = (_state); \ 975 critical_enter(); \ 976 _NOTE(CONSTANTCONDITION) \ 977 } while (B_FALSE) 978 979 #define EFSYS_PREEMPT_ENABLE(_state) \ 980 do { \ 981 (_state) = (_state); \ 982 critical_exit(_state); \ 983 _NOTE(CONSTANTCONDITION) \ 984 } while (B_FALSE) 985 986 /* STAT */ 987 988 typedef uint64_t efsys_stat_t; 989 990 #define EFSYS_STAT_INCR(_knp, _delta) \ 991 do { \ 992 *(_knp) += (_delta); \ 993 _NOTE(CONSTANTCONDITION) \ 994 } while (B_FALSE) 995 996 #define EFSYS_STAT_DECR(_knp, _delta) \ 997 do { \ 998 *(_knp) -= (_delta); \ 999 _NOTE(CONSTANTCONDITION) \ 1000 } while (B_FALSE) 1001 1002 #define EFSYS_STAT_SET(_knp, _val) \ 1003 do { \ 1004 *(_knp) = (_val); \ 1005 _NOTE(CONSTANTCONDITION) \ 1006 } while (B_FALSE) 1007 1008 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1009 do { \ 1010 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1011 _NOTE(CONSTANTCONDITION) \ 1012 } while (B_FALSE) 1013 1014 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1015 do { \ 1016 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1017 _NOTE(CONSTANTCONDITION) \ 1018 } while (B_FALSE) 1019 1020 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1021 do { \ 1022 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1023 _NOTE(CONSTANTCONDITION) \ 1024 } while (B_FALSE) 1025 1026 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1027 do { \ 1028 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1029 _NOTE(CONSTANTCONDITION) \ 1030 } while (B_FALSE) 1031 1032 /* ERR */ 1033 1034 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1035 uint32_t, uint32_t); 1036 1037 #if EFSYS_OPT_DECODE_INTR_FATAL 1038 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1039 do { \ 1040 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1041 _NOTE(CONSTANTCONDITION) \ 1042 } while (B_FALSE) 1043 #endif 1044 1045 /* ASSERT */ 1046 1047 #define EFSYS_ASSERT(_exp) do { \ 1048 if (!(_exp)) \ 1049 panic(#_exp); \ 1050 } while (0) 1051 1052 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1053 const _t __x = (_t)(_x); \ 1054 const _t __y = (_t)(_y); \ 1055 if (!(__x _op __y)) \ 1056 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1057 } while(0) 1058 1059 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1060 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1061 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1062 1063 #ifdef __cplusplus 1064 } 1065 #endif 1066 1067 #endif /* _SYS_EFSYS_H */ 1068