1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _SYS_EFSYS_H 33 #define _SYS_EFSYS_H 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/endian.h> 42 #include <sys/lock.h> 43 #include <sys/malloc.h> 44 #include <sys/mbuf.h> 45 #include <sys/mutex.h> 46 #include <sys/rwlock.h> 47 #include <sys/sdt.h> 48 #include <sys/systm.h> 49 50 #include <machine/bus.h> 51 #include <machine/endian.h> 52 53 #define EFSYS_HAS_UINT64 1 54 #if defined(__x86_64__) 55 #define EFSYS_USE_UINT64 1 56 #else 57 #define EFSYS_USE_UINT64 0 58 #endif 59 #if _BYTE_ORDER == _BIG_ENDIAN 60 #define EFSYS_IS_BIG_ENDIAN 1 61 #define EFSYS_IS_LITTLE_ENDIAN 0 62 #elif _BYTE_ORDER == _LITTLE_ENDIAN 63 #define EFSYS_IS_BIG_ENDIAN 0 64 #define EFSYS_IS_LITTLE_ENDIAN 1 65 #endif 66 #include "efx_types.h" 67 68 /* Common code requires this */ 69 #if __FreeBSD_version < 800068 70 #define memmove(d, s, l) bcopy(s, d, l) 71 #endif 72 73 /* FreeBSD equivalents of Solaris things */ 74 #ifndef _NOTE 75 #define _NOTE(s) 76 #endif 77 78 #ifndef B_FALSE 79 #define B_FALSE FALSE 80 #endif 81 #ifndef B_TRUE 82 #define B_TRUE TRUE 83 #endif 84 85 #ifndef IS_P2ALIGNED 86 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 87 #endif 88 89 #ifndef P2ROUNDUP 90 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 91 #endif 92 93 #ifndef IS2P 94 #define ISP2(x) (((x) & ((x) - 1)) == 0) 95 #endif 96 97 #if defined(__x86_64__) 98 #if !defined(bus_space_read_stream_8) 99 #define bus_space_read_stream_8(t, h, o) \ 100 bus_space_read_8((t), (h), (o)) 101 #define bus_space_write_stream_8(t, h, o, v) \ 102 bus_space_write_8((t), (h), (o), (v)) 103 #endif 104 #endif 105 106 #define ENOTACTIVE EINVAL 107 108 /* Memory type to use on FreeBSD */ 109 MALLOC_DECLARE(M_SFXGE); 110 111 /* Machine dependend prefetch wrappers */ 112 #if defined(__i386__) || defined(__amd64__) 113 static __inline void 114 prefetch_read_many(void *addr) 115 { 116 117 __asm__( 118 "prefetcht0 (%0)" 119 : 120 : "r" (addr)); 121 } 122 123 static __inline void 124 prefetch_read_once(void *addr) 125 { 126 127 __asm__( 128 "prefetchnta (%0)" 129 : 130 : "r" (addr)); 131 } 132 #elif defined(__sparc64__) 133 static __inline void 134 prefetch_read_many(void *addr) 135 { 136 137 __asm__( 138 "prefetch [%0], 0" 139 : 140 : "r" (addr)); 141 } 142 143 static __inline void 144 prefetch_read_once(void *addr) 145 { 146 147 __asm__( 148 "prefetch [%0], 1" 149 : 150 : "r" (addr)); 151 } 152 #else 153 static __inline void 154 prefetch_read_many(void *addr) 155 { 156 157 } 158 159 static __inline void 160 prefetch_read_once(void *addr) 161 { 162 163 } 164 #endif 165 166 #if defined(__i386__) || defined(__amd64__) 167 #include <vm/vm.h> 168 #include <vm/pmap.h> 169 #endif 170 static __inline void 171 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 172 struct mbuf *m, bus_dma_segment_t *seg) 173 { 174 #if defined(__i386__) || defined(__amd64__) 175 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 176 seg->ds_len = m->m_len; 177 #else 178 int nsegstmp; 179 180 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 181 #endif 182 } 183 184 /* Modifiers used for DOS builds */ 185 #define __cs 186 #define __far 187 188 /* Modifiers used for Windows builds */ 189 #define __in 190 #define __in_opt 191 #define __in_ecount(_n) 192 #define __in_ecount_opt(_n) 193 #define __in_bcount(_n) 194 #define __in_bcount_opt(_n) 195 196 #define __out 197 #define __out_opt 198 #define __out_ecount(_n) 199 #define __out_ecount_opt(_n) 200 #define __out_bcount(_n) 201 #define __out_bcount_opt(_n) 202 203 #define __deref_out 204 205 #define __inout 206 #define __inout_opt 207 #define __inout_ecount(_n) 208 #define __inout_ecount_opt(_n) 209 #define __inout_bcount(_n) 210 #define __inout_bcount_opt(_n) 211 #define __inout_bcount_full_opt(_n) 212 213 #define __deref_out_bcount_opt(n) 214 215 #define __checkReturn 216 217 #define __drv_when(_p, _c) 218 219 /* Code inclusion options */ 220 221 222 #define EFSYS_OPT_NAMES 1 223 224 #define EFSYS_OPT_FALCON 0 225 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 226 #define EFSYS_OPT_SIENA 1 227 #ifdef DEBUG 228 #define EFSYS_OPT_CHECK_REG 1 229 #else 230 #define EFSYS_OPT_CHECK_REG 0 231 #endif 232 233 #define EFSYS_OPT_MCDI 1 234 235 #define EFSYS_OPT_MAC_FALCON_GMAC 0 236 #define EFSYS_OPT_MAC_FALCON_XMAC 0 237 #define EFSYS_OPT_MAC_STATS 1 238 239 #define EFSYS_OPT_LOOPBACK 0 240 241 #define EFSYS_OPT_MON_NULL 0 242 #define EFSYS_OPT_MON_LM87 0 243 #define EFSYS_OPT_MON_MAX6647 0 244 #define EFSYS_OPT_MON_SIENA 0 245 #define EFSYS_OPT_MON_STATS 0 246 247 #define EFSYS_OPT_PHY_NULL 0 248 #define EFSYS_OPT_PHY_QT2022C2 0 249 #define EFSYS_OPT_PHY_SFX7101 0 250 #define EFSYS_OPT_PHY_TXC43128 0 251 #define EFSYS_OPT_PHY_PM8358 0 252 #define EFSYS_OPT_PHY_SFT9001 0 253 #define EFSYS_OPT_PHY_QT2025C 0 254 #define EFSYS_OPT_PHY_STATS 1 255 #define EFSYS_OPT_PHY_PROPS 0 256 #define EFSYS_OPT_PHY_BIST 1 257 #define EFSYS_OPT_PHY_LED_CONTROL 1 258 #define EFSYS_OPT_PHY_FLAGS 0 259 260 #define EFSYS_OPT_VPD 1 261 #define EFSYS_OPT_NVRAM 1 262 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 263 #define EFSYS_OPT_NVRAM_SFT9001 0 264 #define EFSYS_OPT_NVRAM_SFX7101 0 265 #define EFSYS_OPT_BOOTCFG 0 266 267 #define EFSYS_OPT_PCIE_TUNE 0 268 #define EFSYS_OPT_DIAG 0 269 #define EFSYS_OPT_WOL 1 270 #define EFSYS_OPT_RX_SCALE 1 271 #define EFSYS_OPT_QSTATS 1 272 #define EFSYS_OPT_FILTER 0 273 #define EFSYS_OPT_RX_SCATTER 0 274 #define EFSYS_OPT_RX_HDR_SPLIT 0 275 276 #define EFSYS_OPT_EV_PREFETCH 0 277 278 #define EFSYS_OPT_DECODE_INTR_FATAL 1 279 280 /* ID */ 281 282 typedef struct __efsys_identifier_s efsys_identifier_t; 283 284 /* PROBE */ 285 286 #ifndef DTRACE_PROBE 287 288 #define EFSYS_PROBE(_name) 289 290 #define EFSYS_PROBE1(_name, _type1, _arg1) 291 292 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 293 294 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 295 _type3, _arg3) 296 297 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 298 _type3, _arg3, _type4, _arg4) 299 300 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 301 _type3, _arg3, _type4, _arg4, _type5, _arg5) 302 303 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 304 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 305 _type6, _arg6) 306 307 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 308 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 309 _type6, _arg6, _type7, _arg7) 310 311 #else /* DTRACE_PROBE */ 312 313 #define EFSYS_PROBE(_name) \ 314 DTRACE_PROBE(_name) 315 316 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 317 DTRACE_PROBE1(_name, _type1, _arg1) 318 319 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 320 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 321 322 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 323 _type3, _arg3) \ 324 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 325 _type3, _arg3) 326 327 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 328 _type3, _arg3, _type4, _arg4) \ 329 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 330 _type3, _arg3, _type4, _arg4) 331 332 #ifdef DTRACE_PROBE5 333 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 334 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 335 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 336 _type3, _arg3, _type4, _arg4, _type5, _arg5) 337 #else 338 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 339 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 340 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 341 _type3, _arg3, _type4, _arg4) 342 #endif 343 344 #ifdef DTRACE_PROBE6 345 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 346 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 347 _type6, _arg6) \ 348 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 349 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 350 _type6, _arg6) 351 #else 352 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 353 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 354 _type6, _arg6) \ 355 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 356 _type3, _arg3, _type4, _arg4, _type5, _arg5) 357 #endif 358 359 #ifdef DTRACE_PROBE7 360 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 361 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 362 _type6, _arg6, _type7, _arg7) \ 363 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 364 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 365 _type6, _arg6, _type7, _arg7) 366 #else 367 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 368 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 369 _type6, _arg6, _type7, _arg7) \ 370 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 371 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 372 _type6, _arg6) 373 #endif 374 375 #endif /* DTRACE_PROBE */ 376 377 /* DMA */ 378 379 typedef uint64_t efsys_dma_addr_t; 380 381 typedef struct efsys_mem_s { 382 bus_dma_tag_t esm_tag; 383 bus_dmamap_t esm_map; 384 caddr_t esm_base; 385 efsys_dma_addr_t esm_addr; 386 } efsys_mem_t; 387 388 389 #define EFSYS_MEM_ZERO(_esmp, _size) \ 390 do { \ 391 (void) memset((_esmp)->esm_base, 0, (_size)); \ 392 \ 393 _NOTE(CONSTANTCONDITION) \ 394 } while (B_FALSE) 395 396 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 397 do { \ 398 uint32_t *addr; \ 399 \ 400 _NOTE(CONSTANTCONDITION) \ 401 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 402 ("not power of 2 aligned")); \ 403 \ 404 addr = (void *)((_esmp)->esm_base + (_offset)); \ 405 \ 406 (_edp)->ed_u32[0] = *addr; \ 407 \ 408 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 409 uint32_t, (_edp)->ed_u32[0]); \ 410 \ 411 _NOTE(CONSTANTCONDITION) \ 412 } while (B_FALSE) 413 414 #if defined(__x86_64__) 415 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 416 do { \ 417 uint64_t *addr; \ 418 \ 419 _NOTE(CONSTANTCONDITION) \ 420 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 421 ("not power of 2 aligned")); \ 422 \ 423 addr = (void *)((_esmp)->esm_base + (_offset)); \ 424 \ 425 (_eqp)->eq_u64[0] = *addr; \ 426 \ 427 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 428 uint32_t, (_eqp)->eq_u32[1], \ 429 uint32_t, (_eqp)->eq_u32[0]); \ 430 \ 431 _NOTE(CONSTANTCONDITION) \ 432 } while (B_FALSE) 433 #else 434 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 435 do { \ 436 uint32_t *addr; \ 437 \ 438 _NOTE(CONSTANTCONDITION) \ 439 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 440 ("not power of 2 aligned")); \ 441 \ 442 addr = (void *)((_esmp)->esm_base + (_offset)); \ 443 \ 444 (_eqp)->eq_u32[0] = *addr++; \ 445 (_eqp)->eq_u32[1] = *addr; \ 446 \ 447 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 448 uint32_t, (_eqp)->eq_u32[1], \ 449 uint32_t, (_eqp)->eq_u32[0]); \ 450 \ 451 _NOTE(CONSTANTCONDITION) \ 452 } while (B_FALSE) 453 #endif 454 455 #if defined(__x86_64__) 456 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 457 do { \ 458 uint64_t *addr; \ 459 \ 460 _NOTE(CONSTANTCONDITION) \ 461 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 462 ("not power of 2 aligned")); \ 463 \ 464 addr = (void *)((_esmp)->esm_base + (_offset)); \ 465 \ 466 (_eop)->eo_u64[0] = *addr++; \ 467 (_eop)->eo_u64[1] = *addr; \ 468 \ 469 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 470 uint32_t, (_eop)->eo_u32[3], \ 471 uint32_t, (_eop)->eo_u32[2], \ 472 uint32_t, (_eop)->eo_u32[1], \ 473 uint32_t, (_eop)->eo_u32[0]); \ 474 \ 475 _NOTE(CONSTANTCONDITION) \ 476 } while (B_FALSE) 477 #else 478 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 479 do { \ 480 uint32_t *addr; \ 481 \ 482 _NOTE(CONSTANTCONDITION) \ 483 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 484 ("not power of 2 aligned")); \ 485 \ 486 addr = (void *)((_esmp)->esm_base + (_offset)); \ 487 \ 488 (_eop)->eo_u32[0] = *addr++; \ 489 (_eop)->eo_u32[1] = *addr++; \ 490 (_eop)->eo_u32[2] = *addr++; \ 491 (_eop)->eo_u32[3] = *addr; \ 492 \ 493 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 494 uint32_t, (_eop)->eo_u32[3], \ 495 uint32_t, (_eop)->eo_u32[2], \ 496 uint32_t, (_eop)->eo_u32[1], \ 497 uint32_t, (_eop)->eo_u32[0]); \ 498 \ 499 _NOTE(CONSTANTCONDITION) \ 500 } while (B_FALSE) 501 #endif 502 503 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 504 do { \ 505 uint32_t *addr; \ 506 \ 507 _NOTE(CONSTANTCONDITION) \ 508 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 509 ("not power of 2 aligned")); \ 510 \ 511 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 512 uint32_t, (_edp)->ed_u32[0]); \ 513 \ 514 addr = (void *)((_esmp)->esm_base + (_offset)); \ 515 \ 516 *addr = (_edp)->ed_u32[0]; \ 517 \ 518 _NOTE(CONSTANTCONDITION) \ 519 } while (B_FALSE) 520 521 #if defined(__x86_64__) 522 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 523 do { \ 524 uint64_t *addr; \ 525 \ 526 _NOTE(CONSTANTCONDITION) \ 527 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 528 ("not power of 2 aligned")); \ 529 \ 530 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 531 uint32_t, (_eqp)->eq_u32[1], \ 532 uint32_t, (_eqp)->eq_u32[0]); \ 533 \ 534 addr = (void *)((_esmp)->esm_base + (_offset)); \ 535 \ 536 *addr = (_eqp)->eq_u64[0]; \ 537 \ 538 _NOTE(CONSTANTCONDITION) \ 539 } while (B_FALSE) 540 541 #else 542 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 543 do { \ 544 uint32_t *addr; \ 545 \ 546 _NOTE(CONSTANTCONDITION) \ 547 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 548 ("not power of 2 aligned")); \ 549 \ 550 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 551 uint32_t, (_eqp)->eq_u32[1], \ 552 uint32_t, (_eqp)->eq_u32[0]); \ 553 \ 554 addr = (void *)((_esmp)->esm_base + (_offset)); \ 555 \ 556 *addr++ = (_eqp)->eq_u32[0]; \ 557 *addr = (_eqp)->eq_u32[1]; \ 558 \ 559 _NOTE(CONSTANTCONDITION) \ 560 } while (B_FALSE) 561 #endif 562 563 #if defined(__x86_64__) 564 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 565 do { \ 566 uint64_t *addr; \ 567 \ 568 _NOTE(CONSTANTCONDITION) \ 569 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 570 ("not power of 2 aligned")); \ 571 \ 572 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 573 uint32_t, (_eop)->eo_u32[3], \ 574 uint32_t, (_eop)->eo_u32[2], \ 575 uint32_t, (_eop)->eo_u32[1], \ 576 uint32_t, (_eop)->eo_u32[0]); \ 577 \ 578 addr = (void *)((_esmp)->esm_base + (_offset)); \ 579 \ 580 *addr++ = (_eop)->eo_u64[0]; \ 581 *addr = (_eop)->eo_u64[1]; \ 582 \ 583 _NOTE(CONSTANTCONDITION) \ 584 } while (B_FALSE) 585 #else 586 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 587 do { \ 588 uint32_t *addr; \ 589 \ 590 _NOTE(CONSTANTCONDITION) \ 591 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 592 ("not power of 2 aligned")); \ 593 \ 594 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 595 uint32_t, (_eop)->eo_u32[3], \ 596 uint32_t, (_eop)->eo_u32[2], \ 597 uint32_t, (_eop)->eo_u32[1], \ 598 uint32_t, (_eop)->eo_u32[0]); \ 599 \ 600 addr = (void *)((_esmp)->esm_base + (_offset)); \ 601 \ 602 *addr++ = (_eop)->eo_u32[0]; \ 603 *addr++ = (_eop)->eo_u32[1]; \ 604 *addr++ = (_eop)->eo_u32[2]; \ 605 *addr = (_eop)->eo_u32[3]; \ 606 \ 607 _NOTE(CONSTANTCONDITION) \ 608 } while (B_FALSE) 609 #endif 610 611 #define EFSYS_MEM_ADDR(_esmp) \ 612 ((_esmp)->esm_addr) 613 614 /* BAR */ 615 616 #define SFXGE_LOCK_NAME_MAX 16 617 618 typedef struct efsys_bar_s { 619 struct mtx esb_lock; 620 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 621 bus_space_tag_t esb_tag; 622 bus_space_handle_t esb_handle; 623 int esb_rid; 624 struct resource *esb_res; 625 } efsys_bar_t; 626 627 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 628 do { \ 629 snprintf((_esbp)->esb_lock_name, \ 630 sizeof((_esbp)->esb_lock_name), \ 631 "%s:bar", (_ifname)); \ 632 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 633 NULL, MTX_DEF); \ 634 _NOTE(CONSTANTCONDITION) \ 635 } while (B_FALSE) 636 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 637 mtx_destroy(&(_esbp)->esb_lock) 638 #define SFXGE_BAR_LOCK(_esbp) \ 639 mtx_lock(&(_esbp)->esb_lock) 640 #define SFXGE_BAR_UNLOCK(_esbp) \ 641 mtx_unlock(&(_esbp)->esb_lock) 642 643 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 644 do { \ 645 _NOTE(CONSTANTCONDITION) \ 646 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 647 ("not power of 2 aligned")); \ 648 \ 649 _NOTE(CONSTANTCONDITION) \ 650 if (_lock) \ 651 SFXGE_BAR_LOCK(_esbp); \ 652 \ 653 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 654 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 655 (_offset)); \ 656 \ 657 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 658 uint32_t, (_edp)->ed_u32[0]); \ 659 \ 660 _NOTE(CONSTANTCONDITION) \ 661 if (_lock) \ 662 SFXGE_BAR_UNLOCK(_esbp); \ 663 _NOTE(CONSTANTCONDITION) \ 664 } while (B_FALSE) 665 666 #if defined(__x86_64__) 667 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 668 do { \ 669 _NOTE(CONSTANTCONDITION) \ 670 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 671 ("not power of 2 aligned")); \ 672 \ 673 SFXGE_BAR_LOCK(_esbp); \ 674 \ 675 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 676 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 677 (_offset)); \ 678 \ 679 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 680 uint32_t, (_eqp)->eq_u32[1], \ 681 uint32_t, (_eqp)->eq_u32[0]); \ 682 \ 683 SFXGE_BAR_UNLOCK(_esbp); \ 684 _NOTE(CONSTANTCONDITION) \ 685 } while (B_FALSE) 686 687 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 688 do { \ 689 _NOTE(CONSTANTCONDITION) \ 690 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 691 ("not power of 2 aligned")); \ 692 \ 693 _NOTE(CONSTANTCONDITION) \ 694 if (_lock) \ 695 SFXGE_BAR_LOCK(_esbp); \ 696 \ 697 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 698 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 699 (_offset)); \ 700 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 701 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 702 (_offset) + 8); \ 703 \ 704 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 705 uint32_t, (_eop)->eo_u32[3], \ 706 uint32_t, (_eop)->eo_u32[2], \ 707 uint32_t, (_eop)->eo_u32[1], \ 708 uint32_t, (_eop)->eo_u32[0]); \ 709 \ 710 _NOTE(CONSTANTCONDITION) \ 711 if (_lock) \ 712 SFXGE_BAR_UNLOCK(_esbp); \ 713 _NOTE(CONSTANTCONDITION) \ 714 } while (B_FALSE) 715 716 #else 717 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 718 do { \ 719 _NOTE(CONSTANTCONDITION) \ 720 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 721 ("not power of 2 aligned")); \ 722 \ 723 SFXGE_BAR_LOCK(_esbp); \ 724 \ 725 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 726 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 727 (_offset)); \ 728 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 729 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 730 (_offset) + 4); \ 731 \ 732 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 733 uint32_t, (_eqp)->eq_u32[1], \ 734 uint32_t, (_eqp)->eq_u32[0]); \ 735 \ 736 SFXGE_BAR_UNLOCK(_esbp); \ 737 _NOTE(CONSTANTCONDITION) \ 738 } while (B_FALSE) 739 740 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 741 do { \ 742 _NOTE(CONSTANTCONDITION) \ 743 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 744 ("not power of 2 aligned")); \ 745 \ 746 _NOTE(CONSTANTCONDITION) \ 747 if (_lock) \ 748 SFXGE_BAR_LOCK(_esbp); \ 749 \ 750 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 751 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 752 (_offset)); \ 753 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 754 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 755 (_offset) + 4); \ 756 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 757 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 758 (_offset) + 8); \ 759 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 760 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 761 (_offset) + 12); \ 762 \ 763 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 764 uint32_t, (_eop)->eo_u32[3], \ 765 uint32_t, (_eop)->eo_u32[2], \ 766 uint32_t, (_eop)->eo_u32[1], \ 767 uint32_t, (_eop)->eo_u32[0]); \ 768 \ 769 _NOTE(CONSTANTCONDITION) \ 770 if (_lock) \ 771 SFXGE_BAR_UNLOCK(_esbp); \ 772 _NOTE(CONSTANTCONDITION) \ 773 } while (B_FALSE) 774 #endif 775 776 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 777 do { \ 778 _NOTE(CONSTANTCONDITION) \ 779 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 780 ("not power of 2 aligned")); \ 781 \ 782 _NOTE(CONSTANTCONDITION) \ 783 if (_lock) \ 784 SFXGE_BAR_LOCK(_esbp); \ 785 \ 786 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 787 uint32_t, (_edp)->ed_u32[0]); \ 788 \ 789 bus_space_write_stream_4((_esbp)->esb_tag, \ 790 (_esbp)->esb_handle, \ 791 (_offset), (_edp)->ed_u32[0]); \ 792 \ 793 _NOTE(CONSTANTCONDITION) \ 794 if (_lock) \ 795 SFXGE_BAR_UNLOCK(_esbp); \ 796 _NOTE(CONSTANTCONDITION) \ 797 } while (B_FALSE) 798 799 #if defined(__x86_64__) 800 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 801 do { \ 802 _NOTE(CONSTANTCONDITION) \ 803 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 804 ("not power of 2 aligned")); \ 805 \ 806 SFXGE_BAR_LOCK(_esbp); \ 807 \ 808 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 809 uint32_t, (_eqp)->eq_u32[1], \ 810 uint32_t, (_eqp)->eq_u32[0]); \ 811 \ 812 bus_space_write_stream_8((_esbp)->esb_tag, \ 813 (_esbp)->esb_handle, \ 814 (_offset), (_eqp)->eq_u64[0]); \ 815 \ 816 SFXGE_BAR_UNLOCK(_esbp); \ 817 _NOTE(CONSTANTCONDITION) \ 818 } while (B_FALSE) 819 #else 820 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 821 do { \ 822 _NOTE(CONSTANTCONDITION) \ 823 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 824 ("not power of 2 aligned")); \ 825 \ 826 SFXGE_BAR_LOCK(_esbp); \ 827 \ 828 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 829 uint32_t, (_eqp)->eq_u32[1], \ 830 uint32_t, (_eqp)->eq_u32[0]); \ 831 \ 832 bus_space_write_stream_4((_esbp)->esb_tag, \ 833 (_esbp)->esb_handle, \ 834 (_offset), (_eqp)->eq_u32[0]); \ 835 bus_space_write_stream_4((_esbp)->esb_tag, \ 836 (_esbp)->esb_handle, \ 837 (_offset) + 4, (_eqp)->eq_u32[1]); \ 838 \ 839 SFXGE_BAR_UNLOCK(_esbp); \ 840 _NOTE(CONSTANTCONDITION) \ 841 } while (B_FALSE) 842 #endif 843 844 #if defined(__x86_64__) 845 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 846 do { \ 847 _NOTE(CONSTANTCONDITION) \ 848 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 849 ("not power of 2 aligned")); \ 850 \ 851 _NOTE(CONSTANTCONDITION) \ 852 if (_lock) \ 853 SFXGE_BAR_LOCK(_esbp); \ 854 \ 855 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 856 uint32_t, (_eop)->eo_u32[3], \ 857 uint32_t, (_eop)->eo_u32[2], \ 858 uint32_t, (_eop)->eo_u32[1], \ 859 uint32_t, (_eop)->eo_u32[0]); \ 860 \ 861 bus_space_write_stream_8((_esbp)->esb_tag, \ 862 (_esbp)->esb_handle, \ 863 (_offset), (_eop)->eo_u64[0]); \ 864 bus_space_write_stream_8((_esbp)->esb_tag, \ 865 (_esbp)->esb_handle, \ 866 (_offset) + 8, (_eop)->eo_u64[1]); \ 867 \ 868 _NOTE(CONSTANTCONDITION) \ 869 if (_lock) \ 870 SFXGE_BAR_UNLOCK(_esbp); \ 871 _NOTE(CONSTANTCONDITION) \ 872 } while (B_FALSE) 873 874 #else 875 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 876 do { \ 877 _NOTE(CONSTANTCONDITION) \ 878 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 879 ("not power of 2 aligned")); \ 880 \ 881 _NOTE(CONSTANTCONDITION) \ 882 if (_lock) \ 883 SFXGE_BAR_LOCK(_esbp); \ 884 \ 885 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 886 uint32_t, (_eop)->eo_u32[3], \ 887 uint32_t, (_eop)->eo_u32[2], \ 888 uint32_t, (_eop)->eo_u32[1], \ 889 uint32_t, (_eop)->eo_u32[0]); \ 890 \ 891 bus_space_write_stream_4((_esbp)->esb_tag, \ 892 (_esbp)->esb_handle, \ 893 (_offset), (_eop)->eo_u32[0]); \ 894 bus_space_write_stream_4((_esbp)->esb_tag, \ 895 (_esbp)->esb_handle, \ 896 (_offset) + 4, (_eop)->eo_u32[1]); \ 897 bus_space_write_stream_4((_esbp)->esb_tag, \ 898 (_esbp)->esb_handle, \ 899 (_offset) + 8, (_eop)->eo_u32[2]); \ 900 bus_space_write_stream_4((_esbp)->esb_tag, \ 901 (_esbp)->esb_handle, \ 902 (_offset) + 12, (_eop)->eo_u32[3]); \ 903 \ 904 _NOTE(CONSTANTCONDITION) \ 905 if (_lock) \ 906 SFXGE_BAR_UNLOCK(_esbp); \ 907 _NOTE(CONSTANTCONDITION) \ 908 } while (B_FALSE) 909 #endif 910 911 /* SPIN */ 912 913 #define EFSYS_SPIN(_us) \ 914 do { \ 915 DELAY(_us); \ 916 _NOTE(CONSTANTCONDITION) \ 917 } while (B_FALSE) 918 919 #define EFSYS_SLEEP EFSYS_SPIN 920 921 /* BARRIERS */ 922 923 #define EFSYS_MEM_READ_BARRIER() rmb() 924 #define EFSYS_PIO_WRITE_BARRIER() 925 926 /* TIMESTAMP */ 927 928 typedef clock_t efsys_timestamp_t; 929 930 #define EFSYS_TIMESTAMP(_usp) \ 931 do { \ 932 clock_t now; \ 933 \ 934 now = ticks; \ 935 *(_usp) = now * hz / 1000000; \ 936 _NOTE(CONSTANTCONDITION) \ 937 } while (B_FALSE) 938 939 /* KMEM */ 940 941 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 942 do { \ 943 (_esip) = (_esip); \ 944 /* \ 945 * The macro is used in non-sleepable contexts, for \ 946 * example, holding a mutex. \ 947 */ \ 948 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 949 _NOTE(CONSTANTCONDITION) \ 950 } while (B_FALSE) 951 952 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 953 do { \ 954 (void) (_esip); \ 955 (void) (_size); \ 956 free((_p), M_SFXGE); \ 957 _NOTE(CONSTANTCONDITION) \ 958 } while (B_FALSE) 959 960 /* LOCK */ 961 962 typedef struct efsys_lock_s { 963 struct mtx lock; 964 char lock_name[SFXGE_LOCK_NAME_MAX]; 965 } efsys_lock_t; 966 967 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 968 do { \ 969 efsys_lock_t *__eslp = (_eslp); \ 970 \ 971 snprintf((__eslp)->lock_name, \ 972 sizeof((__eslp)->lock_name), \ 973 "%s:%s", (_ifname), (_label)); \ 974 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 975 NULL, MTX_DEF); \ 976 } while (B_FALSE) 977 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 978 mtx_destroy(&(_eslp)->lock) 979 #define SFXGE_EFSYS_LOCK(_eslp) \ 980 mtx_lock(&(_eslp)->lock) 981 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 982 mtx_unlock(&(_eslp)->lock) 983 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 984 mtx_assert(&(_eslp)->lock, MA_OWNED) 985 986 #define EFSYS_LOCK_MAGIC 0x000010c4 987 988 #define EFSYS_LOCK(_lockp, _state) \ 989 do { \ 990 SFXGE_EFSYS_LOCK(_lockp); \ 991 (_state) = EFSYS_LOCK_MAGIC; \ 992 _NOTE(CONSTANTCONDITION) \ 993 } while (B_FALSE) 994 995 #define EFSYS_UNLOCK(_lockp, _state) \ 996 do { \ 997 if ((_state) != EFSYS_LOCK_MAGIC) \ 998 KASSERT(B_FALSE, ("not locked")); \ 999 SFXGE_EFSYS_UNLOCK(_lockp); \ 1000 _NOTE(CONSTANTCONDITION) \ 1001 } while (B_FALSE) 1002 1003 /* PREEMPT */ 1004 1005 #define EFSYS_PREEMPT_DISABLE(_state) \ 1006 do { \ 1007 (_state) = (_state); \ 1008 critical_enter(); \ 1009 _NOTE(CONSTANTCONDITION) \ 1010 } while (B_FALSE) 1011 1012 #define EFSYS_PREEMPT_ENABLE(_state) \ 1013 do { \ 1014 (_state) = (_state); \ 1015 critical_exit(_state); \ 1016 _NOTE(CONSTANTCONDITION) \ 1017 } while (B_FALSE) 1018 1019 /* STAT */ 1020 1021 typedef uint64_t efsys_stat_t; 1022 1023 #define EFSYS_STAT_INCR(_knp, _delta) \ 1024 do { \ 1025 *(_knp) += (_delta); \ 1026 _NOTE(CONSTANTCONDITION) \ 1027 } while (B_FALSE) 1028 1029 #define EFSYS_STAT_DECR(_knp, _delta) \ 1030 do { \ 1031 *(_knp) -= (_delta); \ 1032 _NOTE(CONSTANTCONDITION) \ 1033 } while (B_FALSE) 1034 1035 #define EFSYS_STAT_SET(_knp, _val) \ 1036 do { \ 1037 *(_knp) = (_val); \ 1038 _NOTE(CONSTANTCONDITION) \ 1039 } while (B_FALSE) 1040 1041 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1042 do { \ 1043 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1044 _NOTE(CONSTANTCONDITION) \ 1045 } while (B_FALSE) 1046 1047 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1048 do { \ 1049 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1050 _NOTE(CONSTANTCONDITION) \ 1051 } while (B_FALSE) 1052 1053 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1054 do { \ 1055 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1056 _NOTE(CONSTANTCONDITION) \ 1057 } while (B_FALSE) 1058 1059 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1060 do { \ 1061 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1062 _NOTE(CONSTANTCONDITION) \ 1063 } while (B_FALSE) 1064 1065 /* ERR */ 1066 1067 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1068 uint32_t, uint32_t); 1069 1070 #if EFSYS_OPT_DECODE_INTR_FATAL 1071 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1072 do { \ 1073 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1074 _NOTE(CONSTANTCONDITION) \ 1075 } while (B_FALSE) 1076 #endif 1077 1078 /* ASSERT */ 1079 1080 #define EFSYS_ASSERT(_exp) do { \ 1081 if (!(_exp)) \ 1082 panic(#_exp); \ 1083 } while (0) 1084 1085 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1086 const _t __x = (_t)(_x); \ 1087 const _t __y = (_t)(_y); \ 1088 if (!(__x _op __y)) \ 1089 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1090 } while(0) 1091 1092 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1093 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1094 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1095 1096 #ifdef __cplusplus 1097 } 1098 #endif 1099 1100 #endif /* _SYS_EFSYS_H */ 1101