1 /* 2 * Copyright (c) 2008-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #ifndef _SYS_SFXGE_H 32 #define _SYS_SFXGE_H 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 #include <sys/types.h> 39 #include <sys/ddi.h> 40 #include <sys/sunddi.h> 41 #include <sys/stream.h> 42 #include <sys/ethernet.h> 43 #include <sys/cpuvar.h> 44 45 #include <sys/mac.h> 46 #include <sys/mac_ether.h> 47 #include <sys/mac_provider.h> 48 49 #include "sfxge_ioc.h" 50 #include "sfxge_debug.h" 51 52 #include "efx.h" 53 #include "efx_regs.h" 54 55 #ifdef _KERNEL 56 57 #define SFXGE_DRIVER_NAME "sfxge" 58 59 #define SFXGE_CPU_CACHE_SIZE 64 60 61 typedef struct sfxge_s sfxge_t; 62 63 typedef enum sfxge_intr_state_e { 64 SFXGE_INTR_UNINITIALIZED = 0, 65 SFXGE_INTR_INITIALIZED, 66 SFXGE_INTR_TESTING, 67 SFXGE_INTR_STARTED 68 } sfxge_intr_state_t; 69 70 typedef struct sfxge_intr_s { 71 ddi_intr_handle_t *si_table; 72 int si_table_size; 73 int si_nalloc; 74 int si_type; 75 int si_cap; 76 efsys_mem_t si_mem; 77 uint64_t si_mask; 78 sfxge_intr_state_t si_state; 79 uint32_t si_zero_count; 80 int si_intr_pri; 81 } sfxge_intr_t; 82 83 typedef enum sfxge_promisc_type_e { 84 SFXGE_PROMISC_OFF = 0, 85 SFXGE_PROMISC_ALL_MULTI, 86 SFXGE_PROMISC_ALL_PHYS 87 } sfxge_promisc_type_t; 88 89 typedef enum sfxge_link_duplex_e { 90 SFXGE_LINK_DUPLEX_UNKNOWN = 0, 91 SFXGE_LINK_DUPLEX_HALF, 92 SFXGE_LINK_DUPLEX_FULL 93 } sfxge_link_duplex_t; 94 95 typedef enum sfxge_unicst_type_e { 96 SFXGE_UNICST_BIA = 0, 97 SFXGE_UNICST_LAA, 98 SFXGE_UNICST_NTYPES 99 } sfxge_unicst_type_t; 100 101 typedef struct sfxge_phy_s { 102 kstat_t *sp_ksp; 103 kstat_named_t *sp_stat; 104 uint32_t *sp_statbuf; 105 efsys_mem_t sp_mem; 106 } sfxge_phy_t; 107 108 typedef enum sfxge_mac_state_e { 109 SFXGE_MAC_UNINITIALIZED = 0, 110 SFXGE_MAC_INITIALIZED, 111 SFXGE_MAC_STARTED 112 } sfxge_mac_state_t; 113 114 typedef struct sfxge_mac_s { 115 sfxge_t *sm_sp; 116 efsys_mem_t sm_mem; 117 kstat_t *sm_ksp; 118 kstat_named_t *sm_stat; 119 uint8_t sm_bia[ETHERADDRL]; 120 uint8_t sm_laa[ETHERADDRL]; 121 boolean_t sm_laa_valid; 122 unsigned int sm_fcntl; 123 sfxge_promisc_type_t sm_promisc; 124 uint8_t sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX * 125 ETHERADDRL]; /* List of multicast addresses to filter on */ 126 int sm_mcast_count; 127 clock_t sm_lbolt; 128 kmutex_t sm_lock; 129 efx_link_mode_t sm_link_mode; 130 unsigned int sm_link_speed; 131 sfxge_link_duplex_t sm_link_duplex; 132 boolean_t sm_link_up; 133 boolean_t sm_link_poll_reqd; 134 kcondvar_t sm_link_poll_kv; 135 boolean_t sm_mac_stats_timer_reqd; 136 boolean_t sm_mac_stats_pend; 137 ddi_taskq_t *sm_tqp; 138 sfxge_mac_state_t sm_state; 139 sfxge_phy_t sm_phy; 140 uint32_t sm_phy_cap_to_set; 141 uint32_t sm_phy_cap_to_unset; 142 } sfxge_mac_t; 143 144 typedef enum sfxge_mon_state_e { 145 SFXGE_MON_UNINITIALIZED = 0, 146 SFXGE_MON_INITIALIZED, 147 SFXGE_MON_STARTED 148 } sfxge_mon_state_t; 149 150 typedef struct sfxge_mon_s { 151 sfxge_t *sm_sp; 152 efx_mon_type_t sm_type; 153 unsigned int sm_devid; 154 kstat_t *sm_ksp; 155 kstat_named_t *sm_stat; 156 efx_mon_stat_value_t *sm_statbuf; 157 kmutex_t sm_lock; 158 sfxge_mon_state_t sm_state; 159 efsys_mem_t sm_mem; 160 int sm_polling; 161 } sfxge_mon_t; 162 163 typedef enum sfxge_sram_state_e { 164 SFXGE_SRAM_UNINITIALIZED = 0, 165 SFXGE_SRAM_INITIALIZED, 166 SFXGE_SRAM_STARTED 167 } sfxge_sram_state_t; 168 169 typedef struct sfxge_sram_s { 170 sfxge_t *ss_sp; 171 kmutex_t ss_lock; 172 struct map *ss_buf_tbl_map; 173 unsigned int ss_count; 174 sfxge_sram_state_t ss_state; 175 } sfxge_sram_t; 176 177 typedef enum sfxge_mcdi_state_e { 178 SFXGE_MCDI_UNINITIALIZED = 0, 179 SFXGE_MCDI_INITIALIZED, 180 SFXGE_MCDI_BUSY, 181 SFXGE_MCDI_COMPLETED 182 } sfxge_mcdi_state_t; 183 184 typedef struct sfxge_mcdi_s { 185 sfxge_t *sm_sp; 186 kmutex_t sm_lock; 187 sfxge_mcdi_state_t sm_state; 188 efx_mcdi_transport_t sm_emt; 189 efsys_mem_t sm_mem; 190 kcondvar_t sm_kv; /* MCDI poll complete */ 191 } sfxge_mcdi_t; 192 193 #define SFXGE_NEVS 4096 194 #define SFXGE_RX_NDESCS 1024 195 #define SFXGE_TX_NDESCS 1024 196 #define SFXGE_TX_NLABELS EFX_EV_TX_NLABELS 197 198 #define SFXGE_DEFAULT_RXQ_SIZE 1024 199 #define SFXGE_DEFAULT_MODERATION 30 200 201 typedef enum sfxge_evq_state_e { 202 SFXGE_EVQ_UNINITIALIZED = 0, 203 SFXGE_EVQ_INITIALIZED, 204 SFXGE_EVQ_STARTING, 205 SFXGE_EVQ_STARTED 206 } sfxge_evq_state_t; 207 208 #define SFXGE_EV_BATCH (SFXGE_NEVS / 4) 209 210 typedef struct sfxge_txq_s sfxge_txq_t; 211 212 typedef struct sfxge_evq_s { 213 union { 214 struct { 215 sfxge_t *__se_sp; 216 unsigned int __se_index; 217 efsys_mem_t __se_mem; 218 unsigned int __se_id; 219 kstat_t *__se_ksp; 220 kstat_named_t *__se_stat; 221 efx_ev_callbacks_t __se_eec; 222 sfxge_evq_state_t __se_state; 223 boolean_t __se_exception; 224 } __se_s1; 225 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4]; 226 } __se_u1; 227 union { 228 struct { 229 kmutex_t __se_lock; 230 kcondvar_t __se_init_kv; 231 efx_evq_t *__se_eep; 232 unsigned int __se_count; 233 unsigned int __se_rx; 234 unsigned int __se_tx; 235 sfxge_txq_t *__se_stp; 236 sfxge_txq_t **__se_stpp; 237 processorid_t __se_cpu_id; 238 uint16_t __se_ev_batch; 239 } __se_s2; 240 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE]; 241 } __se_u2; 242 union { 243 struct { 244 sfxge_txq_t *__se_label_stp[SFXGE_TX_NLABELS]; 245 } __se_s3; 246 uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4]; 247 } __se_u3; 248 } sfxge_evq_t; 249 250 #define se_sp __se_u1.__se_s1.__se_sp 251 #define se_index __se_u1.__se_s1.__se_index 252 #define se_mem __se_u1.__se_s1.__se_mem 253 #define se_id __se_u1.__se_s1.__se_id 254 #define se_ksp __se_u1.__se_s1.__se_ksp 255 #define se_stat __se_u1.__se_s1.__se_stat 256 #define se_eec __se_u1.__se_s1.__se_eec 257 #define se_state __se_u1.__se_s1.__se_state 258 #define se_exception __se_u1.__se_s1.__se_exception 259 260 #define se_lock __se_u2.__se_s2.__se_lock 261 #define se_init_kv __se_u2.__se_s2.__se_init_kv 262 #define se_eep __se_u2.__se_s2.__se_eep 263 #define se_count __se_u2.__se_s2.__se_count 264 #define se_rx __se_u2.__se_s2.__se_rx 265 #define se_tx __se_u2.__se_s2.__se_tx 266 #define se_stp __se_u2.__se_s2.__se_stp 267 #define se_stpp __se_u2.__se_s2.__se_stpp 268 #define se_cpu_id __se_u2.__se_s2.__se_cpu_id 269 #define se_ev_batch __se_u2.__se_s2.__se_ev_batch 270 271 #define se_label_stp __se_u3.__se_s3.__se_label_stp 272 273 274 #define SFXGE_MAGIC_RESERVED 0x8000 275 276 #define SFXGE_MAGIC_DMAQ_LABEL_WIDTH 5 277 #define SFXGE_MAGIC_DMAQ_LABEL_MASK ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1) 278 279 #define SFXGE_MAGIC_RX_QFLUSH_DONE \ 280 (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH)) 281 282 #define SFXGE_MAGIC_RX_QFLUSH_FAILED \ 283 (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH)) 284 285 #define SFXGE_MAGIC_RX_QFPP_TRIM \ 286 (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH)) 287 288 #define SFXGE_MAGIC_TX_QFLUSH_DONE \ 289 (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH)) 290 291 typedef struct sfxge_rxq_s sfxge_rxq_t; 292 293 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */ 294 295 typedef struct sfxge_rx_packet_s sfxge_rx_packet_t; 296 297 struct sfxge_rx_packet_s { 298 union { 299 struct { 300 frtn_t __srp_free; 301 uint16_t __srp_flags; 302 uint16_t __srp_size; 303 mblk_t *__srp_mp; 304 struct ether_header *__srp_etherhp; 305 struct ip *__srp_iphp; 306 struct tcphdr *__srp_thp; 307 size_t __srp_off; 308 } __srp_s1; 309 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE]; 310 } __srp_u1; 311 union { 312 struct { 313 sfxge_rxq_t *__srp_srp; 314 ddi_dma_handle_t __srp_dma_handle; 315 ddi_acc_handle_t __srp_acc_handle; 316 unsigned char *__srp_base; 317 size_t __srp_mblksize; 318 uint64_t __srp_addr; 319 boolean_t __srp_recycle; 320 caddr_t __srp_putp; 321 } __srp_s2; 322 uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2]; 323 } __srp_u2; 324 }; 325 326 #define srp_free __srp_u1.__srp_s1.__srp_free 327 #define srp_flags __srp_u1.__srp_s1.__srp_flags 328 #define srp_size __srp_u1.__srp_s1.__srp_size 329 #define srp_mp __srp_u1.__srp_s1.__srp_mp 330 #define srp_etherhp __srp_u1.__srp_s1.__srp_etherhp 331 #define srp_iphp __srp_u1.__srp_s1.__srp_iphp 332 #define srp_thp __srp_u1.__srp_s1.__srp_thp 333 #define srp_off __srp_u1.__srp_s1.__srp_off 334 335 #define srp_srp __srp_u2.__srp_s2.__srp_srp 336 #define srp_dma_handle __srp_u2.__srp_s2.__srp_dma_handle 337 #define srp_acc_handle __srp_u2.__srp_s2.__srp_acc_handle 338 #define srp_base __srp_u2.__srp_s2.__srp_base 339 #define srp_mblksize __srp_u2.__srp_s2.__srp_mblksize 340 #define srp_addr __srp_u2.__srp_s2.__srp_addr 341 #define srp_recycle __srp_u2.__srp_s2.__srp_recycle 342 #define srp_putp __srp_u2.__srp_s2.__srp_putp 343 344 #define SFXGE_RX_FPP_NSLOTS 8 345 #define SFXGE_RX_FPP_MASK (SFXGE_RX_FPP_NSLOTS - 1) 346 347 /* Free packet pool putlist (dynamically allocated) */ 348 typedef struct sfxge_rx_fpp_putlist_s { 349 kmutex_t srfpl_lock; 350 unsigned int srfpl_count; 351 mblk_t *srfpl_putp; 352 mblk_t **srfpl_putpp; 353 } sfxge_rx_fpp_putlist_t; 354 355 /* Free packet pool */ 356 typedef struct sfxge_rx_fpp_s { 357 caddr_t srfpp_putp; 358 unsigned int srfpp_loaned; 359 mblk_t *srfpp_get; 360 unsigned int srfpp_count; 361 unsigned int srfpp_min; 362 /* Low water mark: Don't trim to below this */ 363 unsigned int srfpp_lowat; 364 } sfxge_rx_fpp_t; 365 366 typedef struct sfxge_rx_flow_s sfxge_rx_flow_t; 367 368 struct sfxge_rx_flow_s { 369 uint32_t srf_tag; 370 /* in-order segment count */ 371 unsigned int srf_count; 372 uint16_t srf_tci; 373 uint32_t srf_saddr; 374 uint32_t srf_daddr; 375 uint16_t srf_sport; 376 uint16_t srf_dport; 377 /* sequence number */ 378 uint32_t srf_seq; 379 clock_t srf_lbolt; 380 mblk_t *srf_mp; 381 mblk_t **srf_mpp; 382 struct ether_header *srf_etherhp; 383 struct ip *srf_iphp; 384 struct tcphdr *srf_first_thp; 385 struct tcphdr *srf_last_thp; 386 size_t srf_len; 387 sfxge_rx_flow_t *srf_next; 388 }; 389 390 #define SFXGE_MAX_FLOW 1024 391 #define SFXGE_SLOW_START 20 392 393 typedef enum sfxge_flush_state_e { 394 SFXGE_FLUSH_INACTIVE = 0, 395 SFXGE_FLUSH_DONE, 396 SFXGE_FLUSH_PENDING, 397 SFXGE_FLUSH_FAILED 398 } sfxge_flush_state_t; 399 400 typedef enum sfxge_rxq_state_e { 401 SFXGE_RXQ_UNINITIALIZED = 0, 402 SFXGE_RXQ_INITIALIZED, 403 SFXGE_RXQ_STARTED 404 } sfxge_rxq_state_t; 405 406 407 #define SFXGE_RX_BATCH 128 408 #define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */ 409 410 struct sfxge_rxq_s { 411 union { 412 struct { 413 sfxge_t *__sr_sp; 414 unsigned int __sr_index; 415 efsys_mem_t __sr_mem; 416 unsigned int __sr_id; 417 unsigned int __sr_lowat; 418 unsigned int __sr_hiwat; 419 volatile timeout_id_t __sr_tid; 420 sfxge_rxq_state_t __sr_state; 421 } __sr_s1; 422 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2]; 423 } __sr_u1; 424 union { 425 struct { 426 sfxge_rx_packet_t **__sr_srpp; 427 unsigned int __sr_added; 428 unsigned int __sr_pushed; 429 unsigned int __sr_pending; 430 unsigned int __sr_completed; 431 unsigned int __sr_loopback; 432 mblk_t *__sr_mp; 433 mblk_t **__sr_mpp; 434 sfxge_rx_flow_t *__sr_flow; 435 sfxge_rx_flow_t *__sr_srfp; 436 sfxge_rx_flow_t **__sr_srfpp; 437 clock_t __sr_rto; 438 } __sr_s2; 439 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2]; 440 } __sr_u2; 441 union { 442 struct { 443 sfxge_rx_fpp_t __sr_fpp; 444 efx_rxq_t *__sr_erp; 445 volatile sfxge_flush_state_t __sr_flush; 446 kcondvar_t __sr_flush_kv; 447 kstat_t *__sr_ksp; 448 } __sr_s3; 449 uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE]; 450 } __sr_u3; 451 struct { 452 /* NB must match SFXGE_RX_NSTATS */ 453 uint32_t srk_rx_pkt_mem_limit; 454 uint32_t srk_kcache_alloc_nomem; 455 uint32_t srk_dma_alloc_nomem; 456 uint32_t srk_dma_alloc_fail; 457 uint32_t srk_dma_bind_nomem; 458 uint32_t srk_dma_bind_fail; 459 uint32_t srk_desballoc_fail; 460 uint32_t srk_rxq_empty_discard; 461 } sr_kstat; 462 }; 463 464 #define sr_sp __sr_u1.__sr_s1.__sr_sp 465 #define sr_index __sr_u1.__sr_s1.__sr_index 466 #define sr_mem __sr_u1.__sr_s1.__sr_mem 467 #define sr_id __sr_u1.__sr_s1.__sr_id 468 #define sr_mrh __sr_u1.__sr_s1.__sr_mrh 469 #define sr_lowat __sr_u1.__sr_s1.__sr_lowat 470 #define sr_hiwat __sr_u1.__sr_s1.__sr_hiwat 471 #define sr_tid __sr_u1.__sr_s1.__sr_tid 472 #define sr_state __sr_u1.__sr_s1.__sr_state 473 474 #define sr_srpp __sr_u2.__sr_s2.__sr_srpp 475 #define sr_added __sr_u2.__sr_s2.__sr_added 476 #define sr_pushed __sr_u2.__sr_s2.__sr_pushed 477 #define sr_pending __sr_u2.__sr_s2.__sr_pending 478 #define sr_completed __sr_u2.__sr_s2.__sr_completed 479 #define sr_loopback __sr_u2.__sr_s2.__sr_loopback 480 #define sr_mp __sr_u2.__sr_s2.__sr_mp 481 #define sr_mpp __sr_u2.__sr_s2.__sr_mpp 482 #define sr_flow __sr_u2.__sr_s2.__sr_flow 483 #define sr_srfp __sr_u2.__sr_s2.__sr_srfp 484 #define sr_srfpp __sr_u2.__sr_s2.__sr_srfpp 485 #define sr_rto __sr_u2.__sr_s2.__sr_rto 486 487 #define sr_fpp __sr_u3.__sr_s3.__sr_fpp 488 #define sr_erp __sr_u3.__sr_s3.__sr_erp 489 #define sr_flush __sr_u3.__sr_s3.__sr_flush 490 #define sr_flush_kv __sr_u3.__sr_s3.__sr_flush_kv 491 #define sr_ksp __sr_u3.__sr_s3.__sr_ksp 492 493 typedef struct sfxge_tx_packet_s sfxge_tx_packet_t; 494 495 /* Packet type from parsing transmit packet */ 496 typedef enum sfxge_packet_type_e { 497 SFXGE_PACKET_TYPE_UNKNOWN = 0, 498 SFXGE_PACKET_TYPE_IPV4_TCP, 499 SFXGE_PACKET_TYPE_IPV4_UDP, 500 SFXGE_PACKET_TYPE_IPV4_SCTP, 501 SFXGE_PACKET_TYPE_IPV4_OTHER, 502 SFXGE_PACKET_NTYPES 503 } sfxge_packet_type_t; 504 505 struct sfxge_tx_packet_s { 506 sfxge_tx_packet_t *stp_next; 507 mblk_t *stp_mp; 508 struct ether_header *stp_etherhp; 509 struct ip *stp_iphp; 510 struct tcphdr *stp_thp; 511 size_t stp_off; 512 size_t stp_size; 513 size_t stp_mss; 514 uint32_t stp_dpl_put_len; 515 }; 516 517 #define SFXGE_TX_FPP_MAX 64 518 519 typedef struct sfxge_tx_fpp_s { 520 sfxge_tx_packet_t *stf_stpp; 521 unsigned int stf_count; 522 } sfxge_tx_fpp_t; 523 524 typedef struct sfxge_tx_mapping_s sfxge_tx_mapping_t; 525 526 #define SFXGE_TX_MAPPING_NADDR (((1 << 16) >> 12) + 2) 527 528 struct sfxge_tx_mapping_s { 529 sfxge_tx_mapping_t *stm_next; 530 sfxge_t *stm_sp; 531 mblk_t *stm_mp; 532 ddi_dma_handle_t stm_dma_handle; 533 caddr_t stm_base; 534 size_t stm_size; 535 size_t stm_off; 536 uint64_t stm_addr[SFXGE_TX_MAPPING_NADDR]; 537 }; 538 539 typedef struct sfxge_tx_fmp_s { 540 sfxge_tx_mapping_t *stf_stmp; 541 unsigned int stf_count; 542 } sfxge_tx_fmp_t; 543 544 typedef struct sfxge_tx_buffer_s sfxge_tx_buffer_t; 545 546 struct sfxge_tx_buffer_s { 547 sfxge_tx_buffer_t *stb_next; 548 size_t stb_off; 549 efsys_mem_t stb_esm; 550 }; 551 552 #define SFXGE_TX_BUFFER_SIZE 0x400 553 #define SFXGE_TX_HEADER_SIZE 0x100 554 #define SFXGE_TX_COPY_THRESHOLD 0x200 555 556 typedef struct sfxge_tx_fbp_s { 557 sfxge_tx_buffer_t *stf_stbp; 558 unsigned int stf_count; 559 } sfxge_tx_fbp_t; 560 561 typedef struct sfxge_tx_dpl_s { 562 uintptr_t std_put; 563 sfxge_tx_packet_t *std_get; 564 sfxge_tx_packet_t **std_getp; 565 unsigned int std_count; /* only get list count */ 566 unsigned int get_pkt_limit; 567 unsigned int put_pkt_limit; 568 unsigned int get_full_count; 569 unsigned int put_full_count; 570 } sfxge_tx_dpl_t; 571 572 typedef enum sfxge_txq_state_e { 573 SFXGE_TXQ_UNINITIALIZED = 0, 574 SFXGE_TXQ_INITIALIZED, 575 SFXGE_TXQ_STARTED, 576 SFXGE_TXQ_FLUSH_PENDING, 577 SFXGE_TXQ_FLUSH_DONE, 578 SFXGE_TXQ_FLUSH_FAILED 579 } sfxge_txq_state_t; 580 581 typedef enum sfxge_txq_type_e { 582 SFXGE_TXQ_NON_CKSUM = 0, 583 SFXGE_TXQ_IP_CKSUM, 584 SFXGE_TXQ_IP_TCP_UDP_CKSUM, 585 SFXGE_TXQ_NTYPES 586 } sfxge_txq_type_t; 587 588 #define SFXGE_TXQ_UNBLOCK_LEVEL1 (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4) 589 #define SFXGE_TXQ_UNBLOCK_LEVEL2 0 590 #define SFXGE_TXQ_NOT_BLOCKED -1 591 592 #define SFXGE_TX_BATCH 64 593 594 struct sfxge_txq_s { 595 union { 596 struct { 597 sfxge_t *__st_sp; 598 unsigned int __st_index; 599 unsigned int __st_label; 600 sfxge_txq_type_t __st_type; 601 unsigned int __st_evq; 602 efsys_mem_t __st_mem; 603 unsigned int __st_id; 604 kstat_t *__st_ksp; 605 kstat_named_t *__st_stat; 606 sfxge_txq_state_t __st_state; 607 } __st_s1; 608 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2]; 609 } __st_u1; 610 union { 611 struct { 612 sfxge_tx_dpl_t __st_dpl; 613 } __st_s2; 614 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE]; 615 } __st_u2; 616 union { 617 struct { 618 kmutex_t __st_lock; 619 /* mapping pool - sfxge_tx_mapping_t */ 620 sfxge_tx_fmp_t __st_fmp; 621 /* buffer pool - sfxge_tx_buffer_t */ 622 sfxge_tx_fbp_t __st_fbp; 623 /* packet pool - sfxge_tx_packet_t */ 624 sfxge_tx_fpp_t __st_fpp; 625 efx_buffer_t *__st_eb; 626 unsigned int __st_n; 627 efx_txq_t *__st_etp; 628 sfxge_tx_mapping_t **__st_stmp; 629 sfxge_tx_buffer_t **__st_stbp; 630 mblk_t **__st_mp; 631 unsigned int __st_added; 632 unsigned int __st_reaped; 633 int __st_unblock; 634 } __st_s3; 635 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3]; 636 } __st_u3; 637 union { 638 struct { 639 sfxge_txq_t *__st_next; 640 unsigned int __st_pending; 641 unsigned int __st_completed; 642 643 } __st_s4; 644 uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE]; 645 } __st_u4; 646 }; 647 648 #define st_sp __st_u1.__st_s1.__st_sp 649 #define st_index __st_u1.__st_s1.__st_index 650 #define st_label __st_u1.__st_s1.__st_label 651 #define st_type __st_u1.__st_s1.__st_type 652 #define st_evq __st_u1.__st_s1.__st_evq 653 #define st_mem __st_u1.__st_s1.__st_mem 654 #define st_id __st_u1.__st_s1.__st_id 655 #define st_ksp __st_u1.__st_s1.__st_ksp 656 #define st_stat __st_u1.__st_s1.__st_stat 657 #define st_state __st_u1.__st_s1.__st_state 658 659 #define st_dpl __st_u2.__st_s2.__st_dpl 660 661 #define st_lock __st_u3.__st_s3.__st_lock 662 #define st_fmp __st_u3.__st_s3.__st_fmp 663 #define st_fbp __st_u3.__st_s3.__st_fbp 664 #define st_fpp __st_u3.__st_s3.__st_fpp 665 #define st_eb __st_u3.__st_s3.__st_eb 666 #define st_n __st_u3.__st_s3.__st_n 667 #define st_etp __st_u3.__st_s3.__st_etp 668 #define st_stmp __st_u3.__st_s3.__st_stmp 669 #define st_stbp __st_u3.__st_s3.__st_stbp 670 #define st_mp __st_u3.__st_s3.__st_mp 671 #define st_added __st_u3.__st_s3.__st_added 672 #define st_reaped __st_u3.__st_s3.__st_reaped 673 #define st_unblock __st_u3.__st_s3.__st_unblock 674 675 #define st_next __st_u4.__st_s4.__st_next 676 #define st_pending __st_u4.__st_s4.__st_pending 677 #define st_completed __st_u4.__st_s4.__st_completed 678 679 typedef enum sfxge_rx_scale_state_e { 680 SFXGE_RX_SCALE_UNINITIALIZED = 0, 681 SFXGE_RX_SCALE_INITIALIZED, 682 SFXGE_RX_SCALE_STARTED 683 } sfxge_rx_scale_state_t; 684 685 #define SFXGE_RX_SCALE_MAX EFX_RSS_TBL_SIZE 686 687 typedef struct sfxge_rx_scale_s { 688 kmutex_t srs_lock; 689 unsigned int *srs_cpu; 690 unsigned int srs_tbl[SFXGE_RX_SCALE_MAX]; 691 unsigned int srs_count; 692 kstat_t *srs_ksp; 693 sfxge_rx_scale_state_t srs_state; 694 } sfxge_rx_scale_t; 695 696 697 typedef enum sfxge_rx_coalesce_mode_e { 698 SFXGE_RX_COALESCE_OFF = 0, 699 SFXGE_RX_COALESCE_DISALLOW_PUSH = 1, 700 SFXGE_RX_COALESCE_ALLOW_PUSH = 2 701 } sfxge_rx_coalesce_mode_t; 702 703 typedef enum sfxge_vpd_type_e { 704 SFXGE_VPD_ID = 0, 705 SFXGE_VPD_PN = 1, 706 SFXGE_VPD_SN = 2, 707 SFXGE_VPD_EC = 3, 708 SFXGE_VPD_MN = 4, 709 SFXGE_VPD_VD = 5, 710 SFXGE_VPD_VE = 6, 711 SFXGE_VPD_MAX = 7, 712 } sfxge_vpd_type_t; 713 714 typedef struct sfxge_vpd_kstat_s { 715 kstat_t *svk_ksp; 716 kstat_named_t svk_stat[SFXGE_VPD_MAX]; 717 efx_vpd_value_t *svk_vv; 718 } sfxge_vpd_kstat_t; 719 720 typedef struct sfxge_cfg_kstat_s { 721 struct { 722 kstat_named_t sck_mac; 723 kstat_named_t sck_version; 724 } kstat; 725 struct { 726 char sck_mac[64 + 1]; 727 } buf; 728 } sfxge_cfg_kstat_t; 729 730 typedef enum sfxge_state_e { 731 SFXGE_UNINITIALIZED = 0, 732 SFXGE_INITIALIZED, 733 SFXGE_REGISTERED, 734 SFXGE_STARTING, 735 SFXGE_STARTED, 736 SFXGE_STOPPING 737 } sfxge_state_t; 738 739 typedef enum sfxge_hw_err_e { 740 SFXGE_HW_OK = 0, 741 SFXGE_HW_ERR, 742 } sfxge_hw_err_t; 743 744 typedef enum sfxge_action_on_hw_err_e { 745 SFXGE_RECOVER = 0, 746 SFXGE_INVISIBLE = 1, 747 SFXGE_LEAVE_DEAD = 2, 748 } sfxge_action_on_hw_err_t; 749 750 typedef char *sfxge_mac_priv_prop_t; 751 752 #define SFXGE_TOEPLITZ_KEY_LEN 40 753 754 struct sfxge_s { 755 kmutex_t s_state_lock; 756 sfxge_state_t s_state; 757 dev_info_t *s_dip; 758 ddi_taskq_t *s_tqp; 759 ddi_acc_handle_t s_pci_handle; 760 uint16_t s_pci_venid; 761 uint16_t s_pci_devid; 762 #if EFSYS_OPT_MCDI_LOGGING 763 unsigned int s_bus_addr; 764 #endif 765 efx_family_t s_family; 766 unsigned int s_pcie_nlanes; 767 unsigned int s_pcie_linkspeed; 768 kmutex_t s_nic_lock; 769 efsys_bar_t s_bar; 770 sfxge_intr_t s_intr; 771 sfxge_mac_t s_mac; 772 sfxge_mon_t s_mon; 773 sfxge_sram_t s_sram; 774 sfxge_mcdi_t s_mcdi; 775 kmem_cache_t *s_eq0c; /* eventQ 0 */ 776 kmem_cache_t *s_eqXc; /* all other eventQs */ 777 sfxge_evq_t *s_sep[SFXGE_RX_SCALE_MAX]; 778 unsigned int s_ev_moderation; 779 kmem_cache_t *s_rqc; 780 sfxge_rxq_t *s_srp[SFXGE_RX_SCALE_MAX]; 781 sfxge_rx_scale_t s_rx_scale; 782 size_t s_rx_prefix_size; 783 size_t s_rx_buffer_size; 784 size_t s_rx_buffer_align; 785 sfxge_rx_coalesce_mode_t s_rx_coalesce_mode; 786 int64_t s_rx_pkt_mem_max; 787 volatile uint64_t s_rx_pkt_mem_alloc; 788 kmem_cache_t *s_rpc; 789 kmem_cache_t *s_tqc; 790 unsigned int s_tx_scale_base[SFXGE_TXQ_NTYPES]; 791 unsigned int s_tx_scale_max[SFXGE_TXQ_NTYPES]; 792 int s_tx_qcount; 793 sfxge_txq_t *s_stp[SFXGE_RX_SCALE_MAX * 794 SFXGE_TXQ_NTYPES]; /* Sparse array */ 795 kmem_cache_t *s_tpc; 796 int s_tx_flush_pending; 797 kmutex_t s_tx_flush_lock; 798 kcondvar_t s_tx_flush_kv; 799 kmem_cache_t *s_tbc; 800 kmem_cache_t *s_tmc; 801 efx_nic_t *s_enp; 802 sfxge_vpd_kstat_t s_vpd_kstat; 803 sfxge_cfg_kstat_t s_cfg_kstat; 804 kstat_t *s_cfg_ksp; 805 size_t s_mtu; 806 int s_rxq_poll_usec; 807 mac_callbacks_t s_mc; 808 mac_handle_t s_mh; 809 sfxge_mac_priv_prop_t *s_mac_priv_props; 810 int s_mac_priv_props_alloc; 811 volatile uint32_t s_nested_restarts; 812 uint32_t s_num_restarts; 813 uint32_t s_num_restarts_hw_err; 814 sfxge_hw_err_t s_hw_err; 815 sfxge_action_on_hw_err_t s_action_on_hw_err; 816 uint16_t s_rxq_size; 817 uint16_t s_evq0_size; 818 uint16_t s_evqX_size; 819 #if EFSYS_OPT_MCDI_LOGGING 820 int s_mcdi_logging; 821 #endif 822 const uint32_t *s_toeplitz_cache; 823 }; 824 825 typedef struct sfxge_dma_buffer_attr_s { 826 dev_info_t *sdba_dip; 827 ddi_dma_attr_t *sdba_dattrp; 828 int (*sdba_callback) (caddr_t); 829 size_t sdba_length; 830 uint_t sdba_memflags; 831 ddi_device_acc_attr_t *sdba_devaccp; 832 uint_t sdba_bindflags; 833 int sdba_maxcookies; 834 boolean_t sdba_zeroinit; 835 } sfxge_dma_buffer_attr_t; 836 837 extern const char sfxge_ident[]; 838 extern uint8_t sfxge_brdcst[]; 839 840 extern kmutex_t sfxge_global_lock; 841 842 extern unsigned int *sfxge_cpu; 843 844 extern int sfxge_start(sfxge_t *, boolean_t); 845 extern void sfxge_stop(sfxge_t *); 846 extern void sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *); 847 extern int sfxge_restart_dispatch(sfxge_t *, uint_t, 848 sfxge_hw_err_t, const char *, uint32_t); 849 850 extern void sfxge_gld_link_update(sfxge_t *); 851 extern void sfxge_gld_mtu_update(sfxge_t *); 852 extern void sfxge_gld_rx_post(sfxge_t *, unsigned int, 853 mblk_t *); 854 extern void sfxge_gld_rx_push(sfxge_t *); 855 extern int sfxge_gld_register(sfxge_t *); 856 extern int sfxge_gld_unregister(sfxge_t *); 857 858 extern int sfxge_dma_buffer_create(efsys_mem_t *, 859 const sfxge_dma_buffer_attr_t *); 860 extern void sfxge_dma_buffer_destroy(efsys_mem_t *); 861 862 extern int sfxge_intr_init(sfxge_t *); 863 extern int sfxge_intr_start(sfxge_t *); 864 extern void sfxge_intr_stop(sfxge_t *); 865 extern void sfxge_intr_fini(sfxge_t *); 866 extern void sfxge_intr_fatal(sfxge_t *); 867 868 extern int sfxge_ev_init(sfxge_t *); 869 extern int sfxge_ev_start(sfxge_t *); 870 extern void sfxge_ev_moderation_get(sfxge_t *, 871 unsigned int *); 872 extern int sfxge_ev_moderation_set(sfxge_t *, 873 unsigned int); 874 extern int sfxge_ev_qmoderate(sfxge_t *, unsigned int, 875 unsigned int); 876 extern int sfxge_ev_qpoll(sfxge_t *, unsigned int); 877 extern int sfxge_ev_qprime(sfxge_t *, unsigned int); 878 extern void sfxge_ev_stop(sfxge_t *); 879 extern void sfxge_ev_fini(sfxge_t *); 880 extern int sfxge_ev_txlabel_alloc(sfxge_t *sp, 881 unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp); 882 extern int sfxge_ev_txlabel_free(sfxge_t *sp, 883 unsigned int evq, sfxge_txq_t *stp, unsigned int label); 884 885 extern int sfxge_mon_init(sfxge_t *); 886 extern int sfxge_mon_start(sfxge_t *); 887 extern void sfxge_mon_stop(sfxge_t *); 888 extern void sfxge_mon_fini(sfxge_t *); 889 890 extern int sfxge_mac_init(sfxge_t *); 891 extern int sfxge_mac_start(sfxge_t *, boolean_t); 892 extern void sfxge_mac_stat_get(sfxge_t *, unsigned int, 893 uint64_t *); 894 extern void sfxge_mac_link_check(sfxge_t *, boolean_t *); 895 extern void sfxge_mac_link_speed_get(sfxge_t *, 896 unsigned int *); 897 extern void sfxge_mac_link_duplex_get(sfxge_t *, 898 sfxge_link_duplex_t *); 899 extern void sfxge_mac_fcntl_get(sfxge_t *, unsigned int *); 900 extern int sfxge_mac_fcntl_set(sfxge_t *, unsigned int); 901 extern int sfxge_mac_unicst_get(sfxge_t *, 902 sfxge_unicst_type_t, uint8_t *); 903 extern int sfxge_mac_unicst_set(sfxge_t *, 904 uint8_t *); 905 extern int sfxge_mac_promisc_set(sfxge_t *, 906 sfxge_promisc_type_t); 907 extern int sfxge_mac_multicst_add(sfxge_t *, 908 uint8_t const *addr); 909 extern int sfxge_mac_multicst_remove(sfxge_t *, 910 uint8_t const *addr); 911 extern void sfxge_mac_stop(sfxge_t *); 912 extern void sfxge_mac_fini(sfxge_t *); 913 extern void sfxge_mac_link_update(sfxge_t *sp, 914 efx_link_mode_t mode); 915 916 extern int sfxge_mcdi_init(sfxge_t *sp); 917 extern void sfxge_mcdi_fini(sfxge_t *sp); 918 extern int sfxge_mcdi_ioctl(sfxge_t *sp, 919 sfxge_mcdi_ioc_t *smip); 920 extern int sfxge_mcdi2_ioctl(sfxge_t *sp, 921 sfxge_mcdi2_ioc_t *smip); 922 923 extern int sfxge_phy_init(sfxge_t *); 924 extern void sfxge_phy_link_mode_get(sfxge_t *, 925 efx_link_mode_t *); 926 extern void sfxge_phy_fini(sfxge_t *); 927 extern int sfxge_phy_kstat_init(sfxge_t *sp); 928 extern void sfxge_phy_kstat_fini(sfxge_t *sp); 929 extern uint8_t sfxge_phy_lp_cap_test(sfxge_t *sp, 930 uint32_t field); 931 extern int sfxge_phy_cap_apply(sfxge_t *sp, 932 boolean_t use_default); 933 extern uint8_t sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags, 934 uint32_t field, boolean_t *mutablep); 935 extern int sfxge_phy_cap_set(sfxge_t *sp, uint32_t field, 936 int set); 937 938 extern int sfxge_rx_init(sfxge_t *); 939 extern int sfxge_rx_start(sfxge_t *); 940 extern void sfxge_rx_coalesce_mode_get(sfxge_t *, 941 sfxge_rx_coalesce_mode_t *); 942 extern int sfxge_rx_coalesce_mode_set(sfxge_t *, 943 sfxge_rx_coalesce_mode_t); 944 extern unsigned int sfxge_rx_scale_prop_get(sfxge_t *); 945 extern void sfxge_rx_scale_update(void *); 946 extern int sfxge_rx_scale_count_get(sfxge_t *, 947 unsigned int *); 948 extern int sfxge_rx_scale_count_set(sfxge_t *, 949 unsigned int); 950 extern void sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t); 951 extern void sfxge_rx_qflush_done(sfxge_rxq_t *); 952 extern void sfxge_rx_qflush_failed(sfxge_rxq_t *); 953 extern void sfxge_rx_qfpp_trim(sfxge_rxq_t *); 954 extern void sfxge_rx_stop(sfxge_t *); 955 extern unsigned int sfxge_rx_loaned(sfxge_t *); 956 extern void sfxge_rx_fini(sfxge_t *); 957 958 extern int sfxge_tx_init(sfxge_t *); 959 extern int sfxge_tx_start(sfxge_t *); 960 extern int sfxge_tx_packet_add(sfxge_t *, mblk_t *); 961 extern void sfxge_tx_qcomplete(sfxge_txq_t *); 962 extern void sfxge_tx_qflush_done(sfxge_txq_t *); 963 extern void sfxge_tx_stop(sfxge_t *); 964 extern void sfxge_tx_fini(sfxge_t *); 965 extern void sfxge_tx_qdpl_flush(sfxge_txq_t *stp); 966 967 extern void sfxge_sram_init(sfxge_t *); 968 extern int sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t, 969 uint32_t *); 970 extern int sfxge_sram_start(sfxge_t *); 971 extern int sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t, 972 efsys_mem_t *, size_t); 973 extern void sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t, 974 size_t); 975 extern void sfxge_sram_stop(sfxge_t *); 976 extern void sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t, 977 size_t); 978 extern void sfxge_sram_fini(sfxge_t *); 979 980 extern sfxge_packet_type_t sfxge_pkthdr_parse(mblk_t *, 981 struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *, 982 uint16_t *, uint16_t *); 983 984 extern int sfxge_toeplitz_hash_init(sfxge_t *); 985 extern void sfxge_toeplitz_hash_fini(sfxge_t *); 986 extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int, 987 uint8_t *, uint16_t, uint8_t *, uint16_t); 988 989 /* 990 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection. 991 * To compute the same hash value as Siena/Huntington hardware, the inputs 992 * must be in big endian (network) byte order. 993 */ 994 #define SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \ 995 do { \ 996 (_hash) = sfxge_toeplitz_hash(_sp, \ 997 sizeof (struct in_addr), \ 998 (uint8_t *)(_raddr), \ 999 (_rport), \ 1000 (uint8_t *)(_laddr), \ 1001 (_lport)); \ 1002 _NOTE(CONSTANTCONDITION) \ 1003 } while (B_FALSE) 1004 1005 /* 1006 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection. 1007 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers. 1008 * For other IPv4 non-TCP packets, use zero for the port numbers. 1009 */ 1010 #define SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash) \ 1011 SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash)) 1012 1013 1014 extern int sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *); 1015 1016 extern int sfxge_pci_init(sfxge_t *); 1017 extern void sfxge_pcie_check_link(sfxge_t *, unsigned int, 1018 unsigned int); 1019 extern void sfxge_pci_fini(sfxge_t *); 1020 1021 extern int sfxge_bar_init(sfxge_t *); 1022 extern void sfxge_bar_fini(sfxge_t *); 1023 1024 extern int sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *); 1025 1026 1027 #endif /* _KERNEL */ 1028 1029 #ifdef __cplusplus 1030 } 1031 #endif 1032 1033 #endif /* _SYS_SFXGE_H */ 1034