1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 * 32 */ 33 34 #ifndef ENA_H 35 #define ENA_H 36 37 #include <sys/types.h> 38 39 #include "ena-com/ena_com.h" 40 #include "ena-com/ena_eth_com.h" 41 42 #define DRV_MODULE_VER_MAJOR 0 43 #define DRV_MODULE_VER_MINOR 8 44 #define DRV_MODULE_VER_SUBMINOR 4 45 46 #define DRV_MODULE_NAME "ena" 47 48 #ifndef DRV_MODULE_VERSION 49 #define DRV_MODULE_VERSION \ 50 __XSTRING(DRV_MODULE_VER_MAJOR) "." \ 51 __XSTRING(DRV_MODULE_VER_MINOR) "." \ 52 __XSTRING(DRV_MODULE_VER_SUBMINOR) 53 #endif 54 #define DEVICE_NAME "Elastic Network Adapter (ENA)" 55 #define DEVICE_DESC "ENA adapter" 56 57 /* Calculate DMA mask - width for ena cannot exceed 48, so it is safe */ 58 #define ENA_DMA_BIT_MASK(x) ((1ULL << (x)) - 1ULL) 59 60 /* 1 for AENQ + ADMIN */ 61 #define ENA_ADMIN_MSIX_VEC 1 62 #define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues)) 63 64 #define ENA_REG_BAR 0 65 #define ENA_MEM_BAR 2 66 67 #define ENA_BUS_DMA_SEGS 32 68 69 #define ENA_DEFAULT_BUF_RING_SIZE 4096 70 71 #define ENA_DEFAULT_RING_SIZE 1024 72 73 #define ENA_RX_REFILL_THRESH_DIVIDER 8 74 75 #define ENA_IRQNAME_SIZE 40 76 77 #define ENA_PKT_MAX_BUFS 19 78 79 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 80 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 81 82 #define ENA_HASH_KEY_SIZE 40 83 84 #define ENA_MAX_FRAME_LEN 10000 85 #define ENA_MIN_FRAME_LEN 60 86 87 #define ENA_TX_RESUME_THRESH (ENA_PKT_MAX_BUFS + 2) 88 89 #define DB_THRESHOLD 64 90 91 #define TX_COMMIT 32 92 /* 93 * TX budget for cleaning. It should be half of the RX budget to reduce amount 94 * of TCP retransmissions. 95 */ 96 #define TX_BUDGET 128 97 /* RX cleanup budget. -1 stands for infinity. */ 98 #define RX_BUDGET 256 99 /* 100 * How many times we can repeat cleanup in the io irq handling routine if the 101 * RX or TX budget was depleted. 102 */ 103 #define CLEAN_BUDGET 8 104 105 #define RX_IRQ_INTERVAL 20 106 #define TX_IRQ_INTERVAL 50 107 108 #define ENA_MIN_MTU 128 109 110 #define ENA_TSO_MAXSIZE 65536 111 112 #define ENA_MMIO_DISABLE_REG_READ BIT(0) 113 114 #define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) 115 116 #define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) 117 118 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 119 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 120 121 #define ENA_MGMNT_IRQ_IDX 0 122 #define ENA_IO_IRQ_FIRST_IDX 1 123 #define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q)) 124 125 #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3 126 127 /* 128 * ENA device should send keep alive msg every 1 sec. 129 * We wait for 6 sec just to be on the safe side. 130 */ 131 #define DEFAULT_KEEP_ALIVE_TO (SBT_1S * 6) 132 133 /* Time in jiffies before concluding the transmitter is hung. */ 134 #define DEFAULT_TX_CMP_TO (SBT_1S * 5) 135 136 /* Number of queues to check for missing queues per timer tick */ 137 #define DEFAULT_TX_MONITORED_QUEUES (4) 138 139 /* Max number of timeouted packets before device reset */ 140 #define DEFAULT_TX_CMP_THRESHOLD (128) 141 142 /* 143 * Supported PCI vendor and devices IDs 144 */ 145 #define PCI_VENDOR_ID_AMAZON 0x1d0f 146 147 #define PCI_DEV_ID_ENA_PF 0x0ec2 148 #define PCI_DEV_ID_ENA_LLQ_PF 0x1ec2 149 #define PCI_DEV_ID_ENA_VF 0xec20 150 #define PCI_DEV_ID_ENA_LLQ_VF 0xec21 151 152 struct msix_entry { 153 int entry; 154 int vector; 155 }; 156 157 typedef struct _ena_vendor_info_t { 158 uint16_t vendor_id; 159 uint16_t device_id; 160 unsigned int index; 161 } ena_vendor_info_t; 162 163 struct ena_irq { 164 /* Interrupt resources */ 165 struct resource *res; 166 driver_filter_t *handler; 167 void *data; 168 void *cookie; 169 unsigned int vector; 170 bool requested; 171 int cpu; 172 char name[ENA_IRQNAME_SIZE]; 173 }; 174 175 struct ena_que { 176 struct ena_adapter *adapter; 177 struct ena_ring *tx_ring; 178 struct ena_ring *rx_ring; 179 180 struct task cleanup_task; 181 struct taskqueue *cleanup_tq; 182 183 uint32_t id; 184 int cpu; 185 }; 186 187 struct ena_calc_queue_size_ctx { 188 struct ena_com_dev_get_features_ctx *get_feat_ctx; 189 struct ena_com_dev *ena_dev; 190 device_t pdev; 191 uint16_t rx_queue_size; 192 uint16_t tx_queue_size; 193 uint16_t max_tx_sgl_size; 194 uint16_t max_rx_sgl_size; 195 }; 196 197 struct ena_tx_buffer { 198 struct mbuf *mbuf; 199 /* # of ena desc for this specific mbuf 200 * (includes data desc and metadata desc) */ 201 unsigned int tx_descs; 202 /* # of buffers used by this mbuf */ 203 unsigned int num_of_bufs; 204 bus_dmamap_t map_head; 205 bus_dmamap_t map_seg; 206 207 /* Indicate if segments of the mbuf were mapped */ 208 bool seg_mapped; 209 /* Indicate if bufs[0] maps the linear data of the mbuf */ 210 bool head_mapped; 211 212 /* Used to detect missing tx packets */ 213 struct bintime timestamp; 214 bool print_once; 215 216 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; 217 } __aligned(CACHE_LINE_SIZE); 218 219 struct ena_rx_buffer { 220 struct mbuf *mbuf; 221 bus_dmamap_t map; 222 struct ena_com_buf ena_buf; 223 } __aligned(CACHE_LINE_SIZE); 224 225 struct ena_stats_tx { 226 counter_u64_t cnt; 227 counter_u64_t bytes; 228 counter_u64_t prepare_ctx_err; 229 counter_u64_t dma_mapping_err; 230 counter_u64_t doorbells; 231 counter_u64_t missing_tx_comp; 232 counter_u64_t bad_req_id; 233 counter_u64_t collapse; 234 counter_u64_t collapse_err; 235 counter_u64_t queue_wakeup; 236 counter_u64_t queue_stop; 237 counter_u64_t llq_buffer_copy; 238 }; 239 240 struct ena_stats_rx { 241 counter_u64_t cnt; 242 counter_u64_t bytes; 243 counter_u64_t refil_partial; 244 counter_u64_t bad_csum; 245 counter_u64_t mjum_alloc_fail; 246 counter_u64_t mbuf_alloc_fail; 247 counter_u64_t dma_mapping_err; 248 counter_u64_t bad_desc_num; 249 counter_u64_t bad_req_id; 250 counter_u64_t empty_rx_ring; 251 }; 252 253 struct ena_ring { 254 /* Holds the empty requests for TX/RX out of order completions */ 255 union { 256 uint16_t *free_tx_ids; 257 uint16_t *free_rx_ids; 258 }; 259 struct ena_com_dev *ena_dev; 260 struct ena_adapter *adapter; 261 struct ena_com_io_cq *ena_com_io_cq; 262 struct ena_com_io_sq *ena_com_io_sq; 263 264 uint16_t qid; 265 266 /* Determines if device will use LLQ or normal mode for TX */ 267 enum ena_admin_placement_policy_type tx_mem_queue_type; 268 /* The maximum length the driver can push to the device (For LLQ) */ 269 uint8_t tx_max_header_size; 270 271 bool first_interrupt; 272 uint16_t no_interrupt_event_cnt; 273 274 struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; 275 276 /* 277 * Fields used for Adaptive Interrupt Modulation - to be implemented in 278 * the future releases 279 */ 280 uint32_t smoothed_interval; 281 enum ena_intr_moder_level moder_tbl_idx; 282 283 struct ena_que *que; 284 struct lro_ctrl lro; 285 286 uint16_t next_to_use; 287 uint16_t next_to_clean; 288 289 union { 290 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */ 291 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */ 292 }; 293 int ring_size; /* number of tx/rx_buffer_info's entries */ 294 295 struct buf_ring *br; /* only for TX */ 296 uint32_t buf_ring_size; 297 298 struct mtx ring_mtx; 299 char mtx_name[16]; 300 301 struct { 302 struct task enqueue_task; 303 struct taskqueue *enqueue_tq; 304 }; 305 306 union { 307 struct ena_stats_tx tx_stats; 308 struct ena_stats_rx rx_stats; 309 }; 310 311 union { 312 int empty_rx_queue; 313 /* For Tx ring to indicate if it's running or not */ 314 bool running; 315 }; 316 317 /* Used for LLQ */ 318 uint8_t *push_buf_intermediate_buf; 319 } __aligned(CACHE_LINE_SIZE); 320 321 struct ena_stats_dev { 322 counter_u64_t wd_expired; 323 counter_u64_t interface_up; 324 counter_u64_t interface_down; 325 counter_u64_t admin_q_pause; 326 }; 327 328 struct ena_hw_stats { 329 counter_u64_t rx_packets; 330 counter_u64_t tx_packets; 331 332 counter_u64_t rx_bytes; 333 counter_u64_t tx_bytes; 334 335 counter_u64_t rx_drops; 336 }; 337 338 /* Board specific private data structure */ 339 struct ena_adapter { 340 struct ena_com_dev *ena_dev; 341 342 /* OS defined structs */ 343 if_t ifp; 344 device_t pdev; 345 struct ifmedia media; 346 347 /* OS resources */ 348 struct resource *memory; 349 struct resource *registers; 350 351 struct mtx global_mtx; 352 struct sx ioctl_sx; 353 354 /* MSI-X */ 355 uint32_t msix_enabled; 356 struct msix_entry *msix_entries; 357 int msix_vecs; 358 359 /* DMA tags used throughout the driver adapter for Tx and Rx */ 360 bus_dma_tag_t tx_buf_tag; 361 bus_dma_tag_t rx_buf_tag; 362 int dma_width; 363 364 uint32_t max_mtu; 365 366 uint16_t max_tx_sgl_size; 367 uint16_t max_rx_sgl_size; 368 369 uint32_t tx_offload_cap; 370 371 /* Tx fast path data */ 372 int num_queues; 373 374 unsigned int tx_ring_size; 375 unsigned int rx_ring_size; 376 377 uint16_t buf_ring_size; 378 379 /* RSS*/ 380 uint8_t rss_ind_tbl[ENA_RX_RSS_TABLE_SIZE]; 381 bool rss_support; 382 383 uint8_t mac_addr[ETHER_ADDR_LEN]; 384 /* mdio and phy*/ 385 386 bool link_status; 387 bool trigger_reset; 388 bool up; 389 bool running; 390 391 /* Queue will represent one TX and one RX ring */ 392 struct ena_que que[ENA_MAX_NUM_IO_QUEUES] 393 __aligned(CACHE_LINE_SIZE); 394 395 /* TX */ 396 struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES] 397 __aligned(CACHE_LINE_SIZE); 398 399 /* RX */ 400 struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES] 401 __aligned(CACHE_LINE_SIZE); 402 403 struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)]; 404 405 /* Timer service */ 406 struct callout timer_service; 407 sbintime_t keep_alive_timestamp; 408 uint32_t next_monitored_tx_qid; 409 struct task reset_task; 410 struct taskqueue *reset_tq; 411 int wd_active; 412 sbintime_t keep_alive_timeout; 413 sbintime_t missing_tx_timeout; 414 uint32_t missing_tx_max_queues; 415 uint32_t missing_tx_threshold; 416 417 /* Statistics */ 418 struct ena_stats_dev dev_stats; 419 struct ena_hw_stats hw_stats; 420 421 enum ena_regs_reset_reason_types reset_reason; 422 }; 423 424 #define ENA_RING_MTX_LOCK(_ring) mtx_lock(&(_ring)->ring_mtx) 425 #define ENA_RING_MTX_TRYLOCK(_ring) mtx_trylock(&(_ring)->ring_mtx) 426 #define ENA_RING_MTX_UNLOCK(_ring) mtx_unlock(&(_ring)->ring_mtx) 427 428 static inline int ena_mbuf_count(struct mbuf *mbuf) 429 { 430 int count = 1; 431 432 while ((mbuf = mbuf->m_next) != NULL) 433 ++count; 434 435 return count; 436 } 437 438 #endif /* !(ENA_H) */ 439