1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #ifndef ENA_H
33 #define ENA_H
34
35 #include "opt_rss.h"
36
37 #include "ena-com/ena_com.h"
38 #include "ena-com/ena_eth_com.h"
39
40 #define ENA_DRV_MODULE_VER_MAJOR 2
41 #define ENA_DRV_MODULE_VER_MINOR 8
42 #define ENA_DRV_MODULE_VER_SUBMINOR 0
43
44 #define ENA_DRV_MODULE_NAME "ena"
45
46 #ifndef ENA_DRV_MODULE_VERSION
47 #define ENA_DRV_MODULE_VERSION \
48 __XSTRING(ENA_DRV_MODULE_VER_MAJOR) "." \
49 __XSTRING(ENA_DRV_MODULE_VER_MINOR) "." \
50 __XSTRING(ENA_DRV_MODULE_VER_SUBMINOR)
51 #endif
52 #define ENA_DEVICE_NAME "Elastic Network Adapter (ENA)"
53 #define ENA_DEVICE_DESC "ENA adapter"
54
55 /* Calculate DMA mask - width for ena cannot exceed 48, so it is safe */
56 #define ENA_DMA_BIT_MASK(x) ((1ULL << (x)) - 1ULL)
57
58 /* 1 for AENQ + ADMIN */
59 #define ENA_ADMIN_MSIX_VEC 1
60 #define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues))
61
62 #define ENA_REG_BAR 0
63 #define ENA_MEM_BAR 2
64
65 #define ENA_BUS_DMA_SEGS 32
66
67 #define ENA_DEFAULT_BUF_RING_SIZE 4096
68
69 #define ENA_DEFAULT_RING_SIZE 1024
70 #define ENA_MIN_RING_SIZE 256
71
72 #define ENA_BASE_CPU_UNSPECIFIED -1
73 /*
74 * Refill Rx queue when number of required descriptors is above
75 * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET
76 */
77 #define ENA_RX_REFILL_THRESH_DIVIDER 8
78 #define ENA_RX_REFILL_THRESH_PACKET 256
79
80 #define ENA_IRQNAME_SIZE 40
81
82 #define ENA_PKT_MAX_BUFS 19
83
84 #define ENA_RX_RSS_TABLE_LOG_SIZE 7
85 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE)
86
87 #define ENA_HASH_KEY_SIZE 40
88
89 #define ENA_MAX_FRAME_LEN 10000
90 #define ENA_MIN_FRAME_LEN 60
91
92 #define ENA_TX_RESUME_THRESH (ENA_PKT_MAX_BUFS + 2)
93
94 #define ENA_DB_THRESHOLD 64
95
96 #define ENA_TX_COMMIT 32
97 /*
98 * TX budget for cleaning. It should be half of the RX budget to reduce amount
99 * of TCP retransmissions.
100 */
101 #define ENA_TX_BUDGET 128
102 /* RX cleanup budget. -1 stands for infinity. */
103 #define ENA_RX_BUDGET 256
104 /*
105 * How many times we can repeat cleanup in the io irq handling routine if the
106 * RX or TX budget was depleted.
107 */
108 #define ENA_CLEAN_BUDGET 8
109
110 #define ENA_RX_IRQ_INTERVAL 20
111 #define ENA_TX_IRQ_INTERVAL 50
112
113 #define ENA_MIN_MTU 128
114
115 #define ENA_TSO_MAXSIZE 65536
116
117 #define ENA_MMIO_DISABLE_REG_READ BIT(0)
118
119 #define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
120
121 #define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
122
123 #define ENA_IO_TXQ_IDX(q) (2 * (q))
124 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1)
125 #define ENA_IO_TXQ_IDX_TO_COMBINED_IDX(q) ((q) / 2)
126 #define ENA_IO_RXQ_IDX_TO_COMBINED_IDX(q) (((q) - 1) / 2)
127
128 #define ENA_MGMNT_IRQ_IDX 0
129 #define ENA_IO_IRQ_FIRST_IDX 1
130 #define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q))
131
132 #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3
133
134 /*
135 * ENA device should send keep alive msg every 1 sec.
136 * We wait for 6 sec just to be on the safe side.
137 */
138 #define ENA_DEFAULT_KEEP_ALIVE_TO (SBT_1S * 6)
139
140 /* Time in jiffies before concluding the transmitter is hung. */
141 #define ENA_DEFAULT_TX_CMP_TO (SBT_1S * 5)
142
143 /* Number of queues to check for missing queues per timer tick */
144 #define ENA_DEFAULT_TX_MONITORED_QUEUES (4)
145
146 /* Max number of timeouted packets before device reset */
147 #define ENA_DEFAULT_TX_CMP_THRESHOLD (128)
148
149 #define ENA_ADMIN_POLL_DELAY_US 100
150
151 /*
152 * Supported PCI vendor and devices IDs
153 */
154 #define PCI_VENDOR_ID_AMAZON 0x1d0f
155
156 #define PCI_DEV_ID_ENA_PF 0x0ec2
157 #define PCI_DEV_ID_ENA_PF_RSERV0 0x1ec2
158 #define PCI_DEV_ID_ENA_VF 0xec20
159 #define PCI_DEV_ID_ENA_VF_RSERV0 0xec21
160
161 /*
162 * Flags indicating current ENA driver state
163 */
164 enum ena_flags_t {
165 ENA_FLAG_DEVICE_RUNNING,
166 ENA_FLAG_DEV_UP,
167 ENA_FLAG_LINK_UP,
168 ENA_FLAG_MSIX_ENABLED,
169 ENA_FLAG_TRIGGER_RESET,
170 ENA_FLAG_ONGOING_RESET,
171 ENA_FLAG_DEV_UP_BEFORE_RESET,
172 ENA_FLAG_RSS_ACTIVE,
173 ENA_FLAGS_NUMBER = ENA_FLAG_RSS_ACTIVE
174 };
175
176 enum ena_llq_header_size_policy_t {
177 /* Policy for Regular LLQ entry size (128B) */
178 ENA_LLQ_HEADER_SIZE_POLICY_REGULAR,
179 /* Policy for Large LLQ entry size (256B) */
180 ENA_LLQ_HEADER_SIZE_POLICY_LARGE,
181 /* Policy for device recommended LLQ entry size */
182 ENA_LLQ_HEADER_SIZE_POLICY_DEFAULT
183 };
184
185 BITSET_DEFINE(_ena_state, ENA_FLAGS_NUMBER);
186 typedef struct _ena_state ena_state_t;
187
188 #define ENA_FLAG_ZERO(adapter) \
189 BIT_ZERO(ENA_FLAGS_NUMBER, &(adapter)->flags)
190 #define ENA_FLAG_ISSET(bit, adapter) \
191 BIT_ISSET(ENA_FLAGS_NUMBER, (bit), &(adapter)->flags)
192 #define ENA_FLAG_SET_ATOMIC(bit, adapter) \
193 BIT_SET_ATOMIC(ENA_FLAGS_NUMBER, (bit), &(adapter)->flags)
194 #define ENA_FLAG_CLEAR_ATOMIC(bit, adapter) \
195 BIT_CLR_ATOMIC(ENA_FLAGS_NUMBER, (bit), &(adapter)->flags)
196
197 struct msix_entry {
198 int entry;
199 int vector;
200 };
201
202 typedef struct _ena_vendor_info_t {
203 uint16_t vendor_id;
204 uint16_t device_id;
205 unsigned int index;
206 } ena_vendor_info_t;
207
208 struct ena_irq {
209 /* Interrupt resources */
210 struct resource *res;
211 driver_filter_t *handler;
212 void *data;
213 void *cookie;
214 unsigned int vector;
215 bool requested;
216 int cpu;
217 char name[ENA_IRQNAME_SIZE];
218 };
219
220 struct ena_que {
221 struct ena_adapter *adapter;
222 struct ena_ring *tx_ring;
223 struct ena_ring *rx_ring;
224
225 struct task cleanup_task;
226 struct taskqueue *cleanup_tq;
227
228 uint32_t id;
229 int cpu;
230 cpuset_t cpu_mask;
231 int domain;
232 struct sysctl_oid *oid;
233 };
234
235 struct ena_calc_queue_size_ctx {
236 struct ena_com_dev_get_features_ctx *get_feat_ctx;
237 struct ena_com_dev *ena_dev;
238 device_t pdev;
239 uint32_t tx_queue_size;
240 uint32_t rx_queue_size;
241 uint32_t max_tx_queue_size;
242 uint32_t max_rx_queue_size;
243 uint16_t max_tx_sgl_size;
244 uint16_t max_rx_sgl_size;
245 };
246
247 #ifdef DEV_NETMAP
248 struct ena_netmap_tx_info {
249 uint32_t socket_buf_idx[ENA_PKT_MAX_BUFS];
250 bus_dmamap_t map_seg[ENA_PKT_MAX_BUFS];
251 unsigned int sockets_used;
252 };
253 #endif
254
255 struct ena_tx_buffer {
256 struct mbuf *mbuf;
257 /* # of ena desc for this specific mbuf
258 * (includes data desc and metadata desc) */
259 unsigned int tx_descs;
260 /* # of buffers used by this mbuf */
261 unsigned int num_of_bufs;
262
263 bus_dmamap_t dmamap;
264
265 /* Used to detect missing tx packets */
266 struct bintime timestamp;
267 bool print_once;
268
269 #ifdef DEV_NETMAP
270 struct ena_netmap_tx_info nm_info;
271 #endif /* DEV_NETMAP */
272
273 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
274 } __aligned(CACHE_LINE_SIZE);
275
276 struct ena_rx_buffer {
277 struct mbuf *mbuf;
278 bus_dmamap_t map;
279 struct ena_com_buf ena_buf;
280 #ifdef DEV_NETMAP
281 uint32_t netmap_buf_idx;
282 #endif /* DEV_NETMAP */
283 } __aligned(CACHE_LINE_SIZE);
284
285 struct ena_stats_tx {
286 counter_u64_t cnt;
287 counter_u64_t bytes;
288 counter_u64_t prepare_ctx_err;
289 counter_u64_t dma_mapping_err;
290 counter_u64_t doorbells;
291 counter_u64_t missing_tx_comp;
292 counter_u64_t bad_req_id;
293 counter_u64_t collapse;
294 counter_u64_t collapse_err;
295 counter_u64_t queue_wakeup;
296 counter_u64_t queue_stop;
297 counter_u64_t llq_buffer_copy;
298 counter_u64_t unmask_interrupt_num;
299 };
300
301 struct ena_stats_rx {
302 counter_u64_t cnt;
303 counter_u64_t bytes;
304 counter_u64_t refil_partial;
305 counter_u64_t csum_bad;
306 counter_u64_t mjum_alloc_fail;
307 counter_u64_t mbuf_alloc_fail;
308 counter_u64_t dma_mapping_err;
309 counter_u64_t bad_desc_num;
310 counter_u64_t bad_req_id;
311 counter_u64_t empty_rx_ring;
312 counter_u64_t csum_good;
313 };
314
315 struct ena_ring {
316 /* Holds the empty requests for TX/RX out of order completions */
317 union {
318 uint16_t *free_tx_ids;
319 uint16_t *free_rx_ids;
320 };
321 struct ena_com_dev *ena_dev;
322 struct ena_adapter *adapter;
323 struct ena_com_io_cq *ena_com_io_cq;
324 struct ena_com_io_sq *ena_com_io_sq;
325
326 uint16_t qid;
327
328 /* Determines if device will use LLQ or normal mode for TX */
329 enum ena_admin_placement_policy_type tx_mem_queue_type;
330 union {
331 /* The maximum length the driver can push to the device (For LLQ) */
332 uint8_t tx_max_header_size;
333 /* The maximum (and default) mbuf size for the Rx descriptor. */
334 uint16_t rx_mbuf_sz;
335
336 };
337
338 uint8_t first_interrupt;
339 uint8_t cleanup_running;
340 uint16_t no_interrupt_event_cnt;
341
342 struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];
343
344 struct ena_que *que;
345 struct lro_ctrl lro;
346
347 uint16_t next_to_use;
348 uint16_t next_to_clean;
349
350 union {
351 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */
352 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */
353 };
354 int ring_size; /* number of tx/rx_buffer_info's entries */
355
356 struct buf_ring *br; /* only for TX */
357 uint32_t buf_ring_size;
358
359 struct mtx ring_mtx;
360 char mtx_name[16];
361
362 struct {
363 struct task enqueue_task;
364 struct taskqueue *enqueue_tq;
365 };
366
367 union {
368 struct ena_stats_tx tx_stats;
369 struct ena_stats_rx rx_stats;
370 };
371
372 union {
373 int empty_rx_queue;
374 /* For Tx ring to indicate if it's running or not */
375 bool running;
376 };
377
378 /* How many packets are sent in one Tx loop, used for doorbells */
379 uint32_t acum_pkts;
380
381 /* Used for LLQ */
382 uint8_t *push_buf_intermediate_buf;
383
384 int tx_last_cleanup_ticks;
385
386 #ifdef DEV_NETMAP
387 bool initialized;
388 #endif /* DEV_NETMAP */
389 } __aligned(CACHE_LINE_SIZE);
390
391 struct ena_stats_dev {
392 counter_u64_t wd_expired;
393 counter_u64_t interface_up;
394 counter_u64_t interface_down;
395 counter_u64_t admin_q_pause;
396 counter_u64_t total_resets;
397 counter_u64_t os_trigger;
398 counter_u64_t missing_tx_cmpl;
399 counter_u64_t bad_rx_req_id;
400 counter_u64_t bad_tx_req_id;
401 counter_u64_t bad_rx_desc_num;
402 counter_u64_t invalid_state;
403 counter_u64_t missing_intr;
404 counter_u64_t tx_desc_malformed;
405 counter_u64_t rx_desc_malformed;
406 counter_u64_t missing_admin_interrupt;
407 counter_u64_t admin_to;
408 counter_u64_t device_request_reset;
409 };
410
411 struct ena_hw_stats {
412 counter_u64_t rx_packets;
413 counter_u64_t tx_packets;
414
415 counter_u64_t rx_bytes;
416 counter_u64_t tx_bytes;
417
418 counter_u64_t rx_drops;
419 counter_u64_t tx_drops;
420 };
421
422 /* Board specific private data structure */
423 struct ena_adapter {
424 struct ena_com_dev *ena_dev;
425
426 /* OS defined structs */
427 if_t ifp;
428 device_t pdev;
429 struct ifmedia media;
430
431 /* OS resources */
432 struct resource *memory;
433 struct resource *registers;
434 struct resource *msix;
435 int msix_rid;
436
437 /* MSI-X */
438 struct msix_entry *msix_entries;
439 int msix_vecs;
440
441 /* DMA tags used throughout the driver adapter for Tx and Rx */
442 bus_dma_tag_t tx_buf_tag;
443 bus_dma_tag_t rx_buf_tag;
444 int dma_width;
445
446 uint32_t max_mtu;
447
448 uint32_t num_io_queues;
449 uint32_t max_num_io_queues;
450
451 uint32_t requested_tx_ring_size;
452 uint32_t requested_rx_ring_size;
453
454 uint32_t max_tx_ring_size;
455 uint32_t max_rx_ring_size;
456
457 uint16_t max_tx_sgl_size;
458 uint16_t max_rx_sgl_size;
459
460 uint32_t tx_offload_cap;
461
462 uint32_t buf_ring_size;
463
464 /* RSS*/
465 int first_bind;
466 struct ena_indir *rss_indir;
467
468 uint8_t mac_addr[ETHER_ADDR_LEN];
469 /* mdio and phy*/
470
471 uint8_t llq_policy;
472
473 ena_state_t flags;
474
475 /* IRQ CPU affinity */
476 int irq_cpu_base;
477 uint32_t irq_cpu_stride;
478
479 uint8_t rss_enabled;
480
481 /* Queue will represent one TX and one RX ring */
482 struct ena_que que[ENA_MAX_NUM_IO_QUEUES]
483 __aligned(CACHE_LINE_SIZE);
484
485 /* TX */
486 struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES]
487 __aligned(CACHE_LINE_SIZE);
488
489 /* RX */
490 struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES]
491 __aligned(CACHE_LINE_SIZE);
492
493 struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)];
494
495 /* Timer service */
496 struct callout timer_service;
497 sbintime_t keep_alive_timestamp;
498 uint32_t next_monitored_tx_qid;
499 struct task reset_task;
500 struct taskqueue *reset_tq;
501 struct task metrics_task;
502 struct taskqueue *metrics_tq;
503 int wd_active;
504 sbintime_t keep_alive_timeout;
505 sbintime_t missing_tx_timeout;
506 uint32_t missing_tx_max_queues;
507 uint32_t missing_tx_threshold;
508 bool disable_meta_caching;
509
510 uint16_t metrics_sample_interval;
511 uint16_t metrics_sample_interval_cnt;
512
513 /* Statistics */
514 struct ena_stats_dev dev_stats;
515 struct ena_hw_stats hw_stats;
516 struct ena_admin_eni_stats eni_metrics;
517 struct ena_admin_ena_srd_info ena_srd_info;
518 uint64_t *customer_metrics_array;
519
520 enum ena_regs_reset_reason_types reset_reason;
521 };
522
523 #define ENA_RING_MTX_LOCK(_ring) mtx_lock(&(_ring)->ring_mtx)
524 #define ENA_RING_MTX_TRYLOCK(_ring) mtx_trylock(&(_ring)->ring_mtx)
525 #define ENA_RING_MTX_UNLOCK(_ring) mtx_unlock(&(_ring)->ring_mtx)
526 #define ENA_RING_MTX_ASSERT(_ring) \
527 mtx_assert(&(_ring)->ring_mtx, MA_OWNED)
528
529 #define ENA_LOCK_INIT() \
530 sx_init(&ena_global_lock, "ENA global lock")
531 #define ENA_LOCK_DESTROY() sx_destroy(&ena_global_lock)
532 #define ENA_LOCK_LOCK() sx_xlock(&ena_global_lock)
533 #define ENA_LOCK_UNLOCK() sx_unlock(&ena_global_lock)
534 #define ENA_LOCK_ASSERT() sx_assert(&ena_global_lock, SA_XLOCKED)
535
536 #define ENA_TIMER_INIT(_adapter) \
537 callout_init(&(_adapter)->timer_service, true)
538 #define ENA_TIMER_DRAIN(_adapter) \
539 callout_drain(&(_adapter)->timer_service)
540 #define ENA_TIMER_RESET(_adapter) \
541 callout_reset_sbt(&(_adapter)->timer_service, SBT_1S, SBT_1S, \
542 ena_timer_service, (void*)(_adapter), 0)
543
544 #define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max)
545 #define clamp_val(val, lo, hi) clamp_t(__typeof(val), val, lo, hi)
546
547 extern struct sx ena_global_lock;
548
549 #define ENA_RESET_STATS_ENTRY(reset_reason, stat) \
550 [reset_reason] = { \
551 .stat_offset = offsetof(struct ena_stats_dev, stat) / sizeof(u64), \
552 .has_counter = true \
553 }
554
555 struct ena_reset_stats_offset {
556 int stat_offset;
557 bool has_counter;
558 };
559
560 static const struct ena_reset_stats_offset resets_to_stats_offset_map[ENA_REGS_RESET_LAST] = {
561 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_KEEP_ALIVE_TO, wd_expired),
562 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_ADMIN_TO, admin_to),
563 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_OS_TRIGGER, os_trigger),
564 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_MISS_TX_CMPL, missing_tx_cmpl),
565 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_INV_RX_REQ_ID, bad_rx_req_id),
566 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_INV_TX_REQ_ID, bad_tx_req_id),
567 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_TOO_MANY_RX_DESCS, bad_rx_desc_num),
568 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_DRIVER_INVALID_STATE, invalid_state),
569 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_MISS_INTERRUPT, missing_intr),
570 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_TX_DESCRIPTOR_MALFORMED, tx_desc_malformed),
571 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED, rx_desc_malformed),
572 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_MISSING_ADMIN_INTERRUPT, missing_admin_interrupt),
573 ENA_RESET_STATS_ENTRY(ENA_REGS_RESET_DEVICE_REQUEST, device_request_reset),
574 };
575
576 int ena_up(struct ena_adapter *adapter);
577 void ena_down(struct ena_adapter *adapter);
578 int ena_restore_device(struct ena_adapter *adapter);
579 void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
580 int ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num);
581 int ena_update_buf_ring_size(struct ena_adapter *adapter,
582 uint32_t new_buf_ring_size);
583 int ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size,
584 uint32_t new_rx_size);
585 int ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num);
586 int ena_update_base_cpu(struct ena_adapter *adapter, int new_num);
587 int ena_update_cpu_stride(struct ena_adapter *adapter, uint32_t new_num);
588 int validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id, int tx_req_id_rc);
589 static inline int
ena_mbuf_count(struct mbuf * mbuf)590 ena_mbuf_count(struct mbuf *mbuf)
591 {
592 int count = 1;
593
594 while ((mbuf = mbuf->m_next) != NULL)
595 ++count;
596
597 return count;
598 }
599
600 static inline void
ena_increment_reset_counter(struct ena_adapter * adapter)601 ena_increment_reset_counter(struct ena_adapter *adapter)
602 {
603 enum ena_regs_reset_reason_types reset_reason = adapter->reset_reason;
604 const struct ena_reset_stats_offset *ena_reset_stats_offset =
605 &resets_to_stats_offset_map[reset_reason];
606
607 if (ena_reset_stats_offset->has_counter) {
608 uint64_t *stat_ptr = (uint64_t *)&adapter->dev_stats +
609 ena_reset_stats_offset->stat_offset;
610
611 counter_u64_add((counter_u64_t)(*stat_ptr), 1);
612 }
613
614 counter_u64_add(adapter->dev_stats.total_resets, 1);
615 }
616
617 static inline void
ena_trigger_reset(struct ena_adapter * adapter,enum ena_regs_reset_reason_types reset_reason)618 ena_trigger_reset(struct ena_adapter *adapter,
619 enum ena_regs_reset_reason_types reset_reason)
620 {
621 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
622 adapter->reset_reason = reset_reason;
623 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
624 }
625 }
626
627 static inline void
ena_ring_tx_doorbell(struct ena_ring * tx_ring)628 ena_ring_tx_doorbell(struct ena_ring *tx_ring)
629 {
630 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
631 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
632 tx_ring->acum_pkts = 0;
633 }
634
635 #endif /* !(ENA_H) */
636