1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 #include <sys/cdefs.h>
31 #include "opt_rss.h"
32 #include "ena.h"
33 #include "ena_datapath.h"
34 #ifdef DEV_NETMAP
35 #include "ena_netmap.h"
36 #endif /* DEV_NETMAP */
37 #include <net/rss_config.h>
38
39 #include <netinet6/ip6_var.h>
40
41 /*********************************************************************
42 * Static functions prototypes
43 *********************************************************************/
44
45 static int ena_tx_cleanup(struct ena_ring *);
46 static int ena_rx_cleanup(struct ena_ring *);
47 static inline int ena_get_tx_req_id(struct ena_ring *tx_ring,
48 struct ena_com_io_cq *io_cq, uint16_t *req_id);
49 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
50 struct mbuf *);
51 static struct mbuf *ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
52 struct ena_com_rx_ctx *, uint16_t *);
53 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
54 struct mbuf *);
55 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool);
56 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
57 struct mbuf **mbuf);
58 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
59 static void ena_start_xmit(struct ena_ring *);
60
61 /*********************************************************************
62 * Global functions
63 *********************************************************************/
64
65 void
ena_cleanup(void * arg,int pending)66 ena_cleanup(void *arg, int pending)
67 {
68 struct ena_que *que = arg;
69 struct ena_adapter *adapter = que->adapter;
70 if_t ifp = adapter->ifp;
71 struct ena_ring *tx_ring;
72 struct ena_ring *rx_ring;
73 struct ena_com_io_cq *io_cq;
74 struct ena_eth_io_intr_reg intr_reg;
75 int qid, ena_qid;
76 int txc, rxc, i;
77
78 tx_ring = que->tx_ring;
79 rx_ring = que->rx_ring;
80 qid = que->id;
81 ena_qid = ENA_IO_TXQ_IDX(qid);
82 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
83
84 atomic_store_8(&tx_ring->cleanup_running, 1);
85 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and
86 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to
87 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq()
88 */
89 mb();
90 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
91 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))))
92 return;
93
94 ena_log_io(adapter->pdev, DBG, "MSI-X TX/RX routine\n");
95
96 atomic_store_8(&tx_ring->first_interrupt, 1);
97 atomic_store_8(&rx_ring->first_interrupt, 1);
98
99 for (i = 0; i < ENA_CLEAN_BUDGET; ++i) {
100 rxc = ena_rx_cleanup(rx_ring);
101 txc = ena_tx_cleanup(tx_ring);
102
103 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
104 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))))
105 return;
106
107 if ((txc != ENA_TX_BUDGET) && (rxc != ENA_RX_BUDGET))
108 break;
109 }
110
111 /* Signal that work is done and unmask interrupt */
112 ena_com_update_intr_reg(&intr_reg, ENA_RX_IRQ_INTERVAL,
113 ENA_TX_IRQ_INTERVAL, true, false);
114 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
115 ena_com_unmask_intr(io_cq, &intr_reg);
116 atomic_store_8(&tx_ring->cleanup_running, 0);
117 }
118
119 void
ena_deferred_mq_start(void * arg,int pending)120 ena_deferred_mq_start(void *arg, int pending)
121 {
122 struct ena_ring *tx_ring = (struct ena_ring *)arg;
123 if_t ifp = tx_ring->adapter->ifp;
124
125 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running &&
126 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
127 ENA_RING_MTX_LOCK(tx_ring);
128 ena_start_xmit(tx_ring);
129 ENA_RING_MTX_UNLOCK(tx_ring);
130 }
131 }
132
133 int
ena_mq_start(if_t ifp,struct mbuf * m)134 ena_mq_start(if_t ifp, struct mbuf *m)
135 {
136 struct ena_adapter *adapter = if_getsoftc(ifp);
137 struct ena_ring *tx_ring;
138 int ret, is_drbr_empty;
139 uint32_t i;
140 #ifdef RSS
141 uint32_t bucket_id;
142 #endif
143
144 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
145 return (ENODEV);
146
147 /* Which queue to use */
148 /*
149 * If everything is setup correctly, it should be the
150 * same bucket that the current CPU we're on is.
151 * It should improve performance.
152 */
153 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
154 #ifdef RSS
155 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
156 &bucket_id) == 0)
157 i = bucket_id % adapter->num_io_queues;
158 else
159 #endif
160 i = m->m_pkthdr.flowid % adapter->num_io_queues;
161 } else {
162 i = curcpu % adapter->num_io_queues;
163 }
164 tx_ring = &adapter->tx_ring[i];
165
166 /* Check if drbr is empty before putting packet */
167 is_drbr_empty = drbr_empty(ifp, tx_ring->br);
168 ret = drbr_enqueue(ifp, tx_ring->br, m);
169 if (unlikely(ret != 0)) {
170 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
171 return (ret);
172 }
173
174 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) {
175 ena_start_xmit(tx_ring);
176 ENA_RING_MTX_UNLOCK(tx_ring);
177 } else {
178 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
179 }
180
181 return (0);
182 }
183
184 void
ena_qflush(if_t ifp)185 ena_qflush(if_t ifp)
186 {
187 struct ena_adapter *adapter = if_getsoftc(ifp);
188 struct ena_ring *tx_ring = adapter->tx_ring;
189 int i;
190
191 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring)
192 if (!drbr_empty(ifp, tx_ring->br)) {
193 ENA_RING_MTX_LOCK(tx_ring);
194 drbr_flush(ifp, tx_ring->br);
195 ENA_RING_MTX_UNLOCK(tx_ring);
196 }
197
198 if_qflush(ifp);
199 }
200
201 /*********************************************************************
202 * Static functions
203 *********************************************************************/
204
205 static inline int
ena_get_tx_req_id(struct ena_ring * tx_ring,struct ena_com_io_cq * io_cq,uint16_t * req_id)206 ena_get_tx_req_id(struct ena_ring *tx_ring, struct ena_com_io_cq *io_cq,
207 uint16_t *req_id)
208 {
209 struct ena_adapter *adapter = tx_ring->adapter;
210 int rc = ena_com_tx_comp_req_id_get(io_cq, req_id);
211
212 if (unlikely(rc == ENA_COM_TRY_AGAIN))
213 return (EAGAIN);
214
215 rc = validate_tx_req_id(tx_ring, *req_id, rc);
216
217 if (unlikely(tx_ring->tx_buffer_info[*req_id].mbuf == NULL)) {
218 ena_log(adapter->pdev, ERR,
219 "tx_info doesn't have valid mbuf. req_id %hu qid %hu\n",
220 *req_id, tx_ring->qid);
221 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
222 rc = EFAULT;
223 }
224
225 return (rc);
226 }
227
228 /**
229 * ena_tx_cleanup - clear sent packets and corresponding descriptors
230 * @tx_ring: ring for which we want to clean packets
231 *
232 * Once packets are sent, we ask the device in a loop for no longer used
233 * descriptors. We find the related mbuf chain in a map (index in an array)
234 * and free it, then update ring state.
235 * This is performed in "endless" loop, updating ring pointers every
236 * TX_COMMIT. The first check of free descriptor is performed before the actual
237 * loop, then repeated at the loop end.
238 **/
239 static int
ena_tx_cleanup(struct ena_ring * tx_ring)240 ena_tx_cleanup(struct ena_ring *tx_ring)
241 {
242 struct ena_adapter *adapter;
243 struct ena_com_io_cq *io_cq;
244 uint16_t next_to_clean;
245 uint16_t req_id;
246 uint16_t ena_qid;
247 unsigned int total_done = 0;
248 int rc;
249 int commit = ENA_TX_COMMIT;
250 int budget = ENA_TX_BUDGET;
251 int work_done;
252 bool above_thresh;
253
254 adapter = tx_ring->que->adapter;
255 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
256 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
257 next_to_clean = tx_ring->next_to_clean;
258
259 #ifdef DEV_NETMAP
260 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)
261 return (0);
262 #endif /* DEV_NETMAP */
263
264 do {
265 struct ena_tx_buffer *tx_info;
266 struct mbuf *mbuf;
267
268 rc = ena_get_tx_req_id(tx_ring, io_cq, &req_id);
269 if (unlikely(rc != 0))
270 break;
271
272 tx_info = &tx_ring->tx_buffer_info[req_id];
273
274 mbuf = tx_info->mbuf;
275
276 tx_info->mbuf = NULL;
277 bintime_clear(&tx_info->timestamp);
278
279 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
280 BUS_DMASYNC_POSTWRITE);
281 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
282
283 ena_log_io(adapter->pdev, DBG, "tx: q %d mbuf %p completed\n",
284 tx_ring->qid, mbuf);
285
286 m_freem(mbuf);
287
288 total_done += tx_info->tx_descs;
289
290 tx_ring->free_tx_ids[next_to_clean] = req_id;
291 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
292 tx_ring->ring_size);
293
294 if (unlikely(--commit == 0)) {
295 commit = ENA_TX_COMMIT;
296 /* update ring state every ENA_TX_COMMIT descriptor */
297 tx_ring->next_to_clean = next_to_clean;
298 ena_com_comp_ack(
299 &adapter->ena_dev->io_sq_queues[ena_qid],
300 total_done);
301 total_done = 0;
302 }
303 } while (likely(--budget));
304
305 work_done = ENA_TX_BUDGET - budget;
306
307 ena_log_io(adapter->pdev, DBG, "tx: q %d done. total pkts: %d\n",
308 tx_ring->qid, work_done);
309
310 /* If there is still something to commit update ring state */
311 if (likely(commit != ENA_TX_COMMIT)) {
312 tx_ring->next_to_clean = next_to_clean;
313 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
314 total_done);
315 }
316
317 /*
318 * Need to make the rings circular update visible to
319 * ena_xmit_mbuf() before checking for tx_ring->running.
320 */
321 mb();
322
323 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
324 ENA_TX_RESUME_THRESH);
325 if (unlikely(!tx_ring->running && above_thresh)) {
326 ENA_RING_MTX_LOCK(tx_ring);
327 above_thresh = ena_com_sq_have_enough_space(
328 tx_ring->ena_com_io_sq, ENA_TX_RESUME_THRESH);
329 if (!tx_ring->running && above_thresh) {
330 tx_ring->running = true;
331 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
332 taskqueue_enqueue(tx_ring->enqueue_tq,
333 &tx_ring->enqueue_task);
334 }
335 ENA_RING_MTX_UNLOCK(tx_ring);
336 }
337
338 tx_ring->tx_last_cleanup_ticks = ticks;
339
340 return (work_done);
341 }
342
343 static void
ena_rx_hash_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)344 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
345 struct mbuf *mbuf)
346 {
347 struct ena_adapter *adapter = rx_ring->adapter;
348
349 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
350 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
351
352 /*
353 * Hardware and software RSS are in agreement only when both are
354 * configured to Toeplitz algorithm. This driver configures
355 * that algorithm only when software RSS is enabled and uses it.
356 */
357 if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ &&
358 ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) {
359 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
360 return;
361 }
362
363 if (ena_rx_ctx->frag &&
364 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
365 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
366 return;
367 }
368
369 switch (ena_rx_ctx->l3_proto) {
370 case ENA_ETH_IO_L3_PROTO_IPV4:
371 switch (ena_rx_ctx->l4_proto) {
372 case ENA_ETH_IO_L4_PROTO_TCP:
373 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
374 break;
375 case ENA_ETH_IO_L4_PROTO_UDP:
376 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
377 break;
378 default:
379 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
380 }
381 break;
382 case ENA_ETH_IO_L3_PROTO_IPV6:
383 switch (ena_rx_ctx->l4_proto) {
384 case ENA_ETH_IO_L4_PROTO_TCP:
385 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
386 break;
387 case ENA_ETH_IO_L4_PROTO_UDP:
388 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
389 break;
390 default:
391 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
392 }
393 break;
394 case ENA_ETH_IO_L3_PROTO_UNKNOWN:
395 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
396 break;
397 default:
398 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
399 }
400 } else {
401 mbuf->m_pkthdr.flowid = rx_ring->qid;
402 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
403 }
404 }
405
406 /**
407 * ena_rx_mbuf - assemble mbuf from descriptors
408 * @rx_ring: ring for which we want to clean packets
409 * @ena_bufs: buffer info
410 * @ena_rx_ctx: metadata for this packet(s)
411 * @next_to_clean: ring pointer, will be updated only upon success
412 *
413 **/
414 static struct mbuf *
ena_rx_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_buf_info * ena_bufs,struct ena_com_rx_ctx * ena_rx_ctx,uint16_t * next_to_clean)415 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
416 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
417 {
418 struct mbuf *mbuf;
419 struct ena_rx_buffer *rx_info;
420 struct ena_adapter *adapter;
421 device_t pdev;
422 unsigned int descs = ena_rx_ctx->descs;
423 uint16_t ntc, len, req_id, buf = 0;
424
425 ntc = *next_to_clean;
426 adapter = rx_ring->adapter;
427 pdev = adapter->pdev;
428
429 len = ena_bufs[buf].len;
430 req_id = ena_bufs[buf].req_id;
431 rx_info = &rx_ring->rx_buffer_info[req_id];
432 if (unlikely(rx_info->mbuf == NULL)) {
433 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n",
434 rx_ring->qid, req_id);
435 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
436 return (NULL);
437 }
438
439 ena_log_io(pdev, DBG, "rx_info %p, mbuf %p, paddr %jx\n", rx_info,
440 rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
441
442 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
443 BUS_DMASYNC_POSTREAD);
444 mbuf = rx_info->mbuf;
445 mbuf->m_flags |= M_PKTHDR;
446 mbuf->m_pkthdr.len = len;
447 mbuf->m_len = len;
448 /* Only for the first segment the data starts at specific offset */
449 mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset);
450 ena_log_io(pdev, DBG, "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset);
451 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
452
453 /* Fill mbuf with hash key and it's interpretation for optimization */
454 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
455
456 ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf,
457 mbuf->m_flags, mbuf->m_pkthdr.len);
458
459 /* DMA address is not needed anymore, unmap it */
460 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
461
462 rx_info->mbuf = NULL;
463 rx_ring->free_rx_ids[ntc] = req_id;
464 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
465
466 /*
467 * While we have more than 1 descriptors for one rcvd packet, append
468 * other mbufs to the main one
469 */
470 while (--descs) {
471 ++buf;
472 len = ena_bufs[buf].len;
473 req_id = ena_bufs[buf].req_id;
474 rx_info = &rx_ring->rx_buffer_info[req_id];
475
476 if (unlikely(rx_info->mbuf == NULL)) {
477 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n",
478 rx_ring->qid, req_id);
479 /*
480 * If one of the required mbufs was not allocated yet,
481 * we can break there.
482 * All earlier used descriptors will be reallocated
483 * later and not used mbufs can be reused.
484 * The next_to_clean pointer will not be updated in case
485 * of an error, so caller should advance it manually
486 * in error handling routine to keep it up to date
487 * with hw ring.
488 */
489 m_freem(mbuf);
490 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
491 return (NULL);
492 }
493
494 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
495 BUS_DMASYNC_POSTREAD);
496 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
497 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
498 ena_log_io(pdev, WARN, "Failed to append Rx mbuf %p\n",
499 mbuf);
500 }
501
502 ena_log_io(pdev, DBG, "rx mbuf updated. len %d\n",
503 mbuf->m_pkthdr.len);
504
505 /* Free already appended mbuf, it won't be useful anymore */
506 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
507 m_freem(rx_info->mbuf);
508 rx_info->mbuf = NULL;
509
510 rx_ring->free_rx_ids[ntc] = req_id;
511 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
512 }
513
514 *next_to_clean = ntc;
515
516 return (mbuf);
517 }
518
519 /**
520 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
521 **/
522 static inline void
ena_rx_checksum(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)523 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
524 struct mbuf *mbuf)
525 {
526 device_t pdev = rx_ring->adapter->pdev;
527
528 /* if IP and error */
529 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
530 ena_rx_ctx->l3_csum_err)) {
531 /* ipv4 checksum error */
532 mbuf->m_pkthdr.csum_flags = 0;
533 counter_u64_add(rx_ring->rx_stats.csum_bad, 1);
534 ena_log_io(pdev, DBG, "RX IPv4 header checksum error\n");
535 return;
536 }
537
538 /* if TCP/UDP */
539 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
540 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
541 if (ena_rx_ctx->l4_csum_err) {
542 /* TCP/UDP checksum error */
543 mbuf->m_pkthdr.csum_flags = 0;
544 counter_u64_add(rx_ring->rx_stats.csum_bad, 1);
545 ena_log_io(pdev, DBG, "RX L4 checksum error\n");
546 } else {
547 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
548 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
549 counter_u64_add(rx_ring->rx_stats.csum_good, 1);
550 }
551 }
552 }
553
554 /**
555 * ena_rx_cleanup - handle rx irq
556 * @arg: ring for which irq is being handled
557 **/
558 static int
ena_rx_cleanup(struct ena_ring * rx_ring)559 ena_rx_cleanup(struct ena_ring *rx_ring)
560 {
561 struct ena_adapter *adapter;
562 device_t pdev;
563 struct mbuf *mbuf;
564 struct ena_com_rx_ctx ena_rx_ctx;
565 struct ena_com_io_cq *io_cq;
566 struct ena_com_io_sq *io_sq;
567 enum ena_regs_reset_reason_types reset_reason;
568 if_t ifp;
569 uint16_t ena_qid;
570 uint16_t next_to_clean;
571 uint32_t refill_required;
572 uint32_t refill_threshold;
573 uint32_t do_if_input = 0;
574 unsigned int qid;
575 int rc, i;
576 int budget = ENA_RX_BUDGET;
577 #ifdef DEV_NETMAP
578 int done;
579 #endif /* DEV_NETMAP */
580
581 adapter = rx_ring->que->adapter;
582 pdev = adapter->pdev;
583 ifp = adapter->ifp;
584 qid = rx_ring->que->id;
585 ena_qid = ENA_IO_RXQ_IDX(qid);
586 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
587 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
588 next_to_clean = rx_ring->next_to_clean;
589
590 #ifdef DEV_NETMAP
591 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS)
592 return (0);
593 #endif /* DEV_NETMAP */
594
595 ena_log_io(pdev, DBG, "rx: qid %d\n", qid);
596
597 do {
598 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
599 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
600 ena_rx_ctx.descs = 0;
601 ena_rx_ctx.pkt_offset = 0;
602
603 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
604 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
605 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
606 if (unlikely(rc != 0)) {
607 if (rc == ENA_COM_NO_SPACE) {
608 counter_u64_add(rx_ring->rx_stats.bad_desc_num,
609 1);
610 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
611 } else if (rc == ENA_COM_FAULT) {
612 reset_reason = ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED;
613 } else {
614 counter_u64_add(rx_ring->rx_stats.bad_req_id,
615 1);
616 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
617 }
618 ena_trigger_reset(adapter, reset_reason);
619 return (0);
620 }
621
622 if (unlikely(ena_rx_ctx.descs == 0))
623 break;
624
625 ena_log_io(pdev, DBG,
626 "rx: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
627 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
628 ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
629
630 /* Receive mbuf from the ring */
631 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, &ena_rx_ctx,
632 &next_to_clean);
633 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
634 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
635 /* Exit if we failed to retrieve a buffer */
636 if (unlikely(mbuf == NULL)) {
637 for (i = 0; i < ena_rx_ctx.descs; ++i) {
638 rx_ring->free_rx_ids[next_to_clean] =
639 rx_ring->ena_bufs[i].req_id;
640 next_to_clean = ENA_RX_RING_IDX_NEXT(
641 next_to_clean, rx_ring->ring_size);
642 }
643 break;
644 }
645
646 if (((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) ||
647 ((if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) != 0)) {
648 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
649 }
650
651 counter_enter();
652 counter_u64_add_protected(rx_ring->rx_stats.bytes,
653 mbuf->m_pkthdr.len);
654 counter_u64_add_protected(adapter->hw_stats.rx_bytes,
655 mbuf->m_pkthdr.len);
656 counter_exit();
657 /*
658 * LRO is only for IP/TCP packets and TCP checksum of the packet
659 * should be computed by hardware.
660 */
661 do_if_input = 1;
662 if (((if_getcapenable(ifp) & IFCAP_LRO) != 0) &&
663 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
664 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
665 /*
666 * Send to the stack if:
667 * - LRO not enabled, or
668 * - no LRO resources, or
669 * - lro enqueue fails
670 */
671 if ((rx_ring->lro.lro_cnt != 0) &&
672 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
673 do_if_input = 0;
674 }
675 if (do_if_input != 0) {
676 ena_log_io(pdev, DBG,
677 "calling if_input() with mbuf %p\n", mbuf);
678 if_input(ifp, mbuf);
679 }
680
681 counter_enter();
682 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
683 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
684 counter_exit();
685 } while (--budget);
686
687 rx_ring->next_to_clean = next_to_clean;
688
689 refill_required = ena_com_free_q_entries(io_sq);
690 refill_threshold = min_t(int,
691 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
692 ENA_RX_REFILL_THRESH_PACKET);
693
694 if (refill_required > refill_threshold) {
695 ena_refill_rx_bufs(rx_ring, refill_required);
696 }
697
698 tcp_lro_flush_all(&rx_ring->lro);
699
700 return (ENA_RX_BUDGET - budget);
701 }
702
703 static void
ena_tx_csum(struct ena_com_tx_ctx * ena_tx_ctx,struct mbuf * mbuf,bool disable_meta_caching)704 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf,
705 bool disable_meta_caching)
706 {
707 struct ena_com_tx_meta *ena_meta;
708 struct ether_vlan_header *eh;
709 struct mbuf *mbuf_next;
710 u32 mss;
711 bool offload;
712 uint16_t etype;
713 int ehdrlen;
714 struct ip *ip;
715 int ipproto;
716 int iphlen;
717 struct tcphdr *th;
718 int offset;
719
720 offload = false;
721 ena_meta = &ena_tx_ctx->ena_meta;
722 mss = mbuf->m_pkthdr.tso_segsz;
723
724 if (mss != 0)
725 offload = true;
726
727 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
728 offload = true;
729
730 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
731 offload = true;
732
733 if ((mbuf->m_pkthdr.csum_flags & CSUM6_OFFLOAD) != 0)
734 offload = true;
735
736 if (!offload) {
737 if (disable_meta_caching) {
738 memset(ena_meta, 0, sizeof(*ena_meta));
739 ena_tx_ctx->meta_valid = 1;
740 } else {
741 ena_tx_ctx->meta_valid = 0;
742 }
743 return;
744 }
745
746 /* Determine where frame payload starts. */
747 eh = mtod(mbuf, struct ether_vlan_header *);
748 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
749 etype = ntohs(eh->evl_proto);
750 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
751 } else {
752 etype = ntohs(eh->evl_encap_proto);
753 ehdrlen = ETHER_HDR_LEN;
754 }
755
756 mbuf_next = m_getptr(mbuf, ehdrlen, &offset);
757
758 switch (etype) {
759 case ETHERTYPE_IP:
760 ip = (struct ip *)(mtodo(mbuf_next, offset));
761 iphlen = ip->ip_hl << 2;
762 ipproto = ip->ip_p;
763 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
764 if ((ip->ip_off & htons(IP_DF)) != 0)
765 ena_tx_ctx->df = 1;
766 break;
767 case ETHERTYPE_IPV6:
768 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
769 iphlen = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &ipproto);
770 iphlen -= ehdrlen;
771 ena_tx_ctx->df = 1;
772 break;
773 default:
774 iphlen = 0;
775 ipproto = 0;
776 break;
777 }
778
779 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset);
780 th = (struct tcphdr *)(mtodo(mbuf_next, offset));
781
782 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
783 ena_tx_ctx->l3_csum_enable = 1;
784 }
785 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
786 ena_tx_ctx->tso_enable = 1;
787 ena_meta->l4_hdr_len = (th->th_off);
788 }
789
790 if (ipproto == IPPROTO_TCP) {
791 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
792 if ((mbuf->m_pkthdr.csum_flags &
793 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0)
794 ena_tx_ctx->l4_csum_enable = 1;
795 else
796 ena_tx_ctx->l4_csum_enable = 0;
797 } else if (ipproto == IPPROTO_UDP) {
798 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
799 if ((mbuf->m_pkthdr.csum_flags &
800 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0)
801 ena_tx_ctx->l4_csum_enable = 1;
802 else
803 ena_tx_ctx->l4_csum_enable = 0;
804 } else {
805 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
806 ena_tx_ctx->l4_csum_enable = 0;
807 }
808
809 ena_meta->mss = mss;
810 ena_meta->l3_hdr_len = iphlen;
811 ena_meta->l3_hdr_offset = ehdrlen;
812 ena_tx_ctx->meta_valid = 1;
813 }
814
815 static int
ena_check_and_collapse_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)816 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
817 {
818 struct ena_adapter *adapter;
819 struct mbuf *collapsed_mbuf;
820 int num_frags;
821
822 adapter = tx_ring->adapter;
823 num_frags = ena_mbuf_count(*mbuf);
824
825 /* One segment must be reserved for configuration descriptor. */
826 if (num_frags < adapter->max_tx_sgl_size)
827 return (0);
828
829 if ((num_frags == adapter->max_tx_sgl_size) &&
830 ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size))
831 return (0);
832
833 counter_u64_add(tx_ring->tx_stats.collapse, 1);
834
835 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,
836 adapter->max_tx_sgl_size - 1);
837 if (unlikely(collapsed_mbuf == NULL)) {
838 counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
839 return (ENOMEM);
840 }
841
842 /* If mbuf was collapsed succesfully, original mbuf is released. */
843 *mbuf = collapsed_mbuf;
844
845 return (0);
846 }
847
848 static int
ena_tx_map_mbuf(struct ena_ring * tx_ring,struct ena_tx_buffer * tx_info,struct mbuf * mbuf,void ** push_hdr,u16 * header_len)849 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
850 struct mbuf *mbuf, void **push_hdr, u16 *header_len)
851 {
852 struct ena_adapter *adapter = tx_ring->adapter;
853 struct ena_com_buf *ena_buf;
854 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
855 size_t iseg = 0;
856 uint32_t mbuf_head_len;
857 uint16_t offset;
858 int rc, nsegs;
859
860 mbuf_head_len = mbuf->m_len;
861 tx_info->mbuf = mbuf;
862 ena_buf = tx_info->bufs;
863
864 /*
865 * For easier maintaining of the DMA map, map the whole mbuf even if
866 * the LLQ is used. The descriptors will be filled using the segments.
867 */
868 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag,
869 tx_info->dmamap, mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
870 if (unlikely((rc != 0) || (nsegs == 0))) {
871 ena_log_io(adapter->pdev, WARN,
872 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs);
873 goto dma_error;
874 }
875
876 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
877 /*
878 * When the device is LLQ mode, the driver will copy
879 * the header into the device memory space.
880 * the ena_com layer assumes the header is in a linear
881 * memory space.
882 * This assumption might be wrong since part of the header
883 * can be in the fragmented buffers.
884 * First check if header fits in the mbuf. If not, copy it to
885 * separate buffer that will be holding linearized data.
886 */
887 *header_len = min_t(uint32_t, mbuf->m_pkthdr.len,
888 tx_ring->tx_max_header_size);
889
890 /* If header is in linear space, just point into mbuf's data. */
891 if (likely(*header_len <= mbuf_head_len)) {
892 *push_hdr = mbuf->m_data;
893 /*
894 * Otherwise, copy whole portion of header from multiple
895 * mbufs to intermediate buffer.
896 */
897 } else {
898 m_copydata(mbuf, 0, *header_len,
899 tx_ring->push_buf_intermediate_buf);
900 *push_hdr = tx_ring->push_buf_intermediate_buf;
901
902 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
903 }
904
905 ena_log_io(adapter->pdev, DBG,
906 "mbuf: %p header_buf->vaddr: %p push_len: %d\n",
907 mbuf, *push_hdr, *header_len);
908
909 /* If packet is fitted in LLQ header, no need for DMA segments. */
910 if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) {
911 return (0);
912 } else {
913 offset = tx_ring->tx_max_header_size;
914 /*
915 * As Header part is mapped to LLQ header, we can skip
916 * it and just map the residuum of the mbuf to DMA
917 * Segments.
918 */
919 while (offset > 0) {
920 if (offset >= segs[iseg].ds_len) {
921 offset -= segs[iseg].ds_len;
922 } else {
923 ena_buf->paddr = segs[iseg].ds_addr +
924 offset;
925 ena_buf->len = segs[iseg].ds_len -
926 offset;
927 ena_buf++;
928 tx_info->num_of_bufs++;
929 offset = 0;
930 }
931 iseg++;
932 }
933 }
934 } else {
935 *push_hdr = NULL;
936 /*
937 * header_len is just a hint for the device. Because FreeBSD is
938 * not giving us information about packet header length and it
939 * is not guaranteed that all packet headers will be in the 1st
940 * mbuf, setting header_len to 0 is making the device ignore
941 * this value and resolve header on it's own.
942 */
943 *header_len = 0;
944 }
945
946 /* Map rest of the mbuf */
947 while (iseg < nsegs) {
948 ena_buf->paddr = segs[iseg].ds_addr;
949 ena_buf->len = segs[iseg].ds_len;
950 ena_buf++;
951 iseg++;
952 tx_info->num_of_bufs++;
953 }
954
955 return (0);
956
957 dma_error:
958 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
959 tx_info->mbuf = NULL;
960 return (rc);
961 }
962
963 static int
ena_xmit_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)964 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
965 {
966 struct ena_adapter *adapter;
967 device_t pdev;
968 struct ena_tx_buffer *tx_info;
969 struct ena_com_tx_ctx ena_tx_ctx;
970 struct ena_com_dev *ena_dev;
971 struct ena_com_io_sq *io_sq;
972 void *push_hdr;
973 uint16_t next_to_use;
974 uint16_t req_id;
975 uint16_t ena_qid;
976 uint16_t header_len;
977 int rc;
978 int nb_hw_desc;
979
980 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
981 adapter = tx_ring->que->adapter;
982 pdev = adapter->pdev;
983 ena_dev = adapter->ena_dev;
984 io_sq = &ena_dev->io_sq_queues[ena_qid];
985
986 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
987 if (unlikely(rc != 0)) {
988 ena_log_io(pdev, WARN, "Failed to collapse mbuf! err: %d\n",
989 rc);
990 return (rc);
991 }
992
993 ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
994
995 next_to_use = tx_ring->next_to_use;
996 req_id = tx_ring->free_tx_ids[next_to_use];
997 tx_info = &tx_ring->tx_buffer_info[req_id];
998 tx_info->num_of_bufs = 0;
999
1000 ENA_WARN(tx_info->mbuf != NULL, adapter->ena_dev,
1001 "mbuf isn't NULL for req_id %d\n", req_id);
1002
1003 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);
1004 if (unlikely(rc != 0)) {
1005 ena_log_io(pdev, WARN, "Failed to map TX mbuf\n");
1006 return (rc);
1007 }
1008 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
1009 ena_tx_ctx.ena_bufs = tx_info->bufs;
1010 ena_tx_ctx.push_header = push_hdr;
1011 ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
1012 ena_tx_ctx.req_id = req_id;
1013 ena_tx_ctx.header_len = header_len;
1014
1015 /* Set flags and meta data */
1016 ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching);
1017
1018 if (tx_ring->acum_pkts == ENA_DB_THRESHOLD ||
1019 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {
1020 ena_log_io(pdev, DBG,
1021 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
1022 tx_ring->que->id);
1023 ena_ring_tx_doorbell(tx_ring);
1024 }
1025
1026 /* Prepare the packet's descriptors and send them to device */
1027 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
1028 if (unlikely(rc != 0)) {
1029 if (likely(rc == ENA_COM_NO_MEM)) {
1030 ena_log_io(pdev, DBG, "tx ring[%d] is out of space\n",
1031 tx_ring->que->id);
1032 } else {
1033 ena_log(pdev, ERR, "failed to prepare tx bufs\n");
1034 ena_trigger_reset(adapter,
1035 ENA_REGS_RESET_DRIVER_INVALID_STATE);
1036 }
1037 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
1038 goto dma_error;
1039 }
1040
1041 counter_enter();
1042 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
1043 counter_u64_add_protected(tx_ring->tx_stats.bytes,
1044 (*mbuf)->m_pkthdr.len);
1045
1046 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
1047 counter_u64_add_protected(adapter->hw_stats.tx_bytes,
1048 (*mbuf)->m_pkthdr.len);
1049 counter_exit();
1050
1051 tx_info->tx_descs = nb_hw_desc;
1052 getbinuptime(&tx_info->timestamp);
1053 tx_info->print_once = true;
1054
1055 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
1056 tx_ring->ring_size);
1057
1058 /* stop the queue when no more space available, the packet can have up
1059 * to sgl_size + 2. one for the meta descriptor and one for header
1060 * (if the header is larger than tx_max_header_size).
1061 */
1062 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1063 adapter->max_tx_sgl_size + 2))) {
1064 ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id);
1065
1066 tx_ring->running = false;
1067 counter_u64_add(tx_ring->tx_stats.queue_stop, 1);
1068
1069 /* There is a rare condition where this function decides to
1070 * stop the queue but meanwhile tx_cleanup() updates
1071 * next_to_completion and terminates.
1072 * The queue will remain stopped forever.
1073 * To solve this issue this function performs mb(), checks
1074 * the wakeup condition and wakes up the queue if needed.
1075 */
1076 mb();
1077
1078 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1079 ENA_TX_RESUME_THRESH)) {
1080 tx_ring->running = true;
1081 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
1082 }
1083 }
1084
1085 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
1086 BUS_DMASYNC_PREWRITE);
1087
1088 return (0);
1089
1090 dma_error:
1091 tx_info->mbuf = NULL;
1092 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
1093
1094 return (rc);
1095 }
1096
1097 static void
ena_start_xmit(struct ena_ring * tx_ring)1098 ena_start_xmit(struct ena_ring *tx_ring)
1099 {
1100 struct mbuf *mbuf;
1101 struct ena_adapter *adapter = tx_ring->adapter;
1102 int ret = 0;
1103
1104 ENA_RING_MTX_ASSERT(tx_ring);
1105
1106 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1107 return;
1108
1109 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)))
1110 return;
1111
1112 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
1113 ena_log_io(adapter->pdev, DBG,
1114 "\ndequeued mbuf %p with flags %#x and header csum flags %#jx\n",
1115 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);
1116
1117 if (unlikely(!tx_ring->running)) {
1118 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1119 break;
1120 }
1121
1122 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {
1123 if (ret == ENA_COM_NO_MEM) {
1124 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1125 } else if (ret == ENA_COM_NO_SPACE) {
1126 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1127 } else {
1128 m_freem(mbuf);
1129 drbr_advance(adapter->ifp, tx_ring->br);
1130 }
1131
1132 break;
1133 }
1134
1135 drbr_advance(adapter->ifp, tx_ring->br);
1136
1137 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1138 return;
1139
1140 tx_ring->acum_pkts++;
1141
1142 BPF_MTAP(adapter->ifp, mbuf);
1143 }
1144
1145 if (likely(tx_ring->acum_pkts != 0)) {
1146 /* Trigger the dma engine */
1147 ena_ring_tx_doorbell(tx_ring);
1148 }
1149
1150 if (unlikely(!tx_ring->running))
1151 taskqueue_enqueue(tx_ring->que->cleanup_tq,
1152 &tx_ring->que->cleanup_task);
1153 }
1154