xref: /freebsd/sys/dev/ena/ena_datapath.c (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "ena.h"
34 #include "ena_datapath.h"
35 #ifdef DEV_NETMAP
36 #include "ena_netmap.h"
37 #endif /* DEV_NETMAP */
38 
39 /*********************************************************************
40  *  Static functions prototypes
41  *********************************************************************/
42 
43 static int	ena_tx_cleanup(struct ena_ring *);
44 static int	ena_rx_cleanup(struct ena_ring *);
45 static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
46 static void	ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
47     struct mbuf *);
48 static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
49     struct ena_com_rx_ctx *, uint16_t *);
50 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
51     struct mbuf *);
52 static void	ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
53 static int	ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
54     struct mbuf **mbuf);
55 static void	ena_dmamap_llq(void *, bus_dma_segment_t *, int, int);
56 static int	ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
57 static void	ena_start_xmit(struct ena_ring *);
58 
59 /*********************************************************************
60  *  Global functions
61  *********************************************************************/
62 
63 void
64 ena_cleanup(void *arg, int pending)
65 {
66 	struct ena_que	*que = arg;
67 	struct ena_adapter *adapter = que->adapter;
68 	if_t ifp = adapter->ifp;
69 	struct ena_ring *tx_ring;
70 	struct ena_ring *rx_ring;
71 	struct ena_com_io_cq* io_cq;
72 	struct ena_eth_io_intr_reg intr_reg;
73 	int qid, ena_qid;
74 	int txc, rxc, i;
75 
76 	if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
77 		return;
78 
79 	ena_trace(ENA_DBG, "MSI-X TX/RX routine\n");
80 
81 	tx_ring = que->tx_ring;
82 	rx_ring = que->rx_ring;
83 	qid = que->id;
84 	ena_qid = ENA_IO_TXQ_IDX(qid);
85 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
86 
87 	tx_ring->first_interrupt = true;
88 	rx_ring->first_interrupt = true;
89 
90 	for (i = 0; i < CLEAN_BUDGET; ++i) {
91 		rxc = ena_rx_cleanup(rx_ring);
92 		txc = ena_tx_cleanup(tx_ring);
93 
94 		if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
95 			return;
96 
97 		if ((txc != TX_BUDGET) && (rxc != RX_BUDGET))
98 		       break;
99 	}
100 
101 	/* Signal that work is done and unmask interrupt */
102 	ena_com_update_intr_reg(&intr_reg,
103 	    RX_IRQ_INTERVAL,
104 	    TX_IRQ_INTERVAL,
105 	    true);
106 	ena_com_unmask_intr(io_cq, &intr_reg);
107 }
108 
109 void
110 ena_deferred_mq_start(void *arg, int pending)
111 {
112 	struct ena_ring *tx_ring = (struct ena_ring *)arg;
113 	struct ifnet *ifp = tx_ring->adapter->ifp;
114 
115 	while (!drbr_empty(ifp, tx_ring->br) &&
116 	    tx_ring->running &&
117 	    (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
118 		ENA_RING_MTX_LOCK(tx_ring);
119 		ena_start_xmit(tx_ring);
120 		ENA_RING_MTX_UNLOCK(tx_ring);
121 	}
122 }
123 
124 int
125 ena_mq_start(if_t ifp, struct mbuf *m)
126 {
127 	struct ena_adapter *adapter = ifp->if_softc;
128 	struct ena_ring *tx_ring;
129 	int ret, is_drbr_empty;
130 	uint32_t i;
131 
132 	if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
133 		return (ENODEV);
134 
135 	/* Which queue to use */
136 	/*
137 	 * If everything is setup correctly, it should be the
138 	 * same bucket that the current CPU we're on is.
139 	 * It should improve performance.
140 	 */
141 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
142 		i = m->m_pkthdr.flowid % adapter->num_queues;
143 	} else {
144 		i = curcpu % adapter->num_queues;
145 	}
146 	tx_ring = &adapter->tx_ring[i];
147 
148 	/* Check if drbr is empty before putting packet */
149 	is_drbr_empty = drbr_empty(ifp, tx_ring->br);
150 	ret = drbr_enqueue(ifp, tx_ring->br, m);
151 	if (unlikely(ret != 0)) {
152 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
153 		return (ret);
154 	}
155 
156 	if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) {
157 		ena_start_xmit(tx_ring);
158 		ENA_RING_MTX_UNLOCK(tx_ring);
159 	} else {
160 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
161 	}
162 
163 	return (0);
164 }
165 
166 void
167 ena_qflush(if_t ifp)
168 {
169 	struct ena_adapter *adapter = ifp->if_softc;
170 	struct ena_ring *tx_ring = adapter->tx_ring;
171 	int i;
172 
173 	for(i = 0; i < adapter->num_queues; ++i, ++tx_ring)
174 		if (!drbr_empty(ifp, tx_ring->br)) {
175 			ENA_RING_MTX_LOCK(tx_ring);
176 			drbr_flush(ifp, tx_ring->br);
177 			ENA_RING_MTX_UNLOCK(tx_ring);
178 		}
179 
180 	if_qflush(ifp);
181 }
182 
183 /*********************************************************************
184  *  Static functions
185  *********************************************************************/
186 
187 static inline int
188 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
189 {
190 	struct ena_adapter *adapter = tx_ring->adapter;
191 	struct ena_tx_buffer *tx_info = NULL;
192 
193 	if (likely(req_id < tx_ring->ring_size)) {
194 		tx_info = &tx_ring->tx_buffer_info[req_id];
195 		if (tx_info->mbuf != NULL)
196 			return (0);
197 		device_printf(adapter->pdev,
198 		    "tx_info doesn't have valid mbuf\n");
199 	}
200 
201 	device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id);
202 	counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
203 
204 	/* Trigger device reset */
205 	adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
206 	ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
207 
208 	return (EFAULT);
209 }
210 
211 /**
212  * ena_tx_cleanup - clear sent packets and corresponding descriptors
213  * @tx_ring: ring for which we want to clean packets
214  *
215  * Once packets are sent, we ask the device in a loop for no longer used
216  * descriptors. We find the related mbuf chain in a map (index in an array)
217  * and free it, then update ring state.
218  * This is performed in "endless" loop, updating ring pointers every
219  * TX_COMMIT. The first check of free descriptor is performed before the actual
220  * loop, then repeated at the loop end.
221  **/
222 static int
223 ena_tx_cleanup(struct ena_ring *tx_ring)
224 {
225 	struct ena_adapter *adapter;
226 	struct ena_com_io_cq* io_cq;
227 	uint16_t next_to_clean;
228 	uint16_t req_id;
229 	uint16_t ena_qid;
230 	unsigned int total_done = 0;
231 	int rc;
232 	int commit = TX_COMMIT;
233 	int budget = TX_BUDGET;
234 	int work_done;
235 	bool above_thresh;
236 
237 	adapter = tx_ring->que->adapter;
238 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
239 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
240 	next_to_clean = tx_ring->next_to_clean;
241 
242 #ifdef DEV_NETMAP
243 	if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)
244 		return (0);
245 #endif /* DEV_NETMAP */
246 
247 	do {
248 		struct ena_tx_buffer *tx_info;
249 		struct mbuf *mbuf;
250 
251 		rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
252 		if (unlikely(rc != 0))
253 			break;
254 
255 		rc = validate_tx_req_id(tx_ring, req_id);
256 		if (unlikely(rc != 0))
257 			break;
258 
259 		tx_info = &tx_ring->tx_buffer_info[req_id];
260 
261 		mbuf = tx_info->mbuf;
262 
263 		tx_info->mbuf = NULL;
264 		bintime_clear(&tx_info->timestamp);
265 
266 		/* Map is no longer required */
267 		if (tx_info->head_mapped == true) {
268 			bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
269 			    BUS_DMASYNC_POSTWRITE);
270 			bus_dmamap_unload(adapter->tx_buf_tag,
271 			    tx_info->map_head);
272 			tx_info->head_mapped = false;
273 		}
274 		if (tx_info->seg_mapped == true) {
275 			bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
276 			    BUS_DMASYNC_POSTWRITE);
277 			bus_dmamap_unload(adapter->tx_buf_tag,
278 			    tx_info->map_seg);
279 			tx_info->seg_mapped = false;
280 		}
281 
282 		ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
283 		    tx_ring->qid, mbuf);
284 
285 		m_freem(mbuf);
286 
287 		total_done += tx_info->tx_descs;
288 
289 		tx_ring->free_tx_ids[next_to_clean] = req_id;
290 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
291 		    tx_ring->ring_size);
292 
293 		if (unlikely(--commit == 0)) {
294 			commit = TX_COMMIT;
295 			/* update ring state every TX_COMMIT descriptor */
296 			tx_ring->next_to_clean = next_to_clean;
297 			ena_com_comp_ack(
298 			    &adapter->ena_dev->io_sq_queues[ena_qid],
299 			    total_done);
300 			ena_com_update_dev_comp_head(io_cq);
301 			total_done = 0;
302 		}
303 	} while (likely(--budget));
304 
305 	work_done = TX_BUDGET - budget;
306 
307 	ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
308 	tx_ring->qid, work_done);
309 
310 	/* If there is still something to commit update ring state */
311 	if (likely(commit != TX_COMMIT)) {
312 		tx_ring->next_to_clean = next_to_clean;
313 		ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
314 		    total_done);
315 		ena_com_update_dev_comp_head(io_cq);
316 	}
317 
318 	/*
319 	 * Need to make the rings circular update visible to
320 	 * ena_xmit_mbuf() before checking for tx_ring->running.
321 	 */
322 	mb();
323 
324 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
325 	    ENA_TX_RESUME_THRESH);
326 	if (unlikely(!tx_ring->running && above_thresh)) {
327 		ENA_RING_MTX_LOCK(tx_ring);
328 		above_thresh =
329 		    ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
330 		    ENA_TX_RESUME_THRESH);
331 		if (!tx_ring->running && above_thresh) {
332 			tx_ring->running = true;
333 			counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
334 			taskqueue_enqueue(tx_ring->enqueue_tq,
335 			    &tx_ring->enqueue_task);
336 		}
337 		ENA_RING_MTX_UNLOCK(tx_ring);
338 	}
339 
340 	return (work_done);
341 }
342 
343 static void
344 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
345     struct mbuf *mbuf)
346 {
347 	struct ena_adapter *adapter = rx_ring->adapter;
348 
349 	if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
350 		mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
351 
352 		if (ena_rx_ctx->frag &&
353 		    (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
354 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
355 			return;
356 		}
357 
358 		switch (ena_rx_ctx->l3_proto) {
359 		case ENA_ETH_IO_L3_PROTO_IPV4:
360 			switch (ena_rx_ctx->l4_proto) {
361 			case ENA_ETH_IO_L4_PROTO_TCP:
362 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
363 				break;
364 			case ENA_ETH_IO_L4_PROTO_UDP:
365 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
366 				break;
367 			default:
368 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
369 			}
370 			break;
371 		case ENA_ETH_IO_L3_PROTO_IPV6:
372 			switch (ena_rx_ctx->l4_proto) {
373 			case ENA_ETH_IO_L4_PROTO_TCP:
374 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
375 				break;
376 			case ENA_ETH_IO_L4_PROTO_UDP:
377 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
378 				break;
379 			default:
380 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
381 			}
382 			break;
383 		case ENA_ETH_IO_L3_PROTO_UNKNOWN:
384 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
385 			break;
386 		default:
387 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
388 		}
389 	} else {
390 		mbuf->m_pkthdr.flowid = rx_ring->qid;
391 		M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
392 	}
393 }
394 
395 /**
396  * ena_rx_mbuf - assemble mbuf from descriptors
397  * @rx_ring: ring for which we want to clean packets
398  * @ena_bufs: buffer info
399  * @ena_rx_ctx: metadata for this packet(s)
400  * @next_to_clean: ring pointer, will be updated only upon success
401  *
402  **/
403 static struct mbuf*
404 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
405     struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
406 {
407 	struct mbuf *mbuf;
408 	struct ena_rx_buffer *rx_info;
409 	struct ena_adapter *adapter;
410 	unsigned int descs = ena_rx_ctx->descs;
411 	int rc;
412 	uint16_t ntc, len, req_id, buf = 0;
413 
414 	ntc = *next_to_clean;
415 	adapter = rx_ring->adapter;
416 
417 	len = ena_bufs[buf].len;
418 	req_id = ena_bufs[buf].req_id;
419 	rc = validate_rx_req_id(rx_ring, req_id);
420 	if (unlikely(rc != 0))
421 		return (NULL);
422 
423 	rx_info = &rx_ring->rx_buffer_info[req_id];
424 	if (unlikely(rx_info->mbuf == NULL)) {
425 		device_printf(adapter->pdev, "NULL mbuf in rx_info");
426 		return (NULL);
427 	}
428 
429 	ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
430 	    rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
431 
432 	bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
433 	    BUS_DMASYNC_POSTREAD);
434 	mbuf = rx_info->mbuf;
435 	mbuf->m_flags |= M_PKTHDR;
436 	mbuf->m_pkthdr.len = len;
437 	mbuf->m_len = len;
438 	mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
439 
440 	/* Fill mbuf with hash key and it's interpretation for optimization */
441 	ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
442 
443 	ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
444 	    mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
445 
446 	/* DMA address is not needed anymore, unmap it */
447 	bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
448 
449 	rx_info->mbuf = NULL;
450 	rx_ring->free_rx_ids[ntc] = req_id;
451 	ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
452 
453 	/*
454 	 * While we have more than 1 descriptors for one rcvd packet, append
455 	 * other mbufs to the main one
456 	 */
457 	while (--descs) {
458 		++buf;
459 		len = ena_bufs[buf].len;
460 		req_id = ena_bufs[buf].req_id;
461 		rc = validate_rx_req_id(rx_ring, req_id);
462 		if (unlikely(rc != 0)) {
463 			/*
464 			 * If the req_id is invalid, then the device will be
465 			 * reset. In that case we must free all mbufs that
466 			 * were already gathered.
467 			 */
468 			m_freem(mbuf);
469 			return (NULL);
470 		}
471 		rx_info = &rx_ring->rx_buffer_info[req_id];
472 
473 		if (unlikely(rx_info->mbuf == NULL)) {
474 			device_printf(adapter->pdev, "NULL mbuf in rx_info");
475 			/*
476 			 * If one of the required mbufs was not allocated yet,
477 			 * we can break there.
478 			 * All earlier used descriptors will be reallocated
479 			 * later and not used mbufs can be reused.
480 			 * The next_to_clean pointer will not be updated in case
481 			 * of an error, so caller should advance it manually
482 			 * in error handling routine to keep it up to date
483 			 * with hw ring.
484 			 */
485 			m_freem(mbuf);
486 			return (NULL);
487 		}
488 
489 		bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
490 		    BUS_DMASYNC_POSTREAD);
491 		if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
492 			counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
493 			ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n",
494 			    mbuf);
495 		}
496 
497 		ena_trace(ENA_DBG | ENA_RXPTH,
498 		    "rx mbuf updated. len %d\n", mbuf->m_pkthdr.len);
499 
500 		/* Free already appended mbuf, it won't be useful anymore */
501 		bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
502 		m_freem(rx_info->mbuf);
503 		rx_info->mbuf = NULL;
504 
505 		rx_ring->free_rx_ids[ntc] = req_id;
506 		ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
507 	}
508 
509 	*next_to_clean = ntc;
510 
511 	return (mbuf);
512 }
513 
514 /**
515  * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
516  **/
517 static inline void
518 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
519     struct mbuf *mbuf)
520 {
521 
522 	/* if IP and error */
523 	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
524 	    ena_rx_ctx->l3_csum_err)) {
525 		/* ipv4 checksum error */
526 		mbuf->m_pkthdr.csum_flags = 0;
527 		counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
528 		ena_trace(ENA_DBG, "RX IPv4 header checksum error\n");
529 		return;
530 	}
531 
532 	/* if TCP/UDP */
533 	if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
534 	    (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
535 		if (ena_rx_ctx->l4_csum_err) {
536 			/* TCP/UDP checksum error */
537 			mbuf->m_pkthdr.csum_flags = 0;
538 			counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
539 			ena_trace(ENA_DBG, "RX L4 checksum error\n");
540 		} else {
541 			mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
542 			mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
543 		}
544 	}
545 }
546 
547 /**
548  * ena_rx_cleanup - handle rx irq
549  * @arg: ring for which irq is being handled
550  **/
551 static int
552 ena_rx_cleanup(struct ena_ring *rx_ring)
553 {
554 	struct ena_adapter *adapter;
555 	struct mbuf *mbuf;
556 	struct ena_com_rx_ctx ena_rx_ctx;
557 	struct ena_com_io_cq* io_cq;
558 	struct ena_com_io_sq* io_sq;
559 	if_t ifp;
560 	uint16_t ena_qid;
561 	uint16_t next_to_clean;
562 	uint32_t refill_required;
563 	uint32_t refill_threshold;
564 	uint32_t do_if_input = 0;
565 	unsigned int qid;
566 	int rc, i;
567 	int budget = RX_BUDGET;
568 #ifdef DEV_NETMAP
569 	int done;
570 #endif /* DEV_NETMAP */
571 
572 	adapter = rx_ring->que->adapter;
573 	ifp = adapter->ifp;
574 	qid = rx_ring->que->id;
575 	ena_qid = ENA_IO_RXQ_IDX(qid);
576 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
577 	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
578 	next_to_clean = rx_ring->next_to_clean;
579 
580 #ifdef DEV_NETMAP
581 	if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS)
582 		return (0);
583 #endif /* DEV_NETMAP */
584 
585 	ena_trace(ENA_DBG, "rx: qid %d\n", qid);
586 
587 	do {
588 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
589 		ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
590 		ena_rx_ctx.descs = 0;
591 		bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
592 		    io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
593 		rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
594 
595 		if (unlikely(rc != 0))
596 			goto error;
597 
598 		if (unlikely(ena_rx_ctx.descs == 0))
599 			break;
600 
601 		ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
602 		    "descs #: %d l3 proto %d l4 proto %d hash: %x\n",
603 		    rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
604 		    ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
605 
606 		/* Receive mbuf from the ring */
607 		mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
608 		    &ena_rx_ctx, &next_to_clean);
609 		bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
610 		    io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
611 		/* Exit if we failed to retrieve a buffer */
612 		if (unlikely(mbuf == NULL)) {
613 			for (i = 0; i < ena_rx_ctx.descs; ++i) {
614 				rx_ring->free_rx_ids[next_to_clean] =
615 				    rx_ring->ena_bufs[i].req_id;
616 				next_to_clean =
617 				    ENA_RX_RING_IDX_NEXT(next_to_clean,
618 				    rx_ring->ring_size);
619 
620 			}
621 			break;
622 		}
623 
624 		if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) ||
625 		    ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) {
626 			ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
627 		}
628 
629 		counter_enter();
630 		counter_u64_add_protected(rx_ring->rx_stats.bytes,
631 		    mbuf->m_pkthdr.len);
632 		counter_u64_add_protected(adapter->hw_stats.rx_bytes,
633 		    mbuf->m_pkthdr.len);
634 		counter_exit();
635 		/*
636 		 * LRO is only for IP/TCP packets and TCP checksum of the packet
637 		 * should be computed by hardware.
638 		 */
639 		do_if_input = 1;
640 		if (((ifp->if_capenable & IFCAP_LRO) != 0)  &&
641 		    ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
642 		    (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
643 			/*
644 			 * Send to the stack if:
645 			 *  - LRO not enabled, or
646 			 *  - no LRO resources, or
647 			 *  - lro enqueue fails
648 			 */
649 			if ((rx_ring->lro.lro_cnt != 0) &&
650 			    (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
651 					do_if_input = 0;
652 		}
653 		if (do_if_input != 0) {
654 			ena_trace(ENA_DBG | ENA_RXPTH,
655 			    "calling if_input() with mbuf %p\n", mbuf);
656 			(*ifp->if_input)(ifp, mbuf);
657 		}
658 
659 		counter_enter();
660 		counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
661 		counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
662 		counter_exit();
663 	} while (--budget);
664 
665 	rx_ring->next_to_clean = next_to_clean;
666 
667 	refill_required = ena_com_free_desc(io_sq);
668 	refill_threshold = min_t(int,
669 	    rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
670 	    ENA_RX_REFILL_THRESH_PACKET);
671 
672 	if (refill_required > refill_threshold) {
673 		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
674 		ena_refill_rx_bufs(rx_ring, refill_required);
675 	}
676 
677 	tcp_lro_flush_all(&rx_ring->lro);
678 
679 	return (RX_BUDGET - budget);
680 
681 error:
682 	counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
683 
684 	/* Too many desc from the device. Trigger reset */
685 	if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
686 		adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
687 		ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
688 	}
689 
690 	return (0);
691 }
692 
693 static void
694 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf)
695 {
696 	struct ena_com_tx_meta *ena_meta;
697 	struct ether_vlan_header *eh;
698 	struct mbuf *mbuf_next;
699 	u32 mss;
700 	bool offload;
701 	uint16_t etype;
702 	int ehdrlen;
703 	struct ip *ip;
704 	int iphlen;
705 	struct tcphdr *th;
706 	int offset;
707 
708 	offload = false;
709 	ena_meta = &ena_tx_ctx->ena_meta;
710 	mss = mbuf->m_pkthdr.tso_segsz;
711 
712 	if (mss != 0)
713 		offload = true;
714 
715 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
716 		offload = true;
717 
718 	if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
719 		offload = true;
720 
721 	if (!offload) {
722 		ena_tx_ctx->meta_valid = 0;
723 		return;
724 	}
725 
726 	/* Determine where frame payload starts. */
727 	eh = mtod(mbuf, struct ether_vlan_header *);
728 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
729 		etype = ntohs(eh->evl_proto);
730 		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
731 	} else {
732 		etype = ntohs(eh->evl_encap_proto);
733 		ehdrlen = ETHER_HDR_LEN;
734 	}
735 
736 	mbuf_next = m_getptr(mbuf, ehdrlen, &offset);
737 	ip = (struct ip *)(mtodo(mbuf_next, offset));
738 	iphlen = ip->ip_hl << 2;
739 
740 	mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset);
741 	th = (struct tcphdr *)(mtodo(mbuf_next, offset));
742 
743 	if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
744 		ena_tx_ctx->l3_csum_enable = 1;
745 	}
746 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
747 		ena_tx_ctx->tso_enable = 1;
748 		ena_meta->l4_hdr_len = (th->th_off);
749 	}
750 
751 	switch (etype) {
752 	case ETHERTYPE_IP:
753 		ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
754 		if ((ip->ip_off & htons(IP_DF)) != 0)
755 			ena_tx_ctx->df = 1;
756 		break;
757 	case ETHERTYPE_IPV6:
758 		ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
759 
760 	default:
761 		break;
762 	}
763 
764 	if (ip->ip_p == IPPROTO_TCP) {
765 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
766 		if ((mbuf->m_pkthdr.csum_flags &
767 		    (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0)
768 			ena_tx_ctx->l4_csum_enable = 1;
769 		else
770 			ena_tx_ctx->l4_csum_enable = 0;
771 	} else if (ip->ip_p == IPPROTO_UDP) {
772 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
773 		if ((mbuf->m_pkthdr.csum_flags &
774 		    (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0)
775 			ena_tx_ctx->l4_csum_enable = 1;
776 		else
777 			ena_tx_ctx->l4_csum_enable = 0;
778 	} else {
779 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
780 		ena_tx_ctx->l4_csum_enable = 0;
781 	}
782 
783 	ena_meta->mss = mss;
784 	ena_meta->l3_hdr_len = iphlen;
785 	ena_meta->l3_hdr_offset = ehdrlen;
786 	ena_tx_ctx->meta_valid = 1;
787 }
788 
789 static int
790 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
791 {
792 	struct ena_adapter *adapter;
793 	struct mbuf *collapsed_mbuf;
794 	int num_frags;
795 
796 	adapter = tx_ring->adapter;
797 	num_frags = ena_mbuf_count(*mbuf);
798 
799 	/* One segment must be reserved for configuration descriptor. */
800 	if (num_frags < adapter->max_tx_sgl_size)
801 		return (0);
802 	counter_u64_add(tx_ring->tx_stats.collapse, 1);
803 
804 	collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,
805 	    adapter->max_tx_sgl_size - 1);
806 	if (unlikely(collapsed_mbuf == NULL)) {
807 		counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
808 		return (ENOMEM);
809 	}
810 
811 	/* If mbuf was collapsed succesfully, original mbuf is released. */
812 	*mbuf = collapsed_mbuf;
813 
814 	return (0);
815 }
816 
817 static void
818 ena_dmamap_llq(void *arg, bus_dma_segment_t *segs, int nseg, int error)
819 {
820 	struct ena_com_buf *ena_buf = arg;
821 
822 	if (unlikely(error != 0)) {
823 		ena_buf->paddr = 0;
824 		return;
825 	}
826 
827 	KASSERT(nseg == 1, ("Invalid num of segments for LLQ dma"));
828 
829 	ena_buf->paddr = segs->ds_addr;
830 	ena_buf->len = segs->ds_len;
831 }
832 
833 static int
834 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
835     struct mbuf *mbuf, void **push_hdr, u16 *header_len)
836 {
837 	struct ena_adapter *adapter = tx_ring->adapter;
838 	struct ena_com_buf *ena_buf;
839 	bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
840 	uint32_t mbuf_head_len, frag_len;
841 	uint16_t push_len = 0;
842 	uint16_t delta = 0;
843 	int i, rc, nsegs;
844 
845 	mbuf_head_len = mbuf->m_len;
846 	tx_info->mbuf = mbuf;
847 	ena_buf = tx_info->bufs;
848 
849 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
850 		/*
851 		 * When the device is LLQ mode, the driver will copy
852 		 * the header into the device memory space.
853 		 * the ena_com layer assumes the header is in a linear
854 		 * memory space.
855 		 * This assumption might be wrong since part of the header
856 		 * can be in the fragmented buffers.
857 		 * First check if header fits in the mbuf. If not, copy it to
858 		 * separate buffer that will be holding linearized data.
859 		 */
860 		push_len = min_t(uint32_t, mbuf->m_pkthdr.len,
861 		    tx_ring->tx_max_header_size);
862 		*header_len = push_len;
863 		/* If header is in linear space, just point into mbuf's data. */
864 		if (likely(push_len <= mbuf_head_len)) {
865 			*push_hdr = mbuf->m_data;
866 		/*
867 		 * Otherwise, copy whole portion of header from multiple mbufs
868 		 * to intermediate buffer.
869 		 */
870 		} else {
871 			m_copydata(mbuf, 0, push_len,
872 			    tx_ring->push_buf_intermediate_buf);
873 			*push_hdr = tx_ring->push_buf_intermediate_buf;
874 
875 			counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
876 			delta = push_len - mbuf_head_len;
877 		}
878 
879 		ena_trace(ENA_DBG | ENA_TXPTH,
880 		    "mbuf: %p header_buf->vaddr: %p push_len: %d\n",
881 		    mbuf, *push_hdr, push_len);
882 
883 		/*
884 		* If header was in linear memory space, map for the dma rest of the data
885 		* in the first mbuf of the mbuf chain.
886 		*/
887 		if (mbuf_head_len > push_len) {
888 			rc = bus_dmamap_load(adapter->tx_buf_tag,
889 			    tx_info->map_head,
890 			mbuf->m_data + push_len, mbuf_head_len - push_len,
891 			ena_dmamap_llq, ena_buf, BUS_DMA_NOWAIT);
892 			if (unlikely((rc != 0) || (ena_buf->paddr == 0)))
893 				goto single_dma_error;
894 
895 			ena_buf++;
896 			tx_info->num_of_bufs++;
897 
898 			tx_info->head_mapped = true;
899 		}
900 		mbuf = mbuf->m_next;
901 	} else {
902 		*push_hdr = NULL;
903 		/*
904 		* header_len is just a hint for the device. Because FreeBSD is not
905 		* giving us information about packet header length and it is not
906 		* guaranteed that all packet headers will be in the 1st mbuf, setting
907 		* header_len to 0 is making the device ignore this value and resolve
908 		* header on it's own.
909 		*/
910 		*header_len = 0;
911 	}
912 
913 	/*
914 	 * If header is in non linear space (delta > 0), then skip mbufs
915 	 * containing header and map the last one containing both header and the
916 	 * packet data.
917 	 * The first segment is already counted in.
918 	 * If LLQ is not supported, the loop will be skipped.
919 	 */
920 	while (delta > 0) {
921 		frag_len = mbuf->m_len;
922 
923 		/*
924 		 * If whole segment contains header just move to the
925 		 * next one and reduce delta.
926 		 */
927 		if (unlikely(delta >= frag_len)) {
928 			delta -= frag_len;
929 		} else {
930 			/*
931 			 * Map rest of the packet data that was contained in
932 			 * the mbuf.
933 			 */
934 			rc = bus_dmamap_load(adapter->tx_buf_tag,
935 			    tx_info->map_head, mbuf->m_data + delta,
936 			    frag_len - delta, ena_dmamap_llq, ena_buf,
937 			    BUS_DMA_NOWAIT);
938 			if (unlikely((rc != 0) || (ena_buf->paddr == 0)))
939 				goto single_dma_error;
940 
941 			ena_buf++;
942 			tx_info->num_of_bufs++;
943 			tx_info->head_mapped = true;
944 
945 			delta = 0;
946 		}
947 
948 		mbuf = mbuf->m_next;
949 	}
950 
951 	if (mbuf == NULL) {
952 		return (0);
953 	}
954 
955 	/* Map rest of the mbufs */
956 	rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map_seg, mbuf,
957 	    segs, &nsegs, BUS_DMA_NOWAIT);
958 	if (unlikely((rc != 0) || (nsegs == 0))) {
959 		ena_trace(ENA_WARNING,
960 		    "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs);
961 		goto dma_error;
962 	}
963 
964 	for (i = 0; i < nsegs; i++) {
965 		ena_buf->len = segs[i].ds_len;
966 		ena_buf->paddr = segs[i].ds_addr;
967 		ena_buf++;
968 	}
969 	tx_info->num_of_bufs += nsegs;
970 	tx_info->seg_mapped = true;
971 
972 	return (0);
973 
974 dma_error:
975 	if (tx_info->head_mapped == true)
976 		bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_head);
977 single_dma_error:
978 	counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
979 	tx_info->mbuf = NULL;
980 	return (rc);
981 }
982 
983 static int
984 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
985 {
986 	struct ena_adapter *adapter;
987 	struct ena_tx_buffer *tx_info;
988 	struct ena_com_tx_ctx ena_tx_ctx;
989 	struct ena_com_dev *ena_dev;
990 	struct ena_com_io_sq* io_sq;
991 	void *push_hdr;
992 	uint16_t next_to_use;
993 	uint16_t req_id;
994 	uint16_t ena_qid;
995 	uint16_t header_len;
996 	int rc;
997 	int nb_hw_desc;
998 
999 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1000 	adapter = tx_ring->que->adapter;
1001 	ena_dev = adapter->ena_dev;
1002 	io_sq = &ena_dev->io_sq_queues[ena_qid];
1003 
1004 	rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
1005 	if (unlikely(rc != 0)) {
1006 		ena_trace(ENA_WARNING,
1007 		    "Failed to collapse mbuf! err: %d\n", rc);
1008 		return (rc);
1009 	}
1010 
1011 	ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
1012 
1013 	next_to_use = tx_ring->next_to_use;
1014 	req_id = tx_ring->free_tx_ids[next_to_use];
1015 	tx_info = &tx_ring->tx_buffer_info[req_id];
1016 	tx_info->num_of_bufs = 0;
1017 
1018 	rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);
1019 	if (unlikely(rc != 0)) {
1020 		ena_trace(ENA_WARNING, "Failed to map TX mbuf\n");
1021 		return (rc);
1022 	}
1023 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
1024 	ena_tx_ctx.ena_bufs = tx_info->bufs;
1025 	ena_tx_ctx.push_header = push_hdr;
1026 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
1027 	ena_tx_ctx.req_id = req_id;
1028 	ena_tx_ctx.header_len = header_len;
1029 
1030 	/* Set flags and meta data */
1031 	ena_tx_csum(&ena_tx_ctx, *mbuf);
1032 
1033 	if (tx_ring->acum_pkts == DB_THRESHOLD ||
1034 	    ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {
1035 		ena_trace(ENA_DBG | ENA_TXPTH,
1036 		    "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
1037 		    tx_ring->que->id);
1038 		wmb();
1039 		ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
1040 		counter_u64_add(tx_ring->tx_stats.doorbells, 1);
1041 		tx_ring->acum_pkts = 0;
1042 	}
1043 
1044 	/* Prepare the packet's descriptors and send them to device */
1045 	rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
1046 	if (unlikely(rc != 0)) {
1047 		if (likely(rc == ENA_COM_NO_MEM)) {
1048 			ena_trace(ENA_DBG | ENA_TXPTH,
1049 			    "tx ring[%d] if out of space\n", tx_ring->que->id);
1050 		} else {
1051 			device_printf(adapter->pdev,
1052 			    "failed to prepare tx bufs\n");
1053 		}
1054 		counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
1055 		goto dma_error;
1056 	}
1057 
1058 	counter_enter();
1059 	counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
1060 	counter_u64_add_protected(tx_ring->tx_stats.bytes,
1061 	    (*mbuf)->m_pkthdr.len);
1062 
1063 	counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
1064 	counter_u64_add_protected(adapter->hw_stats.tx_bytes,
1065 	    (*mbuf)->m_pkthdr.len);
1066 	counter_exit();
1067 
1068 	tx_info->tx_descs = nb_hw_desc;
1069 	getbinuptime(&tx_info->timestamp);
1070 	tx_info->print_once = true;
1071 
1072 	tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
1073 	    tx_ring->ring_size);
1074 
1075 	/* stop the queue when no more space available, the packet can have up
1076 	 * to sgl_size + 2. one for the meta descriptor and one for header
1077 	 * (if the header is larger than tx_max_header_size).
1078 	 */
1079 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1080 	    adapter->max_tx_sgl_size + 2))) {
1081 		ena_trace(ENA_DBG | ENA_TXPTH, "Stop queue %d\n",
1082 		    tx_ring->que->id);
1083 
1084 		tx_ring->running = false;
1085 		counter_u64_add(tx_ring->tx_stats.queue_stop, 1);
1086 
1087 		/* There is a rare condition where this function decides to
1088 		 * stop the queue but meanwhile tx_cleanup() updates
1089 		 * next_to_completion and terminates.
1090 		 * The queue will remain stopped forever.
1091 		 * To solve this issue this function performs mb(), checks
1092 		 * the wakeup condition and wakes up the queue if needed.
1093 		 */
1094 		mb();
1095 
1096 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1097 		    ENA_TX_RESUME_THRESH)) {
1098 			tx_ring->running = true;
1099 			counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
1100 		}
1101 	}
1102 
1103 	if (tx_info->head_mapped == true)
1104 		bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
1105 		    BUS_DMASYNC_PREWRITE);
1106 	if (tx_info->seg_mapped == true)
1107 		bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
1108 		    BUS_DMASYNC_PREWRITE);
1109 
1110 	return (0);
1111 
1112 dma_error:
1113 	tx_info->mbuf = NULL;
1114 	if (tx_info->seg_mapped == true) {
1115 		bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_seg);
1116 		tx_info->seg_mapped = false;
1117 	}
1118 	if (tx_info->head_mapped == true) {
1119 		bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_head);
1120 		tx_info->head_mapped = false;
1121 	}
1122 
1123 	return (rc);
1124 }
1125 
1126 static void
1127 ena_start_xmit(struct ena_ring *tx_ring)
1128 {
1129 	struct mbuf *mbuf;
1130 	struct ena_adapter *adapter = tx_ring->adapter;
1131 	struct ena_com_io_sq* io_sq;
1132 	int ena_qid;
1133 	int ret = 0;
1134 
1135 	if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1136 		return;
1137 
1138 	if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)))
1139 		return;
1140 
1141 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1142 	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
1143 
1144 	while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
1145 		ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and"
1146 		    " header csum flags %#jx\n",
1147 		    mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);
1148 
1149 		if (unlikely(!tx_ring->running)) {
1150 			drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1151 			break;
1152 		}
1153 
1154 		if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {
1155 			if (ret == ENA_COM_NO_MEM) {
1156 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1157 			} else if (ret == ENA_COM_NO_SPACE) {
1158 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1159 			} else {
1160 				m_freem(mbuf);
1161 				drbr_advance(adapter->ifp, tx_ring->br);
1162 			}
1163 
1164 			break;
1165 		}
1166 
1167 		drbr_advance(adapter->ifp, tx_ring->br);
1168 
1169 		if (unlikely((if_getdrvflags(adapter->ifp) &
1170 		    IFF_DRV_RUNNING) == 0))
1171 			return;
1172 
1173 		tx_ring->acum_pkts++;
1174 
1175 		BPF_MTAP(adapter->ifp, mbuf);
1176 	}
1177 
1178 	if (likely(tx_ring->acum_pkts != 0)) {
1179 		wmb();
1180 		/* Trigger the dma engine */
1181 		ena_com_write_sq_doorbell(io_sq);
1182 		counter_u64_add(tx_ring->tx_stats.doorbells, 1);
1183 		tx_ring->acum_pkts = 0;
1184 	}
1185 
1186 	if (unlikely(!tx_ring->running))
1187 		taskqueue_enqueue(tx_ring->que->cleanup_tq,
1188 		    &tx_ring->que->cleanup_task);
1189 }
1190