ena_datapath.c (38d036e91a8c29bf97b86d441a9d627c5cef2aea) ena_datapath.c (7583c633e0aa5572715d1c792cdc505c2237e820)
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 101 unchanged lines hidden (view full) ---

110 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
111 ena_com_unmask_intr(io_cq, &intr_reg);
112}
113
114void
115ena_deferred_mq_start(void *arg, int pending)
116{
117 struct ena_ring *tx_ring = (struct ena_ring *)arg;
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 101 unchanged lines hidden (view full) ---

110 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
111 ena_com_unmask_intr(io_cq, &intr_reg);
112}
113
114void
115ena_deferred_mq_start(void *arg, int pending)
116{
117 struct ena_ring *tx_ring = (struct ena_ring *)arg;
118 struct ifnet *ifp = tx_ring->adapter->ifp;
118 if_t ifp = tx_ring->adapter->ifp;
119
120 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running &&
121 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
122 ENA_RING_MTX_LOCK(tx_ring);
123 ena_start_xmit(tx_ring);
124 ENA_RING_MTX_UNLOCK(tx_ring);
125 }
126}
127
128int
129ena_mq_start(if_t ifp, struct mbuf *m)
130{
119
120 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running &&
121 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
122 ENA_RING_MTX_LOCK(tx_ring);
123 ena_start_xmit(tx_ring);
124 ENA_RING_MTX_UNLOCK(tx_ring);
125 }
126}
127
128int
129ena_mq_start(if_t ifp, struct mbuf *m)
130{
131 struct ena_adapter *adapter = ifp->if_softc;
131 struct ena_adapter *adapter = if_getsoftc(ifp);
132 struct ena_ring *tx_ring;
133 int ret, is_drbr_empty;
134 uint32_t i;
135#ifdef RSS
136 uint32_t bucket_id;
137#endif
138
139 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))

--- 34 unchanged lines hidden (view full) ---

174 }
175
176 return (0);
177}
178
179void
180ena_qflush(if_t ifp)
181{
132 struct ena_ring *tx_ring;
133 int ret, is_drbr_empty;
134 uint32_t i;
135#ifdef RSS
136 uint32_t bucket_id;
137#endif
138
139 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))

--- 34 unchanged lines hidden (view full) ---

174 }
175
176 return (0);
177}
178
179void
180ena_qflush(if_t ifp)
181{
182 struct ena_adapter *adapter = ifp->if_softc;
182 struct ena_adapter *adapter = if_getsoftc(ifp);
183 struct ena_ring *tx_ring = adapter->tx_ring;
184 int i;
185
186 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring)
187 if (!drbr_empty(ifp, tx_ring->br)) {
188 ENA_RING_MTX_LOCK(tx_ring);
189 drbr_flush(ifp, tx_ring->br);
190 ENA_RING_MTX_UNLOCK(tx_ring);

--- 447 unchanged lines hidden (view full) ---

638 rx_ring->free_rx_ids[next_to_clean] =
639 rx_ring->ena_bufs[i].req_id;
640 next_to_clean = ENA_RX_RING_IDX_NEXT(
641 next_to_clean, rx_ring->ring_size);
642 }
643 break;
644 }
645
183 struct ena_ring *tx_ring = adapter->tx_ring;
184 int i;
185
186 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring)
187 if (!drbr_empty(ifp, tx_ring->br)) {
188 ENA_RING_MTX_LOCK(tx_ring);
189 drbr_flush(ifp, tx_ring->br);
190 ENA_RING_MTX_UNLOCK(tx_ring);

--- 447 unchanged lines hidden (view full) ---

638 rx_ring->free_rx_ids[next_to_clean] =
639 rx_ring->ena_bufs[i].req_id;
640 next_to_clean = ENA_RX_RING_IDX_NEXT(
641 next_to_clean, rx_ring->ring_size);
642 }
643 break;
644 }
645
646 if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) ||
647 ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) {
646 if (((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) ||
647 ((if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) != 0)) {
648 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
649 }
650
651 counter_enter();
652 counter_u64_add_protected(rx_ring->rx_stats.bytes,
653 mbuf->m_pkthdr.len);
654 counter_u64_add_protected(adapter->hw_stats.rx_bytes,
655 mbuf->m_pkthdr.len);
656 counter_exit();
657 /*
658 * LRO is only for IP/TCP packets and TCP checksum of the packet
659 * should be computed by hardware.
660 */
661 do_if_input = 1;
648 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
649 }
650
651 counter_enter();
652 counter_u64_add_protected(rx_ring->rx_stats.bytes,
653 mbuf->m_pkthdr.len);
654 counter_u64_add_protected(adapter->hw_stats.rx_bytes,
655 mbuf->m_pkthdr.len);
656 counter_exit();
657 /*
658 * LRO is only for IP/TCP packets and TCP checksum of the packet
659 * should be computed by hardware.
660 */
661 do_if_input = 1;
662 if (((ifp->if_capenable & IFCAP_LRO) != 0) &&
662 if (((if_getcapenable(ifp) & IFCAP_LRO) != 0) &&
663 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
664 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
665 /*
666 * Send to the stack if:
667 * - LRO not enabled, or
668 * - no LRO resources, or
669 * - lro enqueue fails
670 */
671 if ((rx_ring->lro.lro_cnt != 0) &&
672 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
673 do_if_input = 0;
674 }
675 if (do_if_input != 0) {
676 ena_log_io(pdev, DBG,
677 "calling if_input() with mbuf %p\n", mbuf);
663 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
664 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
665 /*
666 * Send to the stack if:
667 * - LRO not enabled, or
668 * - no LRO resources, or
669 * - lro enqueue fails
670 */
671 if ((rx_ring->lro.lro_cnt != 0) &&
672 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
673 do_if_input = 0;
674 }
675 if (do_if_input != 0) {
676 ena_log_io(pdev, DBG,
677 "calling if_input() with mbuf %p\n", mbuf);
678 (*ifp->if_input)(ifp, mbuf);
678 if_input(ifp, mbuf);
679 }
680
681 counter_enter();
682 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
683 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
684 counter_exit();
685 } while (--budget);
686

--- 468 unchanged lines hidden --- 		679 }
680
681 counter_enter();
682 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
683 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
684 counter_exit();
685 } while (--budget);
686

--- 468 unchanged lines hidden ---