xref: /freebsd/sys/dev/cxgbe/t4_sge.c (revision 94e6b3fee1e5f1ae52741c367c4a3f93a0c0708d)
154e4ee71SNavdeep Parhar /*-
2718cf2ccSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3718cf2ccSPedro F. Giffuni  *
454e4ee71SNavdeep Parhar  * Copyright (c) 2011 Chelsio Communications, Inc.
554e4ee71SNavdeep Parhar  * All rights reserved.
654e4ee71SNavdeep Parhar  * Written by: Navdeep Parhar <np@FreeBSD.org>
754e4ee71SNavdeep Parhar  *
854e4ee71SNavdeep Parhar  * Redistribution and use in source and binary forms, with or without
954e4ee71SNavdeep Parhar  * modification, are permitted provided that the following conditions
1054e4ee71SNavdeep Parhar  * are met:
1154e4ee71SNavdeep Parhar  * 1. Redistributions of source code must retain the above copyright
1254e4ee71SNavdeep Parhar  *    notice, this list of conditions and the following disclaimer.
1354e4ee71SNavdeep Parhar  * 2. Redistributions in binary form must reproduce the above copyright
1454e4ee71SNavdeep Parhar  *    notice, this list of conditions and the following disclaimer in the
1554e4ee71SNavdeep Parhar  *    documentation and/or other materials provided with the distribution.
1654e4ee71SNavdeep Parhar  *
1754e4ee71SNavdeep Parhar  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1854e4ee71SNavdeep Parhar  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1954e4ee71SNavdeep Parhar  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2054e4ee71SNavdeep Parhar  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2154e4ee71SNavdeep Parhar  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2254e4ee71SNavdeep Parhar  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2354e4ee71SNavdeep Parhar  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2454e4ee71SNavdeep Parhar  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2554e4ee71SNavdeep Parhar  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2654e4ee71SNavdeep Parhar  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2754e4ee71SNavdeep Parhar  * SUCH DAMAGE.
2854e4ee71SNavdeep Parhar  */
2954e4ee71SNavdeep Parhar 
3054e4ee71SNavdeep Parhar #include <sys/cdefs.h>
3154e4ee71SNavdeep Parhar __FBSDID("$FreeBSD$");
3254e4ee71SNavdeep Parhar 
3354e4ee71SNavdeep Parhar #include "opt_inet.h"
34a1ea9a82SNavdeep Parhar #include "opt_inet6.h"
35bddf7343SJohn Baldwin #include "opt_kern_tls.h"
36eff62dbaSNavdeep Parhar #include "opt_ratelimit.h"
3754e4ee71SNavdeep Parhar 
3854e4ee71SNavdeep Parhar #include <sys/types.h>
39c3322cb9SGleb Smirnoff #include <sys/eventhandler.h>
4054e4ee71SNavdeep Parhar #include <sys/mbuf.h>
4154e4ee71SNavdeep Parhar #include <sys/socket.h>
4254e4ee71SNavdeep Parhar #include <sys/kernel.h>
43bddf7343SJohn Baldwin #include <sys/ktls.h>
44ecb79ca4SNavdeep Parhar #include <sys/malloc.h>
4514a634dfSMark Johnston #include <sys/msan.h>
46ecb79ca4SNavdeep Parhar #include <sys/queue.h>
4738035ed6SNavdeep Parhar #include <sys/sbuf.h>
48ecb79ca4SNavdeep Parhar #include <sys/taskqueue.h>
49480e603cSNavdeep Parhar #include <sys/time.h>
507951040fSNavdeep Parhar #include <sys/sglist.h>
5154e4ee71SNavdeep Parhar #include <sys/sysctl.h>
52733b9277SNavdeep Parhar #include <sys/smp.h>
53bddf7343SJohn Baldwin #include <sys/socketvar.h>
5482eff304SNavdeep Parhar #include <sys/counter.h>
5554e4ee71SNavdeep Parhar #include <net/bpf.h>
5654e4ee71SNavdeep Parhar #include <net/ethernet.h>
5754e4ee71SNavdeep Parhar #include <net/if.h>
5854e4ee71SNavdeep Parhar #include <net/if_vlan_var.h>
59a4a4ad2dSNavdeep Parhar #include <net/if_vxlan.h>
6054e4ee71SNavdeep Parhar #include <netinet/in.h>
6154e4ee71SNavdeep Parhar #include <netinet/ip.h>
62a1ea9a82SNavdeep Parhar #include <netinet/ip6.h>
6354e4ee71SNavdeep Parhar #include <netinet/tcp.h>
64786099deSNavdeep Parhar #include <netinet/udp.h>
656af45170SJohn Baldwin #include <machine/in_cksum.h>
6664db8966SDimitry Andric #include <machine/md_var.h>
6738035ed6SNavdeep Parhar #include <vm/vm.h>
6838035ed6SNavdeep Parhar #include <vm/pmap.h>
69298d969cSNavdeep Parhar #ifdef DEV_NETMAP
70298d969cSNavdeep Parhar #include <machine/bus.h>
71298d969cSNavdeep Parhar #include <sys/selinfo.h>
72298d969cSNavdeep Parhar #include <net/if_var.h>
73298d969cSNavdeep Parhar #include <net/netmap.h>
74298d969cSNavdeep Parhar #include <dev/netmap/netmap_kern.h>
75298d969cSNavdeep Parhar #endif
7654e4ee71SNavdeep Parhar 
7754e4ee71SNavdeep Parhar #include "common/common.h"
7854e4ee71SNavdeep Parhar #include "common/t4_regs.h"
7954e4ee71SNavdeep Parhar #include "common/t4_regs_values.h"
8054e4ee71SNavdeep Parhar #include "common/t4_msg.h"
81671bf2b8SNavdeep Parhar #include "t4_l2t.h"
827951040fSNavdeep Parhar #include "t4_mp_ring.h"
8354e4ee71SNavdeep Parhar 
84d14b0ac1SNavdeep Parhar #ifdef T4_PKT_TIMESTAMP
85d14b0ac1SNavdeep Parhar #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
86d14b0ac1SNavdeep Parhar #else
87d14b0ac1SNavdeep Parhar #define RX_COPY_THRESHOLD MINCLSIZE
88d14b0ac1SNavdeep Parhar #endif
89d14b0ac1SNavdeep Parhar 
905cdaef71SJohn Baldwin /* Internal mbuf flags stored in PH_loc.eight[1]. */
91d76bbe17SJohn Baldwin #define	MC_NOMAP		0x01
925cdaef71SJohn Baldwin #define	MC_RAW_WR		0x02
93bddf7343SJohn Baldwin #define	MC_TLS			0x04
945cdaef71SJohn Baldwin 
959fb8886bSNavdeep Parhar /*
969fb8886bSNavdeep Parhar  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
979fb8886bSNavdeep Parhar  * 0-7 are valid values.
989fb8886bSNavdeep Parhar  */
99518bca2cSNavdeep Parhar static int fl_pktshift = 0;
1002d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0,
1012d714dbcSJohn Baldwin     "payload DMA offset in rx buffer (bytes)");
10254e4ee71SNavdeep Parhar 
1039fb8886bSNavdeep Parhar /*
1049fb8886bSNavdeep Parhar  * Pad ethernet payload up to this boundary.
1059fb8886bSNavdeep Parhar  * -1: driver should figure out a good value.
1061458bff9SNavdeep Parhar  *  0: disable padding.
1071458bff9SNavdeep Parhar  *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
1089fb8886bSNavdeep Parhar  */
109298d969cSNavdeep Parhar int fl_pad = -1;
1102d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0,
1112d714dbcSJohn Baldwin     "payload pad boundary (bytes)");
1129fb8886bSNavdeep Parhar 
1139fb8886bSNavdeep Parhar /*
1149fb8886bSNavdeep Parhar  * Status page length.
1159fb8886bSNavdeep Parhar  * -1: driver should figure out a good value.
1169fb8886bSNavdeep Parhar  *  64 or 128 are the only other valid values.
1179fb8886bSNavdeep Parhar  */
11829c229e9SJohn Baldwin static int spg_len = -1;
1192d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0,
1202d714dbcSJohn Baldwin     "status page size (bytes)");
1219fb8886bSNavdeep Parhar 
1229fb8886bSNavdeep Parhar /*
1239fb8886bSNavdeep Parhar  * Congestion drops.
1249fb8886bSNavdeep Parhar  * -1: no congestion feedback (not recommended).
1259fb8886bSNavdeep Parhar  *  0: backpressure the channel instead of dropping packets right away.
1269fb8886bSNavdeep Parhar  *  1: no backpressure, drop packets for the congested queue immediately.
1279fb8886bSNavdeep Parhar  */
1289fb8886bSNavdeep Parhar static int cong_drop = 0;
1292d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0,
1302d714dbcSJohn Baldwin     "Congestion control for RX queues (0 = backpressure, 1 = drop");
13154e4ee71SNavdeep Parhar 
1321458bff9SNavdeep Parhar /*
1331458bff9SNavdeep Parhar  * Deliver multiple frames in the same free list buffer if they fit.
1341458bff9SNavdeep Parhar  * -1: let the driver decide whether to enable buffer packing or not.
1351458bff9SNavdeep Parhar  *  0: disable buffer packing.
1361458bff9SNavdeep Parhar  *  1: enable buffer packing.
1371458bff9SNavdeep Parhar  */
1381458bff9SNavdeep Parhar static int buffer_packing = -1;
1392d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing,
1402d714dbcSJohn Baldwin     0, "Enable buffer packing");
1411458bff9SNavdeep Parhar 
1421458bff9SNavdeep Parhar /*
1431458bff9SNavdeep Parhar  * Start next frame in a packed buffer at this boundary.
1441458bff9SNavdeep Parhar  * -1: driver should figure out a good value.
145e3207e19SNavdeep Parhar  * T4: driver will ignore this and use the same value as fl_pad above.
146e3207e19SNavdeep Parhar  * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
1471458bff9SNavdeep Parhar  */
1481458bff9SNavdeep Parhar static int fl_pack = -1;
1492d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0,
1502d714dbcSJohn Baldwin     "payload pack boundary (bytes)");
1511458bff9SNavdeep Parhar 
15238035ed6SNavdeep Parhar /*
15338035ed6SNavdeep Parhar  * Largest rx cluster size that the driver is allowed to allocate.
15438035ed6SNavdeep Parhar  */
15538035ed6SNavdeep Parhar static int largest_rx_cluster = MJUM16BYTES;
1562d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN,
1572d714dbcSJohn Baldwin     &largest_rx_cluster, 0, "Largest rx cluster (bytes)");
15838035ed6SNavdeep Parhar 
15938035ed6SNavdeep Parhar /*
16038035ed6SNavdeep Parhar  * Size of cluster allocation that's most likely to succeed.  The driver will
16138035ed6SNavdeep Parhar  * fall back to this size if it fails to allocate clusters larger than this.
16238035ed6SNavdeep Parhar  */
16338035ed6SNavdeep Parhar static int safest_rx_cluster = PAGE_SIZE;
1642d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN,
1652d714dbcSJohn Baldwin     &safest_rx_cluster, 0, "Safe rx cluster (bytes)");
16638035ed6SNavdeep Parhar 
167786099deSNavdeep Parhar #ifdef RATELIMIT
168786099deSNavdeep Parhar /*
169786099deSNavdeep Parhar  * Knob to control TCP timestamp rewriting, and the granularity of the tick used
170786099deSNavdeep Parhar  * for rewriting.  -1 and 0-3 are all valid values.
171786099deSNavdeep Parhar  * -1: hardware should leave the TCP timestamps alone.
172786099deSNavdeep Parhar  * 0: 1ms
173786099deSNavdeep Parhar  * 1: 100us
174786099deSNavdeep Parhar  * 2: 10us
175786099deSNavdeep Parhar  * 3: 1us
176786099deSNavdeep Parhar  */
177786099deSNavdeep Parhar static int tsclk = -1;
1782d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0,
1792d714dbcSJohn Baldwin     "Control TCP timestamp rewriting when using pacing");
180786099deSNavdeep Parhar 
181786099deSNavdeep Parhar static int eo_max_backlog = 1024 * 1024;
1822d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog,
1832d714dbcSJohn Baldwin     0, "Maximum backlog of ratelimited data per flow");
184786099deSNavdeep Parhar #endif
185786099deSNavdeep Parhar 
186d491f8caSNavdeep Parhar /*
187d491f8caSNavdeep Parhar  * The interrupt holdoff timers are multiplied by this value on T6+.
188d491f8caSNavdeep Parhar  * 1 and 3-17 (both inclusive) are legal values.
189d491f8caSNavdeep Parhar  */
190d491f8caSNavdeep Parhar static int tscale = 1;
1912d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0,
1922d714dbcSJohn Baldwin     "Interrupt holdoff timer scale on T6+");
193d491f8caSNavdeep Parhar 
19446f48ee5SNavdeep Parhar /*
19546f48ee5SNavdeep Parhar  * Number of LRO entries in the lro_ctrl structure per rx queue.
19646f48ee5SNavdeep Parhar  */
19746f48ee5SNavdeep Parhar static int lro_entries = TCP_LRO_ENTRIES;
1982d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0,
1992d714dbcSJohn Baldwin     "Number of LRO entries per RX queue");
20046f48ee5SNavdeep Parhar 
20146f48ee5SNavdeep Parhar /*
20246f48ee5SNavdeep Parhar  * This enables presorting of frames before they're fed into tcp_lro_rx.
20346f48ee5SNavdeep Parhar  */
20446f48ee5SNavdeep Parhar static int lro_mbufs = 0;
2052d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0,
2062d714dbcSJohn Baldwin     "Enable presorting of LRO frames");
20746f48ee5SNavdeep Parhar 
2087054f6ecSNavdeep Parhar static counter_u64_t pullups;
2097054f6ecSNavdeep Parhar SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups,
2107054f6ecSNavdeep Parhar     "Number of mbuf pullups performed");
2117054f6ecSNavdeep Parhar 
2127054f6ecSNavdeep Parhar static counter_u64_t defrags;
2137054f6ecSNavdeep Parhar SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, defrags, CTLFLAG_RD, &defrags,
2147054f6ecSNavdeep Parhar     "Number of mbuf defrags performed");
2157054f6ecSNavdeep Parhar 
2163447df8bSNavdeep Parhar static int t4_tx_coalesce = 1;
2173447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce, CTLFLAG_RWTUN, &t4_tx_coalesce, 0,
2183447df8bSNavdeep Parhar     "tx coalescing allowed");
2193447df8bSNavdeep Parhar 
2203447df8bSNavdeep Parhar /*
2213447df8bSNavdeep Parhar  * The driver will make aggressive attempts at tx coalescing if it sees these
2223447df8bSNavdeep Parhar  * many packets eligible for coalescing in quick succession, with no more than
2233447df8bSNavdeep Parhar  * the specified gap in between the eth_tx calls that delivered the packets.
2243447df8bSNavdeep Parhar  */
2253447df8bSNavdeep Parhar static int t4_tx_coalesce_pkts = 32;
2263447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_pkts, CTLFLAG_RWTUN,
2273447df8bSNavdeep Parhar     &t4_tx_coalesce_pkts, 0,
2283447df8bSNavdeep Parhar     "# of consecutive packets (1 - 255) that will trigger tx coalescing");
2293447df8bSNavdeep Parhar static int t4_tx_coalesce_gap = 5;
2303447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_gap, CTLFLAG_RWTUN,
2313447df8bSNavdeep Parhar     &t4_tx_coalesce_gap, 0, "tx gap (in microseconds)");
2327054f6ecSNavdeep Parhar 
233733b9277SNavdeep Parhar static int service_iq(struct sge_iq *, int);
2343098bcfcSNavdeep Parhar static int service_iq_fl(struct sge_iq *, int);
2354d6db4e0SNavdeep Parhar static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
2361486d2deSNavdeep Parhar static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *,
2371486d2deSNavdeep Parhar     u_int);
23843bbae19SNavdeep Parhar static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
23943bbae19SNavdeep Parhar     int, int);
240e3207e19SNavdeep Parhar static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
24190e7434aSNavdeep Parhar static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t,
24243bbae19SNavdeep Parhar     struct sge_iq *, char *);
243fe2ebb76SJohn Baldwin static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *,
24443bbae19SNavdeep Parhar     struct sysctl_ctx_list *, struct sysctl_oid *);
24543bbae19SNavdeep Parhar static void free_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *);
246348694daSNavdeep Parhar static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
247348694daSNavdeep Parhar     struct sge_iq *);
248aa93b99aSNavdeep Parhar static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *,
249aa93b99aSNavdeep Parhar     struct sysctl_oid *, struct sge_fl *);
25043bbae19SNavdeep Parhar static int alloc_iq_fl_hwq(struct vi_info *, struct sge_iq *, struct sge_fl *);
25143bbae19SNavdeep Parhar static int free_iq_fl_hwq(struct adapter *, struct sge_iq *, struct sge_fl *);
252733b9277SNavdeep Parhar static int alloc_fwq(struct adapter *);
25343bbae19SNavdeep Parhar static void free_fwq(struct adapter *);
25443bbae19SNavdeep Parhar static int alloc_ctrlq(struct adapter *, int);
25543bbae19SNavdeep Parhar static void free_ctrlq(struct adapter *, int);
25643bbae19SNavdeep Parhar static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, int);
25743bbae19SNavdeep Parhar static void free_rxq(struct vi_info *, struct sge_rxq *);
25843bbae19SNavdeep Parhar static void add_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
25943bbae19SNavdeep Parhar     struct sge_rxq *);
26009fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
261fe2ebb76SJohn Baldwin static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int,
26243bbae19SNavdeep Parhar     int);
26343bbae19SNavdeep Parhar static void free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *);
26443bbae19SNavdeep Parhar static void add_ofld_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
26543bbae19SNavdeep Parhar     struct sge_ofld_rxq *);
266733b9277SNavdeep Parhar #endif
267733b9277SNavdeep Parhar static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
268fe2ebb76SJohn Baldwin static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
269eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
270fe2ebb76SJohn Baldwin static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
271733b9277SNavdeep Parhar #endif
27243bbae19SNavdeep Parhar static int alloc_eq(struct adapter *, struct sge_eq *, struct sysctl_ctx_list *,
27343bbae19SNavdeep Parhar     struct sysctl_oid *);
27443bbae19SNavdeep Parhar static void free_eq(struct adapter *, struct sge_eq *);
27543bbae19SNavdeep Parhar static void add_eq_sysctls(struct adapter *, struct sysctl_ctx_list *,
27643bbae19SNavdeep Parhar     struct sysctl_oid *, struct sge_eq *);
27743bbae19SNavdeep Parhar static int alloc_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
27843bbae19SNavdeep Parhar static int free_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
279fe2ebb76SJohn Baldwin static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *,
28043bbae19SNavdeep Parhar     struct sysctl_ctx_list *, struct sysctl_oid *);
28143bbae19SNavdeep Parhar static void free_wrq(struct adapter *, struct sge_wrq *);
28243bbae19SNavdeep Parhar static void add_wrq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
28343bbae19SNavdeep Parhar     struct sge_wrq *);
28443bbae19SNavdeep Parhar static int alloc_txq(struct vi_info *, struct sge_txq *, int);
28543bbae19SNavdeep Parhar static void free_txq(struct vi_info *, struct sge_txq *);
28643bbae19SNavdeep Parhar static void add_txq_sysctls(struct vi_info *, struct sysctl_ctx_list *,
28743bbae19SNavdeep Parhar     struct sysctl_oid *, struct sge_txq *);
288077ba6a8SJohn Baldwin #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
28943bbae19SNavdeep Parhar static int alloc_ofld_txq(struct vi_info *, struct sge_ofld_txq *, int);
29043bbae19SNavdeep Parhar static void free_ofld_txq(struct vi_info *, struct sge_ofld_txq *);
29143bbae19SNavdeep Parhar static void add_ofld_txq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
29243bbae19SNavdeep Parhar     struct sge_ofld_txq *);
293077ba6a8SJohn Baldwin #endif
29454e4ee71SNavdeep Parhar static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
29554e4ee71SNavdeep Parhar static inline void ring_fl_db(struct adapter *, struct sge_fl *);
296733b9277SNavdeep Parhar static int refill_fl(struct adapter *, struct sge_fl *, int);
297733b9277SNavdeep Parhar static void refill_sfl(void *);
29846e1e307SNavdeep Parhar static int find_refill_source(struct adapter *, int, bool);
299733b9277SNavdeep Parhar static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
30054e4ee71SNavdeep Parhar 
3017951040fSNavdeep Parhar static inline void get_pkt_gl(struct mbuf *, struct sglist *);
302a4a4ad2dSNavdeep Parhar static inline u_int txpkt_len16(u_int, const u_int);
303a4a4ad2dSNavdeep Parhar static inline u_int txpkt_vm_len16(u_int, const u_int);
30430e3f2b4SNavdeep Parhar static inline void calculate_mbuf_len16(struct mbuf *, bool);
3057951040fSNavdeep Parhar static inline u_int txpkts0_len16(u_int);
3067951040fSNavdeep Parhar static inline u_int txpkts1_len16(void);
3075cdaef71SJohn Baldwin static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int);
308d735920dSNavdeep Parhar static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *,
309d735920dSNavdeep Parhar     u_int);
310472a6004SNavdeep Parhar static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *,
311d735920dSNavdeep Parhar     struct mbuf *);
312d735920dSNavdeep Parhar static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *,
313d735920dSNavdeep Parhar     int, bool *);
314d735920dSNavdeep Parhar static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *,
315d735920dSNavdeep Parhar     int, bool *);
316d735920dSNavdeep Parhar static u_int write_txpkts_wr(struct adapter *, struct sge_txq *);
317d735920dSNavdeep Parhar static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *);
3187951040fSNavdeep Parhar static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
31954e4ee71SNavdeep Parhar static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
3207951040fSNavdeep Parhar static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
3217951040fSNavdeep Parhar static inline uint16_t read_hw_cidx(struct sge_eq *);
3227951040fSNavdeep Parhar static inline u_int reclaimable_tx_desc(struct sge_eq *);
3237951040fSNavdeep Parhar static inline u_int total_available_tx_desc(struct sge_eq *);
3247951040fSNavdeep Parhar static u_int reclaim_tx_descs(struct sge_txq *, u_int);
3257951040fSNavdeep Parhar static void tx_reclaim(void *, int);
3267951040fSNavdeep Parhar static __be64 get_flit(struct sglist_seg *, int, int);
327733b9277SNavdeep Parhar static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
328733b9277SNavdeep Parhar     struct mbuf *);
3291b4cc91fSNavdeep Parhar static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
330733b9277SNavdeep Parhar     struct mbuf *);
331069af0ebSJohn Baldwin static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *);
3327951040fSNavdeep Parhar static void wrq_tx_drain(void *, int);
3337951040fSNavdeep Parhar static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *);
33454e4ee71SNavdeep Parhar 
33538035ed6SNavdeep Parhar static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
336786099deSNavdeep Parhar #ifdef RATELIMIT
337ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
338786099deSNavdeep Parhar static inline u_int txpkt_eo_len16(u_int, u_int, u_int);
339ffbb373cSNavdeep Parhar #endif
340786099deSNavdeep Parhar static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *,
341786099deSNavdeep Parhar     struct mbuf *);
342786099deSNavdeep Parhar #endif
343f7dfe243SNavdeep Parhar 
34482eff304SNavdeep Parhar static counter_u64_t extfree_refs;
34582eff304SNavdeep Parhar static counter_u64_t extfree_rels;
34682eff304SNavdeep Parhar 
347671bf2b8SNavdeep Parhar an_handler_t t4_an_handler;
348671bf2b8SNavdeep Parhar fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
349671bf2b8SNavdeep Parhar cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
3504535e804SNavdeep Parhar cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES];
3514535e804SNavdeep Parhar cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES];
352111638bfSNavdeep Parhar cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES];
35389f651e7SNavdeep Parhar cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES];
3549c707b32SNavdeep Parhar cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES];
355671bf2b8SNavdeep Parhar 
3564535e804SNavdeep Parhar void
357671bf2b8SNavdeep Parhar t4_register_an_handler(an_handler_t h)
358671bf2b8SNavdeep Parhar {
3594535e804SNavdeep Parhar 	uintptr_t *loc;
360671bf2b8SNavdeep Parhar 
3614535e804SNavdeep Parhar 	MPASS(h == NULL || t4_an_handler == NULL);
3624535e804SNavdeep Parhar 
363671bf2b8SNavdeep Parhar 	loc = (uintptr_t *)&t4_an_handler;
3644535e804SNavdeep Parhar 	atomic_store_rel_ptr(loc, (uintptr_t)h);
365671bf2b8SNavdeep Parhar }
366671bf2b8SNavdeep Parhar 
3674535e804SNavdeep Parhar void
368671bf2b8SNavdeep Parhar t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
369671bf2b8SNavdeep Parhar {
3704535e804SNavdeep Parhar 	uintptr_t *loc;
371671bf2b8SNavdeep Parhar 
3724535e804SNavdeep Parhar 	MPASS(type < nitems(t4_fw_msg_handler));
3734535e804SNavdeep Parhar 	MPASS(h == NULL || t4_fw_msg_handler[type] == NULL);
374671bf2b8SNavdeep Parhar 	/*
375671bf2b8SNavdeep Parhar 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
376671bf2b8SNavdeep Parhar 	 * handler dispatch table.  Reject any attempt to install a handler for
377671bf2b8SNavdeep Parhar 	 * this subtype.
378671bf2b8SNavdeep Parhar 	 */
3794535e804SNavdeep Parhar 	MPASS(type != FW_TYPE_RSSCPL);
3804535e804SNavdeep Parhar 	MPASS(type != FW6_TYPE_RSSCPL);
381671bf2b8SNavdeep Parhar 
382671bf2b8SNavdeep Parhar 	loc = (uintptr_t *)&t4_fw_msg_handler[type];
3834535e804SNavdeep Parhar 	atomic_store_rel_ptr(loc, (uintptr_t)h);
3844535e804SNavdeep Parhar }
385671bf2b8SNavdeep Parhar 
3864535e804SNavdeep Parhar void
3874535e804SNavdeep Parhar t4_register_cpl_handler(int opcode, cpl_handler_t h)
3884535e804SNavdeep Parhar {
3894535e804SNavdeep Parhar 	uintptr_t *loc;
3904535e804SNavdeep Parhar 
3914535e804SNavdeep Parhar 	MPASS(opcode < nitems(t4_cpl_handler));
3924535e804SNavdeep Parhar 	MPASS(h == NULL || t4_cpl_handler[opcode] == NULL);
3934535e804SNavdeep Parhar 
3944535e804SNavdeep Parhar 	loc = (uintptr_t *)&t4_cpl_handler[opcode];
3954535e804SNavdeep Parhar 	atomic_store_rel_ptr(loc, (uintptr_t)h);
396671bf2b8SNavdeep Parhar }
397671bf2b8SNavdeep Parhar 
398671bf2b8SNavdeep Parhar static int
3994535e804SNavdeep Parhar set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
4004535e804SNavdeep Parhar     struct mbuf *m)
401671bf2b8SNavdeep Parhar {
4024535e804SNavdeep Parhar 	const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
4034535e804SNavdeep Parhar 	u_int tid;
4044535e804SNavdeep Parhar 	int cookie;
405671bf2b8SNavdeep Parhar 
4064535e804SNavdeep Parhar 	MPASS(m == NULL);
4074535e804SNavdeep Parhar 
4084535e804SNavdeep Parhar 	tid = GET_TID(cpl);
4095fc0f72fSNavdeep Parhar 	if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) {
4104535e804SNavdeep Parhar 		/*
4114535e804SNavdeep Parhar 		 * The return code for filter-write is put in the CPL cookie so
4124535e804SNavdeep Parhar 		 * we have to rely on the hardware tid (is_ftid) to determine
4134535e804SNavdeep Parhar 		 * that this is a response to a filter.
4144535e804SNavdeep Parhar 		 */
4154535e804SNavdeep Parhar 		cookie = CPL_COOKIE_FILTER;
4164535e804SNavdeep Parhar 	} else {
4174535e804SNavdeep Parhar 		cookie = G_COOKIE(cpl->cookie);
4184535e804SNavdeep Parhar 	}
4194535e804SNavdeep Parhar 	MPASS(cookie > CPL_COOKIE_RESERVED);
4204535e804SNavdeep Parhar 	MPASS(cookie < nitems(set_tcb_rpl_handlers));
4214535e804SNavdeep Parhar 
4224535e804SNavdeep Parhar 	return (set_tcb_rpl_handlers[cookie](iq, rss, m));
423671bf2b8SNavdeep Parhar }
424671bf2b8SNavdeep Parhar 
4254535e804SNavdeep Parhar static int
4264535e804SNavdeep Parhar l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
4274535e804SNavdeep Parhar     struct mbuf *m)
428671bf2b8SNavdeep Parhar {
4294535e804SNavdeep Parhar 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
4304535e804SNavdeep Parhar 	unsigned int cookie;
431671bf2b8SNavdeep Parhar 
4324535e804SNavdeep Parhar 	MPASS(m == NULL);
433671bf2b8SNavdeep Parhar 
4344535e804SNavdeep Parhar 	cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER;
4354535e804SNavdeep Parhar 	return (l2t_write_rpl_handlers[cookie](iq, rss, m));
4364535e804SNavdeep Parhar }
437671bf2b8SNavdeep Parhar 
438111638bfSNavdeep Parhar static int
439111638bfSNavdeep Parhar act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
440111638bfSNavdeep Parhar     struct mbuf *m)
441111638bfSNavdeep Parhar {
442111638bfSNavdeep Parhar 	const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
443111638bfSNavdeep Parhar 	u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status)));
444111638bfSNavdeep Parhar 
445111638bfSNavdeep Parhar 	MPASS(m == NULL);
446111638bfSNavdeep Parhar 	MPASS(cookie != CPL_COOKIE_RESERVED);
447111638bfSNavdeep Parhar 
448111638bfSNavdeep Parhar 	return (act_open_rpl_handlers[cookie](iq, rss, m));
449111638bfSNavdeep Parhar }
450111638bfSNavdeep Parhar 
45189f651e7SNavdeep Parhar static int
45289f651e7SNavdeep Parhar abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss,
45389f651e7SNavdeep Parhar     struct mbuf *m)
45489f651e7SNavdeep Parhar {
45589f651e7SNavdeep Parhar 	struct adapter *sc = iq->adapter;
45689f651e7SNavdeep Parhar 	u_int cookie;
45789f651e7SNavdeep Parhar 
45889f651e7SNavdeep Parhar 	MPASS(m == NULL);
45989f651e7SNavdeep Parhar 	if (is_hashfilter(sc))
46089f651e7SNavdeep Parhar 		cookie = CPL_COOKIE_HASHFILTER;
46189f651e7SNavdeep Parhar 	else
46289f651e7SNavdeep Parhar 		cookie = CPL_COOKIE_TOM;
46389f651e7SNavdeep Parhar 
46489f651e7SNavdeep Parhar 	return (abort_rpl_rss_handlers[cookie](iq, rss, m));
46589f651e7SNavdeep Parhar }
46689f651e7SNavdeep Parhar 
4679c707b32SNavdeep Parhar static int
4689c707b32SNavdeep Parhar fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4699c707b32SNavdeep Parhar {
4709c707b32SNavdeep Parhar 	struct adapter *sc = iq->adapter;
4719c707b32SNavdeep Parhar 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
4729c707b32SNavdeep Parhar 	unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
4739c707b32SNavdeep Parhar 	u_int cookie;
4749c707b32SNavdeep Parhar 
4759c707b32SNavdeep Parhar 	MPASS(m == NULL);
4769c707b32SNavdeep Parhar 	if (is_etid(sc, tid))
4779c707b32SNavdeep Parhar 		cookie = CPL_COOKIE_ETHOFLD;
4789c707b32SNavdeep Parhar 	else
4799c707b32SNavdeep Parhar 		cookie = CPL_COOKIE_TOM;
4809c707b32SNavdeep Parhar 
4819c707b32SNavdeep Parhar 	return (fw4_ack_handlers[cookie](iq, rss, m));
4829c707b32SNavdeep Parhar }
4839c707b32SNavdeep Parhar 
4844535e804SNavdeep Parhar static void
4854535e804SNavdeep Parhar t4_init_shared_cpl_handlers(void)
4864535e804SNavdeep Parhar {
4874535e804SNavdeep Parhar 
4884535e804SNavdeep Parhar 	t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler);
4894535e804SNavdeep Parhar 	t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler);
490111638bfSNavdeep Parhar 	t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler);
49189f651e7SNavdeep Parhar 	t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler);
4929c707b32SNavdeep Parhar 	t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler);
4934535e804SNavdeep Parhar }
4944535e804SNavdeep Parhar 
4954535e804SNavdeep Parhar void
4964535e804SNavdeep Parhar t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
4974535e804SNavdeep Parhar {
4984535e804SNavdeep Parhar 	uintptr_t *loc;
4994535e804SNavdeep Parhar 
5004535e804SNavdeep Parhar 	MPASS(opcode < nitems(t4_cpl_handler));
5014535e804SNavdeep Parhar 	MPASS(cookie > CPL_COOKIE_RESERVED);
5024535e804SNavdeep Parhar 	MPASS(cookie < NUM_CPL_COOKIES);
5034535e804SNavdeep Parhar 	MPASS(t4_cpl_handler[opcode] != NULL);
5044535e804SNavdeep Parhar 
5054535e804SNavdeep Parhar 	switch (opcode) {
5064535e804SNavdeep Parhar 	case CPL_SET_TCB_RPL:
5074535e804SNavdeep Parhar 		loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie];
5084535e804SNavdeep Parhar 		break;
5094535e804SNavdeep Parhar 	case CPL_L2T_WRITE_RPL:
5104535e804SNavdeep Parhar 		loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie];
5114535e804SNavdeep Parhar 		break;
512111638bfSNavdeep Parhar 	case CPL_ACT_OPEN_RPL:
513111638bfSNavdeep Parhar 		loc = (uintptr_t *)&act_open_rpl_handlers[cookie];
514111638bfSNavdeep Parhar 		break;
51589f651e7SNavdeep Parhar 	case CPL_ABORT_RPL_RSS:
51689f651e7SNavdeep Parhar 		loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie];
51789f651e7SNavdeep Parhar 		break;
5189c707b32SNavdeep Parhar 	case CPL_FW4_ACK:
5199c707b32SNavdeep Parhar 		loc = (uintptr_t *)&fw4_ack_handlers[cookie];
5209c707b32SNavdeep Parhar 		break;
5214535e804SNavdeep Parhar 	default:
5224535e804SNavdeep Parhar 		MPASS(0);
5234535e804SNavdeep Parhar 		return;
5244535e804SNavdeep Parhar 	}
5254535e804SNavdeep Parhar 	MPASS(h == NULL || *loc == (uintptr_t)NULL);
5264535e804SNavdeep Parhar 	atomic_store_rel_ptr(loc, (uintptr_t)h);
527671bf2b8SNavdeep Parhar }
528671bf2b8SNavdeep Parhar 
52994586193SNavdeep Parhar /*
5301458bff9SNavdeep Parhar  * Called on MOD_LOAD.  Validates and calculates the SGE tunables.
53194586193SNavdeep Parhar  */
53294586193SNavdeep Parhar void
53394586193SNavdeep Parhar t4_sge_modload(void)
53494586193SNavdeep Parhar {
5354defc81bSNavdeep Parhar 
5369fb8886bSNavdeep Parhar 	if (fl_pktshift < 0 || fl_pktshift > 7) {
5379fb8886bSNavdeep Parhar 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
538518bca2cSNavdeep Parhar 		    " using 0 instead.\n", fl_pktshift);
539518bca2cSNavdeep Parhar 		fl_pktshift = 0;
5409fb8886bSNavdeep Parhar 	}
5419fb8886bSNavdeep Parhar 
5429fb8886bSNavdeep Parhar 	if (spg_len != 64 && spg_len != 128) {
5439fb8886bSNavdeep Parhar 		int len;
5449fb8886bSNavdeep Parhar 
5459fb8886bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__)
5469fb8886bSNavdeep Parhar 		len = cpu_clflush_line_size > 64 ? 128 : 64;
5479fb8886bSNavdeep Parhar #else
5489fb8886bSNavdeep Parhar 		len = 64;
5499fb8886bSNavdeep Parhar #endif
5509fb8886bSNavdeep Parhar 		if (spg_len != -1) {
5519fb8886bSNavdeep Parhar 			printf("Invalid hw.cxgbe.spg_len value (%d),"
5529fb8886bSNavdeep Parhar 			    " using %d instead.\n", spg_len, len);
5539fb8886bSNavdeep Parhar 		}
5549fb8886bSNavdeep Parhar 		spg_len = len;
5559fb8886bSNavdeep Parhar 	}
5569fb8886bSNavdeep Parhar 
5579fb8886bSNavdeep Parhar 	if (cong_drop < -1 || cong_drop > 1) {
5589fb8886bSNavdeep Parhar 		printf("Invalid hw.cxgbe.cong_drop value (%d),"
5599fb8886bSNavdeep Parhar 		    " using 0 instead.\n", cong_drop);
5609fb8886bSNavdeep Parhar 		cong_drop = 0;
5619fb8886bSNavdeep Parhar 	}
56282eff304SNavdeep Parhar 
563d491f8caSNavdeep Parhar 	if (tscale != 1 && (tscale < 3 || tscale > 17)) {
564d491f8caSNavdeep Parhar 		printf("Invalid hw.cxgbe.tscale value (%d),"
565d491f8caSNavdeep Parhar 		    " using 1 instead.\n", tscale);
566d491f8caSNavdeep Parhar 		tscale = 1;
567d491f8caSNavdeep Parhar 	}
568d491f8caSNavdeep Parhar 
5697676c62aSNavdeep Parhar 	if (largest_rx_cluster != MCLBYTES &&
5707676c62aSNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES
5717676c62aSNavdeep Parhar 	    largest_rx_cluster != MJUMPAGESIZE &&
5727676c62aSNavdeep Parhar #endif
5737676c62aSNavdeep Parhar 	    largest_rx_cluster != MJUM9BYTES &&
5747676c62aSNavdeep Parhar 	    largest_rx_cluster != MJUM16BYTES) {
5757676c62aSNavdeep Parhar 		printf("Invalid hw.cxgbe.largest_rx_cluster value (%d),"
5767676c62aSNavdeep Parhar 		    " using %d instead.\n", largest_rx_cluster, MJUM16BYTES);
5777676c62aSNavdeep Parhar 		largest_rx_cluster = MJUM16BYTES;
5787676c62aSNavdeep Parhar 	}
5797676c62aSNavdeep Parhar 
5807676c62aSNavdeep Parhar 	if (safest_rx_cluster != MCLBYTES &&
5817676c62aSNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES
5827676c62aSNavdeep Parhar 	    safest_rx_cluster != MJUMPAGESIZE &&
5837676c62aSNavdeep Parhar #endif
5847676c62aSNavdeep Parhar 	    safest_rx_cluster != MJUM9BYTES &&
5857676c62aSNavdeep Parhar 	    safest_rx_cluster != MJUM16BYTES) {
5867676c62aSNavdeep Parhar 		printf("Invalid hw.cxgbe.safest_rx_cluster value (%d),"
5877676c62aSNavdeep Parhar 		    " using %d instead.\n", safest_rx_cluster, MJUMPAGESIZE);
5887676c62aSNavdeep Parhar 		safest_rx_cluster = MJUMPAGESIZE;
5897676c62aSNavdeep Parhar 	}
5907676c62aSNavdeep Parhar 
59182eff304SNavdeep Parhar 	extfree_refs = counter_u64_alloc(M_WAITOK);
59282eff304SNavdeep Parhar 	extfree_rels = counter_u64_alloc(M_WAITOK);
5937054f6ecSNavdeep Parhar 	pullups = counter_u64_alloc(M_WAITOK);
5947054f6ecSNavdeep Parhar 	defrags = counter_u64_alloc(M_WAITOK);
59582eff304SNavdeep Parhar 	counter_u64_zero(extfree_refs);
59682eff304SNavdeep Parhar 	counter_u64_zero(extfree_rels);
5977054f6ecSNavdeep Parhar 	counter_u64_zero(pullups);
5987054f6ecSNavdeep Parhar 	counter_u64_zero(defrags);
599671bf2b8SNavdeep Parhar 
6004535e804SNavdeep Parhar 	t4_init_shared_cpl_handlers();
601671bf2b8SNavdeep Parhar 	t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
602671bf2b8SNavdeep Parhar 	t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
603671bf2b8SNavdeep Parhar 	t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
604786099deSNavdeep Parhar #ifdef RATELIMIT
605786099deSNavdeep Parhar 	t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack,
606786099deSNavdeep Parhar 	    CPL_COOKIE_ETHOFLD);
607786099deSNavdeep Parhar #endif
608671bf2b8SNavdeep Parhar 	t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
609069af0ebSJohn Baldwin 	t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl);
61082eff304SNavdeep Parhar }
61182eff304SNavdeep Parhar 
61282eff304SNavdeep Parhar void
61382eff304SNavdeep Parhar t4_sge_modunload(void)
61482eff304SNavdeep Parhar {
61582eff304SNavdeep Parhar 
61682eff304SNavdeep Parhar 	counter_u64_free(extfree_refs);
61782eff304SNavdeep Parhar 	counter_u64_free(extfree_rels);
6187054f6ecSNavdeep Parhar 	counter_u64_free(pullups);
6197054f6ecSNavdeep Parhar 	counter_u64_free(defrags);
62082eff304SNavdeep Parhar }
62182eff304SNavdeep Parhar 
62282eff304SNavdeep Parhar uint64_t
62382eff304SNavdeep Parhar t4_sge_extfree_refs(void)
62482eff304SNavdeep Parhar {
62582eff304SNavdeep Parhar 	uint64_t refs, rels;
62682eff304SNavdeep Parhar 
62782eff304SNavdeep Parhar 	rels = counter_u64_fetch(extfree_rels);
62882eff304SNavdeep Parhar 	refs = counter_u64_fetch(extfree_refs);
62982eff304SNavdeep Parhar 
63082eff304SNavdeep Parhar 	return (refs - rels);
63194586193SNavdeep Parhar }
63294586193SNavdeep Parhar 
63344c6fea8SNavdeep Parhar /* max 4096 */
63444c6fea8SNavdeep Parhar #define MAX_PACK_BOUNDARY 512
63544c6fea8SNavdeep Parhar 
636e3207e19SNavdeep Parhar static inline void
637e3207e19SNavdeep Parhar setup_pad_and_pack_boundaries(struct adapter *sc)
638e3207e19SNavdeep Parhar {
639e3207e19SNavdeep Parhar 	uint32_t v, m;
6400dbc6cfdSNavdeep Parhar 	int pad, pack, pad_shift;
641e3207e19SNavdeep Parhar 
6420dbc6cfdSNavdeep Parhar 	pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT :
6430dbc6cfdSNavdeep Parhar 	    X_INGPADBOUNDARY_SHIFT;
644e3207e19SNavdeep Parhar 	pad = fl_pad;
6450dbc6cfdSNavdeep Parhar 	if (fl_pad < (1 << pad_shift) ||
6460dbc6cfdSNavdeep Parhar 	    fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) ||
6470dbc6cfdSNavdeep Parhar 	    !powerof2(fl_pad)) {
648e3207e19SNavdeep Parhar 		/*
649e3207e19SNavdeep Parhar 		 * If there is any chance that we might use buffer packing and
650e3207e19SNavdeep Parhar 		 * the chip is a T4, then pick 64 as the pad/pack boundary.  Set
6510dbc6cfdSNavdeep Parhar 		 * it to the minimum allowed in all other cases.
652e3207e19SNavdeep Parhar 		 */
6530dbc6cfdSNavdeep Parhar 		pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift;
654e3207e19SNavdeep Parhar 
655e3207e19SNavdeep Parhar 		/*
656e3207e19SNavdeep Parhar 		 * For fl_pad = 0 we'll still write a reasonable value to the
657e3207e19SNavdeep Parhar 		 * register but all the freelists will opt out of padding.
658e3207e19SNavdeep Parhar 		 * We'll complain here only if the user tried to set it to a
659e3207e19SNavdeep Parhar 		 * value greater than 0 that was invalid.
660e3207e19SNavdeep Parhar 		 */
661e3207e19SNavdeep Parhar 		if (fl_pad > 0) {
662e3207e19SNavdeep Parhar 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
663e3207e19SNavdeep Parhar 			    " (%d), using %d instead.\n", fl_pad, pad);
664e3207e19SNavdeep Parhar 		}
665e3207e19SNavdeep Parhar 	}
666e3207e19SNavdeep Parhar 	m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
6670dbc6cfdSNavdeep Parhar 	v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift);
668e3207e19SNavdeep Parhar 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
669e3207e19SNavdeep Parhar 
670e3207e19SNavdeep Parhar 	if (is_t4(sc)) {
671e3207e19SNavdeep Parhar 		if (fl_pack != -1 && fl_pack != pad) {
672e3207e19SNavdeep Parhar 			/* Complain but carry on. */
673e3207e19SNavdeep Parhar 			device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
674e3207e19SNavdeep Parhar 			    " using %d instead.\n", fl_pack, pad);
675e3207e19SNavdeep Parhar 		}
676e3207e19SNavdeep Parhar 		return;
677e3207e19SNavdeep Parhar 	}
678e3207e19SNavdeep Parhar 
679e3207e19SNavdeep Parhar 	pack = fl_pack;
680e3207e19SNavdeep Parhar 	if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
681e3207e19SNavdeep Parhar 	    !powerof2(fl_pack)) {
68244c6fea8SNavdeep Parhar 		if (sc->params.pci.mps > MAX_PACK_BOUNDARY)
68344c6fea8SNavdeep Parhar 			pack = MAX_PACK_BOUNDARY;
68444c6fea8SNavdeep Parhar 		else
685e3207e19SNavdeep Parhar 			pack = max(sc->params.pci.mps, CACHE_LINE_SIZE);
686e3207e19SNavdeep Parhar 		MPASS(powerof2(pack));
687e3207e19SNavdeep Parhar 		if (pack < 16)
688e3207e19SNavdeep Parhar 			pack = 16;
689e3207e19SNavdeep Parhar 		if (pack == 32)
690e3207e19SNavdeep Parhar 			pack = 64;
691e3207e19SNavdeep Parhar 		if (pack > 4096)
692e3207e19SNavdeep Parhar 			pack = 4096;
693e3207e19SNavdeep Parhar 		if (fl_pack != -1) {
694e3207e19SNavdeep Parhar 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value"
695e3207e19SNavdeep Parhar 			    " (%d), using %d instead.\n", fl_pack, pack);
696e3207e19SNavdeep Parhar 		}
697e3207e19SNavdeep Parhar 	}
698e3207e19SNavdeep Parhar 	m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
699e3207e19SNavdeep Parhar 	if (pack == 16)
700e3207e19SNavdeep Parhar 		v = V_INGPACKBOUNDARY(0);
701e3207e19SNavdeep Parhar 	else
702e3207e19SNavdeep Parhar 		v = V_INGPACKBOUNDARY(ilog2(pack) - 5);
703e3207e19SNavdeep Parhar 
704e3207e19SNavdeep Parhar 	MPASS(!is_t4(sc));	/* T4 doesn't have SGE_CONTROL2 */
705e3207e19SNavdeep Parhar 	t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
706e3207e19SNavdeep Parhar }
707e3207e19SNavdeep Parhar 
708cf738022SNavdeep Parhar /*
709cf738022SNavdeep Parhar  * adap->params.vpd.cclk must be set up before this is called.
710cf738022SNavdeep Parhar  */
711d14b0ac1SNavdeep Parhar void
712d14b0ac1SNavdeep Parhar t4_tweak_chip_settings(struct adapter *sc)
713d14b0ac1SNavdeep Parhar {
71446e1e307SNavdeep Parhar 	int i, reg;
715d14b0ac1SNavdeep Parhar 	uint32_t v, m;
716d14b0ac1SNavdeep Parhar 	int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
717cf738022SNavdeep Parhar 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
718d14b0ac1SNavdeep Parhar 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
719d14b0ac1SNavdeep Parhar 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
72046e1e307SNavdeep Parhar 	static int sw_buf_sizes[] = {
7211458bff9SNavdeep Parhar 		MCLBYTES,
7221458bff9SNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES
7231458bff9SNavdeep Parhar 		MJUMPAGESIZE,
7241458bff9SNavdeep Parhar #endif
7251458bff9SNavdeep Parhar 		MJUM9BYTES,
72646e1e307SNavdeep Parhar 		MJUM16BYTES
7271458bff9SNavdeep Parhar 	};
728d14b0ac1SNavdeep Parhar 
729d14b0ac1SNavdeep Parhar 	KASSERT(sc->flags & MASTER_PF,
730d14b0ac1SNavdeep Parhar 	    ("%s: trying to change chip settings when not master.", __func__));
731d14b0ac1SNavdeep Parhar 
7321458bff9SNavdeep Parhar 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
733d14b0ac1SNavdeep Parhar 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
7344defc81bSNavdeep Parhar 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
735d14b0ac1SNavdeep Parhar 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
73654e4ee71SNavdeep Parhar 
737e3207e19SNavdeep Parhar 	setup_pad_and_pack_boundaries(sc);
7381458bff9SNavdeep Parhar 
739d14b0ac1SNavdeep Parhar 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
740733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
741733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
742733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
743733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
744733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
745733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
746733b9277SNavdeep Parhar 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
747d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
748733b9277SNavdeep Parhar 
7499b11a65dSNavdeep Parhar 	t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096);
7509b11a65dSNavdeep Parhar 	t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536);
75146e1e307SNavdeep Parhar 	reg = A_SGE_FL_BUFFER_SIZE2;
75246e1e307SNavdeep Parhar 	for (i = 0; i < nitems(sw_buf_sizes); i++) {
75346e1e307SNavdeep Parhar 		MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
75446e1e307SNavdeep Parhar 		t4_write_reg(sc, reg, sw_buf_sizes[i]);
75546e1e307SNavdeep Parhar 		reg += 4;
75646e1e307SNavdeep Parhar 		MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
75746e1e307SNavdeep Parhar 		t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE);
75846e1e307SNavdeep Parhar 		reg += 4;
75954e4ee71SNavdeep Parhar 	}
76054e4ee71SNavdeep Parhar 
761d14b0ac1SNavdeep Parhar 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
762d14b0ac1SNavdeep Parhar 	    V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
763d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
76454e4ee71SNavdeep Parhar 
765cf738022SNavdeep Parhar 	KASSERT(intr_timer[0] <= timer_max,
766cf738022SNavdeep Parhar 	    ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
767cf738022SNavdeep Parhar 	    timer_max));
768cf738022SNavdeep Parhar 	for (i = 1; i < nitems(intr_timer); i++) {
769cf738022SNavdeep Parhar 		KASSERT(intr_timer[i] >= intr_timer[i - 1],
770cf738022SNavdeep Parhar 		    ("%s: timers not listed in increasing order (%d)",
771cf738022SNavdeep Parhar 		    __func__, i));
772cf738022SNavdeep Parhar 
773cf738022SNavdeep Parhar 		while (intr_timer[i] > timer_max) {
774cf738022SNavdeep Parhar 			if (i == nitems(intr_timer) - 1) {
775cf738022SNavdeep Parhar 				intr_timer[i] = timer_max;
776cf738022SNavdeep Parhar 				break;
777cf738022SNavdeep Parhar 			}
778cf738022SNavdeep Parhar 			intr_timer[i] += intr_timer[i - 1];
779cf738022SNavdeep Parhar 			intr_timer[i] /= 2;
780cf738022SNavdeep Parhar 		}
781cf738022SNavdeep Parhar 	}
782cf738022SNavdeep Parhar 
783d14b0ac1SNavdeep Parhar 	v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
784d14b0ac1SNavdeep Parhar 	    V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
785d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
786d14b0ac1SNavdeep Parhar 	v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
787d14b0ac1SNavdeep Parhar 	    V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
788d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
789d14b0ac1SNavdeep Parhar 	v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
790d14b0ac1SNavdeep Parhar 	    V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
791d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
79286e02bf2SNavdeep Parhar 
793d491f8caSNavdeep Parhar 	if (chip_id(sc) >= CHELSIO_T6) {
794d491f8caSNavdeep Parhar 		m = V_TSCALE(M_TSCALE);
795d491f8caSNavdeep Parhar 		if (tscale == 1)
796d491f8caSNavdeep Parhar 			v = 0;
797d491f8caSNavdeep Parhar 		else
798d491f8caSNavdeep Parhar 			v = V_TSCALE(tscale - 2);
799d491f8caSNavdeep Parhar 		t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v);
8002f318252SNavdeep Parhar 
8012f318252SNavdeep Parhar 		if (sc->debug_flags & DF_DISABLE_TCB_CACHE) {
8022f318252SNavdeep Parhar 			m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN |
8032f318252SNavdeep Parhar 			    V_WRTHRTHRESH(M_WRTHRTHRESH);
8042f318252SNavdeep Parhar 			t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1);
8052f318252SNavdeep Parhar 			v &= ~m;
8062f318252SNavdeep Parhar 			v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN |
8072f318252SNavdeep Parhar 			    V_WRTHRTHRESH(16);
8082f318252SNavdeep Parhar 			t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1);
8092f318252SNavdeep Parhar 		}
810d491f8caSNavdeep Parhar 	}
811d491f8caSNavdeep Parhar 
8127cba15b1SNavdeep Parhar 	/* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */
813d14b0ac1SNavdeep Parhar 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
814d14b0ac1SNavdeep Parhar 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
815d14b0ac1SNavdeep Parhar 
8167cba15b1SNavdeep Parhar 	/*
8177cba15b1SNavdeep Parhar 	 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP.  These have been
8187cba15b1SNavdeep Parhar 	 * chosen with MAXPHYS = 128K in mind.  The largest DDP buffer that we
8197cba15b1SNavdeep Parhar 	 * may have to deal with is MAXPHYS + 1 page.
8207cba15b1SNavdeep Parhar 	 */
8217cba15b1SNavdeep Parhar 	v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4);
8227cba15b1SNavdeep Parhar 	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v);
8237cba15b1SNavdeep Parhar 
8247cba15b1SNavdeep Parhar 	/* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
8257cba15b1SNavdeep Parhar 	m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
826d14b0ac1SNavdeep Parhar 	t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
827d14b0ac1SNavdeep Parhar 
828d14b0ac1SNavdeep Parhar 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
829d14b0ac1SNavdeep Parhar 	    F_RESETDDPOFFSET;
830d14b0ac1SNavdeep Parhar 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
831d14b0ac1SNavdeep Parhar 	t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
832d14b0ac1SNavdeep Parhar }
833d14b0ac1SNavdeep Parhar 
834d14b0ac1SNavdeep Parhar /*
83546e1e307SNavdeep Parhar  * SGE wants the buffer to be at least 64B and then a multiple of 16.  Its
83646e1e307SNavdeep Parhar  * address mut be 16B aligned.  If padding is in use the buffer's start and end
83746e1e307SNavdeep Parhar  * need to be aligned to the pad boundary as well.  We'll just make sure that
83846e1e307SNavdeep Parhar  * the size is a multiple of the pad boundary here, it is up to the buffer
83946e1e307SNavdeep Parhar  * allocation code to make sure the start of the buffer is aligned.
84038035ed6SNavdeep Parhar  */
84138035ed6SNavdeep Parhar static inline int
842e3207e19SNavdeep Parhar hwsz_ok(struct adapter *sc, int hwsz)
84338035ed6SNavdeep Parhar {
84490e7434aSNavdeep Parhar 	int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1;
84538035ed6SNavdeep Parhar 
846b741402cSNavdeep Parhar 	return (hwsz >= 64 && (hwsz & mask) == 0);
84738035ed6SNavdeep Parhar }
84838035ed6SNavdeep Parhar 
84938035ed6SNavdeep Parhar /*
850fae028ddSNavdeep Parhar  * Initialize the rx buffer sizes and figure out which zones the buffers will
851fae028ddSNavdeep Parhar  * be allocated from.
852d14b0ac1SNavdeep Parhar  */
853fae028ddSNavdeep Parhar void
854fae028ddSNavdeep Parhar t4_init_rx_buf_info(struct adapter *sc)
855d14b0ac1SNavdeep Parhar {
856d14b0ac1SNavdeep Parhar 	struct sge *s = &sc->sge;
85790e7434aSNavdeep Parhar 	struct sge_params *sp = &sc->params.sge;
858fae028ddSNavdeep Parhar 	int i, j, n;
85938035ed6SNavdeep Parhar 	static int sw_buf_sizes[] = {	/* Sorted by size */
8601458bff9SNavdeep Parhar 		MCLBYTES,
8611458bff9SNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES
8621458bff9SNavdeep Parhar 		MJUMPAGESIZE,
8631458bff9SNavdeep Parhar #endif
8641458bff9SNavdeep Parhar 		MJUM9BYTES,
8651458bff9SNavdeep Parhar 		MJUM16BYTES
8661458bff9SNavdeep Parhar 	};
86746e1e307SNavdeep Parhar 	struct rx_buf_info *rxb;
868d14b0ac1SNavdeep Parhar 
86946e1e307SNavdeep Parhar 	s->safe_zidx = -1;
87046e1e307SNavdeep Parhar 	rxb = &s->rx_buf_info[0];
87146e1e307SNavdeep Parhar 	for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
87246e1e307SNavdeep Parhar 		rxb->size1 = sw_buf_sizes[i];
87346e1e307SNavdeep Parhar 		rxb->zone = m_getzone(rxb->size1);
87446e1e307SNavdeep Parhar 		rxb->type = m_gettype(rxb->size1);
87546e1e307SNavdeep Parhar 		rxb->size2 = 0;
87646e1e307SNavdeep Parhar 		rxb->hwidx1 = -1;
87746e1e307SNavdeep Parhar 		rxb->hwidx2 = -1;
87846e1e307SNavdeep Parhar 		for (j = 0; j < SGE_FLBUF_SIZES; j++) {
87946e1e307SNavdeep Parhar 			int hwsize = sp->sge_fl_buffer_size[j];
88038035ed6SNavdeep Parhar 
88146e1e307SNavdeep Parhar 			if (!hwsz_ok(sc, hwsize))
882e3207e19SNavdeep Parhar 				continue;
883e3207e19SNavdeep Parhar 
88446e1e307SNavdeep Parhar 			/* hwidx for size1 */
88546e1e307SNavdeep Parhar 			if (rxb->hwidx1 == -1 && rxb->size1 == hwsize)
88646e1e307SNavdeep Parhar 				rxb->hwidx1 = j;
88738035ed6SNavdeep Parhar 
88846e1e307SNavdeep Parhar 			/* hwidx for size2 (buffer packing) */
88946e1e307SNavdeep Parhar 			if (rxb->size1 - CL_METADATA_SIZE < hwsize)
8901458bff9SNavdeep Parhar 				continue;
89146e1e307SNavdeep Parhar 			n = rxb->size1 - hwsize - CL_METADATA_SIZE;
8921458bff9SNavdeep Parhar 			if (n == 0) {
89346e1e307SNavdeep Parhar 				rxb->hwidx2 = j;
89446e1e307SNavdeep Parhar 				rxb->size2 = hwsize;
89546e1e307SNavdeep Parhar 				break;	/* stop looking */
896733b9277SNavdeep Parhar 			}
89746e1e307SNavdeep Parhar 			if (rxb->hwidx2 != -1) {
89846e1e307SNavdeep Parhar 				if (n < sp->sge_fl_buffer_size[rxb->hwidx2] -
89946e1e307SNavdeep Parhar 				    hwsize - CL_METADATA_SIZE) {
90046e1e307SNavdeep Parhar 					rxb->hwidx2 = j;
90146e1e307SNavdeep Parhar 					rxb->size2 = hwsize;
90246e1e307SNavdeep Parhar 				}
90346e1e307SNavdeep Parhar 			} else if (n <= 2 * CL_METADATA_SIZE) {
90446e1e307SNavdeep Parhar 				rxb->hwidx2 = j;
90546e1e307SNavdeep Parhar 				rxb->size2 = hwsize;
90638035ed6SNavdeep Parhar 			}
90738035ed6SNavdeep Parhar 		}
90846e1e307SNavdeep Parhar 		if (rxb->hwidx2 != -1)
90946e1e307SNavdeep Parhar 			sc->flags |= BUF_PACKING_OK;
91046e1e307SNavdeep Parhar 		if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster)
91146e1e307SNavdeep Parhar 			s->safe_zidx = i;
912e3207e19SNavdeep Parhar 	}
913fae028ddSNavdeep Parhar }
914fae028ddSNavdeep Parhar 
915fae028ddSNavdeep Parhar /*
916fae028ddSNavdeep Parhar  * Verify some basic SGE settings for the PF and VF driver, and other
917fae028ddSNavdeep Parhar  * miscellaneous settings for the PF driver.
918fae028ddSNavdeep Parhar  */
919fae028ddSNavdeep Parhar int
920fae028ddSNavdeep Parhar t4_verify_chip_settings(struct adapter *sc)
921fae028ddSNavdeep Parhar {
922fae028ddSNavdeep Parhar 	struct sge_params *sp = &sc->params.sge;
923fae028ddSNavdeep Parhar 	uint32_t m, v, r;
924fae028ddSNavdeep Parhar 	int rc = 0;
925fae028ddSNavdeep Parhar 	const uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
926fae028ddSNavdeep Parhar 
927fae028ddSNavdeep Parhar 	m = F_RXPKTCPLMODE;
928fae028ddSNavdeep Parhar 	v = F_RXPKTCPLMODE;
929fae028ddSNavdeep Parhar 	r = sp->sge_control;
930fae028ddSNavdeep Parhar 	if ((r & m) != v) {
931fae028ddSNavdeep Parhar 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
932fae028ddSNavdeep Parhar 		rc = EINVAL;
933fae028ddSNavdeep Parhar 	}
934fae028ddSNavdeep Parhar 
935fae028ddSNavdeep Parhar 	/*
936fae028ddSNavdeep Parhar 	 * If this changes then every single use of PAGE_SHIFT in the driver
937fae028ddSNavdeep Parhar 	 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift.
938fae028ddSNavdeep Parhar 	 */
939fae028ddSNavdeep Parhar 	if (sp->page_shift != PAGE_SHIFT) {
940fae028ddSNavdeep Parhar 		device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
941fae028ddSNavdeep Parhar 		rc = EINVAL;
942fae028ddSNavdeep Parhar 	}
943733b9277SNavdeep Parhar 
9446af45170SJohn Baldwin 	if (sc->flags & IS_VF)
9456af45170SJohn Baldwin 		return (0);
9466af45170SJohn Baldwin 
947d14b0ac1SNavdeep Parhar 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
948d14b0ac1SNavdeep Parhar 	r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
949d14b0ac1SNavdeep Parhar 	if (r != v) {
950d14b0ac1SNavdeep Parhar 		device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
951fae028ddSNavdeep Parhar 		if (sc->vres.ddp.size != 0)
952d14b0ac1SNavdeep Parhar 			rc = EINVAL;
953d14b0ac1SNavdeep Parhar 	}
954733b9277SNavdeep Parhar 
955d14b0ac1SNavdeep Parhar 	m = v = F_TDDPTAGTCB;
956d14b0ac1SNavdeep Parhar 	r = t4_read_reg(sc, A_ULP_RX_CTL);
957d14b0ac1SNavdeep Parhar 	if ((r & m) != v) {
958d14b0ac1SNavdeep Parhar 		device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
959fae028ddSNavdeep Parhar 		if (sc->vres.ddp.size != 0)
960d14b0ac1SNavdeep Parhar 			rc = EINVAL;
961d14b0ac1SNavdeep Parhar 	}
962d14b0ac1SNavdeep Parhar 
963d14b0ac1SNavdeep Parhar 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
964d14b0ac1SNavdeep Parhar 	    F_RESETDDPOFFSET;
965d14b0ac1SNavdeep Parhar 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
966d14b0ac1SNavdeep Parhar 	r = t4_read_reg(sc, A_TP_PARA_REG5);
967d14b0ac1SNavdeep Parhar 	if ((r & m) != v) {
968d14b0ac1SNavdeep Parhar 		device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
969fae028ddSNavdeep Parhar 		if (sc->vres.ddp.size != 0)
970d14b0ac1SNavdeep Parhar 			rc = EINVAL;
971d14b0ac1SNavdeep Parhar 	}
972d14b0ac1SNavdeep Parhar 
973733b9277SNavdeep Parhar 	return (rc);
97454e4ee71SNavdeep Parhar }
97554e4ee71SNavdeep Parhar 
97654e4ee71SNavdeep Parhar int
97754e4ee71SNavdeep Parhar t4_create_dma_tag(struct adapter *sc)
97854e4ee71SNavdeep Parhar {
97954e4ee71SNavdeep Parhar 	int rc;
98054e4ee71SNavdeep Parhar 
98154e4ee71SNavdeep Parhar 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
98254e4ee71SNavdeep Parhar 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
98354e4ee71SNavdeep Parhar 	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
98454e4ee71SNavdeep Parhar 	    NULL, &sc->dmat);
98554e4ee71SNavdeep Parhar 	if (rc != 0) {
98654e4ee71SNavdeep Parhar 		device_printf(sc->dev,
98754e4ee71SNavdeep Parhar 		    "failed to create main DMA tag: %d\n", rc);
98854e4ee71SNavdeep Parhar 	}
98954e4ee71SNavdeep Parhar 
99054e4ee71SNavdeep Parhar 	return (rc);
99154e4ee71SNavdeep Parhar }
99254e4ee71SNavdeep Parhar 
9936e22f9f3SNavdeep Parhar void
9946e22f9f3SNavdeep Parhar t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
9956e22f9f3SNavdeep Parhar     struct sysctl_oid_list *children)
9966e22f9f3SNavdeep Parhar {
99790e7434aSNavdeep Parhar 	struct sge_params *sp = &sc->params.sge;
9986e22f9f3SNavdeep Parhar 
99938035ed6SNavdeep Parhar 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes",
10008741306bSNavdeep Parhar 	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
10017029da5cSPawel Biernacki 	    sysctl_bufsizes, "A", "freelist buffer sizes");
100238035ed6SNavdeep Parhar 
10036e22f9f3SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD,
100490e7434aSNavdeep Parhar 	    NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)");
10056e22f9f3SNavdeep Parhar 
10066e22f9f3SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD,
100790e7434aSNavdeep Parhar 	    NULL, sp->pad_boundary, "payload pad boundary (bytes)");
10086e22f9f3SNavdeep Parhar 
10096e22f9f3SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD,
101090e7434aSNavdeep Parhar 	    NULL, sp->spg_len, "status page size (bytes)");
10116e22f9f3SNavdeep Parhar 
10126e22f9f3SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
10136e22f9f3SNavdeep Parhar 	    NULL, cong_drop, "congestion drop setting");
10141458bff9SNavdeep Parhar 
10151458bff9SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
101690e7434aSNavdeep Parhar 	    NULL, sp->pack_boundary, "payload pack boundary (bytes)");
10176e22f9f3SNavdeep Parhar }
10186e22f9f3SNavdeep Parhar 
101954e4ee71SNavdeep Parhar int
102054e4ee71SNavdeep Parhar t4_destroy_dma_tag(struct adapter *sc)
102154e4ee71SNavdeep Parhar {
102254e4ee71SNavdeep Parhar 	if (sc->dmat)
102354e4ee71SNavdeep Parhar 		bus_dma_tag_destroy(sc->dmat);
102454e4ee71SNavdeep Parhar 
102554e4ee71SNavdeep Parhar 	return (0);
102654e4ee71SNavdeep Parhar }
102754e4ee71SNavdeep Parhar 
102854e4ee71SNavdeep Parhar /*
102937310a98SNavdeep Parhar  * Allocate and initialize the firmware event queue, control queues, and special
103037310a98SNavdeep Parhar  * purpose rx queues owned by the adapter.
103154e4ee71SNavdeep Parhar  *
103254e4ee71SNavdeep Parhar  * Returns errno on failure.  Resources allocated up to that point may still be
103354e4ee71SNavdeep Parhar  * allocated.  Caller is responsible for cleanup in case this function fails.
103454e4ee71SNavdeep Parhar  */
103554e4ee71SNavdeep Parhar int
1036f7dfe243SNavdeep Parhar t4_setup_adapter_queues(struct adapter *sc)
103754e4ee71SNavdeep Parhar {
103837310a98SNavdeep Parhar 	int rc, i;
103954e4ee71SNavdeep Parhar 
104054e4ee71SNavdeep Parhar 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
104154e4ee71SNavdeep Parhar 
104256599263SNavdeep Parhar 	/*
104356599263SNavdeep Parhar 	 * Firmware event queue
104456599263SNavdeep Parhar 	 */
1045733b9277SNavdeep Parhar 	rc = alloc_fwq(sc);
1046aa95b653SNavdeep Parhar 	if (rc != 0)
1047f7dfe243SNavdeep Parhar 		return (rc);
1048f7dfe243SNavdeep Parhar 
1049f7dfe243SNavdeep Parhar 	/*
105037310a98SNavdeep Parhar 	 * That's all for the VF driver.
1051f7dfe243SNavdeep Parhar 	 */
105237310a98SNavdeep Parhar 	if (sc->flags & IS_VF)
105337310a98SNavdeep Parhar 		return (rc);
105437310a98SNavdeep Parhar 
105537310a98SNavdeep Parhar 	/*
105637310a98SNavdeep Parhar 	 * XXX: General purpose rx queues, one per port.
105737310a98SNavdeep Parhar 	 */
105837310a98SNavdeep Parhar 
105937310a98SNavdeep Parhar 	/*
106037310a98SNavdeep Parhar 	 * Control queues, one per port.
106137310a98SNavdeep Parhar 	 */
106237310a98SNavdeep Parhar 	for_each_port(sc, i) {
106343bbae19SNavdeep Parhar 		rc = alloc_ctrlq(sc, i);
106437310a98SNavdeep Parhar 		if (rc != 0)
106537310a98SNavdeep Parhar 			return (rc);
106637310a98SNavdeep Parhar 	}
106754e4ee71SNavdeep Parhar 
106854e4ee71SNavdeep Parhar 	return (rc);
106954e4ee71SNavdeep Parhar }
107054e4ee71SNavdeep Parhar 
107154e4ee71SNavdeep Parhar /*
107254e4ee71SNavdeep Parhar  * Idempotent
107354e4ee71SNavdeep Parhar  */
107454e4ee71SNavdeep Parhar int
1075f7dfe243SNavdeep Parhar t4_teardown_adapter_queues(struct adapter *sc)
107654e4ee71SNavdeep Parhar {
107737310a98SNavdeep Parhar 	int i;
107854e4ee71SNavdeep Parhar 
107954e4ee71SNavdeep Parhar 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
108054e4ee71SNavdeep Parhar 
1081b99651c5SNavdeep Parhar 	if (sc->sge.ctrlq != NULL) {
1082b99651c5SNavdeep Parhar 		MPASS(!(sc->flags & IS_VF));	/* VFs don't allocate ctrlq. */
108337310a98SNavdeep Parhar 		for_each_port(sc, i)
108443bbae19SNavdeep Parhar 			free_ctrlq(sc, i);
1085b8bfcb71SNavdeep Parhar 	}
1086733b9277SNavdeep Parhar 	free_fwq(sc);
108754e4ee71SNavdeep Parhar 
108854e4ee71SNavdeep Parhar 	return (0);
108954e4ee71SNavdeep Parhar }
109054e4ee71SNavdeep Parhar 
10916a59b994SNavdeep Parhar /* Maximum payload that could arrive with a single iq descriptor. */
10928340ece5SNavdeep Parhar static inline int
10936a59b994SNavdeep Parhar max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld)
10948340ece5SNavdeep Parhar {
10956a59b994SNavdeep Parhar 	int maxp;
10968340ece5SNavdeep Parhar 
109738035ed6SNavdeep Parhar 	/* large enough even when hw VLAN extraction is disabled */
10986a59b994SNavdeep Parhar 	maxp = sc->params.sge.fl_pktshift + ETHER_HDR_LEN +
10996a59b994SNavdeep Parhar 	    ETHER_VLAN_ENCAP_LEN + ifp->if_mtu;
11006a59b994SNavdeep Parhar 	if (ofld && sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS &&
11016a59b994SNavdeep Parhar 	    maxp < sc->params.tp.max_rx_pdu)
11026a59b994SNavdeep Parhar 		maxp = sc->params.tp.max_rx_pdu;
11036a59b994SNavdeep Parhar 	return (maxp);
110438035ed6SNavdeep Parhar }
11056eb3180fSNavdeep Parhar 
1106733b9277SNavdeep Parhar int
1107fe2ebb76SJohn Baldwin t4_setup_vi_queues(struct vi_info *vi)
1108733b9277SNavdeep Parhar {
110943bbae19SNavdeep Parhar 	int rc = 0, i, intr_idx;
1110733b9277SNavdeep Parhar 	struct sge_rxq *rxq;
1111733b9277SNavdeep Parhar 	struct sge_txq *txq;
111209fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
1113733b9277SNavdeep Parhar 	struct sge_ofld_rxq *ofld_rxq;
1114eff62dbaSNavdeep Parhar #endif
1115eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1116077ba6a8SJohn Baldwin 	struct sge_ofld_txq *ofld_txq;
1117298d969cSNavdeep Parhar #endif
1118298d969cSNavdeep Parhar #ifdef DEV_NETMAP
111943bbae19SNavdeep Parhar 	int saved_idx, iqidx;
1120298d969cSNavdeep Parhar 	struct sge_nm_rxq *nm_rxq;
1121298d969cSNavdeep Parhar 	struct sge_nm_txq *nm_txq;
1122733b9277SNavdeep Parhar #endif
112343bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
1124fe2ebb76SJohn Baldwin 	struct ifnet *ifp = vi->ifp;
11256a59b994SNavdeep Parhar 	int maxp;
1126733b9277SNavdeep Parhar 
1127733b9277SNavdeep Parhar 	/* Interrupt vector to start from (when using multiple vectors) */
1128f549e352SNavdeep Parhar 	intr_idx = vi->first_intr;
1129fe2ebb76SJohn Baldwin 
1130fe2ebb76SJohn Baldwin #ifdef DEV_NETMAP
113162291463SNavdeep Parhar 	saved_idx = intr_idx;
113262291463SNavdeep Parhar 	if (ifp->if_capabilities & IFCAP_NETMAP) {
113362291463SNavdeep Parhar 
113462291463SNavdeep Parhar 		/* netmap is supported with direct interrupts only. */
1135f549e352SNavdeep Parhar 		MPASS(!forwarding_intr_to_fwq(sc));
113643bbae19SNavdeep Parhar 		MPASS(vi->first_intr >= 0);
113762291463SNavdeep Parhar 
1138fe2ebb76SJohn Baldwin 		/*
1139fe2ebb76SJohn Baldwin 		 * We don't have buffers to back the netmap rx queues
1140fe2ebb76SJohn Baldwin 		 * right now so we create the queues in a way that
1141fe2ebb76SJohn Baldwin 		 * doesn't set off any congestion signal in the chip.
1142fe2ebb76SJohn Baldwin 		 */
1143fe2ebb76SJohn Baldwin 		for_each_nm_rxq(vi, i, nm_rxq) {
114443bbae19SNavdeep Parhar 			rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i);
1145fe2ebb76SJohn Baldwin 			if (rc != 0)
1146fe2ebb76SJohn Baldwin 				goto done;
1147fe2ebb76SJohn Baldwin 			intr_idx++;
1148fe2ebb76SJohn Baldwin 		}
1149fe2ebb76SJohn Baldwin 
1150fe2ebb76SJohn Baldwin 		for_each_nm_txq(vi, i, nm_txq) {
1151f549e352SNavdeep Parhar 			iqidx = vi->first_nm_rxq + (i % vi->nnmrxq);
115243bbae19SNavdeep Parhar 			rc = alloc_nm_txq(vi, nm_txq, iqidx, i);
1153fe2ebb76SJohn Baldwin 			if (rc != 0)
1154fe2ebb76SJohn Baldwin 				goto done;
1155fe2ebb76SJohn Baldwin 		}
1156fe2ebb76SJohn Baldwin 	}
115762291463SNavdeep Parhar 
115862291463SNavdeep Parhar 	/* Normal rx queues and netmap rx queues share the same interrupts. */
115962291463SNavdeep Parhar 	intr_idx = saved_idx;
1160fe2ebb76SJohn Baldwin #endif
1161733b9277SNavdeep Parhar 
1162733b9277SNavdeep Parhar 	/*
1163f549e352SNavdeep Parhar 	 * Allocate rx queues first because a default iqid is required when
1164f549e352SNavdeep Parhar 	 * creating a tx queue.
1165733b9277SNavdeep Parhar 	 */
11666a59b994SNavdeep Parhar 	maxp = max_rx_payload(sc, ifp, false);
1167fe2ebb76SJohn Baldwin 	for_each_rxq(vi, i, rxq) {
116843bbae19SNavdeep Parhar 		rc = alloc_rxq(vi, rxq, i, intr_idx, maxp);
116954e4ee71SNavdeep Parhar 		if (rc != 0)
117054e4ee71SNavdeep Parhar 			goto done;
117143bbae19SNavdeep Parhar 		if (!forwarding_intr_to_fwq(sc))
1172733b9277SNavdeep Parhar 			intr_idx++;
1173733b9277SNavdeep Parhar 	}
117462291463SNavdeep Parhar #ifdef DEV_NETMAP
117562291463SNavdeep Parhar 	if (ifp->if_capabilities & IFCAP_NETMAP)
117662291463SNavdeep Parhar 		intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq);
117762291463SNavdeep Parhar #endif
117809fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
11796a59b994SNavdeep Parhar 	maxp = max_rx_payload(sc, ifp, true);
1180fe2ebb76SJohn Baldwin 	for_each_ofld_rxq(vi, i, ofld_rxq) {
118143bbae19SNavdeep Parhar 		rc = alloc_ofld_rxq(vi, ofld_rxq, i, intr_idx, maxp);
1182733b9277SNavdeep Parhar 		if (rc != 0)
1183733b9277SNavdeep Parhar 			goto done;
118443bbae19SNavdeep Parhar 		if (!forwarding_intr_to_fwq(sc))
1185733b9277SNavdeep Parhar 			intr_idx++;
1186733b9277SNavdeep Parhar 	}
1187733b9277SNavdeep Parhar #endif
1188733b9277SNavdeep Parhar 
1189733b9277SNavdeep Parhar 	/*
1190f549e352SNavdeep Parhar 	 * Now the tx queues.
1191733b9277SNavdeep Parhar 	 */
1192fe2ebb76SJohn Baldwin 	for_each_txq(vi, i, txq) {
119343bbae19SNavdeep Parhar 		rc = alloc_txq(vi, txq, i);
119454e4ee71SNavdeep Parhar 		if (rc != 0)
119554e4ee71SNavdeep Parhar 			goto done;
119654e4ee71SNavdeep Parhar 	}
1197eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1198fe2ebb76SJohn Baldwin 	for_each_ofld_txq(vi, i, ofld_txq) {
119943bbae19SNavdeep Parhar 		rc = alloc_ofld_txq(vi, ofld_txq, i);
1200298d969cSNavdeep Parhar 		if (rc != 0)
1201298d969cSNavdeep Parhar 			goto done;
1202298d969cSNavdeep Parhar 	}
1203298d969cSNavdeep Parhar #endif
120454e4ee71SNavdeep Parhar done:
120554e4ee71SNavdeep Parhar 	if (rc)
1206fe2ebb76SJohn Baldwin 		t4_teardown_vi_queues(vi);
120754e4ee71SNavdeep Parhar 
120854e4ee71SNavdeep Parhar 	return (rc);
120954e4ee71SNavdeep Parhar }
121054e4ee71SNavdeep Parhar 
121154e4ee71SNavdeep Parhar /*
121254e4ee71SNavdeep Parhar  * Idempotent
121354e4ee71SNavdeep Parhar  */
121454e4ee71SNavdeep Parhar int
1215fe2ebb76SJohn Baldwin t4_teardown_vi_queues(struct vi_info *vi)
121654e4ee71SNavdeep Parhar {
121754e4ee71SNavdeep Parhar 	int i;
121854e4ee71SNavdeep Parhar 	struct sge_rxq *rxq;
121954e4ee71SNavdeep Parhar 	struct sge_txq *txq;
122037310a98SNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1221077ba6a8SJohn Baldwin 	struct sge_ofld_txq *ofld_txq;
122237310a98SNavdeep Parhar #endif
122309fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
1224733b9277SNavdeep Parhar 	struct sge_ofld_rxq *ofld_rxq;
1225eff62dbaSNavdeep Parhar #endif
1226298d969cSNavdeep Parhar #ifdef DEV_NETMAP
1227298d969cSNavdeep Parhar 	struct sge_nm_rxq *nm_rxq;
1228298d969cSNavdeep Parhar 	struct sge_nm_txq *nm_txq;
1229298d969cSNavdeep Parhar #endif
123054e4ee71SNavdeep Parhar 
1231fe2ebb76SJohn Baldwin #ifdef DEV_NETMAP
123262291463SNavdeep Parhar 	if (vi->ifp->if_capabilities & IFCAP_NETMAP) {
1233fe2ebb76SJohn Baldwin 		for_each_nm_txq(vi, i, nm_txq) {
1234fe2ebb76SJohn Baldwin 			free_nm_txq(vi, nm_txq);
1235fe2ebb76SJohn Baldwin 		}
1236fe2ebb76SJohn Baldwin 
1237fe2ebb76SJohn Baldwin 		for_each_nm_rxq(vi, i, nm_rxq) {
1238fe2ebb76SJohn Baldwin 			free_nm_rxq(vi, nm_rxq);
1239fe2ebb76SJohn Baldwin 		}
1240fe2ebb76SJohn Baldwin 	}
1241fe2ebb76SJohn Baldwin #endif
1242fe2ebb76SJohn Baldwin 
1243733b9277SNavdeep Parhar 	/*
1244733b9277SNavdeep Parhar 	 * Take down all the tx queues first, as they reference the rx queues
1245733b9277SNavdeep Parhar 	 * (for egress updates, etc.).
1246733b9277SNavdeep Parhar 	 */
1247733b9277SNavdeep Parhar 
1248fe2ebb76SJohn Baldwin 	for_each_txq(vi, i, txq) {
1249fe2ebb76SJohn Baldwin 		free_txq(vi, txq);
125054e4ee71SNavdeep Parhar 	}
1251eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1252fe2ebb76SJohn Baldwin 	for_each_ofld_txq(vi, i, ofld_txq) {
1253077ba6a8SJohn Baldwin 		free_ofld_txq(vi, ofld_txq);
1254733b9277SNavdeep Parhar 	}
1255733b9277SNavdeep Parhar #endif
1256733b9277SNavdeep Parhar 
1257733b9277SNavdeep Parhar 	/*
1258f549e352SNavdeep Parhar 	 * Then take down the rx queues.
1259733b9277SNavdeep Parhar 	 */
1260733b9277SNavdeep Parhar 
1261fe2ebb76SJohn Baldwin 	for_each_rxq(vi, i, rxq) {
1262fe2ebb76SJohn Baldwin 		free_rxq(vi, rxq);
126354e4ee71SNavdeep Parhar 	}
126409fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
1265fe2ebb76SJohn Baldwin 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1266fe2ebb76SJohn Baldwin 		free_ofld_rxq(vi, ofld_rxq);
1267733b9277SNavdeep Parhar 	}
1268733b9277SNavdeep Parhar #endif
1269733b9277SNavdeep Parhar 
127054e4ee71SNavdeep Parhar 	return (0);
127154e4ee71SNavdeep Parhar }
127254e4ee71SNavdeep Parhar 
1273733b9277SNavdeep Parhar /*
12743098bcfcSNavdeep Parhar  * Interrupt handler when the driver is using only 1 interrupt.  This is a very
12753098bcfcSNavdeep Parhar  * unusual scenario.
12763098bcfcSNavdeep Parhar  *
12773098bcfcSNavdeep Parhar  * a) Deals with errors, if any.
12783098bcfcSNavdeep Parhar  * b) Services firmware event queue, which is taking interrupts for all other
12793098bcfcSNavdeep Parhar  *    queues.
1280733b9277SNavdeep Parhar  */
128154e4ee71SNavdeep Parhar void
128254e4ee71SNavdeep Parhar t4_intr_all(void *arg)
128354e4ee71SNavdeep Parhar {
128454e4ee71SNavdeep Parhar 	struct adapter *sc = arg;
1285733b9277SNavdeep Parhar 	struct sge_iq *fwq = &sc->sge.fwq;
128654e4ee71SNavdeep Parhar 
12873098bcfcSNavdeep Parhar 	MPASS(sc->intr_count == 1);
12883098bcfcSNavdeep Parhar 
12891dca7005SNavdeep Parhar 	if (sc->intr_type == INTR_INTX)
12901dca7005SNavdeep Parhar 		t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
12911dca7005SNavdeep Parhar 
129254e4ee71SNavdeep Parhar 	t4_intr_err(arg);
12933098bcfcSNavdeep Parhar 	t4_intr_evt(fwq);
129454e4ee71SNavdeep Parhar }
129554e4ee71SNavdeep Parhar 
12963098bcfcSNavdeep Parhar /*
12973098bcfcSNavdeep Parhar  * Interrupt handler for errors (installed directly when multiple interrupts are
12983098bcfcSNavdeep Parhar  * being used, or called by t4_intr_all).
12993098bcfcSNavdeep Parhar  */
130054e4ee71SNavdeep Parhar void
130154e4ee71SNavdeep Parhar t4_intr_err(void *arg)
130254e4ee71SNavdeep Parhar {
130354e4ee71SNavdeep Parhar 	struct adapter *sc = arg;
1304dd3b96ecSNavdeep Parhar 	uint32_t v;
1305cb7c3f12SNavdeep Parhar 	const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
130654e4ee71SNavdeep Parhar 
1307cb7c3f12SNavdeep Parhar 	if (sc->flags & ADAP_ERR)
1308cb7c3f12SNavdeep Parhar 		return;
1309cb7c3f12SNavdeep Parhar 
1310dd3b96ecSNavdeep Parhar 	v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE));
1311dd3b96ecSNavdeep Parhar 	if (v & F_PFSW) {
1312dd3b96ecSNavdeep Parhar 		sc->swintr++;
1313dd3b96ecSNavdeep Parhar 		t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v);
1314dd3b96ecSNavdeep Parhar 	}
1315dd3b96ecSNavdeep Parhar 
1316cb7c3f12SNavdeep Parhar 	t4_slow_intr_handler(sc, verbose);
131754e4ee71SNavdeep Parhar }
131854e4ee71SNavdeep Parhar 
13193098bcfcSNavdeep Parhar /*
13203098bcfcSNavdeep Parhar  * Interrupt handler for iq-only queues.  The firmware event queue is the only
13213098bcfcSNavdeep Parhar  * such queue right now.
13223098bcfcSNavdeep Parhar  */
132354e4ee71SNavdeep Parhar void
132454e4ee71SNavdeep Parhar t4_intr_evt(void *arg)
132554e4ee71SNavdeep Parhar {
132654e4ee71SNavdeep Parhar 	struct sge_iq *iq = arg;
13272be67d29SNavdeep Parhar 
1328733b9277SNavdeep Parhar 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1329733b9277SNavdeep Parhar 		service_iq(iq, 0);
1330da6e3387SNavdeep Parhar 		(void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
13312be67d29SNavdeep Parhar 	}
13322be67d29SNavdeep Parhar }
13332be67d29SNavdeep Parhar 
13343098bcfcSNavdeep Parhar /*
13353098bcfcSNavdeep Parhar  * Interrupt handler for iq+fl queues.
13363098bcfcSNavdeep Parhar  */
1337733b9277SNavdeep Parhar void
1338733b9277SNavdeep Parhar t4_intr(void *arg)
13392be67d29SNavdeep Parhar {
13402be67d29SNavdeep Parhar 	struct sge_iq *iq = arg;
1341733b9277SNavdeep Parhar 
1342733b9277SNavdeep Parhar 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
13433098bcfcSNavdeep Parhar 		service_iq_fl(iq, 0);
1344da6e3387SNavdeep Parhar 		(void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1345733b9277SNavdeep Parhar 	}
1346733b9277SNavdeep Parhar }
1347733b9277SNavdeep Parhar 
13483098bcfcSNavdeep Parhar #ifdef DEV_NETMAP
13493098bcfcSNavdeep Parhar /*
13503098bcfcSNavdeep Parhar  * Interrupt handler for netmap rx queues.
13513098bcfcSNavdeep Parhar  */
13523098bcfcSNavdeep Parhar void
13533098bcfcSNavdeep Parhar t4_nm_intr(void *arg)
13543098bcfcSNavdeep Parhar {
13553098bcfcSNavdeep Parhar 	struct sge_nm_rxq *nm_rxq = arg;
13563098bcfcSNavdeep Parhar 
13573098bcfcSNavdeep Parhar 	if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
13583098bcfcSNavdeep Parhar 		service_nm_rxq(nm_rxq);
1359da6e3387SNavdeep Parhar 		(void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
13603098bcfcSNavdeep Parhar 	}
13613098bcfcSNavdeep Parhar }
13623098bcfcSNavdeep Parhar 
13633098bcfcSNavdeep Parhar /*
13643098bcfcSNavdeep Parhar  * Interrupt handler for vectors shared between NIC and netmap rx queues.
13653098bcfcSNavdeep Parhar  */
136662291463SNavdeep Parhar void
136762291463SNavdeep Parhar t4_vi_intr(void *arg)
136862291463SNavdeep Parhar {
136962291463SNavdeep Parhar 	struct irq *irq = arg;
137062291463SNavdeep Parhar 
13713098bcfcSNavdeep Parhar 	MPASS(irq->nm_rxq != NULL);
137262291463SNavdeep Parhar 	t4_nm_intr(irq->nm_rxq);
13733098bcfcSNavdeep Parhar 
13743098bcfcSNavdeep Parhar 	MPASS(irq->rxq != NULL);
137562291463SNavdeep Parhar 	t4_intr(irq->rxq);
137662291463SNavdeep Parhar }
13773098bcfcSNavdeep Parhar #endif
137846f48ee5SNavdeep Parhar 
1379733b9277SNavdeep Parhar /*
13803098bcfcSNavdeep Parhar  * Deals with interrupts on an iq-only (no freelist) queue.
1381733b9277SNavdeep Parhar  */
1382733b9277SNavdeep Parhar static int
1383733b9277SNavdeep Parhar service_iq(struct sge_iq *iq, int budget)
1384733b9277SNavdeep Parhar {
1385733b9277SNavdeep Parhar 	struct sge_iq *q;
138654e4ee71SNavdeep Parhar 	struct adapter *sc = iq->adapter;
1387b2daa9a9SNavdeep Parhar 	struct iq_desc *d = &iq->desc[iq->cidx];
13884d6db4e0SNavdeep Parhar 	int ndescs = 0, limit;
13893098bcfcSNavdeep Parhar 	int rsp_type;
1390733b9277SNavdeep Parhar 	uint32_t lq;
1391733b9277SNavdeep Parhar 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1392733b9277SNavdeep Parhar 
1393733b9277SNavdeep Parhar 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
13943098bcfcSNavdeep Parhar 	KASSERT((iq->flags & IQ_HAS_FL) == 0,
13953098bcfcSNavdeep Parhar 	    ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
13963098bcfcSNavdeep Parhar 	    iq->flags));
13973098bcfcSNavdeep Parhar 	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
13983098bcfcSNavdeep Parhar 	MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
1399733b9277SNavdeep Parhar 
14004d6db4e0SNavdeep Parhar 	limit = budget ? budget : iq->qsize / 16;
14014d6db4e0SNavdeep Parhar 
1402733b9277SNavdeep Parhar 	/*
1403733b9277SNavdeep Parhar 	 * We always come back and check the descriptor ring for new indirect
1404733b9277SNavdeep Parhar 	 * interrupts and other responses after running a single handler.
1405733b9277SNavdeep Parhar 	 */
1406733b9277SNavdeep Parhar 	for (;;) {
1407b2daa9a9SNavdeep Parhar 		while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
140854e4ee71SNavdeep Parhar 
140954e4ee71SNavdeep Parhar 			rmb();
141054e4ee71SNavdeep Parhar 
1411b2daa9a9SNavdeep Parhar 			rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1412b2daa9a9SNavdeep Parhar 			lq = be32toh(d->rsp.pldbuflen_qid);
141354e4ee71SNavdeep Parhar 
1414733b9277SNavdeep Parhar 			switch (rsp_type) {
1415733b9277SNavdeep Parhar 			case X_RSPD_TYPE_FLBUF:
14163098bcfcSNavdeep Parhar 				panic("%s: data for an iq (%p) with no freelist",
14173098bcfcSNavdeep Parhar 				    __func__, iq);
141854e4ee71SNavdeep Parhar 
14193098bcfcSNavdeep Parhar 				/* NOTREACHED */
1420733b9277SNavdeep Parhar 
1421733b9277SNavdeep Parhar 			case X_RSPD_TYPE_CPL:
1422b2daa9a9SNavdeep Parhar 				KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1423733b9277SNavdeep Parhar 				    ("%s: bad opcode %02x.", __func__,
1424b2daa9a9SNavdeep Parhar 				    d->rss.opcode));
14253098bcfcSNavdeep Parhar 				t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL);
1426733b9277SNavdeep Parhar 				break;
1427733b9277SNavdeep Parhar 
1428733b9277SNavdeep Parhar 			case X_RSPD_TYPE_INTR:
142998005176SNavdeep Parhar 				/*
143098005176SNavdeep Parhar 				 * There are 1K interrupt-capable queues (qids 0
143198005176SNavdeep Parhar 				 * through 1023).  A response type indicating a
143298005176SNavdeep Parhar 				 * forwarded interrupt with a qid >= 1K is an
143398005176SNavdeep Parhar 				 * iWARP async notification.
143498005176SNavdeep Parhar 				 */
14353098bcfcSNavdeep Parhar 				if (__predict_true(lq >= 1024)) {
1436671bf2b8SNavdeep Parhar 					t4_an_handler(iq, &d->rsp);
143798005176SNavdeep Parhar 					break;
143898005176SNavdeep Parhar 				}
143998005176SNavdeep Parhar 
1440ec55567cSJohn Baldwin 				q = sc->sge.iqmap[lq - sc->sge.iq_start -
1441ec55567cSJohn Baldwin 				    sc->sge.iq_base];
1442733b9277SNavdeep Parhar 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
1443733b9277SNavdeep Parhar 				    IQS_BUSY)) {
14443098bcfcSNavdeep Parhar 					if (service_iq_fl(q, q->qsize / 16) == 0) {
1445da6e3387SNavdeep Parhar 						(void) atomic_cmpset_int(&q->state,
1446733b9277SNavdeep Parhar 						    IQS_BUSY, IQS_IDLE);
1447733b9277SNavdeep Parhar 					} else {
1448733b9277SNavdeep Parhar 						STAILQ_INSERT_TAIL(&iql, q,
1449733b9277SNavdeep Parhar 						    link);
1450733b9277SNavdeep Parhar 					}
1451733b9277SNavdeep Parhar 				}
1452733b9277SNavdeep Parhar 				break;
1453733b9277SNavdeep Parhar 
1454733b9277SNavdeep Parhar 			default:
145598005176SNavdeep Parhar 				KASSERT(0,
145698005176SNavdeep Parhar 				    ("%s: illegal response type %d on iq %p",
145798005176SNavdeep Parhar 				    __func__, rsp_type, iq));
145898005176SNavdeep Parhar 				log(LOG_ERR,
145998005176SNavdeep Parhar 				    "%s: illegal response type %d on iq %p",
146098005176SNavdeep Parhar 				    device_get_nameunit(sc->dev), rsp_type, iq);
146109fe6320SNavdeep Parhar 				break;
146254e4ee71SNavdeep Parhar 			}
146356599263SNavdeep Parhar 
1464b2daa9a9SNavdeep Parhar 			d++;
1465b2daa9a9SNavdeep Parhar 			if (__predict_false(++iq->cidx == iq->sidx)) {
1466b2daa9a9SNavdeep Parhar 				iq->cidx = 0;
1467b2daa9a9SNavdeep Parhar 				iq->gen ^= F_RSPD_GEN;
1468b2daa9a9SNavdeep Parhar 				d = &iq->desc[0];
1469b2daa9a9SNavdeep Parhar 			}
1470b2daa9a9SNavdeep Parhar 			if (__predict_false(++ndescs == limit)) {
1471315048f2SJohn Baldwin 				t4_write_reg(sc, sc->sge_gts_reg,
1472733b9277SNavdeep Parhar 				    V_CIDXINC(ndescs) |
1473733b9277SNavdeep Parhar 				    V_INGRESSQID(iq->cntxt_id) |
1474733b9277SNavdeep Parhar 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1475733b9277SNavdeep Parhar 				ndescs = 0;
1476733b9277SNavdeep Parhar 
14773098bcfcSNavdeep Parhar 				if (budget) {
14783098bcfcSNavdeep Parhar 					return (EINPROGRESS);
14793098bcfcSNavdeep Parhar 				}
14803098bcfcSNavdeep Parhar 			}
14813098bcfcSNavdeep Parhar 		}
14823098bcfcSNavdeep Parhar 
14833098bcfcSNavdeep Parhar 		if (STAILQ_EMPTY(&iql))
14843098bcfcSNavdeep Parhar 			break;
14853098bcfcSNavdeep Parhar 
14863098bcfcSNavdeep Parhar 		/*
14873098bcfcSNavdeep Parhar 		 * Process the head only, and send it to the back of the list if
14883098bcfcSNavdeep Parhar 		 * it's still not done.
14893098bcfcSNavdeep Parhar 		 */
14903098bcfcSNavdeep Parhar 		q = STAILQ_FIRST(&iql);
14913098bcfcSNavdeep Parhar 		STAILQ_REMOVE_HEAD(&iql, link);
14923098bcfcSNavdeep Parhar 		if (service_iq_fl(q, q->qsize / 8) == 0)
1493da6e3387SNavdeep Parhar 			(void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
14943098bcfcSNavdeep Parhar 		else
14953098bcfcSNavdeep Parhar 			STAILQ_INSERT_TAIL(&iql, q, link);
14963098bcfcSNavdeep Parhar 	}
14973098bcfcSNavdeep Parhar 
14983098bcfcSNavdeep Parhar 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
14993098bcfcSNavdeep Parhar 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
15003098bcfcSNavdeep Parhar 
15013098bcfcSNavdeep Parhar 	return (0);
15023098bcfcSNavdeep Parhar }
15033098bcfcSNavdeep Parhar 
1504ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
15053098bcfcSNavdeep Parhar static inline int
15063098bcfcSNavdeep Parhar sort_before_lro(struct lro_ctrl *lro)
15073098bcfcSNavdeep Parhar {
15083098bcfcSNavdeep Parhar 
15093098bcfcSNavdeep Parhar 	return (lro->lro_mbuf_max != 0);
15103098bcfcSNavdeep Parhar }
1511ffbb373cSNavdeep Parhar #endif
15123098bcfcSNavdeep Parhar 
1513e7e08444SNavdeep Parhar static inline uint64_t
1514e7e08444SNavdeep Parhar last_flit_to_ns(struct adapter *sc, uint64_t lf)
1515e7e08444SNavdeep Parhar {
1516e7e08444SNavdeep Parhar 	uint64_t n = be64toh(lf) & 0xfffffffffffffff;	/* 60b, not 64b. */
1517e7e08444SNavdeep Parhar 
1518e7e08444SNavdeep Parhar 	if (n > UINT64_MAX / 1000000)
1519e7e08444SNavdeep Parhar 		return (n / sc->params.vpd.cclk * 1000000);
1520e7e08444SNavdeep Parhar 	else
1521e7e08444SNavdeep Parhar 		return (n * 1000000 / sc->params.vpd.cclk);
1522e7e08444SNavdeep Parhar }
1523e7e08444SNavdeep Parhar 
152446e1e307SNavdeep Parhar static inline void
152546e1e307SNavdeep Parhar move_to_next_rxbuf(struct sge_fl *fl)
152646e1e307SNavdeep Parhar {
152746e1e307SNavdeep Parhar 
152846e1e307SNavdeep Parhar 	fl->rx_offset = 0;
152946e1e307SNavdeep Parhar 	if (__predict_false((++fl->cidx & 7) == 0)) {
153046e1e307SNavdeep Parhar 		uint16_t cidx = fl->cidx >> 3;
153146e1e307SNavdeep Parhar 
153246e1e307SNavdeep Parhar 		if (__predict_false(cidx == fl->sidx))
153346e1e307SNavdeep Parhar 			fl->cidx = cidx = 0;
153446e1e307SNavdeep Parhar 		fl->hw_cidx = cidx;
153546e1e307SNavdeep Parhar 	}
153646e1e307SNavdeep Parhar }
153746e1e307SNavdeep Parhar 
15383098bcfcSNavdeep Parhar /*
15393098bcfcSNavdeep Parhar  * Deals with interrupts on an iq+fl queue.
15403098bcfcSNavdeep Parhar  */
15413098bcfcSNavdeep Parhar static int
15423098bcfcSNavdeep Parhar service_iq_fl(struct sge_iq *iq, int budget)
15433098bcfcSNavdeep Parhar {
15443098bcfcSNavdeep Parhar 	struct sge_rxq *rxq = iq_to_rxq(iq);
15453098bcfcSNavdeep Parhar 	struct sge_fl *fl;
15463098bcfcSNavdeep Parhar 	struct adapter *sc = iq->adapter;
15473098bcfcSNavdeep Parhar 	struct iq_desc *d = &iq->desc[iq->cidx];
154846e1e307SNavdeep Parhar 	int ndescs, limit;
154946e1e307SNavdeep Parhar 	int rsp_type, starved;
15503098bcfcSNavdeep Parhar 	uint32_t lq;
15513098bcfcSNavdeep Parhar 	uint16_t fl_hw_cidx;
15523098bcfcSNavdeep Parhar 	struct mbuf *m0;
15533098bcfcSNavdeep Parhar #if defined(INET) || defined(INET6)
15543098bcfcSNavdeep Parhar 	const struct timeval lro_timeout = {0, sc->lro_timeout};
15553098bcfcSNavdeep Parhar 	struct lro_ctrl *lro = &rxq->lro;
15563098bcfcSNavdeep Parhar #endif
15573098bcfcSNavdeep Parhar 
15583098bcfcSNavdeep Parhar 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
15593098bcfcSNavdeep Parhar 	MPASS(iq->flags & IQ_HAS_FL);
15603098bcfcSNavdeep Parhar 
156146e1e307SNavdeep Parhar 	ndescs = 0;
15623098bcfcSNavdeep Parhar #if defined(INET) || defined(INET6)
15633098bcfcSNavdeep Parhar 	if (iq->flags & IQ_ADJ_CREDIT) {
15643098bcfcSNavdeep Parhar 		MPASS(sort_before_lro(lro));
15653098bcfcSNavdeep Parhar 		iq->flags &= ~IQ_ADJ_CREDIT;
15663098bcfcSNavdeep Parhar 		if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
15673098bcfcSNavdeep Parhar 			tcp_lro_flush_all(lro);
15683098bcfcSNavdeep Parhar 			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
15693098bcfcSNavdeep Parhar 			    V_INGRESSQID((u32)iq->cntxt_id) |
15703098bcfcSNavdeep Parhar 			    V_SEINTARM(iq->intr_params));
15713098bcfcSNavdeep Parhar 			return (0);
15723098bcfcSNavdeep Parhar 		}
15733098bcfcSNavdeep Parhar 		ndescs = 1;
15743098bcfcSNavdeep Parhar 	}
15753098bcfcSNavdeep Parhar #else
15763098bcfcSNavdeep Parhar 	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
15773098bcfcSNavdeep Parhar #endif
15783098bcfcSNavdeep Parhar 
157946e1e307SNavdeep Parhar 	limit = budget ? budget : iq->qsize / 16;
158046e1e307SNavdeep Parhar 	fl = &rxq->fl;
158146e1e307SNavdeep Parhar 	fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
15823098bcfcSNavdeep Parhar 	while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
15833098bcfcSNavdeep Parhar 
15843098bcfcSNavdeep Parhar 		rmb();
15853098bcfcSNavdeep Parhar 
15863098bcfcSNavdeep Parhar 		m0 = NULL;
15873098bcfcSNavdeep Parhar 		rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
15883098bcfcSNavdeep Parhar 		lq = be32toh(d->rsp.pldbuflen_qid);
15893098bcfcSNavdeep Parhar 
15903098bcfcSNavdeep Parhar 		switch (rsp_type) {
15913098bcfcSNavdeep Parhar 		case X_RSPD_TYPE_FLBUF:
159246e1e307SNavdeep Parhar 			if (lq & F_RSPD_NEWBUF) {
159346e1e307SNavdeep Parhar 				if (fl->rx_offset > 0)
159446e1e307SNavdeep Parhar 					move_to_next_rxbuf(fl);
159546e1e307SNavdeep Parhar 				lq = G_RSPD_LEN(lq);
159646e1e307SNavdeep Parhar 			}
159746e1e307SNavdeep Parhar 			if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) {
159846e1e307SNavdeep Parhar 				FL_LOCK(fl);
159946e1e307SNavdeep Parhar 				refill_fl(sc, fl, 64);
160046e1e307SNavdeep Parhar 				FL_UNLOCK(fl);
160146e1e307SNavdeep Parhar 				fl_hw_cidx = fl->hw_cidx;
160246e1e307SNavdeep Parhar 			}
16033098bcfcSNavdeep Parhar 
16041486d2deSNavdeep Parhar 			if (d->rss.opcode == CPL_RX_PKT) {
16051486d2deSNavdeep Parhar 				if (__predict_true(eth_rx(sc, rxq, d, lq) == 0))
16061486d2deSNavdeep Parhar 					break;
16071486d2deSNavdeep Parhar 				goto out;
16081486d2deSNavdeep Parhar 			}
16093098bcfcSNavdeep Parhar 			m0 = get_fl_payload(sc, fl, lq);
16103098bcfcSNavdeep Parhar 			if (__predict_false(m0 == NULL))
16113098bcfcSNavdeep Parhar 				goto out;
1612e7e08444SNavdeep Parhar 
16133098bcfcSNavdeep Parhar 			/* fall through */
16143098bcfcSNavdeep Parhar 
16153098bcfcSNavdeep Parhar 		case X_RSPD_TYPE_CPL:
16163098bcfcSNavdeep Parhar 			KASSERT(d->rss.opcode < NUM_CPL_CMDS,
16173098bcfcSNavdeep Parhar 			    ("%s: bad opcode %02x.", __func__, d->rss.opcode));
16183098bcfcSNavdeep Parhar 			t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
16193098bcfcSNavdeep Parhar 			break;
16203098bcfcSNavdeep Parhar 
16213098bcfcSNavdeep Parhar 		case X_RSPD_TYPE_INTR:
16223098bcfcSNavdeep Parhar 
16233098bcfcSNavdeep Parhar 			/*
16243098bcfcSNavdeep Parhar 			 * There are 1K interrupt-capable queues (qids 0
16253098bcfcSNavdeep Parhar 			 * through 1023).  A response type indicating a
16263098bcfcSNavdeep Parhar 			 * forwarded interrupt with a qid >= 1K is an
16273098bcfcSNavdeep Parhar 			 * iWARP async notification.  That is the only
16283098bcfcSNavdeep Parhar 			 * acceptable indirect interrupt on this queue.
16293098bcfcSNavdeep Parhar 			 */
16303098bcfcSNavdeep Parhar 			if (__predict_false(lq < 1024)) {
16313098bcfcSNavdeep Parhar 				panic("%s: indirect interrupt on iq_fl %p "
16323098bcfcSNavdeep Parhar 				    "with qid %u", __func__, iq, lq);
16333098bcfcSNavdeep Parhar 			}
16343098bcfcSNavdeep Parhar 
16353098bcfcSNavdeep Parhar 			t4_an_handler(iq, &d->rsp);
16363098bcfcSNavdeep Parhar 			break;
16373098bcfcSNavdeep Parhar 
16383098bcfcSNavdeep Parhar 		default:
16393098bcfcSNavdeep Parhar 			KASSERT(0, ("%s: illegal response type %d on iq %p",
16403098bcfcSNavdeep Parhar 			    __func__, rsp_type, iq));
16413098bcfcSNavdeep Parhar 			log(LOG_ERR, "%s: illegal response type %d on iq %p",
16423098bcfcSNavdeep Parhar 			    device_get_nameunit(sc->dev), rsp_type, iq);
16433098bcfcSNavdeep Parhar 			break;
16443098bcfcSNavdeep Parhar 		}
16453098bcfcSNavdeep Parhar 
16463098bcfcSNavdeep Parhar 		d++;
16473098bcfcSNavdeep Parhar 		if (__predict_false(++iq->cidx == iq->sidx)) {
16483098bcfcSNavdeep Parhar 			iq->cidx = 0;
16493098bcfcSNavdeep Parhar 			iq->gen ^= F_RSPD_GEN;
16503098bcfcSNavdeep Parhar 			d = &iq->desc[0];
16513098bcfcSNavdeep Parhar 		}
16523098bcfcSNavdeep Parhar 		if (__predict_false(++ndescs == limit)) {
16533098bcfcSNavdeep Parhar 			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
16543098bcfcSNavdeep Parhar 			    V_INGRESSQID(iq->cntxt_id) |
16553098bcfcSNavdeep Parhar 			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
16563098bcfcSNavdeep Parhar 
1657480e603cSNavdeep Parhar #if defined(INET) || defined(INET6)
1658480e603cSNavdeep Parhar 			if (iq->flags & IQ_LRO_ENABLED &&
165946f48ee5SNavdeep Parhar 			    !sort_before_lro(lro) &&
1660480e603cSNavdeep Parhar 			    sc->lro_timeout != 0) {
16613098bcfcSNavdeep Parhar 				tcp_lro_flush_inactive(lro, &lro_timeout);
1662480e603cSNavdeep Parhar 			}
1663480e603cSNavdeep Parhar #endif
166446e1e307SNavdeep Parhar 			if (budget)
1665733b9277SNavdeep Parhar 				return (EINPROGRESS);
166646e1e307SNavdeep Parhar 			ndescs = 0;
16674d6db4e0SNavdeep Parhar 		}
1668861e42b2SNavdeep Parhar 	}
16693098bcfcSNavdeep Parhar out:
1670a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6)
1671733b9277SNavdeep Parhar 	if (iq->flags & IQ_LRO_ENABLED) {
167246f48ee5SNavdeep Parhar 		if (ndescs > 0 && lro->lro_mbuf_count > 8) {
167346f48ee5SNavdeep Parhar 			MPASS(sort_before_lro(lro));
167446f48ee5SNavdeep Parhar 			/* hold back one credit and don't flush LRO state */
167546f48ee5SNavdeep Parhar 			iq->flags |= IQ_ADJ_CREDIT;
167646f48ee5SNavdeep Parhar 			ndescs--;
167746f48ee5SNavdeep Parhar 		} else {
16786dd38b87SSepherosa Ziehau 			tcp_lro_flush_all(lro);
1679733b9277SNavdeep Parhar 		}
168046f48ee5SNavdeep Parhar 	}
1681733b9277SNavdeep Parhar #endif
1682733b9277SNavdeep Parhar 
1683315048f2SJohn Baldwin 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1684733b9277SNavdeep Parhar 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1685733b9277SNavdeep Parhar 
1686733b9277SNavdeep Parhar 	FL_LOCK(fl);
168738035ed6SNavdeep Parhar 	starved = refill_fl(sc, fl, 64);
1688733b9277SNavdeep Parhar 	FL_UNLOCK(fl);
1689733b9277SNavdeep Parhar 	if (__predict_false(starved != 0))
1690733b9277SNavdeep Parhar 		add_fl_to_sfl(sc, fl);
1691733b9277SNavdeep Parhar 
1692733b9277SNavdeep Parhar 	return (0);
1693733b9277SNavdeep Parhar }
1694733b9277SNavdeep Parhar 
169538035ed6SNavdeep Parhar static inline struct cluster_metadata *
169646e1e307SNavdeep Parhar cl_metadata(struct fl_sdesc *sd)
16971458bff9SNavdeep Parhar {
16981458bff9SNavdeep Parhar 
169946e1e307SNavdeep Parhar 	return ((void *)(sd->cl + sd->moff));
17001458bff9SNavdeep Parhar }
17011458bff9SNavdeep Parhar 
170215c28f87SGleb Smirnoff static void
1703e8fd18f3SGleb Smirnoff rxb_free(struct mbuf *m)
17041458bff9SNavdeep Parhar {
1705d6f79b27SNavdeep Parhar 	struct cluster_metadata *clm = m->m_ext.ext_arg1;
17061458bff9SNavdeep Parhar 
1707d6f79b27SNavdeep Parhar 	uma_zfree(clm->zone, clm->cl);
170882eff304SNavdeep Parhar 	counter_u64_add(extfree_rels, 1);
17091458bff9SNavdeep Parhar }
17101458bff9SNavdeep Parhar 
171138035ed6SNavdeep Parhar /*
171246e1e307SNavdeep Parhar  * The mbuf returned comes from zone_muf and carries the payload in one of these
171346e1e307SNavdeep Parhar  * ways
171446e1e307SNavdeep Parhar  * a) complete frame inside the mbuf
171546e1e307SNavdeep Parhar  * b) m_cljset (for clusters without metadata)
171646e1e307SNavdeep Parhar  * d) m_extaddref (cluster with metadata)
171738035ed6SNavdeep Parhar  */
17181458bff9SNavdeep Parhar static struct mbuf *
1719b741402cSNavdeep Parhar get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1720b741402cSNavdeep Parhar     int remaining)
172138035ed6SNavdeep Parhar {
172238035ed6SNavdeep Parhar 	struct mbuf *m;
172338035ed6SNavdeep Parhar 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
172446e1e307SNavdeep Parhar 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
172546e1e307SNavdeep Parhar 	struct cluster_metadata *clm;
1726b741402cSNavdeep Parhar 	int len, blen;
172738035ed6SNavdeep Parhar 	caddr_t payload;
172838035ed6SNavdeep Parhar 
1729e3207e19SNavdeep Parhar 	if (fl->flags & FL_BUF_PACKING) {
173046e1e307SNavdeep Parhar 		u_int l, pad;
1731b741402cSNavdeep Parhar 
173246e1e307SNavdeep Parhar 		blen = rxb->size2 - fl->rx_offset;	/* max possible in this buf */
173346e1e307SNavdeep Parhar 		len = min(remaining, blen);
173446e1e307SNavdeep Parhar 		payload = sd->cl + fl->rx_offset;
173546e1e307SNavdeep Parhar 
173646e1e307SNavdeep Parhar 		l = fr_offset + len;
173746e1e307SNavdeep Parhar 		pad = roundup2(l, fl->buf_boundary) - l;
173846e1e307SNavdeep Parhar 		if (fl->rx_offset + len + pad < rxb->size2)
1739b741402cSNavdeep Parhar 			blen = len + pad;
174046e1e307SNavdeep Parhar 		MPASS(fl->rx_offset + blen <= rxb->size2);
1741e3207e19SNavdeep Parhar 	} else {
1742e3207e19SNavdeep Parhar 		MPASS(fl->rx_offset == 0);	/* not packing */
174346e1e307SNavdeep Parhar 		blen = rxb->size1;
174446e1e307SNavdeep Parhar 		len = min(remaining, blen);
174546e1e307SNavdeep Parhar 		payload = sd->cl;
1746e3207e19SNavdeep Parhar 	}
174738035ed6SNavdeep Parhar 
174846e1e307SNavdeep Parhar 	if (fr_offset == 0) {
174946e1e307SNavdeep Parhar 		m = m_gethdr(M_NOWAIT, MT_DATA);
175046e1e307SNavdeep Parhar 		if (__predict_false(m == NULL))
175146e1e307SNavdeep Parhar 			return (NULL);
175246e1e307SNavdeep Parhar 		m->m_pkthdr.len = remaining;
175346e1e307SNavdeep Parhar 	} else {
175446e1e307SNavdeep Parhar 		m = m_get(M_NOWAIT, MT_DATA);
175546e1e307SNavdeep Parhar 		if (__predict_false(m == NULL))
175646e1e307SNavdeep Parhar 			return (NULL);
175746e1e307SNavdeep Parhar 	}
175846e1e307SNavdeep Parhar 	m->m_len = len;
175914a634dfSMark Johnston 	kmsan_mark(payload, len, KMSAN_STATE_INITED);
1760b741402cSNavdeep Parhar 
176138035ed6SNavdeep Parhar 	if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
176238035ed6SNavdeep Parhar 		/* copy data to mbuf */
176338035ed6SNavdeep Parhar 		bcopy(payload, mtod(m, caddr_t), len);
176446e1e307SNavdeep Parhar 		if (fl->flags & FL_BUF_PACKING) {
176546e1e307SNavdeep Parhar 			fl->rx_offset += blen;
176646e1e307SNavdeep Parhar 			MPASS(fl->rx_offset <= rxb->size2);
176746e1e307SNavdeep Parhar 			if (fl->rx_offset < rxb->size2)
176846e1e307SNavdeep Parhar 				return (m);	/* without advancing the cidx */
176946e1e307SNavdeep Parhar 		}
177046e1e307SNavdeep Parhar 	} else if (fl->flags & FL_BUF_PACKING) {
177146e1e307SNavdeep Parhar 		clm = cl_metadata(sd);
1772a9c4062aSNavdeep Parhar 		if (sd->nmbuf++ == 0) {
1773a9c4062aSNavdeep Parhar 			clm->refcount = 1;
177446e1e307SNavdeep Parhar 			clm->zone = rxb->zone;
1775d6f79b27SNavdeep Parhar 			clm->cl = sd->cl;
1776a9c4062aSNavdeep Parhar 			counter_u64_add(extfree_refs, 1);
1777a9c4062aSNavdeep Parhar 		}
1778d6f79b27SNavdeep Parhar 		m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm,
1779d6f79b27SNavdeep Parhar 		    NULL);
178038035ed6SNavdeep Parhar 
178146e1e307SNavdeep Parhar 		fl->rx_offset += blen;
178246e1e307SNavdeep Parhar 		MPASS(fl->rx_offset <= rxb->size2);
178346e1e307SNavdeep Parhar 		if (fl->rx_offset < rxb->size2)
178446e1e307SNavdeep Parhar 			return (m);	/* without advancing the cidx */
1785ccc69b2fSNavdeep Parhar 	} else {
178646e1e307SNavdeep Parhar 		m_cljset(m, sd->cl, rxb->type);
178738035ed6SNavdeep Parhar 		sd->cl = NULL;	/* consumed, not a recycle candidate */
178838035ed6SNavdeep Parhar 	}
178938035ed6SNavdeep Parhar 
179046e1e307SNavdeep Parhar 	move_to_next_rxbuf(fl);
179138035ed6SNavdeep Parhar 
179238035ed6SNavdeep Parhar 	return (m);
179338035ed6SNavdeep Parhar }
179438035ed6SNavdeep Parhar 
179538035ed6SNavdeep Parhar static struct mbuf *
179646e1e307SNavdeep Parhar get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen)
17971458bff9SNavdeep Parhar {
179838035ed6SNavdeep Parhar 	struct mbuf *m0, *m, **pnext;
1799b741402cSNavdeep Parhar 	u_int remaining;
18001458bff9SNavdeep Parhar 
18014d6db4e0SNavdeep Parhar 	if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1802368541baSNavdeep Parhar 		M_ASSERTPKTHDR(fl->m0);
180346e1e307SNavdeep Parhar 		MPASS(fl->m0->m_pkthdr.len == plen);
180446e1e307SNavdeep Parhar 		MPASS(fl->remaining < plen);
18051458bff9SNavdeep Parhar 
180638035ed6SNavdeep Parhar 		m0 = fl->m0;
180738035ed6SNavdeep Parhar 		pnext = fl->pnext;
1808b741402cSNavdeep Parhar 		remaining = fl->remaining;
18094d6db4e0SNavdeep Parhar 		fl->flags &= ~FL_BUF_RESUME;
181038035ed6SNavdeep Parhar 		goto get_segment;
18111458bff9SNavdeep Parhar 	}
18121458bff9SNavdeep Parhar 
18131458bff9SNavdeep Parhar 	/*
181438035ed6SNavdeep Parhar 	 * Payload starts at rx_offset in the current hw buffer.  Its length is
181538035ed6SNavdeep Parhar 	 * 'len' and it may span multiple hw buffers.
18161458bff9SNavdeep Parhar 	 */
18171458bff9SNavdeep Parhar 
181846e1e307SNavdeep Parhar 	m0 = get_scatter_segment(sc, fl, 0, plen);
1819368541baSNavdeep Parhar 	if (m0 == NULL)
18204d6db4e0SNavdeep Parhar 		return (NULL);
182146e1e307SNavdeep Parhar 	remaining = plen - m0->m_len;
182238035ed6SNavdeep Parhar 	pnext = &m0->m_next;
1823b741402cSNavdeep Parhar 	while (remaining > 0) {
182438035ed6SNavdeep Parhar get_segment:
182538035ed6SNavdeep Parhar 		MPASS(fl->rx_offset == 0);
182646e1e307SNavdeep Parhar 		m = get_scatter_segment(sc, fl, plen - remaining, remaining);
18274d6db4e0SNavdeep Parhar 		if (__predict_false(m == NULL)) {
182838035ed6SNavdeep Parhar 			fl->m0 = m0;
182938035ed6SNavdeep Parhar 			fl->pnext = pnext;
1830b741402cSNavdeep Parhar 			fl->remaining = remaining;
18314d6db4e0SNavdeep Parhar 			fl->flags |= FL_BUF_RESUME;
18324d6db4e0SNavdeep Parhar 			return (NULL);
18331458bff9SNavdeep Parhar 		}
183438035ed6SNavdeep Parhar 		*pnext = m;
183538035ed6SNavdeep Parhar 		pnext = &m->m_next;
1836b741402cSNavdeep Parhar 		remaining -= m->m_len;
1837733b9277SNavdeep Parhar 	}
183838035ed6SNavdeep Parhar 	*pnext = NULL;
18394d6db4e0SNavdeep Parhar 
1840dbbf46c4SNavdeep Parhar 	M_ASSERTPKTHDR(m0);
1841733b9277SNavdeep Parhar 	return (m0);
1842733b9277SNavdeep Parhar }
1843733b9277SNavdeep Parhar 
1844733b9277SNavdeep Parhar static int
184587bbb333SNavdeep Parhar skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
184687bbb333SNavdeep Parhar     int remaining)
184787bbb333SNavdeep Parhar {
184887bbb333SNavdeep Parhar 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
184987bbb333SNavdeep Parhar 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
185087bbb333SNavdeep Parhar 	int len, blen;
185187bbb333SNavdeep Parhar 
185287bbb333SNavdeep Parhar 	if (fl->flags & FL_BUF_PACKING) {
185387bbb333SNavdeep Parhar 		u_int l, pad;
185487bbb333SNavdeep Parhar 
185587bbb333SNavdeep Parhar 		blen = rxb->size2 - fl->rx_offset;	/* max possible in this buf */
185687bbb333SNavdeep Parhar 		len = min(remaining, blen);
185787bbb333SNavdeep Parhar 
185887bbb333SNavdeep Parhar 		l = fr_offset + len;
185987bbb333SNavdeep Parhar 		pad = roundup2(l, fl->buf_boundary) - l;
186087bbb333SNavdeep Parhar 		if (fl->rx_offset + len + pad < rxb->size2)
186187bbb333SNavdeep Parhar 			blen = len + pad;
186287bbb333SNavdeep Parhar 		fl->rx_offset += blen;
186387bbb333SNavdeep Parhar 		MPASS(fl->rx_offset <= rxb->size2);
186487bbb333SNavdeep Parhar 		if (fl->rx_offset < rxb->size2)
186587bbb333SNavdeep Parhar 			return (len);	/* without advancing the cidx */
186687bbb333SNavdeep Parhar 	} else {
186787bbb333SNavdeep Parhar 		MPASS(fl->rx_offset == 0);	/* not packing */
186887bbb333SNavdeep Parhar 		blen = rxb->size1;
186987bbb333SNavdeep Parhar 		len = min(remaining, blen);
187087bbb333SNavdeep Parhar 	}
187187bbb333SNavdeep Parhar 	move_to_next_rxbuf(fl);
187287bbb333SNavdeep Parhar 	return (len);
187387bbb333SNavdeep Parhar }
187487bbb333SNavdeep Parhar 
187587bbb333SNavdeep Parhar static inline void
187687bbb333SNavdeep Parhar skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen)
187787bbb333SNavdeep Parhar {
187887bbb333SNavdeep Parhar 	int remaining, fr_offset, len;
187987bbb333SNavdeep Parhar 
188087bbb333SNavdeep Parhar 	fr_offset = 0;
188187bbb333SNavdeep Parhar 	remaining = plen;
188287bbb333SNavdeep Parhar 	while (remaining > 0) {
188387bbb333SNavdeep Parhar 		len = skip_scatter_segment(sc, fl, fr_offset, remaining);
188487bbb333SNavdeep Parhar 		fr_offset += len;
188587bbb333SNavdeep Parhar 		remaining -= len;
188687bbb333SNavdeep Parhar 	}
188787bbb333SNavdeep Parhar }
188887bbb333SNavdeep Parhar 
188987bbb333SNavdeep Parhar static inline int
189087bbb333SNavdeep Parhar get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen)
189187bbb333SNavdeep Parhar {
189287bbb333SNavdeep Parhar 	int len;
189387bbb333SNavdeep Parhar 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
189487bbb333SNavdeep Parhar 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
189587bbb333SNavdeep Parhar 
189687bbb333SNavdeep Parhar 	if (fl->flags & FL_BUF_PACKING)
189787bbb333SNavdeep Parhar 		len = rxb->size2 - fl->rx_offset;
189887bbb333SNavdeep Parhar 	else
189987bbb333SNavdeep Parhar 		len = rxb->size1;
190087bbb333SNavdeep Parhar 
190187bbb333SNavdeep Parhar 	return (min(plen, len));
190287bbb333SNavdeep Parhar }
190387bbb333SNavdeep Parhar 
190487bbb333SNavdeep Parhar static int
19051486d2deSNavdeep Parhar eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d,
19061486d2deSNavdeep Parhar     u_int plen)
1907733b9277SNavdeep Parhar {
19081486d2deSNavdeep Parhar 	struct mbuf *m0;
1909733b9277SNavdeep Parhar 	struct ifnet *ifp = rxq->ifp;
19101486d2deSNavdeep Parhar 	struct sge_fl *fl = &rxq->fl;
191187bbb333SNavdeep Parhar 	struct vi_info *vi = ifp->if_softc;
19121486d2deSNavdeep Parhar 	const struct cpl_rx_pkt *cpl;
1913a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6)
1914733b9277SNavdeep Parhar 	struct lro_ctrl *lro = &rxq->lro;
1915733b9277SNavdeep Parhar #endif
1916a4a4ad2dSNavdeep Parhar 	uint16_t err_vec, tnl_type, tnlhdr_len;
191770ca6229SNavdeep Parhar 	static const int sw_hashtype[4][2] = {
191870ca6229SNavdeep Parhar 		{M_HASHTYPE_NONE, M_HASHTYPE_NONE},
191970ca6229SNavdeep Parhar 		{M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6},
192070ca6229SNavdeep Parhar 		{M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6},
192170ca6229SNavdeep Parhar 		{M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6},
192270ca6229SNavdeep Parhar 	};
1923a4a4ad2dSNavdeep Parhar 	static const int sw_csum_flags[2][2] = {
1924a4a4ad2dSNavdeep Parhar 		{
1925a4a4ad2dSNavdeep Parhar 			/* IP, inner IP */
1926a4a4ad2dSNavdeep Parhar 			CSUM_ENCAP_VXLAN |
1927a4a4ad2dSNavdeep Parhar 			    CSUM_L3_CALC | CSUM_L3_VALID |
1928a4a4ad2dSNavdeep Parhar 			    CSUM_L4_CALC | CSUM_L4_VALID |
1929a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1930a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1931a4a4ad2dSNavdeep Parhar 
1932a4a4ad2dSNavdeep Parhar 			/* IP, inner IP6 */
1933a4a4ad2dSNavdeep Parhar 			CSUM_ENCAP_VXLAN |
1934a4a4ad2dSNavdeep Parhar 			    CSUM_L3_CALC | CSUM_L3_VALID |
1935a4a4ad2dSNavdeep Parhar 			    CSUM_L4_CALC | CSUM_L4_VALID |
1936a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1937a4a4ad2dSNavdeep Parhar 		},
1938a4a4ad2dSNavdeep Parhar 		{
1939a4a4ad2dSNavdeep Parhar 			/* IP6, inner IP */
1940a4a4ad2dSNavdeep Parhar 			CSUM_ENCAP_VXLAN |
1941a4a4ad2dSNavdeep Parhar 			    CSUM_L4_CALC | CSUM_L4_VALID |
1942a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1943a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1944a4a4ad2dSNavdeep Parhar 
1945a4a4ad2dSNavdeep Parhar 			/* IP6, inner IP6 */
1946a4a4ad2dSNavdeep Parhar 			CSUM_ENCAP_VXLAN |
1947a4a4ad2dSNavdeep Parhar 			    CSUM_L4_CALC | CSUM_L4_VALID |
1948a4a4ad2dSNavdeep Parhar 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1949a4a4ad2dSNavdeep Parhar 		},
1950a4a4ad2dSNavdeep Parhar 	};
1951733b9277SNavdeep Parhar 
19521486d2deSNavdeep Parhar 	MPASS(plen > sc->params.sge.fl_pktshift);
195387bbb333SNavdeep Parhar 	if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) &&
195487bbb333SNavdeep Parhar 	    __predict_true((fl->flags & FL_BUF_RESUME) == 0)) {
195587bbb333SNavdeep Parhar 		struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
195687bbb333SNavdeep Parhar 		caddr_t frame;
195787bbb333SNavdeep Parhar 		int rc, slen;
195887bbb333SNavdeep Parhar 
195987bbb333SNavdeep Parhar 		slen = get_segment_len(sc, fl, plen) -
196087bbb333SNavdeep Parhar 		    sc->params.sge.fl_pktshift;
196187bbb333SNavdeep Parhar 		frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift;
196287bbb333SNavdeep Parhar 		CURVNET_SET_QUIET(ifp->if_vnet);
196387bbb333SNavdeep Parhar 		rc = pfil_run_hooks(vi->pfil, frame, ifp,
196487bbb333SNavdeep Parhar 		    slen | PFIL_MEMPTR | PFIL_IN, NULL);
196587bbb333SNavdeep Parhar 		CURVNET_RESTORE();
196687bbb333SNavdeep Parhar 		if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) {
196787bbb333SNavdeep Parhar 			skip_fl_payload(sc, fl, plen);
196887bbb333SNavdeep Parhar 			return (0);
196987bbb333SNavdeep Parhar 		}
197087bbb333SNavdeep Parhar 		if (rc == PFIL_REALLOCED) {
197187bbb333SNavdeep Parhar 			skip_fl_payload(sc, fl, plen);
197287bbb333SNavdeep Parhar 			m0 = pfil_mem2mbuf(frame);
197387bbb333SNavdeep Parhar 			goto have_mbuf;
197487bbb333SNavdeep Parhar 		}
197587bbb333SNavdeep Parhar 	}
197687bbb333SNavdeep Parhar 
19771486d2deSNavdeep Parhar 	m0 = get_fl_payload(sc, fl, plen);
19781486d2deSNavdeep Parhar 	if (__predict_false(m0 == NULL))
19791486d2deSNavdeep Parhar 		return (ENOMEM);
1980733b9277SNavdeep Parhar 
198190e7434aSNavdeep Parhar 	m0->m_pkthdr.len -= sc->params.sge.fl_pktshift;
198290e7434aSNavdeep Parhar 	m0->m_len -= sc->params.sge.fl_pktshift;
198390e7434aSNavdeep Parhar 	m0->m_data += sc->params.sge.fl_pktshift;
198454e4ee71SNavdeep Parhar 
198587bbb333SNavdeep Parhar have_mbuf:
198654e4ee71SNavdeep Parhar 	m0->m_pkthdr.rcvif = ifp;
19871486d2deSNavdeep Parhar 	M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]);
19881486d2deSNavdeep Parhar 	m0->m_pkthdr.flowid = be32toh(d->rss.hash_val);
198954e4ee71SNavdeep Parhar 
19901486d2deSNavdeep Parhar 	cpl = (const void *)(&d->rss + 1);
1991a4a4ad2dSNavdeep Parhar 	if (sc->params.tp.rx_pkt_encap) {
1992a4a4ad2dSNavdeep Parhar 		const uint16_t ev = be16toh(cpl->err_vec);
19939600bf00SNavdeep Parhar 
1994a4a4ad2dSNavdeep Parhar 		err_vec = G_T6_COMPR_RXERR_VEC(ev);
1995a4a4ad2dSNavdeep Parhar 		tnl_type = G_T6_RX_TNL_TYPE(ev);
1996a4a4ad2dSNavdeep Parhar 		tnlhdr_len = G_T6_RX_TNLHDR_LEN(ev);
1997a4a4ad2dSNavdeep Parhar 	} else {
1998a4a4ad2dSNavdeep Parhar 		err_vec = be16toh(cpl->err_vec);
1999a4a4ad2dSNavdeep Parhar 		tnl_type = 0;
2000a4a4ad2dSNavdeep Parhar 		tnlhdr_len = 0;
2001a4a4ad2dSNavdeep Parhar 	}
2002a4a4ad2dSNavdeep Parhar 	if (cpl->csum_calc && err_vec == 0) {
2003a4a4ad2dSNavdeep Parhar 		int ipv6 = !!(cpl->l2info & htobe32(F_RXF_IP6));
2004a4a4ad2dSNavdeep Parhar 
2005a4a4ad2dSNavdeep Parhar 		/* checksum(s) calculated and found to be correct. */
2006a4a4ad2dSNavdeep Parhar 
2007a4a4ad2dSNavdeep Parhar 		MPASS((cpl->l2info & htobe32(F_RXF_IP)) ^
2008a4a4ad2dSNavdeep Parhar 		    (cpl->l2info & htobe32(F_RXF_IP6)));
200954e4ee71SNavdeep Parhar 		m0->m_pkthdr.csum_data = be16toh(cpl->csum);
2010a4a4ad2dSNavdeep Parhar 		if (tnl_type == 0) {
2011a4a4ad2dSNavdeep Parhar 	    		if (!ipv6 && ifp->if_capenable & IFCAP_RXCSUM) {
2012a4a4ad2dSNavdeep Parhar 				m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2013a4a4ad2dSNavdeep Parhar 				    CSUM_L3_VALID | CSUM_L4_CALC |
2014a4a4ad2dSNavdeep Parhar 				    CSUM_L4_VALID;
2015a4a4ad2dSNavdeep Parhar 			} else if (ipv6 && ifp->if_capenable & IFCAP_RXCSUM_IPV6) {
2016a4a4ad2dSNavdeep Parhar 				m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2017a4a4ad2dSNavdeep Parhar 				    CSUM_L4_VALID;
2018a4a4ad2dSNavdeep Parhar 			}
2019a4a4ad2dSNavdeep Parhar 			rxq->rxcsum++;
2020a4a4ad2dSNavdeep Parhar 		} else {
2021a4a4ad2dSNavdeep Parhar 			MPASS(tnl_type == RX_PKT_TNL_TYPE_VXLAN);
2022d107ee06SNavdeep Parhar 
2023d107ee06SNavdeep Parhar 			M_HASHTYPE_SETINNER(m0);
2024a4a4ad2dSNavdeep Parhar 			if (__predict_false(cpl->ip_frag)) {
2025a4a4ad2dSNavdeep Parhar 				/*
2026a4a4ad2dSNavdeep Parhar 				 * csum_data is for the inner frame (which is an
2027a4a4ad2dSNavdeep Parhar 				 * IP fragment) and is not 0xffff.  There is no
2028a4a4ad2dSNavdeep Parhar 				 * way to pass the inner csum_data to the stack.
2029a4a4ad2dSNavdeep Parhar 				 * We don't want the stack to use the inner
2030a4a4ad2dSNavdeep Parhar 				 * csum_data to validate the outer frame or it
2031a4a4ad2dSNavdeep Parhar 				 * will get rejected.  So we fix csum_data here
2032a4a4ad2dSNavdeep Parhar 				 * and let sw do the checksum of inner IP
2033a4a4ad2dSNavdeep Parhar 				 * fragments.
2034a4a4ad2dSNavdeep Parhar 				 *
2035a4a4ad2dSNavdeep Parhar 				 * XXX: Need 32b for csum_data2 in an rx mbuf.
2036a4a4ad2dSNavdeep Parhar 				 * Maybe stuff it into rcv_tstmp?
2037a4a4ad2dSNavdeep Parhar 				 */
203854e4ee71SNavdeep Parhar 				m0->m_pkthdr.csum_data = 0xffff;
2039a4a4ad2dSNavdeep Parhar 				if (ipv6) {
2040a4a4ad2dSNavdeep Parhar 					m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2041a4a4ad2dSNavdeep Parhar 					    CSUM_L4_VALID;
2042a4a4ad2dSNavdeep Parhar 				} else {
2043a4a4ad2dSNavdeep Parhar 					m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2044a4a4ad2dSNavdeep Parhar 					    CSUM_L3_VALID | CSUM_L4_CALC |
2045a4a4ad2dSNavdeep Parhar 					    CSUM_L4_VALID;
2046a4a4ad2dSNavdeep Parhar 				}
2047a4a4ad2dSNavdeep Parhar 			} else {
2048a4a4ad2dSNavdeep Parhar 				int outer_ipv6;
2049a4a4ad2dSNavdeep Parhar 
2050a4a4ad2dSNavdeep Parhar 				MPASS(m0->m_pkthdr.csum_data == 0xffff);
2051a4a4ad2dSNavdeep Parhar 
2052a4a4ad2dSNavdeep Parhar 				outer_ipv6 = tnlhdr_len >=
2053a4a4ad2dSNavdeep Parhar 				    sizeof(struct ether_header) +
2054a4a4ad2dSNavdeep Parhar 				    sizeof(struct ip6_hdr);
2055a4a4ad2dSNavdeep Parhar 				m0->m_pkthdr.csum_flags =
2056a4a4ad2dSNavdeep Parhar 				    sw_csum_flags[outer_ipv6][ipv6];
2057a4a4ad2dSNavdeep Parhar 			}
2058a4a4ad2dSNavdeep Parhar 			rxq->vxlan_rxcsum++;
2059a4a4ad2dSNavdeep Parhar 		}
206054e4ee71SNavdeep Parhar 	}
206154e4ee71SNavdeep Parhar 
206254e4ee71SNavdeep Parhar 	if (cpl->vlan_ex) {
206354e4ee71SNavdeep Parhar 		m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
206454e4ee71SNavdeep Parhar 		m0->m_flags |= M_VLANTAG;
206554e4ee71SNavdeep Parhar 		rxq->vlan_extraction++;
206654e4ee71SNavdeep Parhar 	}
206754e4ee71SNavdeep Parhar 
20681486d2deSNavdeep Parhar 	if (rxq->iq.flags & IQ_RX_TIMESTAMP) {
20691486d2deSNavdeep Parhar 		/*
20701486d2deSNavdeep Parhar 		 * Fill up rcv_tstmp but do not set M_TSTMP.
20711486d2deSNavdeep Parhar 		 * rcv_tstmp is not in the format that the
20721486d2deSNavdeep Parhar 		 * kernel expects and we don't want to mislead
20731486d2deSNavdeep Parhar 		 * it.  For now this is only for custom code
20741486d2deSNavdeep Parhar 		 * that knows how to interpret cxgbe's stamp.
20751486d2deSNavdeep Parhar 		 */
20761486d2deSNavdeep Parhar 		m0->m_pkthdr.rcv_tstmp =
20771486d2deSNavdeep Parhar 		    last_flit_to_ns(sc, d->rsp.u.last_flit);
20781486d2deSNavdeep Parhar #ifdef notyet
20791486d2deSNavdeep Parhar 		m0->m_flags |= M_TSTMP;
20801486d2deSNavdeep Parhar #endif
20811486d2deSNavdeep Parhar 	}
20821486d2deSNavdeep Parhar 
208350575ce1SAndrew Gallatin #ifdef NUMA
208450575ce1SAndrew Gallatin 	m0->m_pkthdr.numa_domain = ifp->if_numa_domain;
208550575ce1SAndrew Gallatin #endif
2086a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6)
2087a4a4ad2dSNavdeep Parhar 	if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 &&
20889087a3dfSNavdeep Parhar 	    (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 ||
20899087a3dfSNavdeep Parhar 	    M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) {
209046f48ee5SNavdeep Parhar 		if (sort_before_lro(lro)) {
209146f48ee5SNavdeep Parhar 			tcp_lro_queue_mbuf(lro, m0);
209246f48ee5SNavdeep Parhar 			return (0); /* queued for sort, then LRO */
209346f48ee5SNavdeep Parhar 		}
209446f48ee5SNavdeep Parhar 		if (tcp_lro_rx(lro, m0, 0) == 0)
209546f48ee5SNavdeep Parhar 			return (0); /* queued for LRO */
209646f48ee5SNavdeep Parhar 	}
209754e4ee71SNavdeep Parhar #endif
20987d29df59SNavdeep Parhar 	ifp->if_input(ifp, m0);
209954e4ee71SNavdeep Parhar 
2100733b9277SNavdeep Parhar 	return (0);
210154e4ee71SNavdeep Parhar }
210254e4ee71SNavdeep Parhar 
2103733b9277SNavdeep Parhar /*
21047951040fSNavdeep Parhar  * Must drain the wrq or make sure that someone else will.
21057951040fSNavdeep Parhar  */
21067951040fSNavdeep Parhar static void
21077951040fSNavdeep Parhar wrq_tx_drain(void *arg, int n)
21087951040fSNavdeep Parhar {
21097951040fSNavdeep Parhar 	struct sge_wrq *wrq = arg;
21107951040fSNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
21117951040fSNavdeep Parhar 
21127951040fSNavdeep Parhar 	EQ_LOCK(eq);
21137951040fSNavdeep Parhar 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
21147951040fSNavdeep Parhar 		drain_wrq_wr_list(wrq->adapter, wrq);
21157951040fSNavdeep Parhar 	EQ_UNLOCK(eq);
21167951040fSNavdeep Parhar }
21177951040fSNavdeep Parhar 
21187951040fSNavdeep Parhar static void
21197951040fSNavdeep Parhar drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq)
21207951040fSNavdeep Parhar {
21217951040fSNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
21227951040fSNavdeep Parhar 	u_int available, dbdiff;	/* # of hardware descriptors */
21237951040fSNavdeep Parhar 	u_int n;
21247951040fSNavdeep Parhar 	struct wrqe *wr;
21257951040fSNavdeep Parhar 	struct fw_eth_tx_pkt_wr *dst;	/* any fw WR struct will do */
21267951040fSNavdeep Parhar 
21277951040fSNavdeep Parhar 	EQ_LOCK_ASSERT_OWNED(eq);
21287951040fSNavdeep Parhar 	MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
21297951040fSNavdeep Parhar 	wr = STAILQ_FIRST(&wrq->wr_list);
21307951040fSNavdeep Parhar 	MPASS(wr != NULL);	/* Must be called with something useful to do */
2131cda2ab0eSNavdeep Parhar 	MPASS(eq->pidx == eq->dbidx);
2132cda2ab0eSNavdeep Parhar 	dbdiff = 0;
21337951040fSNavdeep Parhar 
21347951040fSNavdeep Parhar 	do {
21357951040fSNavdeep Parhar 		eq->cidx = read_hw_cidx(eq);
21367951040fSNavdeep Parhar 		if (eq->pidx == eq->cidx)
21377951040fSNavdeep Parhar 			available = eq->sidx - 1;
21387951040fSNavdeep Parhar 		else
21397951040fSNavdeep Parhar 			available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
21407951040fSNavdeep Parhar 
21417951040fSNavdeep Parhar 		MPASS(wr->wrq == wrq);
21427951040fSNavdeep Parhar 		n = howmany(wr->wr_len, EQ_ESIZE);
21437951040fSNavdeep Parhar 		if (available < n)
2144cda2ab0eSNavdeep Parhar 			break;
21457951040fSNavdeep Parhar 
21467951040fSNavdeep Parhar 		dst = (void *)&eq->desc[eq->pidx];
21477951040fSNavdeep Parhar 		if (__predict_true(eq->sidx - eq->pidx > n)) {
21487951040fSNavdeep Parhar 			/* Won't wrap, won't end exactly at the status page. */
21497951040fSNavdeep Parhar 			bcopy(&wr->wr[0], dst, wr->wr_len);
21507951040fSNavdeep Parhar 			eq->pidx += n;
21517951040fSNavdeep Parhar 		} else {
21527951040fSNavdeep Parhar 			int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE;
21537951040fSNavdeep Parhar 
21547951040fSNavdeep Parhar 			bcopy(&wr->wr[0], dst, first_portion);
21557951040fSNavdeep Parhar 			if (wr->wr_len > first_portion) {
21567951040fSNavdeep Parhar 				bcopy(&wr->wr[first_portion], &eq->desc[0],
21577951040fSNavdeep Parhar 				    wr->wr_len - first_portion);
21587951040fSNavdeep Parhar 			}
21597951040fSNavdeep Parhar 			eq->pidx = n - (eq->sidx - eq->pidx);
21607951040fSNavdeep Parhar 		}
21610459a175SNavdeep Parhar 		wrq->tx_wrs_copied++;
21627951040fSNavdeep Parhar 
21637951040fSNavdeep Parhar 		if (available < eq->sidx / 4 &&
21647951040fSNavdeep Parhar 		    atomic_cmpset_int(&eq->equiq, 0, 1)) {
2165ddf09ad6SNavdeep Parhar 				/*
2166ddf09ad6SNavdeep Parhar 				 * XXX: This is not 100% reliable with some
2167ddf09ad6SNavdeep Parhar 				 * types of WRs.  But this is a very unusual
2168ddf09ad6SNavdeep Parhar 				 * situation for an ofld/ctrl queue anyway.
2169ddf09ad6SNavdeep Parhar 				 */
21707951040fSNavdeep Parhar 			dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
21717951040fSNavdeep Parhar 			    F_FW_WR_EQUEQ);
21727951040fSNavdeep Parhar 		}
21737951040fSNavdeep Parhar 
21747951040fSNavdeep Parhar 		dbdiff += n;
21757951040fSNavdeep Parhar 		if (dbdiff >= 16) {
21767951040fSNavdeep Parhar 			ring_eq_db(sc, eq, dbdiff);
21777951040fSNavdeep Parhar 			dbdiff = 0;
21787951040fSNavdeep Parhar 		}
21797951040fSNavdeep Parhar 
21807951040fSNavdeep Parhar 		STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
21817951040fSNavdeep Parhar 		free_wrqe(wr);
21827951040fSNavdeep Parhar 		MPASS(wrq->nwr_pending > 0);
21837951040fSNavdeep Parhar 		wrq->nwr_pending--;
21847951040fSNavdeep Parhar 		MPASS(wrq->ndesc_needed >= n);
21857951040fSNavdeep Parhar 		wrq->ndesc_needed -= n;
21867951040fSNavdeep Parhar 	} while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL);
21877951040fSNavdeep Parhar 
21887951040fSNavdeep Parhar 	if (dbdiff)
21897951040fSNavdeep Parhar 		ring_eq_db(sc, eq, dbdiff);
21907951040fSNavdeep Parhar }
21917951040fSNavdeep Parhar 
21927951040fSNavdeep Parhar /*
2193733b9277SNavdeep Parhar  * Doesn't fail.  Holds on to work requests it can't send right away.
2194733b9277SNavdeep Parhar  */
219509fe6320SNavdeep Parhar void
219609fe6320SNavdeep Parhar t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
2197733b9277SNavdeep Parhar {
2198733b9277SNavdeep Parhar #ifdef INVARIANTS
21997951040fSNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
2200733b9277SNavdeep Parhar #endif
2201733b9277SNavdeep Parhar 
22027951040fSNavdeep Parhar 	EQ_LOCK_ASSERT_OWNED(eq);
22037951040fSNavdeep Parhar 	MPASS(wr != NULL);
22047951040fSNavdeep Parhar 	MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN);
22057951040fSNavdeep Parhar 	MPASS((wr->wr_len & 0x7) == 0);
2206733b9277SNavdeep Parhar 
22077951040fSNavdeep Parhar 	STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
22087951040fSNavdeep Parhar 	wrq->nwr_pending++;
22097951040fSNavdeep Parhar 	wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE);
2210733b9277SNavdeep Parhar 
22117951040fSNavdeep Parhar 	if (!TAILQ_EMPTY(&wrq->incomplete_wrs))
22127951040fSNavdeep Parhar 		return;	/* commit_wrq_wr will drain wr_list as well. */
2213733b9277SNavdeep Parhar 
22147951040fSNavdeep Parhar 	drain_wrq_wr_list(sc, wrq);
2215733b9277SNavdeep Parhar 
22167951040fSNavdeep Parhar 	/* Doorbell must have caught up to the pidx. */
22177951040fSNavdeep Parhar 	MPASS(eq->pidx == eq->dbidx);
221854e4ee71SNavdeep Parhar }
221954e4ee71SNavdeep Parhar 
222054e4ee71SNavdeep Parhar void
222154e4ee71SNavdeep Parhar t4_update_fl_bufsize(struct ifnet *ifp)
222254e4ee71SNavdeep Parhar {
2223fe2ebb76SJohn Baldwin 	struct vi_info *vi = ifp->if_softc;
22247c228be3SNavdeep Parhar 	struct adapter *sc = vi->adapter;
222554e4ee71SNavdeep Parhar 	struct sge_rxq *rxq;
22266eb3180fSNavdeep Parhar #ifdef TCP_OFFLOAD
22276eb3180fSNavdeep Parhar 	struct sge_ofld_rxq *ofld_rxq;
22286eb3180fSNavdeep Parhar #endif
222954e4ee71SNavdeep Parhar 	struct sge_fl *fl;
22306a59b994SNavdeep Parhar 	int i, maxp;
223154e4ee71SNavdeep Parhar 
22326a59b994SNavdeep Parhar 	maxp = max_rx_payload(sc, ifp, false);
2233fe2ebb76SJohn Baldwin 	for_each_rxq(vi, i, rxq) {
223454e4ee71SNavdeep Parhar 		fl = &rxq->fl;
223554e4ee71SNavdeep Parhar 
223654e4ee71SNavdeep Parhar 		FL_LOCK(fl);
223746e1e307SNavdeep Parhar 		fl->zidx = find_refill_source(sc, maxp,
223846e1e307SNavdeep Parhar 		    fl->flags & FL_BUF_PACKING);
223954e4ee71SNavdeep Parhar 		FL_UNLOCK(fl);
224054e4ee71SNavdeep Parhar 	}
22416eb3180fSNavdeep Parhar #ifdef TCP_OFFLOAD
22426a59b994SNavdeep Parhar 	maxp = max_rx_payload(sc, ifp, true);
2243fe2ebb76SJohn Baldwin 	for_each_ofld_rxq(vi, i, ofld_rxq) {
22446eb3180fSNavdeep Parhar 		fl = &ofld_rxq->fl;
22456eb3180fSNavdeep Parhar 
22466eb3180fSNavdeep Parhar 		FL_LOCK(fl);
224746e1e307SNavdeep Parhar 		fl->zidx = find_refill_source(sc, maxp,
224846e1e307SNavdeep Parhar 		    fl->flags & FL_BUF_PACKING);
22496eb3180fSNavdeep Parhar 		FL_UNLOCK(fl);
22506eb3180fSNavdeep Parhar 	}
22516eb3180fSNavdeep Parhar #endif
225254e4ee71SNavdeep Parhar }
225354e4ee71SNavdeep Parhar 
22547951040fSNavdeep Parhar static inline int
22557951040fSNavdeep Parhar mbuf_nsegs(struct mbuf *m)
2256733b9277SNavdeep Parhar {
22570835ddc7SNavdeep Parhar 
22587951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2259a4a4ad2dSNavdeep Parhar 	KASSERT(m->m_pkthdr.inner_l5hlen > 0,
22607951040fSNavdeep Parhar 	    ("%s: mbuf %p missing information on # of segments.", __func__, m));
22617951040fSNavdeep Parhar 
2262a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.inner_l5hlen);
22637951040fSNavdeep Parhar }
22647951040fSNavdeep Parhar 
22657951040fSNavdeep Parhar static inline void
22667951040fSNavdeep Parhar set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
22677951040fSNavdeep Parhar {
22687951040fSNavdeep Parhar 
22697951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2270a4a4ad2dSNavdeep Parhar 	m->m_pkthdr.inner_l5hlen = nsegs;
22717951040fSNavdeep Parhar }
22727951040fSNavdeep Parhar 
22737951040fSNavdeep Parhar static inline int
22745cdaef71SJohn Baldwin mbuf_cflags(struct mbuf *m)
22755cdaef71SJohn Baldwin {
22765cdaef71SJohn Baldwin 
22775cdaef71SJohn Baldwin 	M_ASSERTPKTHDR(m);
22785cdaef71SJohn Baldwin 	return (m->m_pkthdr.PH_loc.eight[4]);
22795cdaef71SJohn Baldwin }
22805cdaef71SJohn Baldwin 
22815cdaef71SJohn Baldwin static inline void
22825cdaef71SJohn Baldwin set_mbuf_cflags(struct mbuf *m, uint8_t flags)
22835cdaef71SJohn Baldwin {
22845cdaef71SJohn Baldwin 
22855cdaef71SJohn Baldwin 	M_ASSERTPKTHDR(m);
22865cdaef71SJohn Baldwin 	m->m_pkthdr.PH_loc.eight[4] = flags;
22875cdaef71SJohn Baldwin }
22885cdaef71SJohn Baldwin 
22895cdaef71SJohn Baldwin static inline int
22907951040fSNavdeep Parhar mbuf_len16(struct mbuf *m)
22917951040fSNavdeep Parhar {
22927951040fSNavdeep Parhar 	int n;
22937951040fSNavdeep Parhar 
22947951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
22957951040fSNavdeep Parhar 	n = m->m_pkthdr.PH_loc.eight[0];
2296bddf7343SJohn Baldwin 	if (!(mbuf_cflags(m) & MC_TLS))
22977951040fSNavdeep Parhar 		MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
22987951040fSNavdeep Parhar 
22997951040fSNavdeep Parhar 	return (n);
23007951040fSNavdeep Parhar }
23017951040fSNavdeep Parhar 
23027951040fSNavdeep Parhar static inline void
23037951040fSNavdeep Parhar set_mbuf_len16(struct mbuf *m, uint8_t len16)
23047951040fSNavdeep Parhar {
23057951040fSNavdeep Parhar 
23067951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
230730e3f2b4SNavdeep Parhar 	if (!(mbuf_cflags(m) & MC_TLS))
230830e3f2b4SNavdeep Parhar 		MPASS(len16 > 0 && len16 <= SGE_MAX_WR_LEN / 16);
23097951040fSNavdeep Parhar 	m->m_pkthdr.PH_loc.eight[0] = len16;
23107951040fSNavdeep Parhar }
23117951040fSNavdeep Parhar 
2312786099deSNavdeep Parhar #ifdef RATELIMIT
2313786099deSNavdeep Parhar static inline int
2314786099deSNavdeep Parhar mbuf_eo_nsegs(struct mbuf *m)
2315786099deSNavdeep Parhar {
2316786099deSNavdeep Parhar 
2317786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2318786099deSNavdeep Parhar 	return (m->m_pkthdr.PH_loc.eight[1]);
2319786099deSNavdeep Parhar }
2320786099deSNavdeep Parhar 
2321ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
2322786099deSNavdeep Parhar static inline void
2323786099deSNavdeep Parhar set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs)
2324786099deSNavdeep Parhar {
2325786099deSNavdeep Parhar 
2326786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2327786099deSNavdeep Parhar 	m->m_pkthdr.PH_loc.eight[1] = nsegs;
2328786099deSNavdeep Parhar }
2329ffbb373cSNavdeep Parhar #endif
2330786099deSNavdeep Parhar 
2331786099deSNavdeep Parhar static inline int
2332786099deSNavdeep Parhar mbuf_eo_len16(struct mbuf *m)
2333786099deSNavdeep Parhar {
2334786099deSNavdeep Parhar 	int n;
2335786099deSNavdeep Parhar 
2336786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2337786099deSNavdeep Parhar 	n = m->m_pkthdr.PH_loc.eight[2];
2338786099deSNavdeep Parhar 	MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2339786099deSNavdeep Parhar 
2340786099deSNavdeep Parhar 	return (n);
2341786099deSNavdeep Parhar }
2342786099deSNavdeep Parhar 
2343ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
2344786099deSNavdeep Parhar static inline void
2345786099deSNavdeep Parhar set_mbuf_eo_len16(struct mbuf *m, uint8_t len16)
2346786099deSNavdeep Parhar {
2347786099deSNavdeep Parhar 
2348786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2349786099deSNavdeep Parhar 	m->m_pkthdr.PH_loc.eight[2] = len16;
2350786099deSNavdeep Parhar }
2351ffbb373cSNavdeep Parhar #endif
2352786099deSNavdeep Parhar 
2353786099deSNavdeep Parhar static inline int
2354786099deSNavdeep Parhar mbuf_eo_tsclk_tsoff(struct mbuf *m)
2355786099deSNavdeep Parhar {
2356786099deSNavdeep Parhar 
2357786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2358786099deSNavdeep Parhar 	return (m->m_pkthdr.PH_loc.eight[3]);
2359786099deSNavdeep Parhar }
2360786099deSNavdeep Parhar 
2361ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
2362786099deSNavdeep Parhar static inline void
2363786099deSNavdeep Parhar set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff)
2364786099deSNavdeep Parhar {
2365786099deSNavdeep Parhar 
2366786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2367786099deSNavdeep Parhar 	m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff;
2368786099deSNavdeep Parhar }
2369ffbb373cSNavdeep Parhar #endif
2370786099deSNavdeep Parhar 
2371786099deSNavdeep Parhar static inline int
237256fb710fSJohn Baldwin needs_eo(struct m_snd_tag *mst)
2373786099deSNavdeep Parhar {
2374786099deSNavdeep Parhar 
2375c782ea8bSJohn Baldwin 	return (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT);
2376786099deSNavdeep Parhar }
2377786099deSNavdeep Parhar #endif
2378786099deSNavdeep Parhar 
23795cdaef71SJohn Baldwin /*
23805cdaef71SJohn Baldwin  * Try to allocate an mbuf to contain a raw work request.  To make it
23815cdaef71SJohn Baldwin  * easy to construct the work request, don't allocate a chain but a
23825cdaef71SJohn Baldwin  * single mbuf.
23835cdaef71SJohn Baldwin  */
23845cdaef71SJohn Baldwin struct mbuf *
23855cdaef71SJohn Baldwin alloc_wr_mbuf(int len, int how)
23865cdaef71SJohn Baldwin {
23875cdaef71SJohn Baldwin 	struct mbuf *m;
23885cdaef71SJohn Baldwin 
23895cdaef71SJohn Baldwin 	if (len <= MHLEN)
23905cdaef71SJohn Baldwin 		m = m_gethdr(how, MT_DATA);
23915cdaef71SJohn Baldwin 	else if (len <= MCLBYTES)
23925cdaef71SJohn Baldwin 		m = m_getcl(how, MT_DATA, M_PKTHDR);
23935cdaef71SJohn Baldwin 	else
23945cdaef71SJohn Baldwin 		m = NULL;
23955cdaef71SJohn Baldwin 	if (m == NULL)
23965cdaef71SJohn Baldwin 		return (NULL);
23975cdaef71SJohn Baldwin 	m->m_pkthdr.len = len;
23985cdaef71SJohn Baldwin 	m->m_len = len;
23995cdaef71SJohn Baldwin 	set_mbuf_cflags(m, MC_RAW_WR);
24005cdaef71SJohn Baldwin 	set_mbuf_len16(m, howmany(len, 16));
24015cdaef71SJohn Baldwin 	return (m);
24025cdaef71SJohn Baldwin }
24035cdaef71SJohn Baldwin 
2404a4a4ad2dSNavdeep Parhar static inline bool
2405c0236bd9SNavdeep Parhar needs_hwcsum(struct mbuf *m)
2406c0236bd9SNavdeep Parhar {
2407a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP |
2408a4a4ad2dSNavdeep Parhar 	    CSUM_IP_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP |
2409a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_IP6_UDP |
2410a4a4ad2dSNavdeep Parhar 	    CSUM_IP6_TCP | CSUM_IP6_TSO | CSUM_INNER_IP6_UDP |
2411a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO;
2412c0236bd9SNavdeep Parhar 
2413c0236bd9SNavdeep Parhar 	M_ASSERTPKTHDR(m);
2414c0236bd9SNavdeep Parhar 
2415a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
2416c0236bd9SNavdeep Parhar }
2417c0236bd9SNavdeep Parhar 
2418a4a4ad2dSNavdeep Parhar static inline bool
24197951040fSNavdeep Parhar needs_tso(struct mbuf *m)
24207951040fSNavdeep Parhar {
2421a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP_TSO | CSUM_IP6_TSO |
2422a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
24237951040fSNavdeep Parhar 
24247951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
24257951040fSNavdeep Parhar 
2426a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
24277951040fSNavdeep Parhar }
24287951040fSNavdeep Parhar 
2429a4a4ad2dSNavdeep Parhar static inline bool
2430a4a4ad2dSNavdeep Parhar needs_vxlan_csum(struct mbuf *m)
2431a4a4ad2dSNavdeep Parhar {
2432a4a4ad2dSNavdeep Parhar 
2433a4a4ad2dSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2434a4a4ad2dSNavdeep Parhar 
2435a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN);
2436a4a4ad2dSNavdeep Parhar }
2437a4a4ad2dSNavdeep Parhar 
2438a4a4ad2dSNavdeep Parhar static inline bool
2439a4a4ad2dSNavdeep Parhar needs_vxlan_tso(struct mbuf *m)
2440a4a4ad2dSNavdeep Parhar {
2441a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_ENCAP_VXLAN | CSUM_INNER_IP_TSO |
2442a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP6_TSO;
2443a4a4ad2dSNavdeep Parhar 
2444a4a4ad2dSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2445a4a4ad2dSNavdeep Parhar 
2446a4a4ad2dSNavdeep Parhar 	return ((m->m_pkthdr.csum_flags & csum_flags) != 0 &&
2447a4a4ad2dSNavdeep Parhar 	    (m->m_pkthdr.csum_flags & csum_flags) != CSUM_ENCAP_VXLAN);
2448a4a4ad2dSNavdeep Parhar }
2449a4a4ad2dSNavdeep Parhar 
2450ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
2451a4a4ad2dSNavdeep Parhar static inline bool
2452a4a4ad2dSNavdeep Parhar needs_inner_tcp_csum(struct mbuf *m)
2453a4a4ad2dSNavdeep Parhar {
2454a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
2455a4a4ad2dSNavdeep Parhar 
2456a4a4ad2dSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2457a4a4ad2dSNavdeep Parhar 
2458a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
2459a4a4ad2dSNavdeep Parhar }
2460ffbb373cSNavdeep Parhar #endif
2461a4a4ad2dSNavdeep Parhar 
2462a4a4ad2dSNavdeep Parhar static inline bool
24637951040fSNavdeep Parhar needs_l3_csum(struct mbuf *m)
24647951040fSNavdeep Parhar {
2465a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP | CSUM_IP_TSO | CSUM_INNER_IP |
2466a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP_TSO;
24677951040fSNavdeep Parhar 
24687951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
24697951040fSNavdeep Parhar 
2470a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
24717951040fSNavdeep Parhar }
24727951040fSNavdeep Parhar 
2473a4a4ad2dSNavdeep Parhar static inline bool
2474a4a4ad2dSNavdeep Parhar needs_outer_tcp_csum(struct mbuf *m)
2475c0236bd9SNavdeep Parhar {
2476a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP_TCP | CSUM_IP_TSO | CSUM_IP6_TCP |
2477a4a4ad2dSNavdeep Parhar 	    CSUM_IP6_TSO;
2478c0236bd9SNavdeep Parhar 
2479c0236bd9SNavdeep Parhar 	M_ASSERTPKTHDR(m);
2480a4a4ad2dSNavdeep Parhar 
2481a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
2482c0236bd9SNavdeep Parhar }
2483c0236bd9SNavdeep Parhar 
2484c0236bd9SNavdeep Parhar #ifdef RATELIMIT
2485a4a4ad2dSNavdeep Parhar static inline bool
2486a4a4ad2dSNavdeep Parhar needs_outer_l4_csum(struct mbuf *m)
24877951040fSNavdeep Parhar {
2488a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_TSO |
2489a4a4ad2dSNavdeep Parhar 	    CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_TSO;
24907951040fSNavdeep Parhar 
24917951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
24927951040fSNavdeep Parhar 
2493a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
24947951040fSNavdeep Parhar }
24957951040fSNavdeep Parhar 
2496a4a4ad2dSNavdeep Parhar static inline bool
2497a4a4ad2dSNavdeep Parhar needs_outer_udp_csum(struct mbuf *m)
2498786099deSNavdeep Parhar {
2499a4a4ad2dSNavdeep Parhar 	const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP6_UDP;
2500786099deSNavdeep Parhar 
2501786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2502a4a4ad2dSNavdeep Parhar 
2503a4a4ad2dSNavdeep Parhar 	return (m->m_pkthdr.csum_flags & csum_flags);
2504786099deSNavdeep Parhar }
2505c3fce948SNavdeep Parhar #endif
2506786099deSNavdeep Parhar 
2507a4a4ad2dSNavdeep Parhar static inline bool
25087951040fSNavdeep Parhar needs_vlan_insertion(struct mbuf *m)
25097951040fSNavdeep Parhar {
25107951040fSNavdeep Parhar 
25117951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
25127951040fSNavdeep Parhar 
2513a6a8ff35SNavdeep Parhar 	return (m->m_flags & M_VLANTAG);
25147951040fSNavdeep Parhar }
25157951040fSNavdeep Parhar 
2516*94e6b3feSNavdeep Parhar #if defined(INET) || defined(INET6)
25177951040fSNavdeep Parhar static void *
25187951040fSNavdeep Parhar m_advance(struct mbuf **pm, int *poffset, int len)
25197951040fSNavdeep Parhar {
25207951040fSNavdeep Parhar 	struct mbuf *m = *pm;
25217951040fSNavdeep Parhar 	int offset = *poffset;
25227951040fSNavdeep Parhar 	uintptr_t p = 0;
25237951040fSNavdeep Parhar 
25247951040fSNavdeep Parhar 	MPASS(len > 0);
25257951040fSNavdeep Parhar 
2526e06ab612SJohn Baldwin 	for (;;) {
25277951040fSNavdeep Parhar 		if (offset + len < m->m_len) {
25287951040fSNavdeep Parhar 			offset += len;
25297951040fSNavdeep Parhar 			p = mtod(m, uintptr_t) + offset;
25307951040fSNavdeep Parhar 			break;
25317951040fSNavdeep Parhar 		}
25327951040fSNavdeep Parhar 		len -= m->m_len - offset;
25337951040fSNavdeep Parhar 		m = m->m_next;
25347951040fSNavdeep Parhar 		offset = 0;
25357951040fSNavdeep Parhar 		MPASS(m != NULL);
25367951040fSNavdeep Parhar 	}
25377951040fSNavdeep Parhar 	*poffset = offset;
25387951040fSNavdeep Parhar 	*pm = m;
25397951040fSNavdeep Parhar 	return ((void *)p);
25407951040fSNavdeep Parhar }
2541*94e6b3feSNavdeep Parhar #endif
25427951040fSNavdeep Parhar 
2543d76bbe17SJohn Baldwin static inline int
2544d76bbe17SJohn Baldwin count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
2545d76bbe17SJohn Baldwin {
2546d76bbe17SJohn Baldwin 	vm_paddr_t paddr;
2547d76bbe17SJohn Baldwin 	int i, len, off, pglen, pgoff, seglen, segoff;
2548d76bbe17SJohn Baldwin 	int nsegs = 0;
2549d76bbe17SJohn Baldwin 
2550365e8da4SGleb Smirnoff 	M_ASSERTEXTPG(m);
2551d76bbe17SJohn Baldwin 	off = mtod(m, vm_offset_t);
2552d76bbe17SJohn Baldwin 	len = m->m_len;
2553d76bbe17SJohn Baldwin 	off += skip;
2554d76bbe17SJohn Baldwin 	len -= skip;
2555d76bbe17SJohn Baldwin 
25567b6c99d0SGleb Smirnoff 	if (m->m_epg_hdrlen != 0) {
25577b6c99d0SGleb Smirnoff 		if (off >= m->m_epg_hdrlen) {
25587b6c99d0SGleb Smirnoff 			off -= m->m_epg_hdrlen;
2559d76bbe17SJohn Baldwin 		} else {
25607b6c99d0SGleb Smirnoff 			seglen = m->m_epg_hdrlen - off;
2561d76bbe17SJohn Baldwin 			segoff = off;
2562d76bbe17SJohn Baldwin 			seglen = min(seglen, len);
2563d76bbe17SJohn Baldwin 			off = 0;
2564d76bbe17SJohn Baldwin 			len -= seglen;
2565d76bbe17SJohn Baldwin 			paddr = pmap_kextract(
25660c103266SGleb Smirnoff 			    (vm_offset_t)&m->m_epg_hdr[segoff]);
2567d76bbe17SJohn Baldwin 			if (*nextaddr != paddr)
2568d76bbe17SJohn Baldwin 				nsegs++;
2569d76bbe17SJohn Baldwin 			*nextaddr = paddr + seglen;
2570d76bbe17SJohn Baldwin 		}
2571d76bbe17SJohn Baldwin 	}
25727b6c99d0SGleb Smirnoff 	pgoff = m->m_epg_1st_off;
25737b6c99d0SGleb Smirnoff 	for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
2574c4ee38f8SGleb Smirnoff 		pglen = m_epg_pagelen(m, i, pgoff);
2575d76bbe17SJohn Baldwin 		if (off >= pglen) {
2576d76bbe17SJohn Baldwin 			off -= pglen;
2577d76bbe17SJohn Baldwin 			pgoff = 0;
2578d76bbe17SJohn Baldwin 			continue;
2579d76bbe17SJohn Baldwin 		}
2580d76bbe17SJohn Baldwin 		seglen = pglen - off;
2581d76bbe17SJohn Baldwin 		segoff = pgoff + off;
2582d76bbe17SJohn Baldwin 		off = 0;
2583d76bbe17SJohn Baldwin 		seglen = min(seglen, len);
2584d76bbe17SJohn Baldwin 		len -= seglen;
25850c103266SGleb Smirnoff 		paddr = m->m_epg_pa[i] + segoff;
2586d76bbe17SJohn Baldwin 		if (*nextaddr != paddr)
2587d76bbe17SJohn Baldwin 			nsegs++;
2588d76bbe17SJohn Baldwin 		*nextaddr = paddr + seglen;
2589d76bbe17SJohn Baldwin 		pgoff = 0;
2590d76bbe17SJohn Baldwin 	};
2591d76bbe17SJohn Baldwin 	if (len != 0) {
25927b6c99d0SGleb Smirnoff 		seglen = min(len, m->m_epg_trllen - off);
2593d76bbe17SJohn Baldwin 		len -= seglen;
25940c103266SGleb Smirnoff 		paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
2595d76bbe17SJohn Baldwin 		if (*nextaddr != paddr)
2596d76bbe17SJohn Baldwin 			nsegs++;
2597d76bbe17SJohn Baldwin 		*nextaddr = paddr + seglen;
2598d76bbe17SJohn Baldwin 	}
2599d76bbe17SJohn Baldwin 
2600d76bbe17SJohn Baldwin 	return (nsegs);
2601d76bbe17SJohn Baldwin }
2602d76bbe17SJohn Baldwin 
2603d76bbe17SJohn Baldwin 
26047951040fSNavdeep Parhar /*
26057951040fSNavdeep Parhar  * Can deal with empty mbufs in the chain that have m_len = 0, but the chain
2606786099deSNavdeep Parhar  * must have at least one mbuf that's not empty.  It is possible for this
2607786099deSNavdeep Parhar  * routine to return 0 if skip accounts for all the contents of the mbuf chain.
26087951040fSNavdeep Parhar  */
26097951040fSNavdeep Parhar static inline int
2610d76bbe17SJohn Baldwin count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags)
26117951040fSNavdeep Parhar {
2612d76bbe17SJohn Baldwin 	vm_paddr_t nextaddr, paddr;
261377e9044cSNavdeep Parhar 	vm_offset_t va;
26147951040fSNavdeep Parhar 	int len, nsegs;
26157951040fSNavdeep Parhar 
2616786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m);
2617786099deSNavdeep Parhar 	MPASS(m->m_pkthdr.len > 0);
2618786099deSNavdeep Parhar 	MPASS(m->m_pkthdr.len >= skip);
26197951040fSNavdeep Parhar 
26207951040fSNavdeep Parhar 	nsegs = 0;
2621d76bbe17SJohn Baldwin 	nextaddr = 0;
26227951040fSNavdeep Parhar 	for (; m; m = m->m_next) {
26237951040fSNavdeep Parhar 		len = m->m_len;
26247951040fSNavdeep Parhar 		if (__predict_false(len == 0))
26257951040fSNavdeep Parhar 			continue;
2626786099deSNavdeep Parhar 		if (skip >= len) {
2627786099deSNavdeep Parhar 			skip -= len;
2628786099deSNavdeep Parhar 			continue;
2629786099deSNavdeep Parhar 		}
26306edfd179SGleb Smirnoff 		if ((m->m_flags & M_EXTPG) != 0) {
2631d76bbe17SJohn Baldwin 			*cflags |= MC_NOMAP;
2632d76bbe17SJohn Baldwin 			nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr);
2633d76bbe17SJohn Baldwin 			skip = 0;
2634d76bbe17SJohn Baldwin 			continue;
2635d76bbe17SJohn Baldwin 		}
2636786099deSNavdeep Parhar 		va = mtod(m, vm_offset_t) + skip;
2637786099deSNavdeep Parhar 		len -= skip;
2638786099deSNavdeep Parhar 		skip = 0;
2639d76bbe17SJohn Baldwin 		paddr = pmap_kextract(va);
2640786099deSNavdeep Parhar 		nsegs += sglist_count((void *)(uintptr_t)va, len);
2641d76bbe17SJohn Baldwin 		if (paddr == nextaddr)
26427951040fSNavdeep Parhar 			nsegs--;
2643d76bbe17SJohn Baldwin 		nextaddr = pmap_kextract(va + len - 1) + 1;
26447951040fSNavdeep Parhar 	}
26457951040fSNavdeep Parhar 
26467951040fSNavdeep Parhar 	return (nsegs);
26477951040fSNavdeep Parhar }
26487951040fSNavdeep Parhar 
26497951040fSNavdeep Parhar /*
2650a4a4ad2dSNavdeep Parhar  * The maximum number of segments that can fit in a WR.
2651a4a4ad2dSNavdeep Parhar  */
2652a4a4ad2dSNavdeep Parhar static int
265330e3f2b4SNavdeep Parhar max_nsegs_allowed(struct mbuf *m, bool vm_wr)
2654a4a4ad2dSNavdeep Parhar {
2655a4a4ad2dSNavdeep Parhar 
265630e3f2b4SNavdeep Parhar 	if (vm_wr) {
265730e3f2b4SNavdeep Parhar 		if (needs_tso(m))
265830e3f2b4SNavdeep Parhar 			return (TX_SGL_SEGS_VM_TSO);
265930e3f2b4SNavdeep Parhar 		return (TX_SGL_SEGS_VM);
266030e3f2b4SNavdeep Parhar 	}
266130e3f2b4SNavdeep Parhar 
2662a4a4ad2dSNavdeep Parhar 	if (needs_tso(m)) {
2663a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_tso(m))
2664a4a4ad2dSNavdeep Parhar 			return (TX_SGL_SEGS_VXLAN_TSO);
2665a4a4ad2dSNavdeep Parhar 		else
2666a4a4ad2dSNavdeep Parhar 			return (TX_SGL_SEGS_TSO);
2667a4a4ad2dSNavdeep Parhar 	}
2668a4a4ad2dSNavdeep Parhar 
2669a4a4ad2dSNavdeep Parhar 	return (TX_SGL_SEGS);
2670a4a4ad2dSNavdeep Parhar }
2671a4a4ad2dSNavdeep Parhar 
2672b9820bcaSNavdeep Parhar static struct timeval txerr_ratecheck = {0};
2673b9820bcaSNavdeep Parhar static const struct timeval txerr_interval = {3, 0};
2674b9820bcaSNavdeep Parhar 
2675a4a4ad2dSNavdeep Parhar /*
26767951040fSNavdeep Parhar  * Analyze the mbuf to determine its tx needs.  The mbuf passed in may change:
26777951040fSNavdeep Parhar  * a) caller can assume it's been freed if this function returns with an error.
26787951040fSNavdeep Parhar  * b) it may get defragged up if the gather list is too long for the hardware.
26797951040fSNavdeep Parhar  */
26807951040fSNavdeep Parhar int
268130e3f2b4SNavdeep Parhar parse_pkt(struct mbuf **mp, bool vm_wr)
26827951040fSNavdeep Parhar {
26837951040fSNavdeep Parhar 	struct mbuf *m0 = *mp, *m;
268439d5cbdcSNavdeep Parhar 	int rc, nsegs, defragged = 0;
26857951040fSNavdeep Parhar 	struct ether_header *eh;
268639d5cbdcSNavdeep Parhar #ifdef INET
26877951040fSNavdeep Parhar 	void *l3hdr;
268839d5cbdcSNavdeep Parhar #endif
26897951040fSNavdeep Parhar #if defined(INET) || defined(INET6)
269039d5cbdcSNavdeep Parhar 	int offset;
26917951040fSNavdeep Parhar 	struct tcphdr *tcp;
26927951040fSNavdeep Parhar #endif
2693bddf7343SJohn Baldwin #if defined(KERN_TLS) || defined(RATELIMIT)
269456fb710fSJohn Baldwin 	struct m_snd_tag *mst;
2695e38a50e8SJohn Baldwin #endif
26967951040fSNavdeep Parhar 	uint16_t eh_type;
2697d76bbe17SJohn Baldwin 	uint8_t cflags;
26987951040fSNavdeep Parhar 
2699d76bbe17SJohn Baldwin 	cflags = 0;
27007951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m0);
27017951040fSNavdeep Parhar 	if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) {
27027951040fSNavdeep Parhar 		rc = EINVAL;
27037951040fSNavdeep Parhar fail:
27047951040fSNavdeep Parhar 		m_freem(m0);
27057951040fSNavdeep Parhar 		*mp = NULL;
27067951040fSNavdeep Parhar 		return (rc);
27077951040fSNavdeep Parhar 	}
27087951040fSNavdeep Parhar restart:
27097951040fSNavdeep Parhar 	/*
27107951040fSNavdeep Parhar 	 * First count the number of gather list segments in the payload.
27117951040fSNavdeep Parhar 	 * Defrag the mbuf if nsegs exceeds the hardware limit.
27127951040fSNavdeep Parhar 	 */
27137951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m0);
27147951040fSNavdeep Parhar 	MPASS(m0->m_pkthdr.len > 0);
2715d76bbe17SJohn Baldwin 	nsegs = count_mbuf_nsegs(m0, 0, &cflags);
2716bddf7343SJohn Baldwin #if defined(KERN_TLS) || defined(RATELIMIT)
2717e38a50e8SJohn Baldwin 	if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG)
271856fb710fSJohn Baldwin 		mst = m0->m_pkthdr.snd_tag;
2719e38a50e8SJohn Baldwin 	else
272056fb710fSJohn Baldwin 		mst = NULL;
2721e38a50e8SJohn Baldwin #endif
2722bddf7343SJohn Baldwin #ifdef KERN_TLS
2723c782ea8bSJohn Baldwin 	if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) {
2724bddf7343SJohn Baldwin 		int len16;
2725bddf7343SJohn Baldwin 
2726bddf7343SJohn Baldwin 		cflags |= MC_TLS;
2727bddf7343SJohn Baldwin 		set_mbuf_cflags(m0, cflags);
2728bddf7343SJohn Baldwin 		rc = t6_ktls_parse_pkt(m0, &nsegs, &len16);
2729bddf7343SJohn Baldwin 		if (rc != 0)
2730bddf7343SJohn Baldwin 			goto fail;
2731bddf7343SJohn Baldwin 		set_mbuf_nsegs(m0, nsegs);
2732bddf7343SJohn Baldwin 		set_mbuf_len16(m0, len16);
2733bddf7343SJohn Baldwin 		return (0);
2734bddf7343SJohn Baldwin 	}
2735bddf7343SJohn Baldwin #endif
273630e3f2b4SNavdeep Parhar 	if (nsegs > max_nsegs_allowed(m0, vm_wr)) {
27377054f6ecSNavdeep Parhar 		if (defragged++ > 0) {
27387951040fSNavdeep Parhar 			rc = EFBIG;
27397951040fSNavdeep Parhar 			goto fail;
27407951040fSNavdeep Parhar 		}
27417054f6ecSNavdeep Parhar 		counter_u64_add(defrags, 1);
27427054f6ecSNavdeep Parhar 		if ((m = m_defrag(m0, M_NOWAIT)) == NULL) {
27437054f6ecSNavdeep Parhar 			rc = ENOMEM;
27447054f6ecSNavdeep Parhar 			goto fail;
27457054f6ecSNavdeep Parhar 		}
27467951040fSNavdeep Parhar 		*mp = m0 = m;	/* update caller's copy after defrag */
27477951040fSNavdeep Parhar 		goto restart;
27487951040fSNavdeep Parhar 	}
27497951040fSNavdeep Parhar 
2750d76bbe17SJohn Baldwin 	if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN &&
2751d76bbe17SJohn Baldwin 	    !(cflags & MC_NOMAP))) {
27527054f6ecSNavdeep Parhar 		counter_u64_add(pullups, 1);
27537951040fSNavdeep Parhar 		m0 = m_pullup(m0, m0->m_pkthdr.len);
27547951040fSNavdeep Parhar 		if (m0 == NULL) {
27557951040fSNavdeep Parhar 			/* Should have left well enough alone. */
27567951040fSNavdeep Parhar 			rc = EFBIG;
27577951040fSNavdeep Parhar 			goto fail;
27587951040fSNavdeep Parhar 		}
27597951040fSNavdeep Parhar 		*mp = m0;	/* update caller's copy after pullup */
27607951040fSNavdeep Parhar 		goto restart;
27617951040fSNavdeep Parhar 	}
27627951040fSNavdeep Parhar 	set_mbuf_nsegs(m0, nsegs);
2763d76bbe17SJohn Baldwin 	set_mbuf_cflags(m0, cflags);
276430e3f2b4SNavdeep Parhar 	calculate_mbuf_len16(m0, vm_wr);
27657951040fSNavdeep Parhar 
2766786099deSNavdeep Parhar #ifdef RATELIMIT
2767786099deSNavdeep Parhar 	/*
2768786099deSNavdeep Parhar 	 * Ethofld is limited to TCP and UDP for now, and only when L4 hw
2769a4a4ad2dSNavdeep Parhar 	 * checksumming is enabled.  needs_outer_l4_csum happens to check for
2770a4a4ad2dSNavdeep Parhar 	 * all the right things.
2771786099deSNavdeep Parhar 	 */
277256fb710fSJohn Baldwin 	if (__predict_false(needs_eo(mst) && !needs_outer_l4_csum(m0))) {
2773fb3bc596SJohn Baldwin 		m_snd_tag_rele(m0->m_pkthdr.snd_tag);
2774786099deSNavdeep Parhar 		m0->m_pkthdr.snd_tag = NULL;
2775fb3bc596SJohn Baldwin 		m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
277656fb710fSJohn Baldwin 		mst = NULL;
2777fb3bc596SJohn Baldwin 	}
2778786099deSNavdeep Parhar #endif
2779786099deSNavdeep Parhar 
2780c0236bd9SNavdeep Parhar 	if (!needs_hwcsum(m0)
2781786099deSNavdeep Parhar #ifdef RATELIMIT
278256fb710fSJohn Baldwin    		 && !needs_eo(mst)
2783786099deSNavdeep Parhar #endif
2784c0236bd9SNavdeep Parhar 	)
27857951040fSNavdeep Parhar 		return (0);
27867951040fSNavdeep Parhar 
27877951040fSNavdeep Parhar 	m = m0;
27887951040fSNavdeep Parhar 	eh = mtod(m, struct ether_header *);
27897951040fSNavdeep Parhar 	eh_type = ntohs(eh->ether_type);
27907951040fSNavdeep Parhar 	if (eh_type == ETHERTYPE_VLAN) {
27917951040fSNavdeep Parhar 		struct ether_vlan_header *evh = (void *)eh;
27927951040fSNavdeep Parhar 
27937951040fSNavdeep Parhar 		eh_type = ntohs(evh->evl_proto);
27947951040fSNavdeep Parhar 		m0->m_pkthdr.l2hlen = sizeof(*evh);
27957951040fSNavdeep Parhar 	} else
27967951040fSNavdeep Parhar 		m0->m_pkthdr.l2hlen = sizeof(*eh);
27977951040fSNavdeep Parhar 
279839d5cbdcSNavdeep Parhar #if defined(INET) || defined(INET6)
27997951040fSNavdeep Parhar 	offset = 0;
280039d5cbdcSNavdeep Parhar #ifdef INET
28017951040fSNavdeep Parhar 	l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
280239d5cbdcSNavdeep Parhar #else
280339d5cbdcSNavdeep Parhar 	m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
280439d5cbdcSNavdeep Parhar #endif
280539d5cbdcSNavdeep Parhar #endif
28067951040fSNavdeep Parhar 
28077951040fSNavdeep Parhar 	switch (eh_type) {
28087951040fSNavdeep Parhar #ifdef INET6
28097951040fSNavdeep Parhar 	case ETHERTYPE_IPV6:
2810a4a4ad2dSNavdeep Parhar 		m0->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
28117951040fSNavdeep Parhar 		break;
28127951040fSNavdeep Parhar #endif
28137951040fSNavdeep Parhar #ifdef INET
28147951040fSNavdeep Parhar 	case ETHERTYPE_IP:
28157951040fSNavdeep Parhar 	{
28167951040fSNavdeep Parhar 		struct ip *ip = l3hdr;
28177951040fSNavdeep Parhar 
2818a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_csum(m0)) {
2819a4a4ad2dSNavdeep Parhar 			/* Driver will do the outer IP hdr checksum. */
2820a4a4ad2dSNavdeep Parhar 			ip->ip_sum = 0;
2821a4a4ad2dSNavdeep Parhar 			if (needs_vxlan_tso(m0)) {
2822a4a4ad2dSNavdeep Parhar 				const uint16_t ipl = ip->ip_len;
2823a4a4ad2dSNavdeep Parhar 
2824a4a4ad2dSNavdeep Parhar 				ip->ip_len = 0;
2825a4a4ad2dSNavdeep Parhar 				ip->ip_sum = ~in_cksum_hdr(ip);
2826a4a4ad2dSNavdeep Parhar 				ip->ip_len = ipl;
2827a4a4ad2dSNavdeep Parhar 			} else
2828a4a4ad2dSNavdeep Parhar 				ip->ip_sum = in_cksum_hdr(ip);
2829a4a4ad2dSNavdeep Parhar 		}
2830a4a4ad2dSNavdeep Parhar 		m0->m_pkthdr.l3hlen = ip->ip_hl << 2;
28317951040fSNavdeep Parhar 		break;
28327951040fSNavdeep Parhar 	}
28337951040fSNavdeep Parhar #endif
28347951040fSNavdeep Parhar 	default:
2835b9820bcaSNavdeep Parhar 		if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2836b9820bcaSNavdeep Parhar 			log(LOG_ERR, "%s: ethertype 0x%04x unknown.  "
2837b9820bcaSNavdeep Parhar 			    "if_cxgbe must be compiled with the same "
2838b9820bcaSNavdeep Parhar 			    "INET/INET6 options as the kernel.\n", __func__,
2839b9820bcaSNavdeep Parhar 			    eh_type);
2840b9820bcaSNavdeep Parhar 		}
2841b9820bcaSNavdeep Parhar 		rc = EINVAL;
2842b9820bcaSNavdeep Parhar 		goto fail;
28437951040fSNavdeep Parhar 	}
28447951040fSNavdeep Parhar 
284539d5cbdcSNavdeep Parhar #if defined(INET) || defined(INET6)
2846a4a4ad2dSNavdeep Parhar 	if (needs_vxlan_csum(m0)) {
2847a4a4ad2dSNavdeep Parhar 		m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2848a4a4ad2dSNavdeep Parhar 		m0->m_pkthdr.l5hlen = sizeof(struct vxlan_header);
2849a4a4ad2dSNavdeep Parhar 
2850a4a4ad2dSNavdeep Parhar 		/* Inner headers. */
2851a4a4ad2dSNavdeep Parhar 		eh = m_advance(&m, &offset, m0->m_pkthdr.l3hlen +
2852a4a4ad2dSNavdeep Parhar 		    sizeof(struct udphdr) + sizeof(struct vxlan_header));
2853a4a4ad2dSNavdeep Parhar 		eh_type = ntohs(eh->ether_type);
2854a4a4ad2dSNavdeep Parhar 		if (eh_type == ETHERTYPE_VLAN) {
2855a4a4ad2dSNavdeep Parhar 			struct ether_vlan_header *evh = (void *)eh;
2856a4a4ad2dSNavdeep Parhar 
2857a4a4ad2dSNavdeep Parhar 			eh_type = ntohs(evh->evl_proto);
2858a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.inner_l2hlen = sizeof(*evh);
2859a4a4ad2dSNavdeep Parhar 		} else
2860a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.inner_l2hlen = sizeof(*eh);
286139d5cbdcSNavdeep Parhar #ifdef INET
2862a4a4ad2dSNavdeep Parhar 		l3hdr = m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen);
286339d5cbdcSNavdeep Parhar #else
286439d5cbdcSNavdeep Parhar 		m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen);
286539d5cbdcSNavdeep Parhar #endif
2866a4a4ad2dSNavdeep Parhar 
2867a4a4ad2dSNavdeep Parhar 		switch (eh_type) {
2868a4a4ad2dSNavdeep Parhar #ifdef INET6
2869a4a4ad2dSNavdeep Parhar 		case ETHERTYPE_IPV6:
2870a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.inner_l3hlen = sizeof(struct ip6_hdr);
2871a4a4ad2dSNavdeep Parhar 			break;
2872a4a4ad2dSNavdeep Parhar #endif
2873a4a4ad2dSNavdeep Parhar #ifdef INET
2874a4a4ad2dSNavdeep Parhar 		case ETHERTYPE_IP:
2875a4a4ad2dSNavdeep Parhar 		{
2876a4a4ad2dSNavdeep Parhar 			struct ip *ip = l3hdr;
2877a4a4ad2dSNavdeep Parhar 
2878a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.inner_l3hlen = ip->ip_hl << 2;
2879a4a4ad2dSNavdeep Parhar 			break;
2880a4a4ad2dSNavdeep Parhar 		}
2881a4a4ad2dSNavdeep Parhar #endif
2882a4a4ad2dSNavdeep Parhar 		default:
2883b9820bcaSNavdeep Parhar 			if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2884b9820bcaSNavdeep Parhar 				log(LOG_ERR, "%s: VXLAN hw offload requested"
2885b9820bcaSNavdeep Parhar 				    "with unknown ethertype 0x%04x.  if_cxgbe "
2886b9820bcaSNavdeep Parhar 				    "must be compiled with the same INET/INET6 "
2887b9820bcaSNavdeep Parhar 				    "options as the kernel.\n", __func__,
2888b9820bcaSNavdeep Parhar 				    eh_type);
2889b9820bcaSNavdeep Parhar 			}
2890b9820bcaSNavdeep Parhar 			rc = EINVAL;
2891b9820bcaSNavdeep Parhar 			goto fail;
2892a4a4ad2dSNavdeep Parhar 		}
2893a4a4ad2dSNavdeep Parhar 		if (needs_inner_tcp_csum(m0)) {
2894a4a4ad2dSNavdeep Parhar 			tcp = m_advance(&m, &offset, m0->m_pkthdr.inner_l3hlen);
2895a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.inner_l4hlen = tcp->th_off * 4;
2896a4a4ad2dSNavdeep Parhar 		}
2897a4a4ad2dSNavdeep Parhar 		MPASS((m0->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
2898a4a4ad2dSNavdeep Parhar 		m0->m_pkthdr.csum_flags &= CSUM_INNER_IP6_UDP |
2899a4a4ad2dSNavdeep Parhar 		    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP |
2900a4a4ad2dSNavdeep Parhar 		    CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO |
2901a4a4ad2dSNavdeep Parhar 		    CSUM_ENCAP_VXLAN;
2902a4a4ad2dSNavdeep Parhar 	}
2903a4a4ad2dSNavdeep Parhar 
2904a4a4ad2dSNavdeep Parhar 	if (needs_outer_tcp_csum(m0)) {
29057951040fSNavdeep Parhar 		tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen);
29067951040fSNavdeep Parhar 		m0->m_pkthdr.l4hlen = tcp->th_off * 4;
2907786099deSNavdeep Parhar #ifdef RATELIMIT
2908786099deSNavdeep Parhar 		if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) {
2909786099deSNavdeep Parhar 			set_mbuf_eo_tsclk_tsoff(m0,
2910786099deSNavdeep Parhar 			    V_FW_ETH_TX_EO_WR_TSCLK(tsclk) |
2911786099deSNavdeep Parhar 			    V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1));
2912786099deSNavdeep Parhar 		} else
2913786099deSNavdeep Parhar 			set_mbuf_eo_tsclk_tsoff(m0, 0);
2914a4a4ad2dSNavdeep Parhar 	} else if (needs_outer_udp_csum(m0)) {
2915786099deSNavdeep Parhar 		m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2916786099deSNavdeep Parhar #endif
29176af45170SJohn Baldwin 	}
2918786099deSNavdeep Parhar #ifdef RATELIMIT
291956fb710fSJohn Baldwin 	if (needs_eo(mst)) {
2920786099deSNavdeep Parhar 		u_int immhdrs;
2921786099deSNavdeep Parhar 
2922786099deSNavdeep Parhar 		/* EO WRs have the headers in the WR and not the GL. */
2923786099deSNavdeep Parhar 		immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen +
2924786099deSNavdeep Parhar 		    m0->m_pkthdr.l4hlen;
2925d76bbe17SJohn Baldwin 		cflags = 0;
2926d76bbe17SJohn Baldwin 		nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags);
2927d76bbe17SJohn Baldwin 		MPASS(cflags == mbuf_cflags(m0));
2928786099deSNavdeep Parhar 		set_mbuf_eo_nsegs(m0, nsegs);
2929786099deSNavdeep Parhar 		set_mbuf_eo_len16(m0,
2930786099deSNavdeep Parhar 		    txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0)));
2931786099deSNavdeep Parhar 	}
2932786099deSNavdeep Parhar #endif
29337951040fSNavdeep Parhar #endif
29347951040fSNavdeep Parhar 	MPASS(m0 == *mp);
29357951040fSNavdeep Parhar 	return (0);
29367951040fSNavdeep Parhar }
29377951040fSNavdeep Parhar 
29387951040fSNavdeep Parhar void *
29397951040fSNavdeep Parhar start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
29407951040fSNavdeep Parhar {
29417951040fSNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
29427951040fSNavdeep Parhar 	struct adapter *sc = wrq->adapter;
29437951040fSNavdeep Parhar 	int ndesc, available;
29447951040fSNavdeep Parhar 	struct wrqe *wr;
29457951040fSNavdeep Parhar 	void *w;
29467951040fSNavdeep Parhar 
29477951040fSNavdeep Parhar 	MPASS(len16 > 0);
29480cadedfcSNavdeep Parhar 	ndesc = tx_len16_to_desc(len16);
29497951040fSNavdeep Parhar 	MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
29507951040fSNavdeep Parhar 
29517951040fSNavdeep Parhar 	EQ_LOCK(eq);
29527951040fSNavdeep Parhar 
29538d6ae10aSNavdeep Parhar 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
29547951040fSNavdeep Parhar 		drain_wrq_wr_list(sc, wrq);
29557951040fSNavdeep Parhar 
29567951040fSNavdeep Parhar 	if (!STAILQ_EMPTY(&wrq->wr_list)) {
29577951040fSNavdeep Parhar slowpath:
29587951040fSNavdeep Parhar 		EQ_UNLOCK(eq);
29597951040fSNavdeep Parhar 		wr = alloc_wrqe(len16 * 16, wrq);
29607951040fSNavdeep Parhar 		if (__predict_false(wr == NULL))
29617951040fSNavdeep Parhar 			return (NULL);
29627951040fSNavdeep Parhar 		cookie->pidx = -1;
29637951040fSNavdeep Parhar 		cookie->ndesc = ndesc;
29647951040fSNavdeep Parhar 		return (&wr->wr);
29657951040fSNavdeep Parhar 	}
29667951040fSNavdeep Parhar 
29677951040fSNavdeep Parhar 	eq->cidx = read_hw_cidx(eq);
29687951040fSNavdeep Parhar 	if (eq->pidx == eq->cidx)
29697951040fSNavdeep Parhar 		available = eq->sidx - 1;
29707951040fSNavdeep Parhar 	else
29717951040fSNavdeep Parhar 		available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
29727951040fSNavdeep Parhar 	if (available < ndesc)
29737951040fSNavdeep Parhar 		goto slowpath;
29747951040fSNavdeep Parhar 
29757951040fSNavdeep Parhar 	cookie->pidx = eq->pidx;
29767951040fSNavdeep Parhar 	cookie->ndesc = ndesc;
29777951040fSNavdeep Parhar 	TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link);
29787951040fSNavdeep Parhar 
29797951040fSNavdeep Parhar 	w = &eq->desc[eq->pidx];
29807951040fSNavdeep Parhar 	IDXINCR(eq->pidx, ndesc, eq->sidx);
2981f50c49ccSNavdeep Parhar 	if (__predict_false(cookie->pidx + ndesc > eq->sidx)) {
29827951040fSNavdeep Parhar 		w = &wrq->ss[0];
29837951040fSNavdeep Parhar 		wrq->ss_pidx = cookie->pidx;
29847951040fSNavdeep Parhar 		wrq->ss_len = len16 * 16;
29857951040fSNavdeep Parhar 	}
29867951040fSNavdeep Parhar 
29877951040fSNavdeep Parhar 	EQ_UNLOCK(eq);
29887951040fSNavdeep Parhar 
29897951040fSNavdeep Parhar 	return (w);
29907951040fSNavdeep Parhar }
29917951040fSNavdeep Parhar 
29927951040fSNavdeep Parhar void
29937951040fSNavdeep Parhar commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
29947951040fSNavdeep Parhar {
29957951040fSNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
29967951040fSNavdeep Parhar 	struct adapter *sc = wrq->adapter;
29977951040fSNavdeep Parhar 	int ndesc, pidx;
29987951040fSNavdeep Parhar 	struct wrq_cookie *prev, *next;
29997951040fSNavdeep Parhar 
30007951040fSNavdeep Parhar 	if (cookie->pidx == -1) {
30017951040fSNavdeep Parhar 		struct wrqe *wr = __containerof(w, struct wrqe, wr);
30027951040fSNavdeep Parhar 
30037951040fSNavdeep Parhar 		t4_wrq_tx(sc, wr);
30047951040fSNavdeep Parhar 		return;
30057951040fSNavdeep Parhar 	}
30067951040fSNavdeep Parhar 
30077951040fSNavdeep Parhar 	if (__predict_false(w == &wrq->ss[0])) {
30087951040fSNavdeep Parhar 		int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE;
30097951040fSNavdeep Parhar 
30107951040fSNavdeep Parhar 		MPASS(wrq->ss_len > n);	/* WR had better wrap around. */
30117951040fSNavdeep Parhar 		bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n);
30127951040fSNavdeep Parhar 		bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n);
30137951040fSNavdeep Parhar 		wrq->tx_wrs_ss++;
30147951040fSNavdeep Parhar 	} else
30157951040fSNavdeep Parhar 		wrq->tx_wrs_direct++;
30167951040fSNavdeep Parhar 
30177951040fSNavdeep Parhar 	EQ_LOCK(eq);
30188d6ae10aSNavdeep Parhar 	ndesc = cookie->ndesc;	/* Can be more than SGE_MAX_WR_NDESC here. */
30198d6ae10aSNavdeep Parhar 	pidx = cookie->pidx;
30208d6ae10aSNavdeep Parhar 	MPASS(pidx >= 0 && pidx < eq->sidx);
30217951040fSNavdeep Parhar 	prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link);
30227951040fSNavdeep Parhar 	next = TAILQ_NEXT(cookie, link);
30237951040fSNavdeep Parhar 	if (prev == NULL) {
30247951040fSNavdeep Parhar 		MPASS(pidx == eq->dbidx);
30252e09fe91SNavdeep Parhar 		if (next == NULL || ndesc >= 16) {
30262e09fe91SNavdeep Parhar 			int available;
30272e09fe91SNavdeep Parhar 			struct fw_eth_tx_pkt_wr *dst;	/* any fw WR struct will do */
30282e09fe91SNavdeep Parhar 
30292e09fe91SNavdeep Parhar 			/*
30302e09fe91SNavdeep Parhar 			 * Note that the WR via which we'll request tx updates
30312e09fe91SNavdeep Parhar 			 * is at pidx and not eq->pidx, which has moved on
30322e09fe91SNavdeep Parhar 			 * already.
30332e09fe91SNavdeep Parhar 			 */
30342e09fe91SNavdeep Parhar 			dst = (void *)&eq->desc[pidx];
30352e09fe91SNavdeep Parhar 			available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
30362e09fe91SNavdeep Parhar 			if (available < eq->sidx / 4 &&
30372e09fe91SNavdeep Parhar 			    atomic_cmpset_int(&eq->equiq, 0, 1)) {
3038ddf09ad6SNavdeep Parhar 				/*
3039ddf09ad6SNavdeep Parhar 				 * XXX: This is not 100% reliable with some
3040ddf09ad6SNavdeep Parhar 				 * types of WRs.  But this is a very unusual
3041ddf09ad6SNavdeep Parhar 				 * situation for an ofld/ctrl queue anyway.
3042ddf09ad6SNavdeep Parhar 				 */
30432e09fe91SNavdeep Parhar 				dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
30442e09fe91SNavdeep Parhar 				    F_FW_WR_EQUEQ);
30452e09fe91SNavdeep Parhar 			}
30462e09fe91SNavdeep Parhar 
30477951040fSNavdeep Parhar 			ring_eq_db(wrq->adapter, eq, ndesc);
30482e09fe91SNavdeep Parhar 		} else {
30497951040fSNavdeep Parhar 			MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
30507951040fSNavdeep Parhar 			next->pidx = pidx;
30517951040fSNavdeep Parhar 			next->ndesc += ndesc;
30527951040fSNavdeep Parhar 		}
30537951040fSNavdeep Parhar 	} else {
30547951040fSNavdeep Parhar 		MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc);
30557951040fSNavdeep Parhar 		prev->ndesc += ndesc;
30567951040fSNavdeep Parhar 	}
30577951040fSNavdeep Parhar 	TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link);
30587951040fSNavdeep Parhar 
30597951040fSNavdeep Parhar 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
30607951040fSNavdeep Parhar 		drain_wrq_wr_list(sc, wrq);
30617951040fSNavdeep Parhar 
30627951040fSNavdeep Parhar #ifdef INVARIANTS
30637951040fSNavdeep Parhar 	if (TAILQ_EMPTY(&wrq->incomplete_wrs)) {
30647951040fSNavdeep Parhar 		/* Doorbell must have caught up to the pidx. */
30657951040fSNavdeep Parhar 		MPASS(wrq->eq.pidx == wrq->eq.dbidx);
30667951040fSNavdeep Parhar 	}
30677951040fSNavdeep Parhar #endif
30687951040fSNavdeep Parhar 	EQ_UNLOCK(eq);
30697951040fSNavdeep Parhar }
30707951040fSNavdeep Parhar 
30717951040fSNavdeep Parhar static u_int
30727951040fSNavdeep Parhar can_resume_eth_tx(struct mp_ring *r)
30737951040fSNavdeep Parhar {
30747951040fSNavdeep Parhar 	struct sge_eq *eq = r->cookie;
30757951040fSNavdeep Parhar 
30767951040fSNavdeep Parhar 	return (total_available_tx_desc(eq) > eq->sidx / 8);
30777951040fSNavdeep Parhar }
30787951040fSNavdeep Parhar 
3079d735920dSNavdeep Parhar static inline bool
30807951040fSNavdeep Parhar cannot_use_txpkts(struct mbuf *m)
30817951040fSNavdeep Parhar {
30827951040fSNavdeep Parhar 	/* maybe put a GL limit too, to avoid silliness? */
30837951040fSNavdeep Parhar 
3084bddf7343SJohn Baldwin 	return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0);
30857951040fSNavdeep Parhar }
30867951040fSNavdeep Parhar 
30871404daa7SNavdeep Parhar static inline int
30881404daa7SNavdeep Parhar discard_tx(struct sge_eq *eq)
30891404daa7SNavdeep Parhar {
30901404daa7SNavdeep Parhar 
30911404daa7SNavdeep Parhar 	return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED);
30921404daa7SNavdeep Parhar }
30931404daa7SNavdeep Parhar 
30945cdaef71SJohn Baldwin static inline int
3095d735920dSNavdeep Parhar wr_can_update_eq(void *p)
30965cdaef71SJohn Baldwin {
3097d735920dSNavdeep Parhar 	struct fw_eth_tx_pkts_wr *wr = p;
30985cdaef71SJohn Baldwin 
30995cdaef71SJohn Baldwin 	switch (G_FW_WR_OP(be32toh(wr->op_pkd))) {
31005cdaef71SJohn Baldwin 	case FW_ULPTX_WR:
31015cdaef71SJohn Baldwin 	case FW_ETH_TX_PKT_WR:
31025cdaef71SJohn Baldwin 	case FW_ETH_TX_PKTS_WR:
3103693a9dfcSNavdeep Parhar 	case FW_ETH_TX_PKTS2_WR:
31045cdaef71SJohn Baldwin 	case FW_ETH_TX_PKT_VM_WR:
3105d735920dSNavdeep Parhar 	case FW_ETH_TX_PKTS_VM_WR:
31065cdaef71SJohn Baldwin 		return (1);
31075cdaef71SJohn Baldwin 	default:
31085cdaef71SJohn Baldwin 		return (0);
31095cdaef71SJohn Baldwin 	}
31105cdaef71SJohn Baldwin }
31115cdaef71SJohn Baldwin 
3112d735920dSNavdeep Parhar static inline void
3113d735920dSNavdeep Parhar set_txupdate_flags(struct sge_txq *txq, u_int avail,
3114d735920dSNavdeep Parhar     struct fw_eth_tx_pkt_wr *wr)
3115d735920dSNavdeep Parhar {
3116d735920dSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
3117d735920dSNavdeep Parhar 	struct txpkts *txp = &txq->txp;
3118d735920dSNavdeep Parhar 
3119d735920dSNavdeep Parhar 	if ((txp->npkt > 0 || avail < eq->sidx / 2) &&
3120d735920dSNavdeep Parhar 	    atomic_cmpset_int(&eq->equiq, 0, 1)) {
3121d735920dSNavdeep Parhar 		wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3122d735920dSNavdeep Parhar 		eq->equeqidx = eq->pidx;
3123d735920dSNavdeep Parhar 	} else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
3124d735920dSNavdeep Parhar 		wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
3125d735920dSNavdeep Parhar 		eq->equeqidx = eq->pidx;
3126d735920dSNavdeep Parhar 	}
3127d735920dSNavdeep Parhar }
3128d735920dSNavdeep Parhar 
31293447df8bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__)
31303447df8bSNavdeep Parhar extern uint64_t tsc_freq;
31313447df8bSNavdeep Parhar #endif
31323447df8bSNavdeep Parhar 
31333447df8bSNavdeep Parhar static inline bool
31343447df8bSNavdeep Parhar record_eth_tx_time(struct sge_txq *txq)
31353447df8bSNavdeep Parhar {
31363447df8bSNavdeep Parhar 	const uint64_t cycles = get_cyclecount();
31373447df8bSNavdeep Parhar 	const uint64_t last_tx = txq->last_tx;
31383447df8bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__)
31393447df8bSNavdeep Parhar 	const uint64_t itg = tsc_freq * t4_tx_coalesce_gap / 1000000;
31403447df8bSNavdeep Parhar #else
31413447df8bSNavdeep Parhar 	const uint64_t itg = 0;
31423447df8bSNavdeep Parhar #endif
31433447df8bSNavdeep Parhar 
31443447df8bSNavdeep Parhar 	MPASS(cycles >= last_tx);
31453447df8bSNavdeep Parhar 	txq->last_tx = cycles;
31463447df8bSNavdeep Parhar 	return (cycles - last_tx < itg);
31473447df8bSNavdeep Parhar }
31483447df8bSNavdeep Parhar 
31497951040fSNavdeep Parhar /*
31507951040fSNavdeep Parhar  * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to
31517951040fSNavdeep Parhar  * be consumed.  Return the actual number consumed.  0 indicates a stall.
31527951040fSNavdeep Parhar  */
31537951040fSNavdeep Parhar static u_int
3154d735920dSNavdeep Parhar eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing)
31557951040fSNavdeep Parhar {
31567951040fSNavdeep Parhar 	struct sge_txq *txq = r->cookie;
31577951040fSNavdeep Parhar 	struct ifnet *ifp = txq->ifp;
3158d735920dSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
3159d735920dSNavdeep Parhar 	struct txpkts *txp = &txq->txp;
3160fe2ebb76SJohn Baldwin 	struct vi_info *vi = ifp->if_softc;
31617c228be3SNavdeep Parhar 	struct adapter *sc = vi->adapter;
31627951040fSNavdeep Parhar 	u_int total, remaining;		/* # of packets */
3163d735920dSNavdeep Parhar 	u_int n, avail, dbdiff;		/* # of hardware descriptors */
3164d735920dSNavdeep Parhar 	int i, rc;
3165d735920dSNavdeep Parhar 	struct mbuf *m0;
31663447df8bSNavdeep Parhar 	bool snd, recent_tx;
3167d735920dSNavdeep Parhar 	void *wr;	/* start of the last WR written to the ring */
3168d735920dSNavdeep Parhar 
3169d735920dSNavdeep Parhar 	TXQ_LOCK_ASSERT_OWNED(txq);
31703447df8bSNavdeep Parhar 	recent_tx = record_eth_tx_time(txq);
31717951040fSNavdeep Parhar 
31727951040fSNavdeep Parhar 	remaining = IDXDIFF(pidx, cidx, r->size);
31731404daa7SNavdeep Parhar 	if (__predict_false(discard_tx(eq))) {
3174d735920dSNavdeep Parhar 		for (i = 0; i < txp->npkt; i++)
3175d735920dSNavdeep Parhar 			m_freem(txp->mb[i]);
3176d735920dSNavdeep Parhar 		txp->npkt = 0;
31777951040fSNavdeep Parhar 		while (cidx != pidx) {
31787951040fSNavdeep Parhar 			m0 = r->items[cidx];
31797951040fSNavdeep Parhar 			m_freem(m0);
31807951040fSNavdeep Parhar 			if (++cidx == r->size)
31817951040fSNavdeep Parhar 				cidx = 0;
31827951040fSNavdeep Parhar 		}
3183d735920dSNavdeep Parhar 		reclaim_tx_descs(txq, eq->sidx);
3184d735920dSNavdeep Parhar 		*coalescing = false;
3185d735920dSNavdeep Parhar 		return (remaining);	/* emptied */
31867951040fSNavdeep Parhar 	}
31877951040fSNavdeep Parhar 
31887951040fSNavdeep Parhar 	/* How many hardware descriptors do we have readily available. */
31893447df8bSNavdeep Parhar 	if (eq->pidx == eq->cidx)
3190d735920dSNavdeep Parhar 		avail = eq->sidx - 1;
31913447df8bSNavdeep Parhar 	else
3192d735920dSNavdeep Parhar 		avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
31937951040fSNavdeep Parhar 
3194d735920dSNavdeep Parhar 	total = 0;
3195d735920dSNavdeep Parhar 	if (remaining == 0) {
31963447df8bSNavdeep Parhar 		txp->score = 0;
31973447df8bSNavdeep Parhar 		txq->txpkts_flush++;
3198d735920dSNavdeep Parhar 		goto send_txpkts;
3199d735920dSNavdeep Parhar 	}
3200d735920dSNavdeep Parhar 
3201d735920dSNavdeep Parhar 	dbdiff = 0;
3202d735920dSNavdeep Parhar 	MPASS(remaining > 0);
32037951040fSNavdeep Parhar 	while (remaining > 0) {
32047951040fSNavdeep Parhar 		m0 = r->items[cidx];
32057951040fSNavdeep Parhar 		M_ASSERTPKTHDR(m0);
32067951040fSNavdeep Parhar 		MPASS(m0->m_nextpkt == NULL);
32077951040fSNavdeep Parhar 
3208d735920dSNavdeep Parhar 		if (avail < 2 * SGE_MAX_WR_NDESC)
3209d735920dSNavdeep Parhar 			avail += reclaim_tx_descs(txq, 64);
3210d735920dSNavdeep Parhar 
32113447df8bSNavdeep Parhar 		if (t4_tx_coalesce == 0 && txp->npkt == 0)
32123447df8bSNavdeep Parhar 			goto skip_coalescing;
32133447df8bSNavdeep Parhar 		if (cannot_use_txpkts(m0))
32143447df8bSNavdeep Parhar 			txp->score = 0;
32153447df8bSNavdeep Parhar 		else if (recent_tx) {
32163447df8bSNavdeep Parhar 			if (++txp->score == 0)
32173447df8bSNavdeep Parhar 				txp->score = UINT8_MAX;
32183447df8bSNavdeep Parhar 		} else
32193447df8bSNavdeep Parhar 			txp->score = 1;
32203447df8bSNavdeep Parhar 		if (txp->npkt > 0 || remaining > 1 ||
32213447df8bSNavdeep Parhar 		    txp->score >= t4_tx_coalesce_pkts ||
3222d735920dSNavdeep Parhar 		    atomic_load_int(&txq->eq.equiq) != 0) {
322330e3f2b4SNavdeep Parhar 			if (vi->flags & TX_USES_VM_WR)
3224d735920dSNavdeep Parhar 				rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd);
3225d735920dSNavdeep Parhar 			else
3226d735920dSNavdeep Parhar 				rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd);
3227d735920dSNavdeep Parhar 		} else {
3228d735920dSNavdeep Parhar 			snd = false;
3229d735920dSNavdeep Parhar 			rc = EINVAL;
3230d735920dSNavdeep Parhar 		}
3231d735920dSNavdeep Parhar 		if (snd) {
3232d735920dSNavdeep Parhar 			MPASS(txp->npkt > 0);
3233d735920dSNavdeep Parhar 			for (i = 0; i < txp->npkt; i++)
3234d735920dSNavdeep Parhar 				ETHER_BPF_MTAP(ifp, txp->mb[i]);
3235d735920dSNavdeep Parhar 			if (txp->npkt > 1) {
3236d735920dSNavdeep Parhar 				MPASS(avail >= tx_len16_to_desc(txp->len16));
323730e3f2b4SNavdeep Parhar 				if (vi->flags & TX_USES_VM_WR)
3238d735920dSNavdeep Parhar 					n = write_txpkts_vm_wr(sc, txq);
3239d735920dSNavdeep Parhar 				else
3240d735920dSNavdeep Parhar 					n = write_txpkts_wr(sc, txq);
3241d735920dSNavdeep Parhar 			} else {
3242d735920dSNavdeep Parhar 				MPASS(avail >=
3243d735920dSNavdeep Parhar 				    tx_len16_to_desc(mbuf_len16(txp->mb[0])));
324430e3f2b4SNavdeep Parhar 				if (vi->flags & TX_USES_VM_WR)
3245d735920dSNavdeep Parhar 					n = write_txpkt_vm_wr(sc, txq,
3246d735920dSNavdeep Parhar 					    txp->mb[0]);
3247d735920dSNavdeep Parhar 				else
3248d735920dSNavdeep Parhar 					n = write_txpkt_wr(sc, txq, txp->mb[0],
3249d735920dSNavdeep Parhar 					    avail);
3250d735920dSNavdeep Parhar 			}
3251d735920dSNavdeep Parhar 			MPASS(n <= SGE_MAX_WR_NDESC);
3252d735920dSNavdeep Parhar 			avail -= n;
3253d735920dSNavdeep Parhar 			dbdiff += n;
3254d735920dSNavdeep Parhar 			wr = &eq->desc[eq->pidx];
3255d735920dSNavdeep Parhar 			IDXINCR(eq->pidx, n, eq->sidx);
3256d735920dSNavdeep Parhar 			txp->npkt = 0;	/* emptied */
3257d735920dSNavdeep Parhar 		}
3258d735920dSNavdeep Parhar 		if (rc == 0) {
3259d735920dSNavdeep Parhar 			/* m0 was coalesced into txq->txpkts. */
3260d735920dSNavdeep Parhar 			goto next_mbuf;
3261d735920dSNavdeep Parhar 		}
3262d735920dSNavdeep Parhar 		if (rc == EAGAIN) {
3263d735920dSNavdeep Parhar 			/*
3264d735920dSNavdeep Parhar 			 * m0 is suitable for tx coalescing but could not be
3265d735920dSNavdeep Parhar 			 * combined with the existing txq->txpkts, which has now
3266d735920dSNavdeep Parhar 			 * been transmitted.  Start a new txpkts with m0.
3267d735920dSNavdeep Parhar 			 */
3268d735920dSNavdeep Parhar 			MPASS(snd);
3269d735920dSNavdeep Parhar 			MPASS(txp->npkt == 0);
3270d735920dSNavdeep Parhar 			continue;
32717951040fSNavdeep Parhar 		}
32727951040fSNavdeep Parhar 
3273d735920dSNavdeep Parhar 		MPASS(rc != 0 && rc != EAGAIN);
3274d735920dSNavdeep Parhar 		MPASS(txp->npkt == 0);
32753447df8bSNavdeep Parhar skip_coalescing:
3276565b8fceSNavdeep Parhar 		n = tx_len16_to_desc(mbuf_len16(m0));
3277565b8fceSNavdeep Parhar 		if (__predict_false(avail < n)) {
3278565b8fceSNavdeep Parhar 			avail += reclaim_tx_descs(txq, min(n, 32));
3279565b8fceSNavdeep Parhar 			if (avail < n)
3280565b8fceSNavdeep Parhar 				break;	/* out of descriptors */
3281565b8fceSNavdeep Parhar 		}
3282565b8fceSNavdeep Parhar 
3283d735920dSNavdeep Parhar 		wr = &eq->desc[eq->pidx];
3284bddf7343SJohn Baldwin 		if (mbuf_cflags(m0) & MC_RAW_WR) {
3285d735920dSNavdeep Parhar 			n = write_raw_wr(txq, wr, m0, avail);
3286bddf7343SJohn Baldwin #ifdef KERN_TLS
3287bddf7343SJohn Baldwin 		} else if (mbuf_cflags(m0) & MC_TLS) {
3288bddf7343SJohn Baldwin 			ETHER_BPF_MTAP(ifp, m0);
3289d735920dSNavdeep Parhar 			n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0),
3290d735920dSNavdeep Parhar 			    avail);
3291bddf7343SJohn Baldwin #endif
32927951040fSNavdeep Parhar 		} else {
32933bbb68f0SNavdeep Parhar 			ETHER_BPF_MTAP(ifp, m0);
329430e3f2b4SNavdeep Parhar 			if (vi->flags & TX_USES_VM_WR)
3295d735920dSNavdeep Parhar 				n = write_txpkt_vm_wr(sc, txq, m0);
3296d735920dSNavdeep Parhar 			else
3297d735920dSNavdeep Parhar 				n = write_txpkt_wr(sc, txq, m0, avail);
3298d735920dSNavdeep Parhar 		}
3299d735920dSNavdeep Parhar 		MPASS(n >= 1 && n <= avail);
3300bddf7343SJohn Baldwin 		if (!(mbuf_cflags(m0) & MC_TLS))
3301bddf7343SJohn Baldwin 			MPASS(n <= SGE_MAX_WR_NDESC);
33027951040fSNavdeep Parhar 
3303d735920dSNavdeep Parhar 		avail -= n;
33047951040fSNavdeep Parhar 		dbdiff += n;
33057951040fSNavdeep Parhar 		IDXINCR(eq->pidx, n, eq->sidx);
33067951040fSNavdeep Parhar 
3307d735920dSNavdeep Parhar 		if (dbdiff >= 512 / EQ_ESIZE) {	/* X_FETCHBURSTMAX_512B */
3308d735920dSNavdeep Parhar 			if (wr_can_update_eq(wr))
3309d735920dSNavdeep Parhar 				set_txupdate_flags(txq, avail, wr);
33107951040fSNavdeep Parhar 			ring_eq_db(sc, eq, dbdiff);
3311d735920dSNavdeep Parhar 			avail += reclaim_tx_descs(txq, 32);
33127951040fSNavdeep Parhar 			dbdiff = 0;
33137951040fSNavdeep Parhar 		}
3314d735920dSNavdeep Parhar next_mbuf:
3315d735920dSNavdeep Parhar 		total++;
3316d735920dSNavdeep Parhar 		remaining--;
3317d735920dSNavdeep Parhar 		if (__predict_false(++cidx == r->size))
3318d735920dSNavdeep Parhar 			cidx = 0;
33197951040fSNavdeep Parhar 	}
33207951040fSNavdeep Parhar 	if (dbdiff != 0) {
3321d735920dSNavdeep Parhar 		if (wr_can_update_eq(wr))
3322d735920dSNavdeep Parhar 			set_txupdate_flags(txq, avail, wr);
33237951040fSNavdeep Parhar 		ring_eq_db(sc, eq, dbdiff);
33247951040fSNavdeep Parhar 		reclaim_tx_descs(txq, 32);
3325d735920dSNavdeep Parhar 	} else if (eq->pidx == eq->cidx && txp->npkt > 0 &&
3326d735920dSNavdeep Parhar 	    atomic_load_int(&txq->eq.equiq) == 0) {
3327d735920dSNavdeep Parhar 		/*
3328d735920dSNavdeep Parhar 		 * If nothing was submitted to the chip for tx (it was coalesced
3329d735920dSNavdeep Parhar 		 * into txpkts instead) and there is no tx update outstanding
3330d735920dSNavdeep Parhar 		 * then we need to send txpkts now.
3331d735920dSNavdeep Parhar 		 */
3332d735920dSNavdeep Parhar send_txpkts:
3333d735920dSNavdeep Parhar 		MPASS(txp->npkt > 0);
3334d735920dSNavdeep Parhar 		for (i = 0; i < txp->npkt; i++)
3335d735920dSNavdeep Parhar 			ETHER_BPF_MTAP(ifp, txp->mb[i]);
3336d735920dSNavdeep Parhar 		if (txp->npkt > 1) {
3337d735920dSNavdeep Parhar 			MPASS(avail >= tx_len16_to_desc(txp->len16));
333830e3f2b4SNavdeep Parhar 			if (vi->flags & TX_USES_VM_WR)
3339d735920dSNavdeep Parhar 				n = write_txpkts_vm_wr(sc, txq);
3340d735920dSNavdeep Parhar 			else
3341d735920dSNavdeep Parhar 				n = write_txpkts_wr(sc, txq);
3342d735920dSNavdeep Parhar 		} else {
3343d735920dSNavdeep Parhar 			MPASS(avail >=
3344d735920dSNavdeep Parhar 			    tx_len16_to_desc(mbuf_len16(txp->mb[0])));
334530e3f2b4SNavdeep Parhar 			if (vi->flags & TX_USES_VM_WR)
3346d735920dSNavdeep Parhar 				n = write_txpkt_vm_wr(sc, txq, txp->mb[0]);
3347d735920dSNavdeep Parhar 			else
3348d735920dSNavdeep Parhar 				n = write_txpkt_wr(sc, txq, txp->mb[0], avail);
33497951040fSNavdeep Parhar 		}
3350d735920dSNavdeep Parhar 		MPASS(n <= SGE_MAX_WR_NDESC);
3351d735920dSNavdeep Parhar 		wr = &eq->desc[eq->pidx];
3352d735920dSNavdeep Parhar 		IDXINCR(eq->pidx, n, eq->sidx);
3353d735920dSNavdeep Parhar 		txp->npkt = 0;	/* emptied */
3354d735920dSNavdeep Parhar 
3355d735920dSNavdeep Parhar 		MPASS(wr_can_update_eq(wr));
3356d735920dSNavdeep Parhar 		set_txupdate_flags(txq, avail - n, wr);
3357d735920dSNavdeep Parhar 		ring_eq_db(sc, eq, n);
3358d735920dSNavdeep Parhar 		reclaim_tx_descs(txq, 32);
3359d735920dSNavdeep Parhar 	}
3360d735920dSNavdeep Parhar 	*coalescing = txp->npkt > 0;
33617951040fSNavdeep Parhar 
33627951040fSNavdeep Parhar 	return (total);
3363733b9277SNavdeep Parhar }
3364733b9277SNavdeep Parhar 
336554e4ee71SNavdeep Parhar static inline void
336654e4ee71SNavdeep Parhar init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
336743bbae19SNavdeep Parhar     int qsize, int intr_idx, int cong)
336854e4ee71SNavdeep Parhar {
3369b2daa9a9SNavdeep Parhar 
337054e4ee71SNavdeep Parhar 	KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
337154e4ee71SNavdeep Parhar 	    ("%s: bad tmr_idx %d", __func__, tmr_idx));
337254e4ee71SNavdeep Parhar 	KASSERT(pktc_idx < SGE_NCOUNTERS,	/* -ve is ok, means don't use */
337354e4ee71SNavdeep Parhar 	    ("%s: bad pktc_idx %d", __func__, pktc_idx));
337443bbae19SNavdeep Parhar 	KASSERT(intr_idx >= -1 && intr_idx < sc->intr_count,
337543bbae19SNavdeep Parhar 	    ("%s: bad intr_idx %d", __func__, intr_idx));
337654e4ee71SNavdeep Parhar 
337754e4ee71SNavdeep Parhar 	iq->flags = 0;
337843bbae19SNavdeep Parhar 	iq->state = IQS_DISABLED;
337954e4ee71SNavdeep Parhar 	iq->adapter = sc;
33807a32954cSNavdeep Parhar 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
33817a32954cSNavdeep Parhar 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
33827a32954cSNavdeep Parhar 	if (pktc_idx >= 0) {
33837a32954cSNavdeep Parhar 		iq->intr_params |= F_QINTR_CNT_EN;
338454e4ee71SNavdeep Parhar 		iq->intr_pktc_idx = pktc_idx;
33857a32954cSNavdeep Parhar 	}
3386d14b0ac1SNavdeep Parhar 	iq->qsize = roundup2(qsize, 16);	/* See FW_IQ_CMD/iqsize */
338790e7434aSNavdeep Parhar 	iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE;
338843bbae19SNavdeep Parhar 	iq->intr_idx = intr_idx;
338943bbae19SNavdeep Parhar 	iq->cong = cong;
339054e4ee71SNavdeep Parhar }
339154e4ee71SNavdeep Parhar 
339254e4ee71SNavdeep Parhar static inline void
3393e3207e19SNavdeep Parhar init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
339454e4ee71SNavdeep Parhar {
339543bbae19SNavdeep Parhar 	struct sge_params *sp = &sc->params.sge;
33961458bff9SNavdeep Parhar 
339754e4ee71SNavdeep Parhar 	fl->qsize = qsize;
339890e7434aSNavdeep Parhar 	fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
339954e4ee71SNavdeep Parhar 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
340043bbae19SNavdeep Parhar 	mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
3401e3207e19SNavdeep Parhar 	if (sc->flags & BUF_PACKING_OK &&
3402e3207e19SNavdeep Parhar 	    ((!is_t4(sc) && buffer_packing) ||	/* T5+: enabled unless 0 */
3403e3207e19SNavdeep Parhar 	    (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */
34041458bff9SNavdeep Parhar 		fl->flags |= FL_BUF_PACKING;
340546e1e307SNavdeep Parhar 	fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING);
340646e1e307SNavdeep Parhar 	fl->safe_zidx = sc->sge.safe_zidx;
340743bbae19SNavdeep Parhar 	if (fl->flags & FL_BUF_PACKING) {
340843bbae19SNavdeep Parhar 		fl->lowat = roundup2(sp->fl_starve_threshold2, 8);
340943bbae19SNavdeep Parhar 		fl->buf_boundary = sp->pack_boundary;
341043bbae19SNavdeep Parhar 	} else {
341143bbae19SNavdeep Parhar 		fl->lowat = roundup2(sp->fl_starve_threshold, 8);
341243bbae19SNavdeep Parhar 		fl->buf_boundary = 16;
341343bbae19SNavdeep Parhar 	}
341443bbae19SNavdeep Parhar 	if (fl_pad && fl->buf_boundary < sp->pad_boundary)
341543bbae19SNavdeep Parhar 		fl->buf_boundary = sp->pad_boundary;
341654e4ee71SNavdeep Parhar }
341754e4ee71SNavdeep Parhar 
341854e4ee71SNavdeep Parhar static inline void
341990e7434aSNavdeep Parhar init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize,
342043bbae19SNavdeep Parhar     uint8_t tx_chan, struct sge_iq *iq, char *name)
342154e4ee71SNavdeep Parhar {
342243bbae19SNavdeep Parhar 	KASSERT(eqtype >= EQ_CTRL && eqtype <= EQ_OFLD,
342343bbae19SNavdeep Parhar 	    ("%s: bad qtype %d", __func__, eqtype));
3424733b9277SNavdeep Parhar 
342543bbae19SNavdeep Parhar 	eq->type = eqtype;
3426733b9277SNavdeep Parhar 	eq->tx_chan = tx_chan;
342743bbae19SNavdeep Parhar 	eq->iq = iq;
342890e7434aSNavdeep Parhar 	eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
3429f7dfe243SNavdeep Parhar 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
343043bbae19SNavdeep Parhar 	mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
343154e4ee71SNavdeep Parhar }
343254e4ee71SNavdeep Parhar 
34338eba75edSNavdeep Parhar int
343454e4ee71SNavdeep Parhar alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
343554e4ee71SNavdeep Parhar     bus_dmamap_t *map, bus_addr_t *pa, void **va)
343654e4ee71SNavdeep Parhar {
343754e4ee71SNavdeep Parhar 	int rc;
343854e4ee71SNavdeep Parhar 
343954e4ee71SNavdeep Parhar 	rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
344054e4ee71SNavdeep Parhar 	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
344154e4ee71SNavdeep Parhar 	if (rc != 0) {
344243bbae19SNavdeep Parhar 		CH_ERR(sc, "cannot allocate DMA tag: %d\n", rc);
344354e4ee71SNavdeep Parhar 		goto done;
344454e4ee71SNavdeep Parhar 	}
344554e4ee71SNavdeep Parhar 
344654e4ee71SNavdeep Parhar 	rc = bus_dmamem_alloc(*tag, va,
344754e4ee71SNavdeep Parhar 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
344854e4ee71SNavdeep Parhar 	if (rc != 0) {
344943bbae19SNavdeep Parhar 		CH_ERR(sc, "cannot allocate DMA memory: %d\n", rc);
345054e4ee71SNavdeep Parhar 		goto done;
345154e4ee71SNavdeep Parhar 	}
345254e4ee71SNavdeep Parhar 
345354e4ee71SNavdeep Parhar 	rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
345454e4ee71SNavdeep Parhar 	if (rc != 0) {
345543bbae19SNavdeep Parhar 		CH_ERR(sc, "cannot load DMA map: %d\n", rc);
345654e4ee71SNavdeep Parhar 		goto done;
345754e4ee71SNavdeep Parhar 	}
345854e4ee71SNavdeep Parhar done:
345954e4ee71SNavdeep Parhar 	if (rc)
346054e4ee71SNavdeep Parhar 		free_ring(sc, *tag, *map, *pa, *va);
346154e4ee71SNavdeep Parhar 
346254e4ee71SNavdeep Parhar 	return (rc);
346354e4ee71SNavdeep Parhar }
346454e4ee71SNavdeep Parhar 
34658eba75edSNavdeep Parhar int
346654e4ee71SNavdeep Parhar free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
346754e4ee71SNavdeep Parhar     bus_addr_t pa, void *va)
346854e4ee71SNavdeep Parhar {
346954e4ee71SNavdeep Parhar 	if (pa)
347054e4ee71SNavdeep Parhar 		bus_dmamap_unload(tag, map);
347154e4ee71SNavdeep Parhar 	if (va)
347254e4ee71SNavdeep Parhar 		bus_dmamem_free(tag, va, map);
347354e4ee71SNavdeep Parhar 	if (tag)
347454e4ee71SNavdeep Parhar 		bus_dma_tag_destroy(tag);
347554e4ee71SNavdeep Parhar 
347654e4ee71SNavdeep Parhar 	return (0);
347754e4ee71SNavdeep Parhar }
347854e4ee71SNavdeep Parhar 
347954e4ee71SNavdeep Parhar /*
348043bbae19SNavdeep Parhar  * Allocates the software resources (mainly memory and sysctl nodes) for an
348143bbae19SNavdeep Parhar  * ingress queue and an optional freelist.
348254e4ee71SNavdeep Parhar  *
348343bbae19SNavdeep Parhar  * Sets IQ_SW_ALLOCATED and returns 0 on success.
348454e4ee71SNavdeep Parhar  */
348554e4ee71SNavdeep Parhar static int
3486fe2ebb76SJohn Baldwin alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,
348743bbae19SNavdeep Parhar     struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
348854e4ee71SNavdeep Parhar {
348943bbae19SNavdeep Parhar 	int rc;
349054e4ee71SNavdeep Parhar 	size_t len;
349143bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
349243bbae19SNavdeep Parhar 
349343bbae19SNavdeep Parhar 	MPASS(!(iq->flags & IQ_SW_ALLOCATED));
349454e4ee71SNavdeep Parhar 
3495b2daa9a9SNavdeep Parhar 	len = iq->qsize * IQ_ESIZE;
349654e4ee71SNavdeep Parhar 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
349754e4ee71SNavdeep Parhar 	    (void **)&iq->desc);
349854e4ee71SNavdeep Parhar 	if (rc != 0)
349954e4ee71SNavdeep Parhar 		return (rc);
350054e4ee71SNavdeep Parhar 
350143bbae19SNavdeep Parhar 	if (fl) {
350243bbae19SNavdeep Parhar 		len = fl->qsize * EQ_ESIZE;
350343bbae19SNavdeep Parhar 		rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
350443bbae19SNavdeep Parhar 		    &fl->ba, (void **)&fl->desc);
350543bbae19SNavdeep Parhar 		if (rc) {
350643bbae19SNavdeep Parhar 			free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba,
350743bbae19SNavdeep Parhar 			    iq->desc);
350843bbae19SNavdeep Parhar 			return (rc);
350943bbae19SNavdeep Parhar 		}
351043bbae19SNavdeep Parhar 
351143bbae19SNavdeep Parhar 		/* Allocate space for one software descriptor per buffer. */
351243bbae19SNavdeep Parhar 		fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc),
351343bbae19SNavdeep Parhar 		    M_CXGBE, M_ZERO | M_WAITOK);
351443bbae19SNavdeep Parhar 
351543bbae19SNavdeep Parhar 		add_fl_sysctls(sc, ctx, oid, fl);
351643bbae19SNavdeep Parhar 		iq->flags |= IQ_HAS_FL;
351743bbae19SNavdeep Parhar 	}
351843bbae19SNavdeep Parhar 	add_iq_sysctls(ctx, oid, iq);
351943bbae19SNavdeep Parhar 	iq->flags |= IQ_SW_ALLOCATED;
352043bbae19SNavdeep Parhar 
352143bbae19SNavdeep Parhar 	return (0);
352243bbae19SNavdeep Parhar }
352343bbae19SNavdeep Parhar 
352443bbae19SNavdeep Parhar /*
352543bbae19SNavdeep Parhar  * Frees all software resources (memory and locks) associated with an ingress
352643bbae19SNavdeep Parhar  * queue and an optional freelist.
352743bbae19SNavdeep Parhar  */
352843bbae19SNavdeep Parhar static void
352943bbae19SNavdeep Parhar free_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
353043bbae19SNavdeep Parhar {
353143bbae19SNavdeep Parhar 	MPASS(iq->flags & IQ_SW_ALLOCATED);
353243bbae19SNavdeep Parhar 
353343bbae19SNavdeep Parhar 	if (fl) {
353443bbae19SNavdeep Parhar 		MPASS(iq->flags & IQ_HAS_FL);
353543bbae19SNavdeep Parhar 		free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc);
353643bbae19SNavdeep Parhar 		free_fl_buffers(sc, fl);
353743bbae19SNavdeep Parhar 		free(fl->sdesc, M_CXGBE);
353843bbae19SNavdeep Parhar 		mtx_destroy(&fl->fl_lock);
353943bbae19SNavdeep Parhar 		bzero(fl, sizeof(*fl));
354043bbae19SNavdeep Parhar 	}
354143bbae19SNavdeep Parhar 	free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
354243bbae19SNavdeep Parhar 	bzero(iq, sizeof(*iq));
354343bbae19SNavdeep Parhar }
354443bbae19SNavdeep Parhar 
354543bbae19SNavdeep Parhar /*
354643bbae19SNavdeep Parhar  * Allocates a hardware ingress queue and an optional freelist that will be
354743bbae19SNavdeep Parhar  * associated with it.
354843bbae19SNavdeep Parhar  *
354943bbae19SNavdeep Parhar  * Returns errno on failure.  Resources allocated up to that point may still be
355043bbae19SNavdeep Parhar  * allocated.  Caller is responsible for cleanup in case this function fails.
355143bbae19SNavdeep Parhar  */
355243bbae19SNavdeep Parhar static int
355343bbae19SNavdeep Parhar alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)
355443bbae19SNavdeep Parhar {
355543bbae19SNavdeep Parhar 	int rc, i, cntxt_id;
355643bbae19SNavdeep Parhar 	struct fw_iq_cmd c;
355743bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
355843bbae19SNavdeep Parhar 	__be32 v = 0;
355943bbae19SNavdeep Parhar 
356043bbae19SNavdeep Parhar 	MPASS (!(iq->flags & IQ_HW_ALLOCATED));
356143bbae19SNavdeep Parhar 
356254e4ee71SNavdeep Parhar 	bzero(&c, sizeof(c));
356354e4ee71SNavdeep Parhar 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
356454e4ee71SNavdeep Parhar 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
356554e4ee71SNavdeep Parhar 	    V_FW_IQ_CMD_VFN(0));
356654e4ee71SNavdeep Parhar 
356754e4ee71SNavdeep Parhar 	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
356854e4ee71SNavdeep Parhar 	    FW_LEN16(c));
356954e4ee71SNavdeep Parhar 
357054e4ee71SNavdeep Parhar 	/* Special handling for firmware event queue */
357154e4ee71SNavdeep Parhar 	if (iq == &sc->sge.fwq)
357254e4ee71SNavdeep Parhar 		v |= F_FW_IQ_CMD_IQASYNCH;
357354e4ee71SNavdeep Parhar 
357443bbae19SNavdeep Parhar 	if (iq->intr_idx < 0) {
3575f549e352SNavdeep Parhar 		/* Forwarded interrupts, all headed to fwq */
3576f549e352SNavdeep Parhar 		v |= F_FW_IQ_CMD_IQANDST;
3577f549e352SNavdeep Parhar 		v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id);
3578f549e352SNavdeep Parhar 	} else {
357943bbae19SNavdeep Parhar 		KASSERT(iq->intr_idx < sc->intr_count,
358043bbae19SNavdeep Parhar 		    ("%s: invalid direct intr_idx %d", __func__, iq->intr_idx));
358143bbae19SNavdeep Parhar 		v |= V_FW_IQ_CMD_IQANDSTINDEX(iq->intr_idx);
3582f549e352SNavdeep Parhar 	}
358354e4ee71SNavdeep Parhar 
358443bbae19SNavdeep Parhar 	bzero(iq->desc, iq->qsize * IQ_ESIZE);
358554e4ee71SNavdeep Parhar 	c.type_to_iqandstindex = htobe32(v |
358654e4ee71SNavdeep Parhar 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
3587fe2ebb76SJohn Baldwin 	    V_FW_IQ_CMD_VIID(vi->viid) |
358854e4ee71SNavdeep Parhar 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
358943bbae19SNavdeep Parhar 	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(vi->pi->tx_chan) |
359054e4ee71SNavdeep Parhar 	    F_FW_IQ_CMD_IQGTSMODE |
359154e4ee71SNavdeep Parhar 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
3592b2daa9a9SNavdeep Parhar 	    V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
359354e4ee71SNavdeep Parhar 	c.iqsize = htobe16(iq->qsize);
359454e4ee71SNavdeep Parhar 	c.iqaddr = htobe64(iq->ba);
359543bbae19SNavdeep Parhar 	if (iq->cong >= 0)
3596bc14b14dSNavdeep Parhar 		c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
359754e4ee71SNavdeep Parhar 
359854e4ee71SNavdeep Parhar 	if (fl) {
359943bbae19SNavdeep Parhar 		bzero(fl->desc, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len);
3600214c3582SNavdeep Parhar 		c.iqns_to_fl0congen |=
3601bc14b14dSNavdeep Parhar 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
3602bc14b14dSNavdeep Parhar 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
36031458bff9SNavdeep Parhar 			(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
36041458bff9SNavdeep Parhar 			(fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
36051458bff9SNavdeep Parhar 			    0));
360643bbae19SNavdeep Parhar 		if (iq->cong >= 0) {
3607bc14b14dSNavdeep Parhar 			c.iqns_to_fl0congen |=
360843bbae19SNavdeep Parhar 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(iq->cong) |
3609bc14b14dSNavdeep Parhar 				    F_FW_IQ_CMD_FL0CONGCIF |
3610bc14b14dSNavdeep Parhar 				    F_FW_IQ_CMD_FL0CONGEN);
3611bc14b14dSNavdeep Parhar 		}
361254e4ee71SNavdeep Parhar 		c.fl0dcaen_to_fl0cidxfthresh =
3613ed7e5640SNavdeep Parhar 		    htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ?
3614adb0cd84SNavdeep Parhar 			X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) |
3615ed7e5640SNavdeep Parhar 			V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ?
3616ed7e5640SNavdeep Parhar 			X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
361754e4ee71SNavdeep Parhar 		c.fl0size = htobe16(fl->qsize);
361854e4ee71SNavdeep Parhar 		c.fl0addr = htobe64(fl->ba);
361954e4ee71SNavdeep Parhar 	}
362054e4ee71SNavdeep Parhar 
362154e4ee71SNavdeep Parhar 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
362254e4ee71SNavdeep Parhar 	if (rc != 0) {
362343bbae19SNavdeep Parhar 		CH_ERR(sc, "failed to create hw ingress queue: %d\n", rc);
362454e4ee71SNavdeep Parhar 		return (rc);
362554e4ee71SNavdeep Parhar 	}
362654e4ee71SNavdeep Parhar 
362754e4ee71SNavdeep Parhar 	iq->cidx = 0;
3628b2daa9a9SNavdeep Parhar 	iq->gen = F_RSPD_GEN;
362954e4ee71SNavdeep Parhar 	iq->cntxt_id = be16toh(c.iqid);
363054e4ee71SNavdeep Parhar 	iq->abs_id = be16toh(c.physiqid);
363154e4ee71SNavdeep Parhar 
363254e4ee71SNavdeep Parhar 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
3633b20b25e7SNavdeep Parhar 	if (cntxt_id >= sc->sge.iqmap_sz) {
3634733b9277SNavdeep Parhar 		panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
3635b20b25e7SNavdeep Parhar 		    cntxt_id, sc->sge.iqmap_sz - 1);
3636733b9277SNavdeep Parhar 	}
363754e4ee71SNavdeep Parhar 	sc->sge.iqmap[cntxt_id] = iq;
363854e4ee71SNavdeep Parhar 
363954e4ee71SNavdeep Parhar 	if (fl) {
36404d6db4e0SNavdeep Parhar 		u_int qid;
364143bbae19SNavdeep Parhar #ifdef INVARIANTS
364243bbae19SNavdeep Parhar 		MPASS(!(fl->flags & FL_BUF_RESUME));
364343bbae19SNavdeep Parhar 		for (i = 0; i < fl->sidx * 8; i++)
364443bbae19SNavdeep Parhar 			MPASS(fl->sdesc[i].cl == NULL);
364543bbae19SNavdeep Parhar #endif
364654e4ee71SNavdeep Parhar 		fl->cntxt_id = be16toh(c.fl0id);
364743bbae19SNavdeep Parhar 		fl->pidx = fl->cidx = fl->hw_cidx = fl->dbidx = 0;
364843bbae19SNavdeep Parhar 		fl->rx_offset = 0;
364943bbae19SNavdeep Parhar 		fl->flags &= ~(FL_STARVING | FL_DOOMED);
365054e4ee71SNavdeep Parhar 
36519f1f7ec9SNavdeep Parhar 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
3652b20b25e7SNavdeep Parhar 		if (cntxt_id >= sc->sge.eqmap_sz) {
3653733b9277SNavdeep Parhar 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
3654b20b25e7SNavdeep Parhar 			    __func__, cntxt_id, sc->sge.eqmap_sz - 1);
3655733b9277SNavdeep Parhar 		}
365654e4ee71SNavdeep Parhar 		sc->sge.eqmap[cntxt_id] = (void *)fl;
365754e4ee71SNavdeep Parhar 
36584d6db4e0SNavdeep Parhar 		qid = fl->cntxt_id;
36594d6db4e0SNavdeep Parhar 		if (isset(&sc->doorbells, DOORBELL_UDB)) {
366090e7434aSNavdeep Parhar 			uint32_t s_qpp = sc->params.sge.eq_s_qpp;
36614d6db4e0SNavdeep Parhar 			uint32_t mask = (1 << s_qpp) - 1;
36624d6db4e0SNavdeep Parhar 			volatile uint8_t *udb;
36634d6db4e0SNavdeep Parhar 
36644d6db4e0SNavdeep Parhar 			udb = sc->udbs_base + UDBS_DB_OFFSET;
36654d6db4e0SNavdeep Parhar 			udb += (qid >> s_qpp) << PAGE_SHIFT;
36664d6db4e0SNavdeep Parhar 			qid &= mask;
36674d6db4e0SNavdeep Parhar 			if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
36684d6db4e0SNavdeep Parhar 				udb += qid << UDBS_SEG_SHIFT;
36694d6db4e0SNavdeep Parhar 				qid = 0;
36704d6db4e0SNavdeep Parhar 			}
36714d6db4e0SNavdeep Parhar 			fl->udb = (volatile void *)udb;
36724d6db4e0SNavdeep Parhar 		}
3673d1205d09SNavdeep Parhar 		fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
36744d6db4e0SNavdeep Parhar 
367554e4ee71SNavdeep Parhar 		FL_LOCK(fl);
3676733b9277SNavdeep Parhar 		/* Enough to make sure the SGE doesn't think it's starved */
3677733b9277SNavdeep Parhar 		refill_fl(sc, fl, fl->lowat);
367854e4ee71SNavdeep Parhar 		FL_UNLOCK(fl);
367954e4ee71SNavdeep Parhar 	}
368054e4ee71SNavdeep Parhar 
368143bbae19SNavdeep Parhar 	if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && iq->cong >= 0) {
3682ba41ec48SNavdeep Parhar 		uint32_t param, val;
3683ba41ec48SNavdeep Parhar 
3684ba41ec48SNavdeep Parhar 		param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
3685ba41ec48SNavdeep Parhar 		    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3686ba41ec48SNavdeep Parhar 		    V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
368743bbae19SNavdeep Parhar 		if (iq->cong == 0)
368873cd9220SNavdeep Parhar 			val = 1 << 19;
368973cd9220SNavdeep Parhar 		else {
369073cd9220SNavdeep Parhar 			val = 2 << 19;
369173cd9220SNavdeep Parhar 			for (i = 0; i < 4; i++) {
369243bbae19SNavdeep Parhar 				if (iq->cong & (1 << i))
369373cd9220SNavdeep Parhar 					val |= 1 << (i << 2);
369473cd9220SNavdeep Parhar 			}
369573cd9220SNavdeep Parhar 		}
369673cd9220SNavdeep Parhar 
3697ba41ec48SNavdeep Parhar 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3698ba41ec48SNavdeep Parhar 		if (rc != 0) {
3699ba41ec48SNavdeep Parhar 			/* report error but carry on */
370043bbae19SNavdeep Parhar 			CH_ERR(sc, "failed to set congestion manager context "
370143bbae19SNavdeep Parhar 			    "for ingress queue %d: %d\n", iq->cntxt_id, rc);
3702ba41ec48SNavdeep Parhar 		}
3703ba41ec48SNavdeep Parhar 	}
3704ba41ec48SNavdeep Parhar 
370554e4ee71SNavdeep Parhar 	/* Enable IQ interrupts */
3706733b9277SNavdeep Parhar 	atomic_store_rel_int(&iq->state, IQS_IDLE);
3707315048f2SJohn Baldwin 	t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) |
370854e4ee71SNavdeep Parhar 	    V_INGRESSQID(iq->cntxt_id));
370954e4ee71SNavdeep Parhar 
371043bbae19SNavdeep Parhar 	iq->flags |= IQ_HW_ALLOCATED;
371143bbae19SNavdeep Parhar 
371254e4ee71SNavdeep Parhar 	return (0);
371354e4ee71SNavdeep Parhar }
371454e4ee71SNavdeep Parhar 
371554e4ee71SNavdeep Parhar static int
371643bbae19SNavdeep Parhar free_iq_fl_hwq(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
371754e4ee71SNavdeep Parhar {
371838035ed6SNavdeep Parhar 	int rc;
371954e4ee71SNavdeep Parhar 
372043bbae19SNavdeep Parhar 	MPASS(iq->flags & IQ_HW_ALLOCATED);
372143bbae19SNavdeep Parhar 	rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
372243bbae19SNavdeep Parhar 	    iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff);
372354e4ee71SNavdeep Parhar 	if (rc != 0) {
372443bbae19SNavdeep Parhar 		CH_ERR(sc, "failed to free iq %p: %d\n", iq, rc);
372554e4ee71SNavdeep Parhar 		return (rc);
372654e4ee71SNavdeep Parhar 	}
372743bbae19SNavdeep Parhar 	iq->flags &= ~IQ_HW_ALLOCATED;
372854e4ee71SNavdeep Parhar 
372954e4ee71SNavdeep Parhar 	return (0);
373054e4ee71SNavdeep Parhar }
373154e4ee71SNavdeep Parhar 
373238035ed6SNavdeep Parhar static void
3733348694daSNavdeep Parhar add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
3734348694daSNavdeep Parhar     struct sge_iq *iq)
3735348694daSNavdeep Parhar {
373643bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
3737348694daSNavdeep Parhar 
373843bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
373943bbae19SNavdeep Parhar 		return;
374043bbae19SNavdeep Parhar 
374143bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
3742348694daSNavdeep Parhar 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba,
3743348694daSNavdeep Parhar 	    "bus address of descriptor ring");
3744348694daSNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3745348694daSNavdeep Parhar 	    iq->qsize * IQ_ESIZE, "descriptor ring size in bytes");
3746473f6163SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
3747473f6163SNavdeep Parhar 	    &iq->abs_id, 0, "absolute id of the queue");
3748473f6163SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3749473f6163SNavdeep Parhar 	    &iq->cntxt_id, 0, "SGE context id of the queue");
3750473f6163SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &iq->cidx,
3751473f6163SNavdeep Parhar 	    0, "consumer index");
3752348694daSNavdeep Parhar }
3753348694daSNavdeep Parhar 
3754348694daSNavdeep Parhar static void
3755aa93b99aSNavdeep Parhar add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
3756aa93b99aSNavdeep Parhar     struct sysctl_oid *oid, struct sge_fl *fl)
375738035ed6SNavdeep Parhar {
375843bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
375938035ed6SNavdeep Parhar 
376043bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
376143bbae19SNavdeep Parhar 		return;
376243bbae19SNavdeep Parhar 
376343bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
37647029da5cSPawel Biernacki 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl",
37657029da5cSPawel Biernacki 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist");
376638035ed6SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
376738035ed6SNavdeep Parhar 
3768aa93b99aSNavdeep Parhar 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD,
3769aa93b99aSNavdeep Parhar 	    &fl->ba, "bus address of descriptor ring");
3770aa93b99aSNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3771aa93b99aSNavdeep Parhar 	    fl->sidx * EQ_ESIZE + sc->params.sge.spg_len,
3772aa93b99aSNavdeep Parhar 	    "desc ring size in bytes");
3773473f6163SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3774473f6163SNavdeep Parhar 	    &fl->cntxt_id, 0, "SGE context id of the freelist");
3775e3207e19SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL,
3776e3207e19SNavdeep Parhar 	    fl_pad ? 1 : 0, "padding enabled");
3777e3207e19SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL,
3778e3207e19SNavdeep Parhar 	    fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled");
377938035ed6SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
378038035ed6SNavdeep Parhar 	    0, "consumer index");
378138035ed6SNavdeep Parhar 	if (fl->flags & FL_BUF_PACKING) {
378238035ed6SNavdeep Parhar 		SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset",
378338035ed6SNavdeep Parhar 		    CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
378438035ed6SNavdeep Parhar 	}
378538035ed6SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
378638035ed6SNavdeep Parhar 	    0, "producer index");
378738035ed6SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated",
378838035ed6SNavdeep Parhar 	    CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
378938035ed6SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled",
379038035ed6SNavdeep Parhar 	    CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
379138035ed6SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled",
379238035ed6SNavdeep Parhar 	    CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
379338035ed6SNavdeep Parhar }
379438035ed6SNavdeep Parhar 
379543bbae19SNavdeep Parhar /*
379643bbae19SNavdeep Parhar  * Idempotent.
379743bbae19SNavdeep Parhar  */
379854e4ee71SNavdeep Parhar static int
3799733b9277SNavdeep Parhar alloc_fwq(struct adapter *sc)
380054e4ee71SNavdeep Parhar {
3801733b9277SNavdeep Parhar 	int rc, intr_idx;
380256599263SNavdeep Parhar 	struct sge_iq *fwq = &sc->sge.fwq;
380343bbae19SNavdeep Parhar 	struct vi_info *vi = &sc->port[0]->vi[0];
380456599263SNavdeep Parhar 
380543bbae19SNavdeep Parhar 	if (!(fwq->flags & IQ_SW_ALLOCATED)) {
380643bbae19SNavdeep Parhar 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
380743bbae19SNavdeep Parhar 
38086af45170SJohn Baldwin 		if (sc->flags & IS_VF)
38096af45170SJohn Baldwin 			intr_idx = 0;
38104535e804SNavdeep Parhar 		else
3811733b9277SNavdeep Parhar 			intr_idx = sc->intr_count > 1 ? 1 : 0;
381243bbae19SNavdeep Parhar 		init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, intr_idx, -1);
381343bbae19SNavdeep Parhar 		rc = alloc_iq_fl(vi, fwq, NULL, &sc->ctx, sc->fwq_oid);
3814733b9277SNavdeep Parhar 		if (rc != 0) {
381543bbae19SNavdeep Parhar 			CH_ERR(sc, "failed to allocate fwq: %d\n", rc);
381656599263SNavdeep Parhar 			return (rc);
3817733b9277SNavdeep Parhar 		}
381843bbae19SNavdeep Parhar 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
381943bbae19SNavdeep Parhar 	}
382056599263SNavdeep Parhar 
382143bbae19SNavdeep Parhar 	if (!(fwq->flags & IQ_HW_ALLOCATED)) {
382243bbae19SNavdeep Parhar 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
382343bbae19SNavdeep Parhar 
382443bbae19SNavdeep Parhar 		rc = alloc_iq_fl_hwq(vi, fwq, NULL);
382543bbae19SNavdeep Parhar 		if (rc != 0) {
382643bbae19SNavdeep Parhar 			CH_ERR(sc, "failed to create hw fwq: %d\n", rc);
382743bbae19SNavdeep Parhar 			return (rc);
382843bbae19SNavdeep Parhar 		}
382943bbae19SNavdeep Parhar 		MPASS(fwq->flags & IQ_HW_ALLOCATED);
383043bbae19SNavdeep Parhar 	}
383156599263SNavdeep Parhar 
3832733b9277SNavdeep Parhar 	return (0);
3833733b9277SNavdeep Parhar }
3834733b9277SNavdeep Parhar 
383543bbae19SNavdeep Parhar /*
383643bbae19SNavdeep Parhar  * Idempotent.
383743bbae19SNavdeep Parhar  */
383843bbae19SNavdeep Parhar static void
3839733b9277SNavdeep Parhar free_fwq(struct adapter *sc)
3840733b9277SNavdeep Parhar {
384143bbae19SNavdeep Parhar 	struct sge_iq *fwq = &sc->sge.fwq;
384243bbae19SNavdeep Parhar 
384343bbae19SNavdeep Parhar 	if (fwq->flags & IQ_HW_ALLOCATED) {
384443bbae19SNavdeep Parhar 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
384543bbae19SNavdeep Parhar 		free_iq_fl_hwq(sc, fwq, NULL);
384643bbae19SNavdeep Parhar 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3847733b9277SNavdeep Parhar 	}
3848733b9277SNavdeep Parhar 
384943bbae19SNavdeep Parhar 	if (fwq->flags & IQ_SW_ALLOCATED) {
385043bbae19SNavdeep Parhar 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
385143bbae19SNavdeep Parhar 		free_iq_fl(sc, fwq, NULL);
385243bbae19SNavdeep Parhar 		MPASS(!(fwq->flags & IQ_SW_ALLOCATED));
385343bbae19SNavdeep Parhar 	}
385443bbae19SNavdeep Parhar }
385543bbae19SNavdeep Parhar 
385643bbae19SNavdeep Parhar /*
385743bbae19SNavdeep Parhar  * Idempotent.
385843bbae19SNavdeep Parhar  */
3859733b9277SNavdeep Parhar static int
386043bbae19SNavdeep Parhar alloc_ctrlq(struct adapter *sc, int idx)
3861733b9277SNavdeep Parhar {
3862733b9277SNavdeep Parhar 	int rc;
3863733b9277SNavdeep Parhar 	char name[16];
386443bbae19SNavdeep Parhar 	struct sysctl_oid *oid;
386543bbae19SNavdeep Parhar 	struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
3866733b9277SNavdeep Parhar 
386743bbae19SNavdeep Parhar 	MPASS(idx < sc->params.nports);
386837310a98SNavdeep Parhar 
386943bbae19SNavdeep Parhar 	if (!(ctrlq->eq.flags & EQ_SW_ALLOCATED)) {
387043bbae19SNavdeep Parhar 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
387143bbae19SNavdeep Parhar 
387237310a98SNavdeep Parhar 		snprintf(name, sizeof(name), "%d", idx);
387343bbae19SNavdeep Parhar 		oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(sc->ctrlq_oid),
387443bbae19SNavdeep Parhar 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
387543bbae19SNavdeep Parhar 		    "ctrl queue");
387637310a98SNavdeep Parhar 
387743bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%s ctrlq%d",
387843bbae19SNavdeep Parhar 		    device_get_nameunit(sc->dev), idx);
387943bbae19SNavdeep Parhar 		init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE,
388043bbae19SNavdeep Parhar 		    sc->port[idx]->tx_chan, &sc->sge.fwq, name);
388143bbae19SNavdeep Parhar 		rc = alloc_wrq(sc, NULL, ctrlq, &sc->ctx, oid);
388243bbae19SNavdeep Parhar 		if (rc != 0) {
388343bbae19SNavdeep Parhar 			CH_ERR(sc, "failed to allocate ctrlq%d: %d\n", idx, rc);
388443bbae19SNavdeep Parhar 			sysctl_remove_oid(oid, 1, 1);
388556599263SNavdeep Parhar 			return (rc);
388656599263SNavdeep Parhar 		}
388743bbae19SNavdeep Parhar 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
388843bbae19SNavdeep Parhar 	}
388943bbae19SNavdeep Parhar 
389043bbae19SNavdeep Parhar 	if (!(ctrlq->eq.flags & EQ_HW_ALLOCATED)) {
389143bbae19SNavdeep Parhar 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
389243bbae19SNavdeep Parhar 
389343bbae19SNavdeep Parhar 		rc = alloc_eq_hwq(sc, NULL, &ctrlq->eq);
389443bbae19SNavdeep Parhar 		if (rc != 0) {
389543bbae19SNavdeep Parhar 			CH_ERR(sc, "failed to create hw ctrlq%d: %d\n", idx, rc);
389643bbae19SNavdeep Parhar 			return (rc);
389743bbae19SNavdeep Parhar 		}
389843bbae19SNavdeep Parhar 		MPASS(ctrlq->eq.flags & EQ_HW_ALLOCATED);
389943bbae19SNavdeep Parhar 	}
390043bbae19SNavdeep Parhar 
390143bbae19SNavdeep Parhar 	return (0);
390243bbae19SNavdeep Parhar }
390343bbae19SNavdeep Parhar 
390443bbae19SNavdeep Parhar /*
390543bbae19SNavdeep Parhar  * Idempotent.
390643bbae19SNavdeep Parhar  */
390743bbae19SNavdeep Parhar static void
390843bbae19SNavdeep Parhar free_ctrlq(struct adapter *sc, int idx)
390943bbae19SNavdeep Parhar {
391043bbae19SNavdeep Parhar 	struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
391143bbae19SNavdeep Parhar 
391243bbae19SNavdeep Parhar 	if (ctrlq->eq.flags & EQ_HW_ALLOCATED) {
391343bbae19SNavdeep Parhar 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
391443bbae19SNavdeep Parhar 		free_eq_hwq(sc, NULL, &ctrlq->eq);
391543bbae19SNavdeep Parhar 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
391643bbae19SNavdeep Parhar 	}
391743bbae19SNavdeep Parhar 
391843bbae19SNavdeep Parhar 	if (ctrlq->eq.flags & EQ_SW_ALLOCATED) {
391943bbae19SNavdeep Parhar 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
392043bbae19SNavdeep Parhar 		free_wrq(sc, ctrlq);
392143bbae19SNavdeep Parhar 		MPASS(!(ctrlq->eq.flags & EQ_SW_ALLOCATED));
392243bbae19SNavdeep Parhar 	}
392343bbae19SNavdeep Parhar }
392456599263SNavdeep Parhar 
39251605bac6SNavdeep Parhar int
39269af71ab3SNavdeep Parhar tnl_cong(struct port_info *pi, int drop)
39279fb8886bSNavdeep Parhar {
39289fb8886bSNavdeep Parhar 
39299af71ab3SNavdeep Parhar 	if (drop == -1)
39309fb8886bSNavdeep Parhar 		return (-1);
39319af71ab3SNavdeep Parhar 	else if (drop == 1)
39329fb8886bSNavdeep Parhar 		return (0);
39339fb8886bSNavdeep Parhar 	else
39345bcae8ddSNavdeep Parhar 		return (pi->rx_e_chan_map);
39359fb8886bSNavdeep Parhar }
39369fb8886bSNavdeep Parhar 
393743bbae19SNavdeep Parhar /*
393843bbae19SNavdeep Parhar  * Idempotent.
393943bbae19SNavdeep Parhar  */
3940733b9277SNavdeep Parhar static int
394143bbae19SNavdeep Parhar alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int idx, int intr_idx,
394243bbae19SNavdeep Parhar     int maxp)
394354e4ee71SNavdeep Parhar {
394454e4ee71SNavdeep Parhar 	int rc;
39457c228be3SNavdeep Parhar 	struct adapter *sc = vi->adapter;
394643bbae19SNavdeep Parhar 	struct ifnet *ifp = vi->ifp;
394743bbae19SNavdeep Parhar 	struct sysctl_oid *oid;
394854e4ee71SNavdeep Parhar 	char name[16];
394954e4ee71SNavdeep Parhar 
395043bbae19SNavdeep Parhar 	if (!(rxq->iq.flags & IQ_SW_ALLOCATED)) {
395143bbae19SNavdeep Parhar 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
395243bbae19SNavdeep Parhar #if defined(INET) || defined(INET6)
395343bbae19SNavdeep Parhar 		rc = tcp_lro_init_args(&rxq->lro, ifp, lro_entries, lro_mbufs);
395454e4ee71SNavdeep Parhar 		if (rc != 0)
395554e4ee71SNavdeep Parhar 			return (rc);
395643bbae19SNavdeep Parhar 		MPASS(rxq->lro.ifp == ifp);	/* also indicates LRO init'ed */
395743bbae19SNavdeep Parhar #endif
395843bbae19SNavdeep Parhar 		rxq->ifp = ifp;
395943bbae19SNavdeep Parhar 
396043bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%d", idx);
396143bbae19SNavdeep Parhar 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->rxq_oid),
396243bbae19SNavdeep Parhar 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
396343bbae19SNavdeep Parhar 		    "rx queue");
396443bbae19SNavdeep Parhar 
396543bbae19SNavdeep Parhar 		init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq,
396643bbae19SNavdeep Parhar 		    intr_idx, tnl_cong(vi->pi, cong_drop));
3967df8437a9SAndrew Gallatin #if defined(INET) || defined(INET6)
3968df8437a9SAndrew Gallatin 		if (ifp->if_capenable & IFCAP_LRO)
3969df8437a9SAndrew Gallatin 			rxq->iq.flags |= IQ_LRO_ENABLED;
3970df8437a9SAndrew Gallatin #endif
3971df8437a9SAndrew Gallatin 		if (ifp->if_capenable & IFCAP_HWRXTSTMP)
3972df8437a9SAndrew Gallatin 			rxq->iq.flags |= IQ_RX_TIMESTAMP;
397343bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%s rxq%d-fl",
397443bbae19SNavdeep Parhar 		    device_get_nameunit(vi->dev), idx);
397543bbae19SNavdeep Parhar 		init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);
397643bbae19SNavdeep Parhar 		rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid);
397743bbae19SNavdeep Parhar 		if (rc != 0) {
397843bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to allocate rxq%d: %d\n", idx, rc);
397943bbae19SNavdeep Parhar 			sysctl_remove_oid(oid, 1, 1);
398043bbae19SNavdeep Parhar #if defined(INET) || defined(INET6)
398143bbae19SNavdeep Parhar 			tcp_lro_free(&rxq->lro);
398243bbae19SNavdeep Parhar 			rxq->lro.ifp = NULL;
398343bbae19SNavdeep Parhar #endif
398443bbae19SNavdeep Parhar 			return (rc);
398543bbae19SNavdeep Parhar 		}
398643bbae19SNavdeep Parhar 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
398743bbae19SNavdeep Parhar 		add_rxq_sysctls(&vi->ctx, oid, rxq);
398843bbae19SNavdeep Parhar 	}
398943bbae19SNavdeep Parhar 
399043bbae19SNavdeep Parhar 	if (!(rxq->iq.flags & IQ_HW_ALLOCATED)) {
399143bbae19SNavdeep Parhar 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
399243bbae19SNavdeep Parhar 		rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl);
399343bbae19SNavdeep Parhar 		if (rc != 0) {
399443bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to create hw rxq%d: %d\n", idx, rc);
399543bbae19SNavdeep Parhar 			return (rc);
399643bbae19SNavdeep Parhar 		}
399743bbae19SNavdeep Parhar 		MPASS(rxq->iq.flags & IQ_HW_ALLOCATED);
399854e4ee71SNavdeep Parhar 
3999ec55567cSJohn Baldwin 		if (idx == 0)
4000ec55567cSJohn Baldwin 			sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id;
4001ec55567cSJohn Baldwin 		else
4002ec55567cSJohn Baldwin 			KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id,
4003ec55567cSJohn Baldwin 			    ("iq_base mismatch"));
4004ec55567cSJohn Baldwin 		KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF,
4005ec55567cSJohn Baldwin 		    ("PF with non-zero iq_base"));
4006ec55567cSJohn Baldwin 
40074d6db4e0SNavdeep Parhar 		/*
400843bbae19SNavdeep Parhar 		 * The freelist is just barely above the starvation threshold
400943bbae19SNavdeep Parhar 		 * right now, fill it up a bit more.
40104d6db4e0SNavdeep Parhar 		 */
40119b4d7b4eSNavdeep Parhar 		FL_LOCK(&rxq->fl);
4012ec55567cSJohn Baldwin 		refill_fl(sc, &rxq->fl, 128);
40139b4d7b4eSNavdeep Parhar 		FL_UNLOCK(&rxq->fl);
401454e4ee71SNavdeep Parhar 	}
401554e4ee71SNavdeep Parhar 
401643bbae19SNavdeep Parhar 	return (0);
401743bbae19SNavdeep Parhar }
401843bbae19SNavdeep Parhar 
401943bbae19SNavdeep Parhar /*
402043bbae19SNavdeep Parhar  * Idempotent.
402143bbae19SNavdeep Parhar  */
402243bbae19SNavdeep Parhar static void
4023fe2ebb76SJohn Baldwin free_rxq(struct vi_info *vi, struct sge_rxq *rxq)
402454e4ee71SNavdeep Parhar {
402543bbae19SNavdeep Parhar 	if (rxq->iq.flags & IQ_HW_ALLOCATED) {
402643bbae19SNavdeep Parhar 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
402743bbae19SNavdeep Parhar 		free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl);
402843bbae19SNavdeep Parhar 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
402954e4ee71SNavdeep Parhar 	}
403043bbae19SNavdeep Parhar 
403143bbae19SNavdeep Parhar 	if (rxq->iq.flags & IQ_SW_ALLOCATED) {
403243bbae19SNavdeep Parhar 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
403343bbae19SNavdeep Parhar #if defined(INET) || defined(INET6)
403443bbae19SNavdeep Parhar 		tcp_lro_free(&rxq->lro);
403554e4ee71SNavdeep Parhar #endif
403643bbae19SNavdeep Parhar 		free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl);
403743bbae19SNavdeep Parhar 		MPASS(!(rxq->iq.flags & IQ_SW_ALLOCATED));
403854e4ee71SNavdeep Parhar 		bzero(rxq, sizeof(*rxq));
403943bbae19SNavdeep Parhar 	}
404043bbae19SNavdeep Parhar }
404154e4ee71SNavdeep Parhar 
404243bbae19SNavdeep Parhar static void
404343bbae19SNavdeep Parhar add_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
404443bbae19SNavdeep Parhar     struct sge_rxq *rxq)
404543bbae19SNavdeep Parhar {
404643bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
404743bbae19SNavdeep Parhar 
404843bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
404943bbae19SNavdeep Parhar 		return;
405043bbae19SNavdeep Parhar 
405143bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
405243bbae19SNavdeep Parhar #if defined(INET) || defined(INET6)
405343bbae19SNavdeep Parhar 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
405443bbae19SNavdeep Parhar 	    &rxq->lro.lro_queued, 0, NULL);
405543bbae19SNavdeep Parhar 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
405643bbae19SNavdeep Parhar 	    &rxq->lro.lro_flushed, 0, NULL);
405743bbae19SNavdeep Parhar #endif
405843bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
405943bbae19SNavdeep Parhar 	    &rxq->rxcsum, "# of times hardware assisted with checksum");
406043bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD,
406143bbae19SNavdeep Parhar 	    &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag");
406243bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_rxcsum", CTLFLAG_RD,
406343bbae19SNavdeep Parhar 	    &rxq->vxlan_rxcsum,
406443bbae19SNavdeep Parhar 	    "# of times hardware assisted with inner checksum (VXLAN)");
406554e4ee71SNavdeep Parhar }
406654e4ee71SNavdeep Parhar 
406709fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
406843bbae19SNavdeep Parhar /*
406943bbae19SNavdeep Parhar  * Idempotent.
407043bbae19SNavdeep Parhar  */
407154e4ee71SNavdeep Parhar static int
407243bbae19SNavdeep Parhar alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,
407343bbae19SNavdeep Parhar     int intr_idx, int maxp)
4074f7dfe243SNavdeep Parhar {
4075733b9277SNavdeep Parhar 	int rc;
407643bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
407743bbae19SNavdeep Parhar 	struct sysctl_oid *oid;
4078733b9277SNavdeep Parhar 	char name[16];
4079f7dfe243SNavdeep Parhar 
408043bbae19SNavdeep Parhar 	if (!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)) {
408143bbae19SNavdeep Parhar 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4082733b9277SNavdeep Parhar 
4083733b9277SNavdeep Parhar 		snprintf(name, sizeof(name), "%d", idx);
408443bbae19SNavdeep Parhar 		oid = SYSCTL_ADD_NODE(&vi->ctx,
408543bbae19SNavdeep Parhar 		    SYSCTL_CHILDREN(vi->ofld_rxq_oid), OID_AUTO, name,
408643bbae19SNavdeep Parhar 		    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload rx queue");
4087733b9277SNavdeep Parhar 
408843bbae19SNavdeep Parhar 		init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx,
408943bbae19SNavdeep Parhar 		    vi->qsize_rxq, intr_idx, 0);
409043bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
409143bbae19SNavdeep Parhar 		    device_get_nameunit(vi->dev), idx);
409243bbae19SNavdeep Parhar 		init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);
409343bbae19SNavdeep Parhar 		rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx,
409443bbae19SNavdeep Parhar 		    oid);
409543bbae19SNavdeep Parhar 		if (rc != 0) {
409643bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to allocate ofld_rxq%d: %d\n", idx,
409743bbae19SNavdeep Parhar 			    rc);
409843bbae19SNavdeep Parhar 			sysctl_remove_oid(oid, 1, 1);
409943bbae19SNavdeep Parhar 			return (rc);
410043bbae19SNavdeep Parhar 		}
410143bbae19SNavdeep Parhar 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4102a9f0cf48SJohn Baldwin 		ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
4103a9f0cf48SJohn Baldwin 		ofld_rxq->rx_iscsi_ddp_setup_error =
4104a9f0cf48SJohn Baldwin 		    counter_u64_alloc(M_WAITOK);
410543bbae19SNavdeep Parhar 		add_ofld_rxq_sysctls(&vi->ctx, oid, ofld_rxq);
410643bbae19SNavdeep Parhar 	}
4107fe496dc0SJohn Baldwin 
410843bbae19SNavdeep Parhar 	if (!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)) {
410943bbae19SNavdeep Parhar 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
411043bbae19SNavdeep Parhar 		rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl);
411143bbae19SNavdeep Parhar 		if (rc != 0) {
411243bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to create hw ofld_rxq%d: %d\n", idx,
411343bbae19SNavdeep Parhar 			    rc);
411443bbae19SNavdeep Parhar 			return (rc);
411543bbae19SNavdeep Parhar 		}
411643bbae19SNavdeep Parhar 		MPASS(ofld_rxq->iq.flags & IQ_HW_ALLOCATED);
411743bbae19SNavdeep Parhar 	}
4118733b9277SNavdeep Parhar 	return (rc);
4119733b9277SNavdeep Parhar }
4120733b9277SNavdeep Parhar 
412143bbae19SNavdeep Parhar /*
412243bbae19SNavdeep Parhar  * Idempotent.
412343bbae19SNavdeep Parhar  */
412443bbae19SNavdeep Parhar static void
4125fe2ebb76SJohn Baldwin free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
4126733b9277SNavdeep Parhar {
412743bbae19SNavdeep Parhar 	if (ofld_rxq->iq.flags & IQ_HW_ALLOCATED) {
412843bbae19SNavdeep Parhar 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
412943bbae19SNavdeep Parhar 		free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
413043bbae19SNavdeep Parhar 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
413143bbae19SNavdeep Parhar 	}
4132733b9277SNavdeep Parhar 
413343bbae19SNavdeep Parhar 	if (ofld_rxq->iq.flags & IQ_SW_ALLOCATED) {
413443bbae19SNavdeep Parhar 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
413543bbae19SNavdeep Parhar 		free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
413643bbae19SNavdeep Parhar 		MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED));
4137a9f0cf48SJohn Baldwin 		counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok);
4138a9f0cf48SJohn Baldwin 		counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error);
4139733b9277SNavdeep Parhar 		bzero(ofld_rxq, sizeof(*ofld_rxq));
414043bbae19SNavdeep Parhar 	}
414143bbae19SNavdeep Parhar }
4142733b9277SNavdeep Parhar 
414343bbae19SNavdeep Parhar static void
414443bbae19SNavdeep Parhar add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
414543bbae19SNavdeep Parhar     struct sge_ofld_rxq *ofld_rxq)
414643bbae19SNavdeep Parhar {
414743bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
414843bbae19SNavdeep Parhar 
414943bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
415043bbae19SNavdeep Parhar 		return;
415143bbae19SNavdeep Parhar 
415243bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
41534b6ed075SJohn Baldwin 	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
415443bbae19SNavdeep Parhar 	    "rx_toe_tls_records", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_records,
415543bbae19SNavdeep Parhar 	    "# of TOE TLS records received");
41564b6ed075SJohn Baldwin 	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
415743bbae19SNavdeep Parhar 	    "rx_toe_tls_octets", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_octets,
415843bbae19SNavdeep Parhar 	    "# of payload octets in received TOE TLS records");
41594b6ed075SJohn Baldwin 
41604b6ed075SJohn Baldwin 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "iscsi",
41614b6ed075SJohn Baldwin 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE iSCSI statistics");
41624b6ed075SJohn Baldwin 	children = SYSCTL_CHILDREN(oid);
41634b6ed075SJohn Baldwin 
41644b6ed075SJohn Baldwin 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok",
41654b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_ok,
41664b6ed075SJohn Baldwin 	    "# of times DDP buffer was setup successfully.");
41674b6ed075SJohn Baldwin 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error",
41684b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_error,
41694b6ed075SJohn Baldwin 	    "# of times DDP buffer setup failed.");
41704b6ed075SJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_octets",
41714b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_octets, 0,
41724b6ed075SJohn Baldwin 	    "# of octets placed directly");
41734b6ed075SJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_pdus",
41744b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_pdus, 0,
41754b6ed075SJohn Baldwin 	    "# of PDUs with data placed directly.");
41764b6ed075SJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_octets",
41774b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_octets, 0,
41784b6ed075SJohn Baldwin 	    "# of data octets delivered in freelist");
41794b6ed075SJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_pdus",
41804b6ed075SJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_pdus, 0,
41814b6ed075SJohn Baldwin 	    "# of PDUs with data delivered in freelist");
41824d4cf62eSJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "padding_errors",
41834d4cf62eSJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_padding_errors, 0,
41844d4cf62eSJohn Baldwin 	    "# of PDUs with invalid padding");
41854d4cf62eSJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "header_digest_errors",
41864d4cf62eSJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_header_digest_errors, 0,
41874d4cf62eSJohn Baldwin 	    "# of PDUs with invalid header digests");
41884d4cf62eSJohn Baldwin 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors",
41894d4cf62eSJohn Baldwin 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0,
41904d4cf62eSJohn Baldwin 	    "# of PDUs with invalid data digests");
4191733b9277SNavdeep Parhar }
4192733b9277SNavdeep Parhar #endif
4193733b9277SNavdeep Parhar 
4194ddf09ad6SNavdeep Parhar /*
4195ddf09ad6SNavdeep Parhar  * Returns a reasonable automatic cidx flush threshold for a given queue size.
4196ddf09ad6SNavdeep Parhar  */
4197ddf09ad6SNavdeep Parhar static u_int
4198ddf09ad6SNavdeep Parhar qsize_to_fthresh(int qsize)
4199ddf09ad6SNavdeep Parhar {
4200ddf09ad6SNavdeep Parhar 	u_int fthresh;
4201ddf09ad6SNavdeep Parhar 
4202ddf09ad6SNavdeep Parhar 	while (!powerof2(qsize))
4203ddf09ad6SNavdeep Parhar 		qsize++;
4204ddf09ad6SNavdeep Parhar 	fthresh = ilog2(qsize);
4205ddf09ad6SNavdeep Parhar 	if (fthresh > X_CIDXFLUSHTHRESH_128)
4206ddf09ad6SNavdeep Parhar 		fthresh = X_CIDXFLUSHTHRESH_128;
4207ddf09ad6SNavdeep Parhar 
4208ddf09ad6SNavdeep Parhar 	return (fthresh);
4209ddf09ad6SNavdeep Parhar }
4210ddf09ad6SNavdeep Parhar 
4211733b9277SNavdeep Parhar static int
4212733b9277SNavdeep Parhar ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
4213733b9277SNavdeep Parhar {
4214733b9277SNavdeep Parhar 	int rc, cntxt_id;
4215733b9277SNavdeep Parhar 	struct fw_eq_ctrl_cmd c;
421690e7434aSNavdeep Parhar 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4217f7dfe243SNavdeep Parhar 
4218f7dfe243SNavdeep Parhar 	bzero(&c, sizeof(c));
4219f7dfe243SNavdeep Parhar 
4220f7dfe243SNavdeep Parhar 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
4221f7dfe243SNavdeep Parhar 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
4222f7dfe243SNavdeep Parhar 	    V_FW_EQ_CTRL_CMD_VFN(0));
4223f7dfe243SNavdeep Parhar 	c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
4224f7dfe243SNavdeep Parhar 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
42257951040fSNavdeep Parhar 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
4226f7dfe243SNavdeep Parhar 	c.physeqid_pkd = htobe32(0);
4227f7dfe243SNavdeep Parhar 	c.fetchszm_to_iqid =
422887b027baSNavdeep Parhar 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4229733b9277SNavdeep Parhar 		V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
423056599263SNavdeep Parhar 		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
4231f7dfe243SNavdeep Parhar 	c.dcaen_to_eqsize =
4232adb0cd84SNavdeep Parhar 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4233adb0cd84SNavdeep Parhar 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4234f7dfe243SNavdeep Parhar 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4235ddf09ad6SNavdeep Parhar 		V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
42367951040fSNavdeep Parhar 		V_FW_EQ_CTRL_CMD_EQSIZE(qsize));
4237f7dfe243SNavdeep Parhar 	c.eqaddr = htobe64(eq->ba);
4238f7dfe243SNavdeep Parhar 
4239f7dfe243SNavdeep Parhar 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4240f7dfe243SNavdeep Parhar 	if (rc != 0) {
424143bbae19SNavdeep Parhar 		CH_ERR(sc, "failed to create hw ctrlq for tx_chan %d: %d\n",
424243bbae19SNavdeep Parhar 		    eq->tx_chan, rc);
4243f7dfe243SNavdeep Parhar 		return (rc);
4244f7dfe243SNavdeep Parhar 	}
4245f7dfe243SNavdeep Parhar 
4246f7dfe243SNavdeep Parhar 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
424776c89022SNavdeep Parhar 	eq->abs_id = G_FW_EQ_CTRL_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4248f7dfe243SNavdeep Parhar 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4249b20b25e7SNavdeep Parhar 	if (cntxt_id >= sc->sge.eqmap_sz)
4250733b9277SNavdeep Parhar 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4251b20b25e7SNavdeep Parhar 		cntxt_id, sc->sge.eqmap_sz - 1);
4252f7dfe243SNavdeep Parhar 	sc->sge.eqmap[cntxt_id] = eq;
4253f7dfe243SNavdeep Parhar 
4254f7dfe243SNavdeep Parhar 	return (rc);
4255f7dfe243SNavdeep Parhar }
4256f7dfe243SNavdeep Parhar 
4257f7dfe243SNavdeep Parhar static int
4258fe2ebb76SJohn Baldwin eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
425954e4ee71SNavdeep Parhar {
426054e4ee71SNavdeep Parhar 	int rc, cntxt_id;
426154e4ee71SNavdeep Parhar 	struct fw_eq_eth_cmd c;
426290e7434aSNavdeep Parhar 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
426354e4ee71SNavdeep Parhar 
426454e4ee71SNavdeep Parhar 	bzero(&c, sizeof(c));
426554e4ee71SNavdeep Parhar 
426654e4ee71SNavdeep Parhar 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
426754e4ee71SNavdeep Parhar 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
426854e4ee71SNavdeep Parhar 	    V_FW_EQ_ETH_CMD_VFN(0));
426954e4ee71SNavdeep Parhar 	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
427054e4ee71SNavdeep Parhar 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
42717951040fSNavdeep Parhar 	c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
4272fe2ebb76SJohn Baldwin 	    F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
427354e4ee71SNavdeep Parhar 	c.fetchszm_to_iqid =
42747951040fSNavdeep Parhar 	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
4275733b9277SNavdeep Parhar 		V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
4276aa2457e1SNavdeep Parhar 		V_FW_EQ_ETH_CMD_IQID(eq->iqid));
4277adb0cd84SNavdeep Parhar 	c.dcaen_to_eqsize =
4278adb0cd84SNavdeep Parhar 	    htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4279adb0cd84SNavdeep Parhar 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
428054e4ee71SNavdeep Parhar 		V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
42817951040fSNavdeep Parhar 		V_FW_EQ_ETH_CMD_EQSIZE(qsize));
428254e4ee71SNavdeep Parhar 	c.eqaddr = htobe64(eq->ba);
428354e4ee71SNavdeep Parhar 
428454e4ee71SNavdeep Parhar 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
428554e4ee71SNavdeep Parhar 	if (rc != 0) {
4286fe2ebb76SJohn Baldwin 		device_printf(vi->dev,
4287733b9277SNavdeep Parhar 		    "failed to create Ethernet egress queue: %d\n", rc);
4288733b9277SNavdeep Parhar 		return (rc);
4289733b9277SNavdeep Parhar 	}
4290733b9277SNavdeep Parhar 
4291733b9277SNavdeep Parhar 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
4292ec55567cSJohn Baldwin 	eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4293733b9277SNavdeep Parhar 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4294b20b25e7SNavdeep Parhar 	if (cntxt_id >= sc->sge.eqmap_sz)
4295733b9277SNavdeep Parhar 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4296b20b25e7SNavdeep Parhar 		cntxt_id, sc->sge.eqmap_sz - 1);
4297733b9277SNavdeep Parhar 	sc->sge.eqmap[cntxt_id] = eq;
4298733b9277SNavdeep Parhar 
429954e4ee71SNavdeep Parhar 	return (rc);
430054e4ee71SNavdeep Parhar }
430154e4ee71SNavdeep Parhar 
4302eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4303733b9277SNavdeep Parhar static int
4304fe2ebb76SJohn Baldwin ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4305733b9277SNavdeep Parhar {
4306733b9277SNavdeep Parhar 	int rc, cntxt_id;
4307733b9277SNavdeep Parhar 	struct fw_eq_ofld_cmd c;
430890e7434aSNavdeep Parhar 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
430954e4ee71SNavdeep Parhar 
4310733b9277SNavdeep Parhar 	bzero(&c, sizeof(c));
4311733b9277SNavdeep Parhar 
4312733b9277SNavdeep Parhar 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
4313733b9277SNavdeep Parhar 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
4314733b9277SNavdeep Parhar 	    V_FW_EQ_OFLD_CMD_VFN(0));
4315733b9277SNavdeep Parhar 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
4316733b9277SNavdeep Parhar 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
4317733b9277SNavdeep Parhar 	c.fetchszm_to_iqid =
4318ddf09ad6SNavdeep Parhar 		htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4319733b9277SNavdeep Parhar 		    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
4320733b9277SNavdeep Parhar 		    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
4321733b9277SNavdeep Parhar 	c.dcaen_to_eqsize =
4322adb0cd84SNavdeep Parhar 	    htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4323adb0cd84SNavdeep Parhar 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4324733b9277SNavdeep Parhar 		V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4325ddf09ad6SNavdeep Parhar 		V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
43267951040fSNavdeep Parhar 		V_FW_EQ_OFLD_CMD_EQSIZE(qsize));
4327733b9277SNavdeep Parhar 	c.eqaddr = htobe64(eq->ba);
4328733b9277SNavdeep Parhar 
4329733b9277SNavdeep Parhar 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4330733b9277SNavdeep Parhar 	if (rc != 0) {
4331fe2ebb76SJohn Baldwin 		device_printf(vi->dev,
4332733b9277SNavdeep Parhar 		    "failed to create egress queue for TCP offload: %d\n", rc);
4333733b9277SNavdeep Parhar 		return (rc);
4334733b9277SNavdeep Parhar 	}
4335733b9277SNavdeep Parhar 
4336733b9277SNavdeep Parhar 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
433776c89022SNavdeep Parhar 	eq->abs_id = G_FW_EQ_OFLD_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
433854e4ee71SNavdeep Parhar 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4339b20b25e7SNavdeep Parhar 	if (cntxt_id >= sc->sge.eqmap_sz)
4340733b9277SNavdeep Parhar 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4341b20b25e7SNavdeep Parhar 		cntxt_id, sc->sge.eqmap_sz - 1);
434254e4ee71SNavdeep Parhar 	sc->sge.eqmap[cntxt_id] = eq;
434354e4ee71SNavdeep Parhar 
4344733b9277SNavdeep Parhar 	return (rc);
4345733b9277SNavdeep Parhar }
4346733b9277SNavdeep Parhar #endif
4347733b9277SNavdeep Parhar 
434843bbae19SNavdeep Parhar /* SW only */
4349733b9277SNavdeep Parhar static int
435043bbae19SNavdeep Parhar alloc_eq(struct adapter *sc, struct sge_eq *eq, struct sysctl_ctx_list *ctx,
435143bbae19SNavdeep Parhar     struct sysctl_oid *oid)
4352733b9277SNavdeep Parhar {
43537951040fSNavdeep Parhar 	int rc, qsize;
4354733b9277SNavdeep Parhar 	size_t len;
4355733b9277SNavdeep Parhar 
435643bbae19SNavdeep Parhar 	MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4357733b9277SNavdeep Parhar 
435890e7434aSNavdeep Parhar 	qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
43597951040fSNavdeep Parhar 	len = qsize * EQ_ESIZE;
436043bbae19SNavdeep Parhar 	rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba,
436143bbae19SNavdeep Parhar 	    (void **)&eq->desc);
4362733b9277SNavdeep Parhar 	if (rc)
4363733b9277SNavdeep Parhar 		return (rc);
436443bbae19SNavdeep Parhar 	if (ctx != NULL && oid != NULL)
436543bbae19SNavdeep Parhar 		add_eq_sysctls(sc, ctx, oid, eq);
436643bbae19SNavdeep Parhar 	eq->flags |= EQ_SW_ALLOCATED;
4367733b9277SNavdeep Parhar 
436843bbae19SNavdeep Parhar 	return (0);
436943bbae19SNavdeep Parhar }
437043bbae19SNavdeep Parhar 
437143bbae19SNavdeep Parhar /* SW only */
437243bbae19SNavdeep Parhar static void
437343bbae19SNavdeep Parhar free_eq(struct adapter *sc, struct sge_eq *eq)
437443bbae19SNavdeep Parhar {
437543bbae19SNavdeep Parhar 	MPASS(eq->flags & EQ_SW_ALLOCATED);
43765ef87bf8SNavdeep Parhar 	if (eq->type == EQ_ETH)
437743bbae19SNavdeep Parhar 		MPASS(eq->pidx == eq->cidx);
437843bbae19SNavdeep Parhar 
437943bbae19SNavdeep Parhar 	free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
438043bbae19SNavdeep Parhar 	mtx_destroy(&eq->eq_lock);
438143bbae19SNavdeep Parhar 	bzero(eq, sizeof(*eq));
438243bbae19SNavdeep Parhar }
438343bbae19SNavdeep Parhar 
438443bbae19SNavdeep Parhar static void
438543bbae19SNavdeep Parhar add_eq_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
438643bbae19SNavdeep Parhar     struct sysctl_oid *oid, struct sge_eq *eq)
438743bbae19SNavdeep Parhar {
438843bbae19SNavdeep Parhar 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
438943bbae19SNavdeep Parhar 
439043bbae19SNavdeep Parhar 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba,
439143bbae19SNavdeep Parhar 	    "bus address of descriptor ring");
439243bbae19SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
439343bbae19SNavdeep Parhar 	    eq->sidx * EQ_ESIZE + sc->params.sge.spg_len,
439443bbae19SNavdeep Parhar 	    "desc ring size in bytes");
439543bbae19SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
439643bbae19SNavdeep Parhar 	    &eq->abs_id, 0, "absolute id of the queue");
439743bbae19SNavdeep Parhar 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
439843bbae19SNavdeep Parhar 	    &eq->cntxt_id, 0, "SGE context id of the queue");
439943bbae19SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &eq->cidx,
440043bbae19SNavdeep Parhar 	    0, "consumer index");
440143bbae19SNavdeep Parhar 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &eq->pidx,
440243bbae19SNavdeep Parhar 	    0, "producer index");
440343bbae19SNavdeep Parhar 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL,
440443bbae19SNavdeep Parhar 	    eq->sidx, "status page index");
440543bbae19SNavdeep Parhar }
440643bbae19SNavdeep Parhar 
440743bbae19SNavdeep Parhar static int
440843bbae19SNavdeep Parhar alloc_eq_hwq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
440943bbae19SNavdeep Parhar {
441043bbae19SNavdeep Parhar 	int rc;
441143bbae19SNavdeep Parhar 
441243bbae19SNavdeep Parhar 	MPASS(!(eq->flags & EQ_HW_ALLOCATED));
441343bbae19SNavdeep Parhar 
441443bbae19SNavdeep Parhar 	eq->iqid = eq->iq->cntxt_id;
4415ddf09ad6SNavdeep Parhar 	eq->pidx = eq->cidx = eq->dbidx = 0;
4416ddf09ad6SNavdeep Parhar 	/* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */
4417ddf09ad6SNavdeep Parhar 	eq->equeqidx = 0;
4418d14b0ac1SNavdeep Parhar 	eq->doorbells = sc->doorbells;
441943bbae19SNavdeep Parhar 	bzero(eq->desc, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len);
4420733b9277SNavdeep Parhar 
442143bbae19SNavdeep Parhar 	switch (eq->type) {
4422733b9277SNavdeep Parhar 	case EQ_CTRL:
4423733b9277SNavdeep Parhar 		rc = ctrl_eq_alloc(sc, eq);
4424733b9277SNavdeep Parhar 		break;
4425733b9277SNavdeep Parhar 
4426733b9277SNavdeep Parhar 	case EQ_ETH:
4427fe2ebb76SJohn Baldwin 		rc = eth_eq_alloc(sc, vi, eq);
4428733b9277SNavdeep Parhar 		break;
4429733b9277SNavdeep Parhar 
4430eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4431733b9277SNavdeep Parhar 	case EQ_OFLD:
4432fe2ebb76SJohn Baldwin 		rc = ofld_eq_alloc(sc, vi, eq);
4433733b9277SNavdeep Parhar 		break;
4434733b9277SNavdeep Parhar #endif
4435733b9277SNavdeep Parhar 
4436733b9277SNavdeep Parhar 	default:
443743bbae19SNavdeep Parhar 		panic("%s: invalid eq type %d.", __func__, eq->type);
4438733b9277SNavdeep Parhar 	}
4439733b9277SNavdeep Parhar 	if (rc != 0) {
444043bbae19SNavdeep Parhar 		CH_ERR(sc, "failed to allocate egress queue(%d): %d\n",
444143bbae19SNavdeep Parhar 		    eq->type, rc);
444243bbae19SNavdeep Parhar 		return (rc);
4443733b9277SNavdeep Parhar 	}
4444733b9277SNavdeep Parhar 
4445d14b0ac1SNavdeep Parhar 	if (isset(&eq->doorbells, DOORBELL_UDB) ||
4446d14b0ac1SNavdeep Parhar 	    isset(&eq->doorbells, DOORBELL_UDBWC) ||
444777ad3c41SNavdeep Parhar 	    isset(&eq->doorbells, DOORBELL_WCWR)) {
444890e7434aSNavdeep Parhar 		uint32_t s_qpp = sc->params.sge.eq_s_qpp;
4449d14b0ac1SNavdeep Parhar 		uint32_t mask = (1 << s_qpp) - 1;
4450d14b0ac1SNavdeep Parhar 		volatile uint8_t *udb;
4451d14b0ac1SNavdeep Parhar 
4452d14b0ac1SNavdeep Parhar 		udb = sc->udbs_base + UDBS_DB_OFFSET;
4453d14b0ac1SNavdeep Parhar 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;	/* pg offset */
4454d14b0ac1SNavdeep Parhar 		eq->udb_qid = eq->cntxt_id & mask;		/* id in page */
4455f10405b3SNavdeep Parhar 		if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
445677ad3c41SNavdeep Parhar 	    		clrbit(&eq->doorbells, DOORBELL_WCWR);
4457d14b0ac1SNavdeep Parhar 		else {
4458d14b0ac1SNavdeep Parhar 			udb += eq->udb_qid << UDBS_SEG_SHIFT;	/* seg offset */
4459d14b0ac1SNavdeep Parhar 			eq->udb_qid = 0;
4460d14b0ac1SNavdeep Parhar 		}
4461d14b0ac1SNavdeep Parhar 		eq->udb = (volatile void *)udb;
4462d14b0ac1SNavdeep Parhar 	}
4463d14b0ac1SNavdeep Parhar 
446443bbae19SNavdeep Parhar 	eq->flags |= EQ_HW_ALLOCATED;
446543bbae19SNavdeep Parhar 	return (0);
4466733b9277SNavdeep Parhar }
4467733b9277SNavdeep Parhar 
4468733b9277SNavdeep Parhar static int
446943bbae19SNavdeep Parhar free_eq_hwq(struct adapter *sc, struct vi_info *vi __unused, struct sge_eq *eq)
4470733b9277SNavdeep Parhar {
4471733b9277SNavdeep Parhar 	int rc;
4472733b9277SNavdeep Parhar 
447343bbae19SNavdeep Parhar 	MPASS(eq->flags & EQ_HW_ALLOCATED);
447443bbae19SNavdeep Parhar 
447543bbae19SNavdeep Parhar 	switch (eq->type) {
4476733b9277SNavdeep Parhar 	case EQ_CTRL:
447743bbae19SNavdeep Parhar 		rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4478733b9277SNavdeep Parhar 		break;
4479733b9277SNavdeep Parhar 	case EQ_ETH:
448043bbae19SNavdeep Parhar 		rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4481733b9277SNavdeep Parhar 		break;
4482eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4483733b9277SNavdeep Parhar 	case EQ_OFLD:
448443bbae19SNavdeep Parhar 		rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4485733b9277SNavdeep Parhar 		break;
4486733b9277SNavdeep Parhar #endif
4487733b9277SNavdeep Parhar 	default:
448843bbae19SNavdeep Parhar 		panic("%s: invalid eq type %d.", __func__, eq->type);
4489733b9277SNavdeep Parhar 	}
4490733b9277SNavdeep Parhar 	if (rc != 0) {
449143bbae19SNavdeep Parhar 		CH_ERR(sc, "failed to free eq (type %d): %d\n", eq->type, rc);
4492733b9277SNavdeep Parhar 		return (rc);
4493733b9277SNavdeep Parhar 	}
449443bbae19SNavdeep Parhar 	eq->flags &= ~EQ_HW_ALLOCATED;
4495733b9277SNavdeep Parhar 
4496733b9277SNavdeep Parhar 	return (0);
4497733b9277SNavdeep Parhar }
4498733b9277SNavdeep Parhar 
4499733b9277SNavdeep Parhar static int
4500fe2ebb76SJohn Baldwin alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq,
450143bbae19SNavdeep Parhar     struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
4502733b9277SNavdeep Parhar {
450343bbae19SNavdeep Parhar 	struct sge_eq *eq = &wrq->eq;
4504733b9277SNavdeep Parhar 	int rc;
4505733b9277SNavdeep Parhar 
450643bbae19SNavdeep Parhar 	MPASS(!(eq->flags & EQ_SW_ALLOCATED));
450743bbae19SNavdeep Parhar 
450843bbae19SNavdeep Parhar 	rc = alloc_eq(sc, eq, ctx, oid);
4509733b9277SNavdeep Parhar 	if (rc)
4510733b9277SNavdeep Parhar 		return (rc);
451143bbae19SNavdeep Parhar 	MPASS(eq->flags & EQ_SW_ALLOCATED);
451243bbae19SNavdeep Parhar 	/* Can't fail after this. */
4513733b9277SNavdeep Parhar 
4514733b9277SNavdeep Parhar 	wrq->adapter = sc;
45157951040fSNavdeep Parhar 	TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq);
45167951040fSNavdeep Parhar 	TAILQ_INIT(&wrq->incomplete_wrs);
451709fe6320SNavdeep Parhar 	STAILQ_INIT(&wrq->wr_list);
45187951040fSNavdeep Parhar 	wrq->nwr_pending = 0;
45197951040fSNavdeep Parhar 	wrq->ndesc_needed = 0;
452043bbae19SNavdeep Parhar 	add_wrq_sysctls(ctx, oid, wrq);
4521733b9277SNavdeep Parhar 
452243bbae19SNavdeep Parhar 	return (0);
452343bbae19SNavdeep Parhar }
452443bbae19SNavdeep Parhar 
452543bbae19SNavdeep Parhar static void
452643bbae19SNavdeep Parhar free_wrq(struct adapter *sc, struct sge_wrq *wrq)
452743bbae19SNavdeep Parhar {
452843bbae19SNavdeep Parhar 	free_eq(sc, &wrq->eq);
452943bbae19SNavdeep Parhar 	MPASS(wrq->nwr_pending == 0);
45305ef87bf8SNavdeep Parhar 	MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
45315ef87bf8SNavdeep Parhar 	MPASS(STAILQ_EMPTY(&wrq->wr_list));
453243bbae19SNavdeep Parhar 	bzero(wrq, sizeof(*wrq));
453343bbae19SNavdeep Parhar }
453443bbae19SNavdeep Parhar 
453543bbae19SNavdeep Parhar static void
453643bbae19SNavdeep Parhar add_wrq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
453743bbae19SNavdeep Parhar     struct sge_wrq *wrq)
453843bbae19SNavdeep Parhar {
453943bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
454043bbae19SNavdeep Parhar 
454143bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
454243bbae19SNavdeep Parhar 		return;
454343bbae19SNavdeep Parhar 
454443bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
45457951040fSNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD,
45467951040fSNavdeep Parhar 	    &wrq->tx_wrs_direct, "# of work requests (direct)");
45477951040fSNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD,
45487951040fSNavdeep Parhar 	    &wrq->tx_wrs_copied, "# of work requests (copied)");
45490459a175SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD,
45500459a175SNavdeep Parhar 	    &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)");
4551733b9277SNavdeep Parhar }
4552733b9277SNavdeep Parhar 
455343bbae19SNavdeep Parhar /*
455443bbae19SNavdeep Parhar  * Idempotent.
455543bbae19SNavdeep Parhar  */
4556733b9277SNavdeep Parhar static int
455743bbae19SNavdeep Parhar alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx)
4558733b9277SNavdeep Parhar {
455943bbae19SNavdeep Parhar 	int rc, iqidx;
4560fe2ebb76SJohn Baldwin 	struct port_info *pi = vi->pi;
456143bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
4562733b9277SNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
4563d735920dSNavdeep Parhar 	struct txpkts *txp;
4564733b9277SNavdeep Parhar 	char name[16];
456543bbae19SNavdeep Parhar 	struct sysctl_oid *oid;
4566733b9277SNavdeep Parhar 
456743bbae19SNavdeep Parhar 	if (!(eq->flags & EQ_SW_ALLOCATED)) {
456843bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
456943bbae19SNavdeep Parhar 
457043bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%d", idx);
457143bbae19SNavdeep Parhar 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->txq_oid),
457243bbae19SNavdeep Parhar 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
457343bbae19SNavdeep Parhar 		    "tx queue");
457443bbae19SNavdeep Parhar 
457543bbae19SNavdeep Parhar 		iqidx = vi->first_rxq + (idx % vi->nrxq);
457643bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%s txq%d",
457743bbae19SNavdeep Parhar 		    device_get_nameunit(vi->dev), idx);
457843bbae19SNavdeep Parhar 		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan,
457943bbae19SNavdeep Parhar 		    &sc->sge.rxq[iqidx].iq, name);
458043bbae19SNavdeep Parhar 
458143bbae19SNavdeep Parhar 		rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx,
458243bbae19SNavdeep Parhar 		    can_resume_eth_tx, M_CXGBE, &eq->eq_lock, M_WAITOK);
45837951040fSNavdeep Parhar 		if (rc != 0) {
458443bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to allocate mp_ring for txq%d: %d\n",
458543bbae19SNavdeep Parhar 			    idx, rc);
458643bbae19SNavdeep Parhar failed:
458743bbae19SNavdeep Parhar 			sysctl_remove_oid(oid, 1, 1);
45887951040fSNavdeep Parhar 			return (rc);
45897951040fSNavdeep Parhar 		}
45907951040fSNavdeep Parhar 
459143bbae19SNavdeep Parhar 		rc = alloc_eq(sc, eq, &vi->ctx, oid);
459243bbae19SNavdeep Parhar 		if (rc) {
459343bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to allocate txq%d: %d\n", idx, rc);
45947951040fSNavdeep Parhar 			mp_ring_free(txq->r);
459543bbae19SNavdeep Parhar 			goto failed;
459643bbae19SNavdeep Parhar 		}
459743bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_SW_ALLOCATED);
459843bbae19SNavdeep Parhar 		/* Can't fail after this point. */
459943bbae19SNavdeep Parhar 
460043bbae19SNavdeep Parhar 		TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
460143bbae19SNavdeep Parhar 		txq->ifp = vi->ifp;
460243bbae19SNavdeep Parhar 		txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
460343bbae19SNavdeep Parhar 		txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
460443bbae19SNavdeep Parhar 		    M_ZERO | M_WAITOK);
460543bbae19SNavdeep Parhar 
460643bbae19SNavdeep Parhar 		add_txq_sysctls(vi, &vi->ctx, oid, txq);
46077951040fSNavdeep Parhar 	}
4608733b9277SNavdeep Parhar 
460943bbae19SNavdeep Parhar 	if (!(eq->flags & EQ_HW_ALLOCATED)) {
461043bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_SW_ALLOCATED);
461143bbae19SNavdeep Parhar 		rc = alloc_eq_hwq(sc, vi, eq);
461243bbae19SNavdeep Parhar 		if (rc != 0) {
461343bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to create hw txq%d: %d\n", idx, rc);
461443bbae19SNavdeep Parhar 			return (rc);
461543bbae19SNavdeep Parhar 		}
461643bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_HW_ALLOCATED);
46177951040fSNavdeep Parhar 		/* Can't fail after this point. */
46187951040fSNavdeep Parhar 
4619ec55567cSJohn Baldwin 		if (idx == 0)
4620ec55567cSJohn Baldwin 			sc->sge.eq_base = eq->abs_id - eq->cntxt_id;
4621ec55567cSJohn Baldwin 		else
4622ec55567cSJohn Baldwin 			KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id,
4623ec55567cSJohn Baldwin 			    ("eq_base mismatch"));
4624ec55567cSJohn Baldwin 		KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF,
4625ec55567cSJohn Baldwin 		    ("PF with non-zero eq_base"));
4626ec55567cSJohn Baldwin 
4627d735920dSNavdeep Parhar 		txp = &txq->txp;
4628d735920dSNavdeep Parhar 		MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr);
4629d735920dSNavdeep Parhar 		txq->txp.max_npkt = min(nitems(txp->mb),
4630d735920dSNavdeep Parhar 		    sc->params.max_pkts_per_eth_tx_pkts_wr);
463130e3f2b4SNavdeep Parhar 		if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF))
463230e3f2b4SNavdeep Parhar 			txq->txp.max_npkt--;
4633d735920dSNavdeep Parhar 
463443bbae19SNavdeep Parhar 		if (vi->flags & TX_USES_VM_WR)
463543bbae19SNavdeep Parhar 			txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
463643bbae19SNavdeep Parhar 			    V_TXPKT_INTF(pi->tx_chan));
463743bbae19SNavdeep Parhar 		else
463843bbae19SNavdeep Parhar 			txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
463943bbae19SNavdeep Parhar 			    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
464043bbae19SNavdeep Parhar 			    V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
464143bbae19SNavdeep Parhar 
464243bbae19SNavdeep Parhar 		txq->tc_idx = -1;
464343bbae19SNavdeep Parhar 	}
464443bbae19SNavdeep Parhar 
464543bbae19SNavdeep Parhar 	return (0);
464643bbae19SNavdeep Parhar }
464743bbae19SNavdeep Parhar 
464843bbae19SNavdeep Parhar /*
464943bbae19SNavdeep Parhar  * Idempotent.
465043bbae19SNavdeep Parhar  */
465143bbae19SNavdeep Parhar static void
465243bbae19SNavdeep Parhar free_txq(struct vi_info *vi, struct sge_txq *txq)
465343bbae19SNavdeep Parhar {
465443bbae19SNavdeep Parhar 	struct adapter *sc = vi->adapter;
465543bbae19SNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
465643bbae19SNavdeep Parhar 
465743bbae19SNavdeep Parhar 	if (eq->flags & EQ_HW_ALLOCATED) {
465843bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_SW_ALLOCATED);
465943bbae19SNavdeep Parhar 		free_eq_hwq(sc, NULL, eq);
466043bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
466143bbae19SNavdeep Parhar 	}
466243bbae19SNavdeep Parhar 
466343bbae19SNavdeep Parhar 	if (eq->flags & EQ_SW_ALLOCATED) {
466443bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
466543bbae19SNavdeep Parhar 		sglist_free(txq->gl);
466643bbae19SNavdeep Parhar 		free(txq->sdesc, M_CXGBE);
466743bbae19SNavdeep Parhar 		mp_ring_free(txq->r);
466843bbae19SNavdeep Parhar 		free_eq(sc, eq);
466943bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_SW_ALLOCATED));
467043bbae19SNavdeep Parhar 		bzero(txq, sizeof(*txq));
467143bbae19SNavdeep Parhar 	}
467243bbae19SNavdeep Parhar }
467343bbae19SNavdeep Parhar 
467443bbae19SNavdeep Parhar static void
467543bbae19SNavdeep Parhar add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx,
467643bbae19SNavdeep Parhar     struct sysctl_oid *oid, struct sge_txq *txq)
467743bbae19SNavdeep Parhar {
467843bbae19SNavdeep Parhar 	struct adapter *sc;
467943bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
468043bbae19SNavdeep Parhar 
468143bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
468243bbae19SNavdeep Parhar 		return;
468343bbae19SNavdeep Parhar 
468443bbae19SNavdeep Parhar 	sc = vi->adapter;
468554e4ee71SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
468654e4ee71SNavdeep Parhar 
468743bbae19SNavdeep Parhar 	mp_ring_sysctls(txq->r, ctx, children);
468859bc8ce0SNavdeep Parhar 
468943bbae19SNavdeep Parhar 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tc",
469043bbae19SNavdeep Parhar 	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, txq - sc->sge.txq,
469143bbae19SNavdeep Parhar 	    sysctl_tc, "I", "traffic class (-1 means none)");
469202f972e8SNavdeep Parhar 
469343bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
469454e4ee71SNavdeep Parhar 	    &txq->txcsum, "# of times hardware assisted with checksum");
469543bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD,
469643bbae19SNavdeep Parhar 	    &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag");
469743bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
4698a1ea9a82SNavdeep Parhar 	    &txq->tso_wrs, "# of TSO work requests");
469943bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
470054e4ee71SNavdeep Parhar 	    &txq->imm_wrs, "# of work requests with immediate data");
470143bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
470254e4ee71SNavdeep Parhar 	    &txq->sgl_wrs, "# of work requests with direct SGL");
470343bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
470454e4ee71SNavdeep Parhar 	    &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
470543bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD,
470643bbae19SNavdeep Parhar 	    &txq->txpkts0_wrs, "# of txpkts (type 0) work requests");
470743bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD,
470843bbae19SNavdeep Parhar 	    &txq->txpkts1_wrs, "# of txpkts (type 1) work requests");
470943bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD,
471043bbae19SNavdeep Parhar 	    &txq->txpkts0_pkts,
47117951040fSNavdeep Parhar 	    "# of frames tx'd using type0 txpkts work requests");
471243bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD,
471343bbae19SNavdeep Parhar 	    &txq->txpkts1_pkts,
47147951040fSNavdeep Parhar 	    "# of frames tx'd using type1 txpkts work requests");
471543bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts_flush", CTLFLAG_RD,
471643bbae19SNavdeep Parhar 	    &txq->txpkts_flush,
47173447df8bSNavdeep Parhar 	    "# of times txpkts had to be flushed out by an egress-update");
471843bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD,
47195cdaef71SJohn Baldwin 	    &txq->raw_wrs, "# of raw work requests (non-packets)");
472043bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_tso_wrs", CTLFLAG_RD,
472143bbae19SNavdeep Parhar 	    &txq->vxlan_tso_wrs, "# of VXLAN TSO work requests");
472243bbae19SNavdeep Parhar 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_txcsum", CTLFLAG_RD,
472343bbae19SNavdeep Parhar 	    &txq->vxlan_txcsum,
4724a4a4ad2dSNavdeep Parhar 	    "# of times hardware assisted with inner checksums (VXLAN)");
4725bddf7343SJohn Baldwin 
4726bddf7343SJohn Baldwin #ifdef KERN_TLS
472715f33555SNavdeep Parhar 	if (is_ktls(sc)) {
472843bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_records",
472943bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_records,
4730bddf7343SJohn Baldwin 		    "# of NIC TLS records transmitted");
473143bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_short",
473243bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_short,
4733bddf7343SJohn Baldwin 		    "# of short NIC TLS records transmitted");
473443bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_partial",
473543bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_partial,
4736bddf7343SJohn Baldwin 		    "# of partial NIC TLS records transmitted");
473743bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_full",
473843bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_full,
4739bddf7343SJohn Baldwin 		    "# of full NIC TLS records transmitted");
474043bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_octets",
474143bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_octets,
4742bddf7343SJohn Baldwin 		    "# of payload octets in transmitted NIC TLS records");
474343bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_waste",
474443bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_waste,
4745bddf7343SJohn Baldwin 		    "# of octets DMAd but not transmitted in NIC TLS records");
474643bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_options",
474743bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_options,
4748bddf7343SJohn Baldwin 		    "# of NIC TLS options-only packets transmitted");
474943bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_header",
475043bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_header,
4751bddf7343SJohn Baldwin 		    "# of NIC TLS header-only packets transmitted");
475243bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin",
475343bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_fin,
4754bddf7343SJohn Baldwin 		    "# of NIC TLS FIN-only packets transmitted");
475543bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin_short",
475643bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_fin_short,
4757bddf7343SJohn Baldwin 		    "# of NIC TLS padded FIN packets on short TLS records");
475843bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_cbc",
475943bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_cbc,
4760bddf7343SJohn Baldwin 		    "# of NIC TLS sessions using AES-CBC");
476143bbae19SNavdeep Parhar 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_gcm",
476243bbae19SNavdeep Parhar 		    CTLFLAG_RD, &txq->kern_tls_gcm,
4763bddf7343SJohn Baldwin 		    "# of NIC TLS sessions using AES-GCM");
4764bddf7343SJohn Baldwin 	}
4765bddf7343SJohn Baldwin #endif
476654e4ee71SNavdeep Parhar }
476754e4ee71SNavdeep Parhar 
4768077ba6a8SJohn Baldwin #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
476943bbae19SNavdeep Parhar /*
477043bbae19SNavdeep Parhar  * Idempotent.
477143bbae19SNavdeep Parhar  */
4772077ba6a8SJohn Baldwin static int
477343bbae19SNavdeep Parhar alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)
4774077ba6a8SJohn Baldwin {
477543bbae19SNavdeep Parhar 	struct sysctl_oid *oid;
477643bbae19SNavdeep Parhar 	struct port_info *pi = vi->pi;
4777077ba6a8SJohn Baldwin 	struct adapter *sc = vi->adapter;
477843bbae19SNavdeep Parhar 	struct sge_eq *eq = &ofld_txq->wrq.eq;
477943bbae19SNavdeep Parhar 	int rc, iqidx;
4780077ba6a8SJohn Baldwin 	char name[16];
4781077ba6a8SJohn Baldwin 
478243bbae19SNavdeep Parhar 	MPASS(idx >= 0);
478343bbae19SNavdeep Parhar 	MPASS(idx < vi->nofldtxq);
4784077ba6a8SJohn Baldwin 
478543bbae19SNavdeep Parhar 	if (!(eq->flags & EQ_SW_ALLOCATED)) {
4786077ba6a8SJohn Baldwin 		snprintf(name, sizeof(name), "%d", idx);
478743bbae19SNavdeep Parhar 		oid = SYSCTL_ADD_NODE(&vi->ctx,
478843bbae19SNavdeep Parhar 		    SYSCTL_CHILDREN(vi->ofld_txq_oid), OID_AUTO, name,
4789077ba6a8SJohn Baldwin 		    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue");
4790077ba6a8SJohn Baldwin 
479143bbae19SNavdeep Parhar 		snprintf(name, sizeof(name), "%s ofld_txq%d",
479243bbae19SNavdeep Parhar 		    device_get_nameunit(vi->dev), idx);
479343bbae19SNavdeep Parhar 		if (vi->nofldrxq > 0) {
479443bbae19SNavdeep Parhar 			iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq);
479543bbae19SNavdeep Parhar 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
479643bbae19SNavdeep Parhar 			    &sc->sge.ofld_rxq[iqidx].iq, name);
479743bbae19SNavdeep Parhar 		} else {
479843bbae19SNavdeep Parhar 			iqidx = vi->first_rxq + (idx % vi->nrxq);
479943bbae19SNavdeep Parhar 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
480043bbae19SNavdeep Parhar 			    &sc->sge.rxq[iqidx].iq, name);
480143bbae19SNavdeep Parhar 		}
480243bbae19SNavdeep Parhar 
480343bbae19SNavdeep Parhar 		rc = alloc_wrq(sc, vi, &ofld_txq->wrq, &vi->ctx, oid);
480443bbae19SNavdeep Parhar 		if (rc != 0) {
480543bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to allocate ofld_txq%d: %d\n", idx,
480643bbae19SNavdeep Parhar 			    rc);
480743bbae19SNavdeep Parhar 			sysctl_remove_oid(oid, 1, 1);
4808077ba6a8SJohn Baldwin 			return (rc);
480943bbae19SNavdeep Parhar 		}
481043bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_SW_ALLOCATED);
481143bbae19SNavdeep Parhar 		/* Can't fail after this point. */
4812077ba6a8SJohn Baldwin 
4813568e69e4SJohn Baldwin 		ofld_txq->tx_iscsi_pdus = counter_u64_alloc(M_WAITOK);
4814568e69e4SJohn Baldwin 		ofld_txq->tx_iscsi_octets = counter_u64_alloc(M_WAITOK);
48155b27e4b2SJohn Baldwin 		ofld_txq->tx_iscsi_iso_wrs = counter_u64_alloc(M_WAITOK);
4816fe496dc0SJohn Baldwin 		ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK);
4817fe496dc0SJohn Baldwin 		ofld_txq->tx_toe_tls_octets = counter_u64_alloc(M_WAITOK);
481843bbae19SNavdeep Parhar 		add_ofld_txq_sysctls(&vi->ctx, oid, ofld_txq);
4819077ba6a8SJohn Baldwin 	}
4820077ba6a8SJohn Baldwin 
482143bbae19SNavdeep Parhar 	if (!(eq->flags & EQ_HW_ALLOCATED)) {
482243bbae19SNavdeep Parhar 		rc = alloc_eq_hwq(sc, vi, eq);
482343bbae19SNavdeep Parhar 		if (rc != 0) {
482443bbae19SNavdeep Parhar 			CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx,
482543bbae19SNavdeep Parhar 			    rc);
482643bbae19SNavdeep Parhar 			return (rc);
482743bbae19SNavdeep Parhar 		}
482843bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_HW_ALLOCATED);
482943bbae19SNavdeep Parhar 	}
483043bbae19SNavdeep Parhar 
483143bbae19SNavdeep Parhar 	return (0);
483243bbae19SNavdeep Parhar }
483343bbae19SNavdeep Parhar 
483443bbae19SNavdeep Parhar /*
483543bbae19SNavdeep Parhar  * Idempotent.
483643bbae19SNavdeep Parhar  */
483743bbae19SNavdeep Parhar static void
4838077ba6a8SJohn Baldwin free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)
4839077ba6a8SJohn Baldwin {
4840077ba6a8SJohn Baldwin 	struct adapter *sc = vi->adapter;
484143bbae19SNavdeep Parhar 	struct sge_eq *eq = &ofld_txq->wrq.eq;
4842077ba6a8SJohn Baldwin 
484343bbae19SNavdeep Parhar 	if (eq->flags & EQ_HW_ALLOCATED) {
484443bbae19SNavdeep Parhar 		MPASS(eq->flags & EQ_SW_ALLOCATED);
484543bbae19SNavdeep Parhar 		free_eq_hwq(sc, NULL, eq);
484643bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
484743bbae19SNavdeep Parhar 	}
4848077ba6a8SJohn Baldwin 
484943bbae19SNavdeep Parhar 	if (eq->flags & EQ_SW_ALLOCATED) {
485043bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4851568e69e4SJohn Baldwin 		counter_u64_free(ofld_txq->tx_iscsi_pdus);
4852568e69e4SJohn Baldwin 		counter_u64_free(ofld_txq->tx_iscsi_octets);
48535b27e4b2SJohn Baldwin 		counter_u64_free(ofld_txq->tx_iscsi_iso_wrs);
4854fe496dc0SJohn Baldwin 		counter_u64_free(ofld_txq->tx_toe_tls_records);
4855fe496dc0SJohn Baldwin 		counter_u64_free(ofld_txq->tx_toe_tls_octets);
485643bbae19SNavdeep Parhar 		free_wrq(sc, &ofld_txq->wrq);
485743bbae19SNavdeep Parhar 		MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4858077ba6a8SJohn Baldwin 		bzero(ofld_txq, sizeof(*ofld_txq));
485943bbae19SNavdeep Parhar 	}
486043bbae19SNavdeep Parhar }
486143bbae19SNavdeep Parhar 
486243bbae19SNavdeep Parhar static void
486343bbae19SNavdeep Parhar add_ofld_txq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
486443bbae19SNavdeep Parhar     struct sge_ofld_txq *ofld_txq)
486543bbae19SNavdeep Parhar {
486643bbae19SNavdeep Parhar 	struct sysctl_oid_list *children;
486743bbae19SNavdeep Parhar 
486843bbae19SNavdeep Parhar 	if (ctx == NULL || oid == NULL)
486943bbae19SNavdeep Parhar 		return;
487043bbae19SNavdeep Parhar 
487143bbae19SNavdeep Parhar 	children = SYSCTL_CHILDREN(oid);
487243bbae19SNavdeep Parhar 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_pdus",
487343bbae19SNavdeep Parhar 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_pdus,
487443bbae19SNavdeep Parhar 	    "# of iSCSI PDUs transmitted");
487543bbae19SNavdeep Parhar 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_octets",
487643bbae19SNavdeep Parhar 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_octets,
487743bbae19SNavdeep Parhar 	    "# of payload octets in transmitted iSCSI PDUs");
48785b27e4b2SJohn Baldwin 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_iso_wrs",
48795b27e4b2SJohn Baldwin 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_iso_wrs,
48805b27e4b2SJohn Baldwin 	    "# of iSCSI segmentation offload work requests");
488143bbae19SNavdeep Parhar 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_records",
488243bbae19SNavdeep Parhar 	    CTLFLAG_RD, &ofld_txq->tx_toe_tls_records,
488343bbae19SNavdeep Parhar 	    "# of TOE TLS records transmitted");
488443bbae19SNavdeep Parhar 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_octets",
488543bbae19SNavdeep Parhar 	    CTLFLAG_RD, &ofld_txq->tx_toe_tls_octets,
488643bbae19SNavdeep Parhar 	    "# of payload octets in transmitted TOE TLS records");
4887077ba6a8SJohn Baldwin }
4888077ba6a8SJohn Baldwin #endif
4889077ba6a8SJohn Baldwin 
489054e4ee71SNavdeep Parhar static void
489154e4ee71SNavdeep Parhar oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
489254e4ee71SNavdeep Parhar {
489354e4ee71SNavdeep Parhar 	bus_addr_t *ba = arg;
489454e4ee71SNavdeep Parhar 
489554e4ee71SNavdeep Parhar 	KASSERT(nseg == 1,
489654e4ee71SNavdeep Parhar 	    ("%s meant for single segment mappings only.", __func__));
489754e4ee71SNavdeep Parhar 
489854e4ee71SNavdeep Parhar 	*ba = error ? 0 : segs->ds_addr;
489954e4ee71SNavdeep Parhar }
490054e4ee71SNavdeep Parhar 
490154e4ee71SNavdeep Parhar static inline void
490254e4ee71SNavdeep Parhar ring_fl_db(struct adapter *sc, struct sge_fl *fl)
490354e4ee71SNavdeep Parhar {
49044d6db4e0SNavdeep Parhar 	uint32_t n, v;
490554e4ee71SNavdeep Parhar 
490646e1e307SNavdeep Parhar 	n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx);
49074d6db4e0SNavdeep Parhar 	MPASS(n > 0);
4908d14b0ac1SNavdeep Parhar 
490954e4ee71SNavdeep Parhar 	wmb();
49104d6db4e0SNavdeep Parhar 	v = fl->dbval | V_PIDX(n);
49114d6db4e0SNavdeep Parhar 	if (fl->udb)
49124d6db4e0SNavdeep Parhar 		*fl->udb = htole32(v);
49134d6db4e0SNavdeep Parhar 	else
4914315048f2SJohn Baldwin 		t4_write_reg(sc, sc->sge_kdoorbell_reg, v);
49154d6db4e0SNavdeep Parhar 	IDXINCR(fl->dbidx, n, fl->sidx);
491654e4ee71SNavdeep Parhar }
491754e4ee71SNavdeep Parhar 
4918fb12416cSNavdeep Parhar /*
49194d6db4e0SNavdeep Parhar  * Fills up the freelist by allocating up to 'n' buffers.  Buffers that are
49204d6db4e0SNavdeep Parhar  * recycled do not count towards this allocation budget.
4921733b9277SNavdeep Parhar  *
49224d6db4e0SNavdeep Parhar  * Returns non-zero to indicate that this freelist should be added to the list
49234d6db4e0SNavdeep Parhar  * of starving freelists.
4924fb12416cSNavdeep Parhar  */
4925733b9277SNavdeep Parhar static int
49264d6db4e0SNavdeep Parhar refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
492754e4ee71SNavdeep Parhar {
49284d6db4e0SNavdeep Parhar 	__be64 *d;
49294d6db4e0SNavdeep Parhar 	struct fl_sdesc *sd;
493038035ed6SNavdeep Parhar 	uintptr_t pa;
493154e4ee71SNavdeep Parhar 	caddr_t cl;
493246e1e307SNavdeep Parhar 	struct rx_buf_info *rxb;
493338035ed6SNavdeep Parhar 	struct cluster_metadata *clm;
4934294e62beSAlexander Motin 	uint16_t max_pidx, zidx = fl->zidx;
49354d6db4e0SNavdeep Parhar 	uint16_t hw_cidx = fl->hw_cidx;		/* stable snapshot */
493654e4ee71SNavdeep Parhar 
493754e4ee71SNavdeep Parhar 	FL_LOCK_ASSERT_OWNED(fl);
493854e4ee71SNavdeep Parhar 
49394d6db4e0SNavdeep Parhar 	/*
4940453130d9SPedro F. Giffuni 	 * We always stop at the beginning of the hardware descriptor that's just
49414d6db4e0SNavdeep Parhar 	 * before the one with the hw cidx.  This is to avoid hw pidx = hw cidx,
49424d6db4e0SNavdeep Parhar 	 * which would mean an empty freelist to the chip.
49434d6db4e0SNavdeep Parhar 	 */
49444d6db4e0SNavdeep Parhar 	max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
49454d6db4e0SNavdeep Parhar 	if (fl->pidx == max_pidx * 8)
49464d6db4e0SNavdeep Parhar 		return (0);
494754e4ee71SNavdeep Parhar 
49484d6db4e0SNavdeep Parhar 	d = &fl->desc[fl->pidx];
49494d6db4e0SNavdeep Parhar 	sd = &fl->sdesc[fl->pidx];
4950294e62beSAlexander Motin 	rxb = &sc->sge.rx_buf_info[zidx];
49514d6db4e0SNavdeep Parhar 
49524d6db4e0SNavdeep Parhar 	while (n > 0) {
495354e4ee71SNavdeep Parhar 
495454e4ee71SNavdeep Parhar 		if (sd->cl != NULL) {
495554e4ee71SNavdeep Parhar 
4956c3fb7725SNavdeep Parhar 			if (sd->nmbuf == 0) {
495738035ed6SNavdeep Parhar 				/*
495838035ed6SNavdeep Parhar 				 * Fast recycle without involving any atomics on
495938035ed6SNavdeep Parhar 				 * the cluster's metadata (if the cluster has
496038035ed6SNavdeep Parhar 				 * metadata).  This happens when all frames
496138035ed6SNavdeep Parhar 				 * received in the cluster were small enough to
496238035ed6SNavdeep Parhar 				 * fit within a single mbuf each.
496338035ed6SNavdeep Parhar 				 */
496438035ed6SNavdeep Parhar 				fl->cl_fast_recycled++;
4965a9c4062aSNavdeep Parhar 				goto recycled;
496638035ed6SNavdeep Parhar 			}
496754e4ee71SNavdeep Parhar 
496838035ed6SNavdeep Parhar 			/*
496938035ed6SNavdeep Parhar 			 * Cluster is guaranteed to have metadata.  Clusters
497038035ed6SNavdeep Parhar 			 * without metadata always take the fast recycle path
497138035ed6SNavdeep Parhar 			 * when they're recycled.
497238035ed6SNavdeep Parhar 			 */
497346e1e307SNavdeep Parhar 			clm = cl_metadata(sd);
497438035ed6SNavdeep Parhar 			MPASS(clm != NULL);
49751458bff9SNavdeep Parhar 
497638035ed6SNavdeep Parhar 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
497738035ed6SNavdeep Parhar 				fl->cl_recycled++;
497882eff304SNavdeep Parhar 				counter_u64_add(extfree_rels, 1);
497954e4ee71SNavdeep Parhar 				goto recycled;
498054e4ee71SNavdeep Parhar 			}
49811458bff9SNavdeep Parhar 			sd->cl = NULL;	/* gave up my reference */
49821458bff9SNavdeep Parhar 		}
498338035ed6SNavdeep Parhar 		MPASS(sd->cl == NULL);
498446e1e307SNavdeep Parhar 		cl = uma_zalloc(rxb->zone, M_NOWAIT);
49852b9010f0SNavdeep Parhar 		if (__predict_false(cl == NULL)) {
4986294e62beSAlexander Motin 			if (zidx != fl->safe_zidx) {
4987294e62beSAlexander Motin 				zidx = fl->safe_zidx;
4988294e62beSAlexander Motin 				rxb = &sc->sge.rx_buf_info[zidx];
498946e1e307SNavdeep Parhar 				cl = uma_zalloc(rxb->zone, M_NOWAIT);
49902b9010f0SNavdeep Parhar 			}
49912b9010f0SNavdeep Parhar 			if (cl == NULL)
499254e4ee71SNavdeep Parhar 				break;
499354e4ee71SNavdeep Parhar 		}
499438035ed6SNavdeep Parhar 		fl->cl_allocated++;
49954d6db4e0SNavdeep Parhar 		n--;
499654e4ee71SNavdeep Parhar 
499738035ed6SNavdeep Parhar 		pa = pmap_kextract((vm_offset_t)cl);
499854e4ee71SNavdeep Parhar 		sd->cl = cl;
4999294e62beSAlexander Motin 		sd->zidx = zidx;
500046e1e307SNavdeep Parhar 
500146e1e307SNavdeep Parhar 		if (fl->flags & FL_BUF_PACKING) {
500246e1e307SNavdeep Parhar 			*d = htobe64(pa | rxb->hwidx2);
500346e1e307SNavdeep Parhar 			sd->moff = rxb->size2;
500446e1e307SNavdeep Parhar 		} else {
500546e1e307SNavdeep Parhar 			*d = htobe64(pa | rxb->hwidx1);
500646e1e307SNavdeep Parhar 			sd->moff = 0;
500746e1e307SNavdeep Parhar 		}
50087d29df59SNavdeep Parhar recycled:
5009c3fb7725SNavdeep Parhar 		sd->nmbuf = 0;
501038035ed6SNavdeep Parhar 		d++;
501154e4ee71SNavdeep Parhar 		sd++;
501246e1e307SNavdeep Parhar 		if (__predict_false((++fl->pidx & 7) == 0)) {
501346e1e307SNavdeep Parhar 			uint16_t pidx = fl->pidx >> 3;
50144d6db4e0SNavdeep Parhar 
50154d6db4e0SNavdeep Parhar 			if (__predict_false(pidx == fl->sidx)) {
501654e4ee71SNavdeep Parhar 				fl->pidx = 0;
50174d6db4e0SNavdeep Parhar 				pidx = 0;
501854e4ee71SNavdeep Parhar 				sd = fl->sdesc;
501954e4ee71SNavdeep Parhar 				d = fl->desc;
502054e4ee71SNavdeep Parhar 			}
502146e1e307SNavdeep Parhar 			if (n < 8 || pidx == max_pidx)
50224d6db4e0SNavdeep Parhar 				break;
50234d6db4e0SNavdeep Parhar 
50244d6db4e0SNavdeep Parhar 			if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
50254d6db4e0SNavdeep Parhar 				ring_fl_db(sc, fl);
50264d6db4e0SNavdeep Parhar 		}
502754e4ee71SNavdeep Parhar 	}
5028fb12416cSNavdeep Parhar 
502946e1e307SNavdeep Parhar 	if ((fl->pidx >> 3) != fl->dbidx)
5030fb12416cSNavdeep Parhar 		ring_fl_db(sc, fl);
5031733b9277SNavdeep Parhar 
5032733b9277SNavdeep Parhar 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
5033733b9277SNavdeep Parhar }
5034733b9277SNavdeep Parhar 
5035733b9277SNavdeep Parhar /*
5036733b9277SNavdeep Parhar  * Attempt to refill all starving freelists.
5037733b9277SNavdeep Parhar  */
5038733b9277SNavdeep Parhar static void
5039733b9277SNavdeep Parhar refill_sfl(void *arg)
5040733b9277SNavdeep Parhar {
5041733b9277SNavdeep Parhar 	struct adapter *sc = arg;
5042733b9277SNavdeep Parhar 	struct sge_fl *fl, *fl_temp;
5043733b9277SNavdeep Parhar 
5044fe2ebb76SJohn Baldwin 	mtx_assert(&sc->sfl_lock, MA_OWNED);
5045733b9277SNavdeep Parhar 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
5046733b9277SNavdeep Parhar 		FL_LOCK(fl);
5047733b9277SNavdeep Parhar 		refill_fl(sc, fl, 64);
5048733b9277SNavdeep Parhar 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
5049733b9277SNavdeep Parhar 			TAILQ_REMOVE(&sc->sfl, fl, link);
5050733b9277SNavdeep Parhar 			fl->flags &= ~FL_STARVING;
5051733b9277SNavdeep Parhar 		}
5052733b9277SNavdeep Parhar 		FL_UNLOCK(fl);
5053733b9277SNavdeep Parhar 	}
5054733b9277SNavdeep Parhar 
5055733b9277SNavdeep Parhar 	if (!TAILQ_EMPTY(&sc->sfl))
5056733b9277SNavdeep Parhar 		callout_schedule(&sc->sfl_callout, hz / 5);
505754e4ee71SNavdeep Parhar }
505854e4ee71SNavdeep Parhar 
505943bbae19SNavdeep Parhar /*
506043bbae19SNavdeep Parhar  * Release the driver's reference on all buffers in the given freelist.  Buffers
506143bbae19SNavdeep Parhar  * with kernel references cannot be freed and will prevent the driver from being
506243bbae19SNavdeep Parhar  * unloaded safely.
506343bbae19SNavdeep Parhar  */
506443bbae19SNavdeep Parhar void
506543bbae19SNavdeep Parhar free_fl_buffers(struct adapter *sc, struct sge_fl *fl)
506654e4ee71SNavdeep Parhar {
506754e4ee71SNavdeep Parhar 	struct fl_sdesc *sd;
506838035ed6SNavdeep Parhar 	struct cluster_metadata *clm;
506954e4ee71SNavdeep Parhar 	int i;
507054e4ee71SNavdeep Parhar 
507154e4ee71SNavdeep Parhar 	sd = fl->sdesc;
50724d6db4e0SNavdeep Parhar 	for (i = 0; i < fl->sidx * 8; i++, sd++) {
507338035ed6SNavdeep Parhar 		if (sd->cl == NULL)
507438035ed6SNavdeep Parhar 			continue;
507554e4ee71SNavdeep Parhar 
507682eff304SNavdeep Parhar 		if (sd->nmbuf == 0)
507746e1e307SNavdeep Parhar 			uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl);
507846e1e307SNavdeep Parhar 		else if (fl->flags & FL_BUF_PACKING) {
507946e1e307SNavdeep Parhar 			clm = cl_metadata(sd);
508046e1e307SNavdeep Parhar 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
508146e1e307SNavdeep Parhar 				uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone,
508246e1e307SNavdeep Parhar 				    sd->cl);
508382eff304SNavdeep Parhar 				counter_u64_add(extfree_rels, 1);
508454e4ee71SNavdeep Parhar 			}
508546e1e307SNavdeep Parhar 		}
508638035ed6SNavdeep Parhar 		sd->cl = NULL;
508754e4ee71SNavdeep Parhar 	}
508854e4ee71SNavdeep Parhar 
508943bbae19SNavdeep Parhar 	if (fl->flags & FL_BUF_RESUME) {
509043bbae19SNavdeep Parhar 		m_freem(fl->m0);
509143bbae19SNavdeep Parhar 		fl->flags &= ~FL_BUF_RESUME;
509243bbae19SNavdeep Parhar 	}
509354e4ee71SNavdeep Parhar }
509454e4ee71SNavdeep Parhar 
50957951040fSNavdeep Parhar static inline void
50967951040fSNavdeep Parhar get_pkt_gl(struct mbuf *m, struct sglist *gl)
509754e4ee71SNavdeep Parhar {
50987951040fSNavdeep Parhar 	int rc;
509954e4ee71SNavdeep Parhar 
51007951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m);
510154e4ee71SNavdeep Parhar 
51027951040fSNavdeep Parhar 	sglist_reset(gl);
51037951040fSNavdeep Parhar 	rc = sglist_append_mbuf(gl, m);
51047951040fSNavdeep Parhar 	if (__predict_false(rc != 0)) {
51057951040fSNavdeep Parhar 		panic("%s: mbuf %p (%d segs) was vetted earlier but now fails "
51067951040fSNavdeep Parhar 		    "with %d.", __func__, m, mbuf_nsegs(m), rc);
510754e4ee71SNavdeep Parhar 	}
510854e4ee71SNavdeep Parhar 
51097951040fSNavdeep Parhar 	KASSERT(gl->sg_nseg == mbuf_nsegs(m),
51107951040fSNavdeep Parhar 	    ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m,
51117951040fSNavdeep Parhar 	    mbuf_nsegs(m), gl->sg_nseg));
511230e3f2b4SNavdeep Parhar #if 0	/* vm_wr not readily available here. */
511330e3f2b4SNavdeep Parhar 	KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= max_nsegs_allowed(m, vm_wr),
51147951040fSNavdeep Parhar 	    ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__,
511530e3f2b4SNavdeep Parhar 		gl->sg_nseg, max_nsegs_allowed(m, vm_wr)));
511630e3f2b4SNavdeep Parhar #endif
511754e4ee71SNavdeep Parhar }
511854e4ee71SNavdeep Parhar 
511954e4ee71SNavdeep Parhar /*
51207951040fSNavdeep Parhar  * len16 for a txpkt WR with a GL.  Includes the firmware work request header.
512154e4ee71SNavdeep Parhar  */
51227951040fSNavdeep Parhar static inline u_int
5123a4a4ad2dSNavdeep Parhar txpkt_len16(u_int nsegs, const u_int extra)
51247951040fSNavdeep Parhar {
51257951040fSNavdeep Parhar 	u_int n;
51267951040fSNavdeep Parhar 
51277951040fSNavdeep Parhar 	MPASS(nsegs > 0);
51287951040fSNavdeep Parhar 
51297951040fSNavdeep Parhar 	nsegs--; /* first segment is part of ulptx_sgl */
5130a4a4ad2dSNavdeep Parhar 	n = extra + sizeof(struct fw_eth_tx_pkt_wr) +
5131a4a4ad2dSNavdeep Parhar 	    sizeof(struct cpl_tx_pkt_core) +
51327951040fSNavdeep Parhar 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
51337951040fSNavdeep Parhar 
51347951040fSNavdeep Parhar 	return (howmany(n, 16));
51357951040fSNavdeep Parhar }
513654e4ee71SNavdeep Parhar 
513754e4ee71SNavdeep Parhar /*
51386af45170SJohn Baldwin  * len16 for a txpkt_vm WR with a GL.  Includes the firmware work
51396af45170SJohn Baldwin  * request header.
51406af45170SJohn Baldwin  */
51416af45170SJohn Baldwin static inline u_int
5142a4a4ad2dSNavdeep Parhar txpkt_vm_len16(u_int nsegs, const u_int extra)
51436af45170SJohn Baldwin {
51446af45170SJohn Baldwin 	u_int n;
51456af45170SJohn Baldwin 
51466af45170SJohn Baldwin 	MPASS(nsegs > 0);
51476af45170SJohn Baldwin 
51486af45170SJohn Baldwin 	nsegs--; /* first segment is part of ulptx_sgl */
5149a4a4ad2dSNavdeep Parhar 	n = extra + sizeof(struct fw_eth_tx_pkt_vm_wr) +
51506af45170SJohn Baldwin 	    sizeof(struct cpl_tx_pkt_core) +
51516af45170SJohn Baldwin 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
51526af45170SJohn Baldwin 
51536af45170SJohn Baldwin 	return (howmany(n, 16));
51546af45170SJohn Baldwin }
51556af45170SJohn Baldwin 
5156a4a4ad2dSNavdeep Parhar static inline void
515730e3f2b4SNavdeep Parhar calculate_mbuf_len16(struct mbuf *m, bool vm_wr)
5158a4a4ad2dSNavdeep Parhar {
5159a4a4ad2dSNavdeep Parhar 	const int lso = sizeof(struct cpl_tx_pkt_lso_core);
5160a4a4ad2dSNavdeep Parhar 	const int tnl_lso = sizeof(struct cpl_tx_tnl_lso);
5161a4a4ad2dSNavdeep Parhar 
516230e3f2b4SNavdeep Parhar 	if (vm_wr) {
5163a4a4ad2dSNavdeep Parhar 		if (needs_tso(m))
5164a4a4ad2dSNavdeep Parhar 			set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), lso));
5165a4a4ad2dSNavdeep Parhar 		else
5166a4a4ad2dSNavdeep Parhar 			set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), 0));
5167a4a4ad2dSNavdeep Parhar 		return;
5168a4a4ad2dSNavdeep Parhar 	}
5169a4a4ad2dSNavdeep Parhar 
5170a4a4ad2dSNavdeep Parhar 	if (needs_tso(m)) {
5171a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_tso(m))
5172a4a4ad2dSNavdeep Parhar 			set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), tnl_lso));
5173a4a4ad2dSNavdeep Parhar 		else
5174a4a4ad2dSNavdeep Parhar 			set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), lso));
5175a4a4ad2dSNavdeep Parhar 	} else
5176a4a4ad2dSNavdeep Parhar 		set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), 0));
5177a4a4ad2dSNavdeep Parhar }
5178a4a4ad2dSNavdeep Parhar 
51796af45170SJohn Baldwin /*
51807951040fSNavdeep Parhar  * len16 for a txpkts type 0 WR with a GL.  Does not include the firmware work
51817951040fSNavdeep Parhar  * request header.
51827951040fSNavdeep Parhar  */
51837951040fSNavdeep Parhar static inline u_int
51847951040fSNavdeep Parhar txpkts0_len16(u_int nsegs)
51857951040fSNavdeep Parhar {
51867951040fSNavdeep Parhar 	u_int n;
51877951040fSNavdeep Parhar 
51887951040fSNavdeep Parhar 	MPASS(nsegs > 0);
51897951040fSNavdeep Parhar 
51907951040fSNavdeep Parhar 	nsegs--; /* first segment is part of ulptx_sgl */
51917951040fSNavdeep Parhar 	n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) +
51927951040fSNavdeep Parhar 	    sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) +
51937951040fSNavdeep Parhar 	    8 * ((3 * nsegs) / 2 + (nsegs & 1));
51947951040fSNavdeep Parhar 
51957951040fSNavdeep Parhar 	return (howmany(n, 16));
51967951040fSNavdeep Parhar }
51977951040fSNavdeep Parhar 
51987951040fSNavdeep Parhar /*
51997951040fSNavdeep Parhar  * len16 for a txpkts type 1 WR with a GL.  Does not include the firmware work
52007951040fSNavdeep Parhar  * request header.
52017951040fSNavdeep Parhar  */
52027951040fSNavdeep Parhar static inline u_int
52037951040fSNavdeep Parhar txpkts1_len16(void)
52047951040fSNavdeep Parhar {
52057951040fSNavdeep Parhar 	u_int n;
52067951040fSNavdeep Parhar 
52077951040fSNavdeep Parhar 	n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl);
52087951040fSNavdeep Parhar 
52097951040fSNavdeep Parhar 	return (howmany(n, 16));
52107951040fSNavdeep Parhar }
52117951040fSNavdeep Parhar 
52127951040fSNavdeep Parhar static inline u_int
52137951040fSNavdeep Parhar imm_payload(u_int ndesc)
52147951040fSNavdeep Parhar {
52157951040fSNavdeep Parhar 	u_int n;
52167951040fSNavdeep Parhar 
52177951040fSNavdeep Parhar 	n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) -
52187951040fSNavdeep Parhar 	    sizeof(struct cpl_tx_pkt_core);
52197951040fSNavdeep Parhar 
52207951040fSNavdeep Parhar 	return (n);
52217951040fSNavdeep Parhar }
52227951040fSNavdeep Parhar 
5223c0236bd9SNavdeep Parhar static inline uint64_t
5224c0236bd9SNavdeep Parhar csum_to_ctrl(struct adapter *sc, struct mbuf *m)
5225c0236bd9SNavdeep Parhar {
5226c0236bd9SNavdeep Parhar 	uint64_t ctrl;
5227a4a4ad2dSNavdeep Parhar 	int csum_type, l2hlen, l3hlen;
5228a4a4ad2dSNavdeep Parhar 	int x, y;
5229a4a4ad2dSNavdeep Parhar 	static const int csum_types[3][2] = {
5230a4a4ad2dSNavdeep Parhar 		{TX_CSUM_TCPIP, TX_CSUM_TCPIP6},
5231a4a4ad2dSNavdeep Parhar 		{TX_CSUM_UDPIP, TX_CSUM_UDPIP6},
5232a4a4ad2dSNavdeep Parhar 		{TX_CSUM_IP, 0}
5233a4a4ad2dSNavdeep Parhar 	};
5234c0236bd9SNavdeep Parhar 
5235c0236bd9SNavdeep Parhar 	M_ASSERTPKTHDR(m);
5236c0236bd9SNavdeep Parhar 
5237a4a4ad2dSNavdeep Parhar 	if (!needs_hwcsum(m))
5238c0236bd9SNavdeep Parhar 		return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS);
5239c0236bd9SNavdeep Parhar 
5240a4a4ad2dSNavdeep Parhar 	MPASS(m->m_pkthdr.l2hlen >= ETHER_HDR_LEN);
5241a4a4ad2dSNavdeep Parhar 	MPASS(m->m_pkthdr.l3hlen >= sizeof(struct ip));
5242a4a4ad2dSNavdeep Parhar 
5243a4a4ad2dSNavdeep Parhar 	if (needs_vxlan_csum(m)) {
5244a4a4ad2dSNavdeep Parhar 		MPASS(m->m_pkthdr.l4hlen > 0);
5245a4a4ad2dSNavdeep Parhar 		MPASS(m->m_pkthdr.l5hlen > 0);
5246a4a4ad2dSNavdeep Parhar 		MPASS(m->m_pkthdr.inner_l2hlen >= ETHER_HDR_LEN);
5247a4a4ad2dSNavdeep Parhar 		MPASS(m->m_pkthdr.inner_l3hlen >= sizeof(struct ip));
5248a4a4ad2dSNavdeep Parhar 
5249a4a4ad2dSNavdeep Parhar 		l2hlen = m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
5250a4a4ad2dSNavdeep Parhar 		    m->m_pkthdr.l4hlen + m->m_pkthdr.l5hlen +
5251a4a4ad2dSNavdeep Parhar 		    m->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN;
5252a4a4ad2dSNavdeep Parhar 		l3hlen = m->m_pkthdr.inner_l3hlen;
5253a4a4ad2dSNavdeep Parhar 	} else {
5254a4a4ad2dSNavdeep Parhar 		l2hlen = m->m_pkthdr.l2hlen - ETHER_HDR_LEN;
5255a4a4ad2dSNavdeep Parhar 		l3hlen = m->m_pkthdr.l3hlen;
5256c0236bd9SNavdeep Parhar 	}
5257c0236bd9SNavdeep Parhar 
5258a4a4ad2dSNavdeep Parhar 	ctrl = 0;
5259a4a4ad2dSNavdeep Parhar 	if (!needs_l3_csum(m))
5260a4a4ad2dSNavdeep Parhar 		ctrl |= F_TXPKT_IPCSUM_DIS;
5261a4a4ad2dSNavdeep Parhar 
5262a4a4ad2dSNavdeep Parhar 	if (m->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_INNER_IP_TCP |
5263a4a4ad2dSNavdeep Parhar 	    CSUM_IP6_TCP | CSUM_INNER_IP6_TCP))
5264a4a4ad2dSNavdeep Parhar 		x = 0;	/* TCP */
5265a4a4ad2dSNavdeep Parhar 	else if (m->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_INNER_IP_UDP |
5266a4a4ad2dSNavdeep Parhar 	    CSUM_IP6_UDP | CSUM_INNER_IP6_UDP))
5267a4a4ad2dSNavdeep Parhar 		x = 1;	/* UDP */
5268c0236bd9SNavdeep Parhar 	else
5269a4a4ad2dSNavdeep Parhar 		x = 2;
5270a4a4ad2dSNavdeep Parhar 
5271a4a4ad2dSNavdeep Parhar 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP |
5272a4a4ad2dSNavdeep Parhar 	    CSUM_INNER_IP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP))
5273a4a4ad2dSNavdeep Parhar 		y = 0;	/* IPv4 */
5274a4a4ad2dSNavdeep Parhar 	else {
5275a4a4ad2dSNavdeep Parhar 		MPASS(m->m_pkthdr.csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP |
5276a4a4ad2dSNavdeep Parhar 		    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP));
5277a4a4ad2dSNavdeep Parhar 		y = 1;	/* IPv6 */
5278a4a4ad2dSNavdeep Parhar 	}
5279a4a4ad2dSNavdeep Parhar 	/*
5280a4a4ad2dSNavdeep Parhar 	 * needs_hwcsum returned true earlier so there must be some kind of
5281a4a4ad2dSNavdeep Parhar 	 * checksum to calculate.
5282a4a4ad2dSNavdeep Parhar 	 */
5283a4a4ad2dSNavdeep Parhar 	csum_type = csum_types[x][y];
5284a4a4ad2dSNavdeep Parhar 	MPASS(csum_type != 0);
5285a4a4ad2dSNavdeep Parhar 	if (csum_type == TX_CSUM_IP)
5286a4a4ad2dSNavdeep Parhar 		ctrl |= F_TXPKT_L4CSUM_DIS;
5287a4a4ad2dSNavdeep Parhar 	ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | V_TXPKT_IPHDR_LEN(l3hlen);
5288a4a4ad2dSNavdeep Parhar 	if (chip_id(sc) <= CHELSIO_T5)
5289a4a4ad2dSNavdeep Parhar 		ctrl |= V_TXPKT_ETHHDR_LEN(l2hlen);
5290a4a4ad2dSNavdeep Parhar 	else
5291a4a4ad2dSNavdeep Parhar 		ctrl |= V_T6_TXPKT_ETHHDR_LEN(l2hlen);
5292c0236bd9SNavdeep Parhar 
5293c0236bd9SNavdeep Parhar 	return (ctrl);
5294c0236bd9SNavdeep Parhar }
5295c0236bd9SNavdeep Parhar 
5296a4a4ad2dSNavdeep Parhar static inline void *
5297a4a4ad2dSNavdeep Parhar write_lso_cpl(void *cpl, struct mbuf *m0)
5298a4a4ad2dSNavdeep Parhar {
5299a4a4ad2dSNavdeep Parhar 	struct cpl_tx_pkt_lso_core *lso;
5300a4a4ad2dSNavdeep Parhar 	uint32_t ctrl;
5301a4a4ad2dSNavdeep Parhar 
5302a4a4ad2dSNavdeep Parhar 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5303a4a4ad2dSNavdeep Parhar 	    m0->m_pkthdr.l4hlen > 0,
5304a4a4ad2dSNavdeep Parhar 	    ("%s: mbuf %p needs TSO but missing header lengths",
5305a4a4ad2dSNavdeep Parhar 		__func__, m0));
5306a4a4ad2dSNavdeep Parhar 
5307a4a4ad2dSNavdeep Parhar 	ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
5308a4a4ad2dSNavdeep Parhar 	    F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
5309a4a4ad2dSNavdeep Parhar 	    V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5310a4a4ad2dSNavdeep Parhar 	    V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
5311a4a4ad2dSNavdeep Parhar 	    V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
5312a4a4ad2dSNavdeep Parhar 	if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5313a4a4ad2dSNavdeep Parhar 		ctrl |= F_LSO_IPV6;
5314a4a4ad2dSNavdeep Parhar 
5315a4a4ad2dSNavdeep Parhar 	lso = cpl;
5316a4a4ad2dSNavdeep Parhar 	lso->lso_ctrl = htobe32(ctrl);
5317a4a4ad2dSNavdeep Parhar 	lso->ipid_ofst = htobe16(0);
5318a4a4ad2dSNavdeep Parhar 	lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
5319a4a4ad2dSNavdeep Parhar 	lso->seqno_offset = htobe32(0);
5320a4a4ad2dSNavdeep Parhar 	lso->len = htobe32(m0->m_pkthdr.len);
5321a4a4ad2dSNavdeep Parhar 
5322a4a4ad2dSNavdeep Parhar 	return (lso + 1);
5323a4a4ad2dSNavdeep Parhar }
5324a4a4ad2dSNavdeep Parhar 
5325a4a4ad2dSNavdeep Parhar static void *
5326a4a4ad2dSNavdeep Parhar write_tnl_lso_cpl(void *cpl, struct mbuf *m0)
5327a4a4ad2dSNavdeep Parhar {
5328a4a4ad2dSNavdeep Parhar 	struct cpl_tx_tnl_lso *tnl_lso = cpl;
5329a4a4ad2dSNavdeep Parhar 	uint32_t ctrl;
5330a4a4ad2dSNavdeep Parhar 
5331a4a4ad2dSNavdeep Parhar 	KASSERT(m0->m_pkthdr.inner_l2hlen > 0 &&
5332a4a4ad2dSNavdeep Parhar 	    m0->m_pkthdr.inner_l3hlen > 0 && m0->m_pkthdr.inner_l4hlen > 0 &&
5333a4a4ad2dSNavdeep Parhar 	    m0->m_pkthdr.inner_l5hlen > 0,
5334a4a4ad2dSNavdeep Parhar 	    ("%s: mbuf %p needs VXLAN_TSO but missing inner header lengths",
5335a4a4ad2dSNavdeep Parhar 		__func__, m0));
5336a4a4ad2dSNavdeep Parhar 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5337a4a4ad2dSNavdeep Parhar 	    m0->m_pkthdr.l4hlen > 0 && m0->m_pkthdr.l5hlen > 0,
5338a4a4ad2dSNavdeep Parhar 	    ("%s: mbuf %p needs VXLAN_TSO but missing outer header lengths",
5339a4a4ad2dSNavdeep Parhar 		__func__, m0));
5340a4a4ad2dSNavdeep Parhar 
5341a4a4ad2dSNavdeep Parhar 	/* Outer headers. */
5342a4a4ad2dSNavdeep Parhar 	ctrl = V_CPL_TX_TNL_LSO_OPCODE(CPL_TX_TNL_LSO) |
5343a4a4ad2dSNavdeep Parhar 	    F_CPL_TX_TNL_LSO_FIRST | F_CPL_TX_TNL_LSO_LAST |
5344a4a4ad2dSNavdeep Parhar 	    V_CPL_TX_TNL_LSO_ETHHDRLENOUT(
5345a4a4ad2dSNavdeep Parhar 		(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5346a4a4ad2dSNavdeep Parhar 	    V_CPL_TX_TNL_LSO_IPHDRLENOUT(m0->m_pkthdr.l3hlen >> 2) |
5347a4a4ad2dSNavdeep Parhar 	    F_CPL_TX_TNL_LSO_IPLENSETOUT;
5348a4a4ad2dSNavdeep Parhar 	if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5349a4a4ad2dSNavdeep Parhar 		ctrl |= F_CPL_TX_TNL_LSO_IPV6OUT;
5350a4a4ad2dSNavdeep Parhar 	else {
5351a4a4ad2dSNavdeep Parhar 		ctrl |= F_CPL_TX_TNL_LSO_IPHDRCHKOUT |
5352a4a4ad2dSNavdeep Parhar 		    F_CPL_TX_TNL_LSO_IPIDINCOUT;
5353a4a4ad2dSNavdeep Parhar 	}
5354a4a4ad2dSNavdeep Parhar 	tnl_lso->op_to_IpIdSplitOut = htobe32(ctrl);
5355a4a4ad2dSNavdeep Parhar 	tnl_lso->IpIdOffsetOut = 0;
5356a4a4ad2dSNavdeep Parhar 	tnl_lso->UdpLenSetOut_to_TnlHdrLen =
5357a4a4ad2dSNavdeep Parhar 		htobe16(F_CPL_TX_TNL_LSO_UDPCHKCLROUT |
5358a4a4ad2dSNavdeep Parhar 		    F_CPL_TX_TNL_LSO_UDPLENSETOUT |
5359a4a4ad2dSNavdeep Parhar 		    V_CPL_TX_TNL_LSO_TNLHDRLEN(m0->m_pkthdr.l2hlen +
5360a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen +
5361a4a4ad2dSNavdeep Parhar 			m0->m_pkthdr.l5hlen) |
5362a4a4ad2dSNavdeep Parhar 		    V_CPL_TX_TNL_LSO_TNLTYPE(TX_TNL_TYPE_VXLAN));
5363a4a4ad2dSNavdeep Parhar 	tnl_lso->r1 = 0;
5364a4a4ad2dSNavdeep Parhar 
5365a4a4ad2dSNavdeep Parhar 	/* Inner headers. */
5366a4a4ad2dSNavdeep Parhar 	ctrl = V_CPL_TX_TNL_LSO_ETHHDRLEN(
5367a4a4ad2dSNavdeep Parhar 	    (m0->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN) >> 2) |
5368a4a4ad2dSNavdeep Parhar 	    V_CPL_TX_TNL_LSO_IPHDRLEN(m0->m_pkthdr.inner_l3hlen >> 2) |
5369a4a4ad2dSNavdeep Parhar 	    V_CPL_TX_TNL_LSO_TCPHDRLEN(m0->m_pkthdr.inner_l4hlen >> 2);
5370a4a4ad2dSNavdeep Parhar 	if (m0->m_pkthdr.inner_l3hlen == sizeof(struct ip6_hdr))
5371a4a4ad2dSNavdeep Parhar 		ctrl |= F_CPL_TX_TNL_LSO_IPV6;
5372a4a4ad2dSNavdeep Parhar 	tnl_lso->Flow_to_TcpHdrLen = htobe32(ctrl);
5373a4a4ad2dSNavdeep Parhar 	tnl_lso->IpIdOffset = 0;
5374a4a4ad2dSNavdeep Parhar 	tnl_lso->IpIdSplit_to_Mss =
5375a4a4ad2dSNavdeep Parhar 	    htobe16(V_CPL_TX_TNL_LSO_MSS(m0->m_pkthdr.tso_segsz));
5376a4a4ad2dSNavdeep Parhar 	tnl_lso->TCPSeqOffset = 0;
5377a4a4ad2dSNavdeep Parhar 	tnl_lso->EthLenOffset_Size =
5378a4a4ad2dSNavdeep Parhar 	    htobe32(V_CPL_TX_TNL_LSO_SIZE(m0->m_pkthdr.len));
5379a4a4ad2dSNavdeep Parhar 
5380a4a4ad2dSNavdeep Parhar 	return (tnl_lso + 1);
5381a4a4ad2dSNavdeep Parhar }
5382a4a4ad2dSNavdeep Parhar 
5383800535c2SNavdeep Parhar #define VM_TX_L2HDR_LEN	16	/* ethmacdst to vlantci */
5384800535c2SNavdeep Parhar 
53857951040fSNavdeep Parhar /*
53866af45170SJohn Baldwin  * Write a VM txpkt WR for this packet to the hardware descriptors, update the
53876af45170SJohn Baldwin  * software descriptor, and advance the pidx.  It is guaranteed that enough
53886af45170SJohn Baldwin  * descriptors are available.
53896af45170SJohn Baldwin  *
53906af45170SJohn Baldwin  * The return value is the # of hardware descriptors used.
53916af45170SJohn Baldwin  */
53926af45170SJohn Baldwin static u_int
5393d735920dSNavdeep Parhar write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0)
53946af45170SJohn Baldwin {
5395d735920dSNavdeep Parhar 	struct sge_eq *eq;
5396d735920dSNavdeep Parhar 	struct fw_eth_tx_pkt_vm_wr *wr;
53976af45170SJohn Baldwin 	struct tx_sdesc *txsd;
53986af45170SJohn Baldwin 	struct cpl_tx_pkt_core *cpl;
53996af45170SJohn Baldwin 	uint32_t ctrl;	/* used in many unrelated places */
54006af45170SJohn Baldwin 	uint64_t ctrl1;
540139d5cbdcSNavdeep Parhar 	int len16, ndesc, pktlen;
54026af45170SJohn Baldwin 	caddr_t dst;
54036af45170SJohn Baldwin 
54046af45170SJohn Baldwin 	TXQ_LOCK_ASSERT_OWNED(txq);
54056af45170SJohn Baldwin 	M_ASSERTPKTHDR(m0);
54066af45170SJohn Baldwin 
54076af45170SJohn Baldwin 	len16 = mbuf_len16(m0);
54086af45170SJohn Baldwin 	pktlen = m0->m_pkthdr.len;
54096af45170SJohn Baldwin 	ctrl = sizeof(struct cpl_tx_pkt_core);
54106af45170SJohn Baldwin 	if (needs_tso(m0))
54116af45170SJohn Baldwin 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
54120cadedfcSNavdeep Parhar 	ndesc = tx_len16_to_desc(len16);
54136af45170SJohn Baldwin 
54146af45170SJohn Baldwin 	/* Firmware work request header */
5415d735920dSNavdeep Parhar 	eq = &txq->eq;
5416d735920dSNavdeep Parhar 	wr = (void *)&eq->desc[eq->pidx];
54176af45170SJohn Baldwin 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
54186af45170SJohn Baldwin 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
54196af45170SJohn Baldwin 
54206af45170SJohn Baldwin 	ctrl = V_FW_WR_LEN16(len16);
54216af45170SJohn Baldwin 	wr->equiq_to_len16 = htobe32(ctrl);
54226af45170SJohn Baldwin 	wr->r3[0] = 0;
54236af45170SJohn Baldwin 	wr->r3[1] = 0;
54246af45170SJohn Baldwin 
54256af45170SJohn Baldwin 	/*
54266af45170SJohn Baldwin 	 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci.
54276af45170SJohn Baldwin 	 * vlantci is ignored unless the ethtype is 0x8100, so it's
54286af45170SJohn Baldwin 	 * simpler to always copy it rather than making it
54296af45170SJohn Baldwin 	 * conditional.  Also, it seems that we do not have to set
54306af45170SJohn Baldwin 	 * vlantci or fake the ethtype when doing VLAN tag insertion.
54316af45170SJohn Baldwin 	 */
5432800535c2SNavdeep Parhar 	m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst);
54336af45170SJohn Baldwin 
54346af45170SJohn Baldwin 	if (needs_tso(m0)) {
5435a4a4ad2dSNavdeep Parhar 		cpl = write_lso_cpl(wr + 1, m0);
54366af45170SJohn Baldwin 		txq->tso_wrs++;
5437c0236bd9SNavdeep Parhar 	} else
54386af45170SJohn Baldwin 		cpl = (void *)(wr + 1);
54396af45170SJohn Baldwin 
54406af45170SJohn Baldwin 	/* Checksum offload */
5441c0236bd9SNavdeep Parhar 	ctrl1 = csum_to_ctrl(sc, m0);
5442c0236bd9SNavdeep Parhar 	if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
54436af45170SJohn Baldwin 		txq->txcsum++;	/* some hardware assistance provided */
54446af45170SJohn Baldwin 
54456af45170SJohn Baldwin 	/* VLAN tag insertion */
54466af45170SJohn Baldwin 	if (needs_vlan_insertion(m0)) {
54476af45170SJohn Baldwin 		ctrl1 |= F_TXPKT_VLAN_VLD |
54486af45170SJohn Baldwin 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
54496af45170SJohn Baldwin 		txq->vlan_insertion++;
54506af45170SJohn Baldwin 	}
54516af45170SJohn Baldwin 
54526af45170SJohn Baldwin 	/* CPL header */
54536af45170SJohn Baldwin 	cpl->ctrl0 = txq->cpl_ctrl0;
54546af45170SJohn Baldwin 	cpl->pack = 0;
54556af45170SJohn Baldwin 	cpl->len = htobe16(pktlen);
54566af45170SJohn Baldwin 	cpl->ctrl1 = htobe64(ctrl1);
54576af45170SJohn Baldwin 
54586af45170SJohn Baldwin 	/* SGL */
54596af45170SJohn Baldwin 	dst = (void *)(cpl + 1);
54606af45170SJohn Baldwin 
54616af45170SJohn Baldwin 	/*
54626af45170SJohn Baldwin 	 * A packet using TSO will use up an entire descriptor for the
54636af45170SJohn Baldwin 	 * firmware work request header, LSO CPL, and TX_PKT_XT CPL.
54646af45170SJohn Baldwin 	 * If this descriptor is the last descriptor in the ring, wrap
54656af45170SJohn Baldwin 	 * around to the front of the ring explicitly for the start of
54666af45170SJohn Baldwin 	 * the sgl.
54676af45170SJohn Baldwin 	 */
54686af45170SJohn Baldwin 	if (dst == (void *)&eq->desc[eq->sidx]) {
54696af45170SJohn Baldwin 		dst = (void *)&eq->desc[0];
54706af45170SJohn Baldwin 		write_gl_to_txd(txq, m0, &dst, 0);
54716af45170SJohn Baldwin 	} else
54726af45170SJohn Baldwin 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
54736af45170SJohn Baldwin 	txq->sgl_wrs++;
54746af45170SJohn Baldwin 	txq->txpkt_wrs++;
54756af45170SJohn Baldwin 
54766af45170SJohn Baldwin 	txsd = &txq->sdesc[eq->pidx];
54776af45170SJohn Baldwin 	txsd->m = m0;
54786af45170SJohn Baldwin 	txsd->desc_used = ndesc;
54796af45170SJohn Baldwin 
54806af45170SJohn Baldwin 	return (ndesc);
54816af45170SJohn Baldwin }
54826af45170SJohn Baldwin 
54836af45170SJohn Baldwin /*
54845cdaef71SJohn Baldwin  * Write a raw WR to the hardware descriptors, update the software
54855cdaef71SJohn Baldwin  * descriptor, and advance the pidx.  It is guaranteed that enough
54865cdaef71SJohn Baldwin  * descriptors are available.
54875cdaef71SJohn Baldwin  *
54885cdaef71SJohn Baldwin  * The return value is the # of hardware descriptors used.
54895cdaef71SJohn Baldwin  */
54905cdaef71SJohn Baldwin static u_int
54915cdaef71SJohn Baldwin write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available)
54925cdaef71SJohn Baldwin {
54935cdaef71SJohn Baldwin 	struct sge_eq *eq = &txq->eq;
54945cdaef71SJohn Baldwin 	struct tx_sdesc *txsd;
54955cdaef71SJohn Baldwin 	struct mbuf *m;
54965cdaef71SJohn Baldwin 	caddr_t dst;
54975cdaef71SJohn Baldwin 	int len16, ndesc;
54985cdaef71SJohn Baldwin 
54995cdaef71SJohn Baldwin 	len16 = mbuf_len16(m0);
55000cadedfcSNavdeep Parhar 	ndesc = tx_len16_to_desc(len16);
55015cdaef71SJohn Baldwin 	MPASS(ndesc <= available);
55025cdaef71SJohn Baldwin 
55035cdaef71SJohn Baldwin 	dst = wr;
55045cdaef71SJohn Baldwin 	for (m = m0; m != NULL; m = m->m_next)
55055cdaef71SJohn Baldwin 		copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
55065cdaef71SJohn Baldwin 
55075cdaef71SJohn Baldwin 	txq->raw_wrs++;
55085cdaef71SJohn Baldwin 
55095cdaef71SJohn Baldwin 	txsd = &txq->sdesc[eq->pidx];
55105cdaef71SJohn Baldwin 	txsd->m = m0;
55115cdaef71SJohn Baldwin 	txsd->desc_used = ndesc;
55125cdaef71SJohn Baldwin 
55135cdaef71SJohn Baldwin 	return (ndesc);
55145cdaef71SJohn Baldwin }
55155cdaef71SJohn Baldwin 
55165cdaef71SJohn Baldwin /*
55177951040fSNavdeep Parhar  * Write a txpkt WR for this packet to the hardware descriptors, update the
55187951040fSNavdeep Parhar  * software descriptor, and advance the pidx.  It is guaranteed that enough
55197951040fSNavdeep Parhar  * descriptors are available.
552054e4ee71SNavdeep Parhar  *
55217951040fSNavdeep Parhar  * The return value is the # of hardware descriptors used.
552254e4ee71SNavdeep Parhar  */
55237951040fSNavdeep Parhar static u_int
5524d735920dSNavdeep Parhar write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0,
5525d735920dSNavdeep Parhar     u_int available)
552654e4ee71SNavdeep Parhar {
5527d735920dSNavdeep Parhar 	struct sge_eq *eq;
5528d735920dSNavdeep Parhar 	struct fw_eth_tx_pkt_wr *wr;
55297951040fSNavdeep Parhar 	struct tx_sdesc *txsd;
553054e4ee71SNavdeep Parhar 	struct cpl_tx_pkt_core *cpl;
553154e4ee71SNavdeep Parhar 	uint32_t ctrl;	/* used in many unrelated places */
553254e4ee71SNavdeep Parhar 	uint64_t ctrl1;
55337951040fSNavdeep Parhar 	int len16, ndesc, pktlen, nsegs;
553454e4ee71SNavdeep Parhar 	caddr_t dst;
553554e4ee71SNavdeep Parhar 
553654e4ee71SNavdeep Parhar 	TXQ_LOCK_ASSERT_OWNED(txq);
55377951040fSNavdeep Parhar 	M_ASSERTPKTHDR(m0);
553854e4ee71SNavdeep Parhar 
55397951040fSNavdeep Parhar 	len16 = mbuf_len16(m0);
55407951040fSNavdeep Parhar 	nsegs = mbuf_nsegs(m0);
55417951040fSNavdeep Parhar 	pktlen = m0->m_pkthdr.len;
554254e4ee71SNavdeep Parhar 	ctrl = sizeof(struct cpl_tx_pkt_core);
5543a4a4ad2dSNavdeep Parhar 	if (needs_tso(m0)) {
5544a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_tso(m0))
5545a4a4ad2dSNavdeep Parhar 			ctrl += sizeof(struct cpl_tx_tnl_lso);
5546a4a4ad2dSNavdeep Parhar 		else
55472a5f6b0eSNavdeep Parhar 			ctrl += sizeof(struct cpl_tx_pkt_lso_core);
5548a4a4ad2dSNavdeep Parhar 	} else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) &&
5549d76bbe17SJohn Baldwin 	    available >= 2) {
55507951040fSNavdeep Parhar 		/* Immediate data.  Recalculate len16 and set nsegs to 0. */
5551ecb79ca4SNavdeep Parhar 		ctrl += pktlen;
55527951040fSNavdeep Parhar 		len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) +
55537951040fSNavdeep Parhar 		    sizeof(struct cpl_tx_pkt_core) + pktlen, 16);
55547951040fSNavdeep Parhar 		nsegs = 0;
555554e4ee71SNavdeep Parhar 	}
55560cadedfcSNavdeep Parhar 	ndesc = tx_len16_to_desc(len16);
55577951040fSNavdeep Parhar 	MPASS(ndesc <= available);
555854e4ee71SNavdeep Parhar 
555954e4ee71SNavdeep Parhar 	/* Firmware work request header */
5560d735920dSNavdeep Parhar 	eq = &txq->eq;
5561d735920dSNavdeep Parhar 	wr = (void *)&eq->desc[eq->pidx];
556254e4ee71SNavdeep Parhar 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
5563733b9277SNavdeep Parhar 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
55646b49a4ecSNavdeep Parhar 
55657951040fSNavdeep Parhar 	ctrl = V_FW_WR_LEN16(len16);
556654e4ee71SNavdeep Parhar 	wr->equiq_to_len16 = htobe32(ctrl);
556754e4ee71SNavdeep Parhar 	wr->r3 = 0;
556854e4ee71SNavdeep Parhar 
55697951040fSNavdeep Parhar 	if (needs_tso(m0)) {
5570a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_tso(m0)) {
5571a4a4ad2dSNavdeep Parhar 			cpl = write_tnl_lso_cpl(wr + 1, m0);
5572a4a4ad2dSNavdeep Parhar 			txq->vxlan_tso_wrs++;
5573a4a4ad2dSNavdeep Parhar 		} else {
5574a4a4ad2dSNavdeep Parhar 			cpl = write_lso_cpl(wr + 1, m0);
557554e4ee71SNavdeep Parhar 			txq->tso_wrs++;
5576a4a4ad2dSNavdeep Parhar 		}
557754e4ee71SNavdeep Parhar 	} else
557854e4ee71SNavdeep Parhar 		cpl = (void *)(wr + 1);
557954e4ee71SNavdeep Parhar 
558054e4ee71SNavdeep Parhar 	/* Checksum offload */
5581c0236bd9SNavdeep Parhar 	ctrl1 = csum_to_ctrl(sc, m0);
5582a4a4ad2dSNavdeep Parhar 	if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5583a4a4ad2dSNavdeep Parhar 		/* some hardware assistance provided */
5584a4a4ad2dSNavdeep Parhar 		if (needs_vxlan_csum(m0))
5585a4a4ad2dSNavdeep Parhar 			txq->vxlan_txcsum++;
5586a4a4ad2dSNavdeep Parhar 		else
5587a4a4ad2dSNavdeep Parhar 			txq->txcsum++;
5588a4a4ad2dSNavdeep Parhar 	}
558954e4ee71SNavdeep Parhar 
559054e4ee71SNavdeep Parhar 	/* VLAN tag insertion */
55917951040fSNavdeep Parhar 	if (needs_vlan_insertion(m0)) {
5592a4a4ad2dSNavdeep Parhar 		ctrl1 |= F_TXPKT_VLAN_VLD |
5593a4a4ad2dSNavdeep Parhar 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
559454e4ee71SNavdeep Parhar 		txq->vlan_insertion++;
559554e4ee71SNavdeep Parhar 	}
559654e4ee71SNavdeep Parhar 
559754e4ee71SNavdeep Parhar 	/* CPL header */
55987951040fSNavdeep Parhar 	cpl->ctrl0 = txq->cpl_ctrl0;
559954e4ee71SNavdeep Parhar 	cpl->pack = 0;
5600ecb79ca4SNavdeep Parhar 	cpl->len = htobe16(pktlen);
560154e4ee71SNavdeep Parhar 	cpl->ctrl1 = htobe64(ctrl1);
560254e4ee71SNavdeep Parhar 
560354e4ee71SNavdeep Parhar 	/* SGL */
560454e4ee71SNavdeep Parhar 	dst = (void *)(cpl + 1);
5605a4a4ad2dSNavdeep Parhar 	if (__predict_false((uintptr_t)dst == (uintptr_t)&eq->desc[eq->sidx]))
5606a4a4ad2dSNavdeep Parhar 		dst = (caddr_t)&eq->desc[0];
56077951040fSNavdeep Parhar 	if (nsegs > 0) {
56087951040fSNavdeep Parhar 
56097951040fSNavdeep Parhar 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
561054e4ee71SNavdeep Parhar 		txq->sgl_wrs++;
561154e4ee71SNavdeep Parhar 	} else {
56127951040fSNavdeep Parhar 		struct mbuf *m;
56137951040fSNavdeep Parhar 
56147951040fSNavdeep Parhar 		for (m = m0; m != NULL; m = m->m_next) {
561554e4ee71SNavdeep Parhar 			copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
5616ecb79ca4SNavdeep Parhar #ifdef INVARIANTS
5617ecb79ca4SNavdeep Parhar 			pktlen -= m->m_len;
5618ecb79ca4SNavdeep Parhar #endif
561954e4ee71SNavdeep Parhar 		}
5620ecb79ca4SNavdeep Parhar #ifdef INVARIANTS
5621ecb79ca4SNavdeep Parhar 		KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
5622ecb79ca4SNavdeep Parhar #endif
56237951040fSNavdeep Parhar 		txq->imm_wrs++;
562454e4ee71SNavdeep Parhar 	}
562554e4ee71SNavdeep Parhar 
562654e4ee71SNavdeep Parhar 	txq->txpkt_wrs++;
562754e4ee71SNavdeep Parhar 
5628f7dfe243SNavdeep Parhar 	txsd = &txq->sdesc[eq->pidx];
56297951040fSNavdeep Parhar 	txsd->m = m0;
563054e4ee71SNavdeep Parhar 	txsd->desc_used = ndesc;
563154e4ee71SNavdeep Parhar 
56327951040fSNavdeep Parhar 	return (ndesc);
563354e4ee71SNavdeep Parhar }
563454e4ee71SNavdeep Parhar 
5635d735920dSNavdeep Parhar static inline bool
5636d735920dSNavdeep Parhar cmp_l2hdr(struct txpkts *txp, struct mbuf *m)
563754e4ee71SNavdeep Parhar {
5638d735920dSNavdeep Parhar 	int len;
56397951040fSNavdeep Parhar 
5640d735920dSNavdeep Parhar 	MPASS(txp->npkt > 0);
5641800535c2SNavdeep Parhar 	MPASS(m->m_len >= VM_TX_L2HDR_LEN);
56427951040fSNavdeep Parhar 
5643d735920dSNavdeep Parhar 	if (txp->ethtype == be16toh(ETHERTYPE_VLAN))
5644800535c2SNavdeep Parhar 		len = VM_TX_L2HDR_LEN;
5645d735920dSNavdeep Parhar 	else
5646d735920dSNavdeep Parhar 		len = sizeof(struct ether_header);
5647d735920dSNavdeep Parhar 
5648d735920dSNavdeep Parhar 	return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0);
56497951040fSNavdeep Parhar }
56507951040fSNavdeep Parhar 
5651d735920dSNavdeep Parhar static inline void
5652d735920dSNavdeep Parhar save_l2hdr(struct txpkts *txp, struct mbuf *m)
5653d735920dSNavdeep Parhar {
5654800535c2SNavdeep Parhar 	MPASS(m->m_len >= VM_TX_L2HDR_LEN);
56557951040fSNavdeep Parhar 
5656800535c2SNavdeep Parhar 	memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN);
5657d735920dSNavdeep Parhar }
56587951040fSNavdeep Parhar 
5659d735920dSNavdeep Parhar static int
5660d735920dSNavdeep Parhar add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5661d735920dSNavdeep Parhar     int avail, bool *send)
5662d735920dSNavdeep Parhar {
5663d735920dSNavdeep Parhar 	struct txpkts *txp = &txq->txp;
5664d735920dSNavdeep Parhar 
5665d735920dSNavdeep Parhar 	/* Cannot have TSO and coalesce at the same time. */
5666d735920dSNavdeep Parhar 	if (cannot_use_txpkts(m)) {
5667d735920dSNavdeep Parhar cannot_coalesce:
5668d735920dSNavdeep Parhar 		*send = txp->npkt > 0;
5669d735920dSNavdeep Parhar 		return (EINVAL);
5670d735920dSNavdeep Parhar 	}
5671d735920dSNavdeep Parhar 
5672d735920dSNavdeep Parhar 	/* VF allows coalescing of type 1 (1 GL) only */
5673d735920dSNavdeep Parhar 	if (mbuf_nsegs(m) > 1)
5674d735920dSNavdeep Parhar 		goto cannot_coalesce;
5675d735920dSNavdeep Parhar 
5676d735920dSNavdeep Parhar 	*send = false;
5677d735920dSNavdeep Parhar 	if (txp->npkt > 0) {
5678d735920dSNavdeep Parhar 		MPASS(tx_len16_to_desc(txp->len16) <= avail);
5679d735920dSNavdeep Parhar 		MPASS(txp->npkt < txp->max_npkt);
5680d735920dSNavdeep Parhar 		MPASS(txp->wr_type == 1);	/* VF supports type 1 only */
5681d735920dSNavdeep Parhar 
5682d735920dSNavdeep Parhar 		if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) {
5683d735920dSNavdeep Parhar retry_after_send:
5684d735920dSNavdeep Parhar 			*send = true;
5685d735920dSNavdeep Parhar 			return (EAGAIN);
5686d735920dSNavdeep Parhar 		}
5687d735920dSNavdeep Parhar 		if (m->m_pkthdr.len + txp->plen > 65535)
5688d735920dSNavdeep Parhar 			goto retry_after_send;
5689d735920dSNavdeep Parhar 		if (cmp_l2hdr(txp, m))
5690d735920dSNavdeep Parhar 			goto retry_after_send;
5691d735920dSNavdeep Parhar 
5692d735920dSNavdeep Parhar 		txp->len16 += txpkts1_len16();
5693d735920dSNavdeep Parhar 		txp->plen += m->m_pkthdr.len;
5694d735920dSNavdeep Parhar 		txp->mb[txp->npkt++] = m;
5695d735920dSNavdeep Parhar 		if (txp->npkt == txp->max_npkt)
5696d735920dSNavdeep Parhar 			*send = true;
5697d735920dSNavdeep Parhar 	} else {
5698d735920dSNavdeep Parhar 		txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) +
5699d735920dSNavdeep Parhar 		    txpkts1_len16();
5700d735920dSNavdeep Parhar 		if (tx_len16_to_desc(txp->len16) > avail)
5701d735920dSNavdeep Parhar 			goto cannot_coalesce;
5702d735920dSNavdeep Parhar 		txp->npkt = 1;
5703d735920dSNavdeep Parhar 		txp->wr_type = 1;
5704d735920dSNavdeep Parhar 		txp->plen = m->m_pkthdr.len;
5705d735920dSNavdeep Parhar 		txp->mb[0] = m;
5706d735920dSNavdeep Parhar 		save_l2hdr(txp, m);
5707d735920dSNavdeep Parhar 	}
57087951040fSNavdeep Parhar 	return (0);
57097951040fSNavdeep Parhar }
57107951040fSNavdeep Parhar 
57117951040fSNavdeep Parhar static int
5712d735920dSNavdeep Parhar add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5713d735920dSNavdeep Parhar     int avail, bool *send)
57147951040fSNavdeep Parhar {
5715d735920dSNavdeep Parhar 	struct txpkts *txp = &txq->txp;
5716d735920dSNavdeep Parhar 	int nsegs;
5717d735920dSNavdeep Parhar 
5718d735920dSNavdeep Parhar 	MPASS(!(sc->flags & IS_VF));
5719d735920dSNavdeep Parhar 
5720d735920dSNavdeep Parhar 	/* Cannot have TSO and coalesce at the same time. */
5721d735920dSNavdeep Parhar 	if (cannot_use_txpkts(m)) {
5722d735920dSNavdeep Parhar cannot_coalesce:
5723d735920dSNavdeep Parhar 		*send = txp->npkt > 0;
5724d735920dSNavdeep Parhar 		return (EINVAL);
5725d735920dSNavdeep Parhar 	}
5726d735920dSNavdeep Parhar 
5727d735920dSNavdeep Parhar 	*send = false;
5728d735920dSNavdeep Parhar 	nsegs = mbuf_nsegs(m);
5729d735920dSNavdeep Parhar 	if (txp->npkt == 0) {
5730d735920dSNavdeep Parhar 		if (m->m_pkthdr.len > 65535)
5731d735920dSNavdeep Parhar 			goto cannot_coalesce;
5732d735920dSNavdeep Parhar 		if (nsegs > 1) {
5733d735920dSNavdeep Parhar 			txp->wr_type = 0;
5734d735920dSNavdeep Parhar 			txp->len16 =
5735d735920dSNavdeep Parhar 			    howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5736d735920dSNavdeep Parhar 			    txpkts0_len16(nsegs);
5737d735920dSNavdeep Parhar 		} else {
5738d735920dSNavdeep Parhar 			txp->wr_type = 1;
5739d735920dSNavdeep Parhar 			txp->len16 =
5740d735920dSNavdeep Parhar 			    howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5741d735920dSNavdeep Parhar 			    txpkts1_len16();
5742d735920dSNavdeep Parhar 		}
5743d735920dSNavdeep Parhar 		if (tx_len16_to_desc(txp->len16) > avail)
5744d735920dSNavdeep Parhar 			goto cannot_coalesce;
5745d735920dSNavdeep Parhar 		txp->npkt = 1;
5746d735920dSNavdeep Parhar 		txp->plen = m->m_pkthdr.len;
5747d735920dSNavdeep Parhar 		txp->mb[0] = m;
5748d735920dSNavdeep Parhar 	} else {
5749d735920dSNavdeep Parhar 		MPASS(tx_len16_to_desc(txp->len16) <= avail);
5750d735920dSNavdeep Parhar 		MPASS(txp->npkt < txp->max_npkt);
5751d735920dSNavdeep Parhar 
5752d735920dSNavdeep Parhar 		if (m->m_pkthdr.len + txp->plen > 65535) {
5753d735920dSNavdeep Parhar retry_after_send:
5754d735920dSNavdeep Parhar 			*send = true;
5755d735920dSNavdeep Parhar 			return (EAGAIN);
5756d735920dSNavdeep Parhar 		}
57577951040fSNavdeep Parhar 
57587951040fSNavdeep Parhar 		MPASS(txp->wr_type == 0 || txp->wr_type == 1);
5759d735920dSNavdeep Parhar 		if (txp->wr_type == 0) {
5760d735920dSNavdeep Parhar 			if (tx_len16_to_desc(txp->len16 +
5761d735920dSNavdeep Parhar 			    txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC))
5762d735920dSNavdeep Parhar 				goto retry_after_send;
5763d735920dSNavdeep Parhar 			txp->len16 += txpkts0_len16(nsegs);
5764d735920dSNavdeep Parhar 		} else {
5765d735920dSNavdeep Parhar 			if (nsegs != 1)
5766d735920dSNavdeep Parhar 				goto retry_after_send;
5767d735920dSNavdeep Parhar 			if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) >
5768d735920dSNavdeep Parhar 			    avail)
5769d735920dSNavdeep Parhar 				goto retry_after_send;
5770d735920dSNavdeep Parhar 			txp->len16 += txpkts1_len16();
5771d735920dSNavdeep Parhar 		}
57727951040fSNavdeep Parhar 
5773d735920dSNavdeep Parhar 		txp->plen += m->m_pkthdr.len;
5774d735920dSNavdeep Parhar 		txp->mb[txp->npkt++] = m;
5775d735920dSNavdeep Parhar 		if (txp->npkt == txp->max_npkt)
5776d735920dSNavdeep Parhar 			*send = true;
5777d735920dSNavdeep Parhar 	}
57787951040fSNavdeep Parhar 	return (0);
57797951040fSNavdeep Parhar }
57807951040fSNavdeep Parhar 
57817951040fSNavdeep Parhar /*
57827951040fSNavdeep Parhar  * Write a txpkts WR for the packets in txp to the hardware descriptors, update
57837951040fSNavdeep Parhar  * the software descriptor, and advance the pidx.  It is guaranteed that enough
57847951040fSNavdeep Parhar  * descriptors are available.
57857951040fSNavdeep Parhar  *
57867951040fSNavdeep Parhar  * The return value is the # of hardware descriptors used.
57877951040fSNavdeep Parhar  */
57887951040fSNavdeep Parhar static u_int
5789d735920dSNavdeep Parhar write_txpkts_wr(struct adapter *sc, struct sge_txq *txq)
57907951040fSNavdeep Parhar {
5791d735920dSNavdeep Parhar 	const struct txpkts *txp = &txq->txp;
57927951040fSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
5793d735920dSNavdeep Parhar 	struct fw_eth_tx_pkts_wr *wr;
57947951040fSNavdeep Parhar 	struct tx_sdesc *txsd;
57957951040fSNavdeep Parhar 	struct cpl_tx_pkt_core *cpl;
57967951040fSNavdeep Parhar 	uint64_t ctrl1;
5797d735920dSNavdeep Parhar 	int ndesc, i, checkwrap;
5798d735920dSNavdeep Parhar 	struct mbuf *m, *last;
57997951040fSNavdeep Parhar 	void *flitp;
58007951040fSNavdeep Parhar 
58017951040fSNavdeep Parhar 	TXQ_LOCK_ASSERT_OWNED(txq);
58027951040fSNavdeep Parhar 	MPASS(txp->npkt > 0);
58037951040fSNavdeep Parhar 	MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
58047951040fSNavdeep Parhar 
5805d735920dSNavdeep Parhar 	wr = (void *)&eq->desc[eq->pidx];
58067951040fSNavdeep Parhar 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
5807d735920dSNavdeep Parhar 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
58087951040fSNavdeep Parhar 	wr->plen = htobe16(txp->plen);
58097951040fSNavdeep Parhar 	wr->npkt = txp->npkt;
58107951040fSNavdeep Parhar 	wr->r3 = 0;
58117951040fSNavdeep Parhar 	wr->type = txp->wr_type;
58127951040fSNavdeep Parhar 	flitp = wr + 1;
58137951040fSNavdeep Parhar 
58147951040fSNavdeep Parhar 	/*
58157951040fSNavdeep Parhar 	 * At this point we are 16B into a hardware descriptor.  If checkwrap is
58167951040fSNavdeep Parhar 	 * set then we know the WR is going to wrap around somewhere.  We'll
58177951040fSNavdeep Parhar 	 * check for that at appropriate points.
58187951040fSNavdeep Parhar 	 */
5819d735920dSNavdeep Parhar 	ndesc = tx_len16_to_desc(txp->len16);
5820d735920dSNavdeep Parhar 	last = NULL;
58217951040fSNavdeep Parhar 	checkwrap = eq->sidx - ndesc < eq->pidx;
5822d735920dSNavdeep Parhar 	for (i = 0; i < txp->npkt; i++) {
5823d735920dSNavdeep Parhar 		m = txp->mb[i];
58247951040fSNavdeep Parhar 		if (txp->wr_type == 0) {
582554e4ee71SNavdeep Parhar 			struct ulp_txpkt *ulpmc;
582654e4ee71SNavdeep Parhar 			struct ulptx_idata *ulpsc;
582754e4ee71SNavdeep Parhar 
58287951040fSNavdeep Parhar 			/* ULP master command */
58297951040fSNavdeep Parhar 			ulpmc = flitp;
58307951040fSNavdeep Parhar 			ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
58317951040fSNavdeep Parhar 			    V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid));
5832d735920dSNavdeep Parhar 			ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m)));
583354e4ee71SNavdeep Parhar 
58347951040fSNavdeep Parhar 			/* ULP subcommand */
58357951040fSNavdeep Parhar 			ulpsc = (void *)(ulpmc + 1);
58367951040fSNavdeep Parhar 			ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
58377951040fSNavdeep Parhar 			    F_ULP_TX_SC_MORE);
58387951040fSNavdeep Parhar 			ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
58397951040fSNavdeep Parhar 
58407951040fSNavdeep Parhar 			cpl = (void *)(ulpsc + 1);
58417951040fSNavdeep Parhar 			if (checkwrap &&
58427951040fSNavdeep Parhar 			    (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx])
58437951040fSNavdeep Parhar 				cpl = (void *)&eq->desc[0];
58447951040fSNavdeep Parhar 		} else {
58457951040fSNavdeep Parhar 			cpl = flitp;
58467951040fSNavdeep Parhar 		}
584754e4ee71SNavdeep Parhar 
584854e4ee71SNavdeep Parhar 		/* Checksum offload */
5849c0236bd9SNavdeep Parhar 		ctrl1 = csum_to_ctrl(sc, m);
5850a4a4ad2dSNavdeep Parhar 		if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5851a4a4ad2dSNavdeep Parhar 			/* some hardware assistance provided */
5852a4a4ad2dSNavdeep Parhar 			if (needs_vxlan_csum(m))
5853a4a4ad2dSNavdeep Parhar 				txq->vxlan_txcsum++;
5854a4a4ad2dSNavdeep Parhar 			else
5855a4a4ad2dSNavdeep Parhar 				txq->txcsum++;
5856a4a4ad2dSNavdeep Parhar 		}
585754e4ee71SNavdeep Parhar 
585854e4ee71SNavdeep Parhar 		/* VLAN tag insertion */
58597951040fSNavdeep Parhar 		if (needs_vlan_insertion(m)) {
58607951040fSNavdeep Parhar 			ctrl1 |= F_TXPKT_VLAN_VLD |
58617951040fSNavdeep Parhar 			    V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
586254e4ee71SNavdeep Parhar 			txq->vlan_insertion++;
586354e4ee71SNavdeep Parhar 		}
586454e4ee71SNavdeep Parhar 
58657951040fSNavdeep Parhar 		/* CPL header */
58667951040fSNavdeep Parhar 		cpl->ctrl0 = txq->cpl_ctrl0;
586754e4ee71SNavdeep Parhar 		cpl->pack = 0;
586854e4ee71SNavdeep Parhar 		cpl->len = htobe16(m->m_pkthdr.len);
58697951040fSNavdeep Parhar 		cpl->ctrl1 = htobe64(ctrl1);
587054e4ee71SNavdeep Parhar 
58717951040fSNavdeep Parhar 		flitp = cpl + 1;
58727951040fSNavdeep Parhar 		if (checkwrap &&
58737951040fSNavdeep Parhar 		    (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
58747951040fSNavdeep Parhar 			flitp = (void *)&eq->desc[0];
587554e4ee71SNavdeep Parhar 
58767951040fSNavdeep Parhar 		write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap);
587754e4ee71SNavdeep Parhar 
5878d735920dSNavdeep Parhar 		if (last != NULL)
5879d735920dSNavdeep Parhar 			last->m_nextpkt = m;
5880d735920dSNavdeep Parhar 		last = m;
58817951040fSNavdeep Parhar 	}
58827951040fSNavdeep Parhar 
5883d735920dSNavdeep Parhar 	txq->sgl_wrs++;
5884a59a1477SNavdeep Parhar 	if (txp->wr_type == 0) {
5885a59a1477SNavdeep Parhar 		txq->txpkts0_pkts += txp->npkt;
5886a59a1477SNavdeep Parhar 		txq->txpkts0_wrs++;
5887a59a1477SNavdeep Parhar 	} else {
5888a59a1477SNavdeep Parhar 		txq->txpkts1_pkts += txp->npkt;
5889a59a1477SNavdeep Parhar 		txq->txpkts1_wrs++;
5890a59a1477SNavdeep Parhar 	}
5891a59a1477SNavdeep Parhar 
58927951040fSNavdeep Parhar 	txsd = &txq->sdesc[eq->pidx];
5893d735920dSNavdeep Parhar 	txsd->m = txp->mb[0];
5894d735920dSNavdeep Parhar 	txsd->desc_used = ndesc;
5895d735920dSNavdeep Parhar 
5896d735920dSNavdeep Parhar 	return (ndesc);
5897d735920dSNavdeep Parhar }
5898d735920dSNavdeep Parhar 
5899d735920dSNavdeep Parhar static u_int
5900d735920dSNavdeep Parhar write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq)
5901d735920dSNavdeep Parhar {
5902d735920dSNavdeep Parhar 	const struct txpkts *txp = &txq->txp;
5903d735920dSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
5904d735920dSNavdeep Parhar 	struct fw_eth_tx_pkts_vm_wr *wr;
5905d735920dSNavdeep Parhar 	struct tx_sdesc *txsd;
5906d735920dSNavdeep Parhar 	struct cpl_tx_pkt_core *cpl;
5907d735920dSNavdeep Parhar 	uint64_t ctrl1;
5908d735920dSNavdeep Parhar 	int ndesc, i;
5909d735920dSNavdeep Parhar 	struct mbuf *m, *last;
5910d735920dSNavdeep Parhar 	void *flitp;
5911d735920dSNavdeep Parhar 
5912d735920dSNavdeep Parhar 	TXQ_LOCK_ASSERT_OWNED(txq);
5913d735920dSNavdeep Parhar 	MPASS(txp->npkt > 0);
5914d735920dSNavdeep Parhar 	MPASS(txp->wr_type == 1);	/* VF supports type 1 only */
5915d735920dSNavdeep Parhar 	MPASS(txp->mb[0] != NULL);
5916d735920dSNavdeep Parhar 	MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
5917d735920dSNavdeep Parhar 
5918d735920dSNavdeep Parhar 	wr = (void *)&eq->desc[eq->pidx];
5919d735920dSNavdeep Parhar 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR));
5920d735920dSNavdeep Parhar 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
5921d735920dSNavdeep Parhar 	wr->r3 = 0;
5922d735920dSNavdeep Parhar 	wr->plen = htobe16(txp->plen);
5923d735920dSNavdeep Parhar 	wr->npkt = txp->npkt;
5924d735920dSNavdeep Parhar 	wr->r4 = 0;
5925d735920dSNavdeep Parhar 	memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16);
5926d735920dSNavdeep Parhar 	flitp = wr + 1;
5927d735920dSNavdeep Parhar 
5928d735920dSNavdeep Parhar 	/*
5929d735920dSNavdeep Parhar 	 * At this point we are 32B into a hardware descriptor.  Each mbuf in
5930d735920dSNavdeep Parhar 	 * the WR will take 32B so we check for the end of the descriptor ring
5931d735920dSNavdeep Parhar 	 * before writing odd mbufs (mb[1], 3, 5, ..)
5932d735920dSNavdeep Parhar 	 */
5933d735920dSNavdeep Parhar 	ndesc = tx_len16_to_desc(txp->len16);
5934d735920dSNavdeep Parhar 	last = NULL;
5935d735920dSNavdeep Parhar 	for (i = 0; i < txp->npkt; i++) {
5936d735920dSNavdeep Parhar 		m = txp->mb[i];
5937d735920dSNavdeep Parhar 		if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
5938d735920dSNavdeep Parhar 			flitp = &eq->desc[0];
5939d735920dSNavdeep Parhar 		cpl = flitp;
5940d735920dSNavdeep Parhar 
5941d735920dSNavdeep Parhar 		/* Checksum offload */
5942d735920dSNavdeep Parhar 		ctrl1 = csum_to_ctrl(sc, m);
5943d735920dSNavdeep Parhar 		if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
5944d735920dSNavdeep Parhar 			txq->txcsum++;	/* some hardware assistance provided */
5945d735920dSNavdeep Parhar 
5946d735920dSNavdeep Parhar 		/* VLAN tag insertion */
5947d735920dSNavdeep Parhar 		if (needs_vlan_insertion(m)) {
5948d735920dSNavdeep Parhar 			ctrl1 |= F_TXPKT_VLAN_VLD |
5949d735920dSNavdeep Parhar 			    V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
5950d735920dSNavdeep Parhar 			txq->vlan_insertion++;
5951d735920dSNavdeep Parhar 		}
5952d735920dSNavdeep Parhar 
5953d735920dSNavdeep Parhar 		/* CPL header */
5954d735920dSNavdeep Parhar 		cpl->ctrl0 = txq->cpl_ctrl0;
5955d735920dSNavdeep Parhar 		cpl->pack = 0;
5956d735920dSNavdeep Parhar 		cpl->len = htobe16(m->m_pkthdr.len);
5957d735920dSNavdeep Parhar 		cpl->ctrl1 = htobe64(ctrl1);
5958d735920dSNavdeep Parhar 
5959d735920dSNavdeep Parhar 		flitp = cpl + 1;
5960d735920dSNavdeep Parhar 		MPASS(mbuf_nsegs(m) == 1);
5961d735920dSNavdeep Parhar 		write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0);
5962d735920dSNavdeep Parhar 
5963d735920dSNavdeep Parhar 		if (last != NULL)
5964d735920dSNavdeep Parhar 			last->m_nextpkt = m;
5965d735920dSNavdeep Parhar 		last = m;
5966d735920dSNavdeep Parhar 	}
5967d735920dSNavdeep Parhar 
5968d735920dSNavdeep Parhar 	txq->sgl_wrs++;
5969d735920dSNavdeep Parhar 	txq->txpkts1_pkts += txp->npkt;
5970d735920dSNavdeep Parhar 	txq->txpkts1_wrs++;
5971d735920dSNavdeep Parhar 
5972d735920dSNavdeep Parhar 	txsd = &txq->sdesc[eq->pidx];
5973d735920dSNavdeep Parhar 	txsd->m = txp->mb[0];
59747951040fSNavdeep Parhar 	txsd->desc_used = ndesc;
59757951040fSNavdeep Parhar 
59767951040fSNavdeep Parhar 	return (ndesc);
597754e4ee71SNavdeep Parhar }
597854e4ee71SNavdeep Parhar 
597954e4ee71SNavdeep Parhar /*
598054e4ee71SNavdeep Parhar  * If the SGL ends on an address that is not 16 byte aligned, this function will
59817951040fSNavdeep Parhar  * add a 0 filled flit at the end.
598254e4ee71SNavdeep Parhar  */
59837951040fSNavdeep Parhar static void
59847951040fSNavdeep Parhar write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap)
598554e4ee71SNavdeep Parhar {
59867951040fSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
59877951040fSNavdeep Parhar 	struct sglist *gl = txq->gl;
59887951040fSNavdeep Parhar 	struct sglist_seg *seg;
59897951040fSNavdeep Parhar 	__be64 *flitp, *wrap;
599054e4ee71SNavdeep Parhar 	struct ulptx_sgl *usgl;
59917951040fSNavdeep Parhar 	int i, nflits, nsegs;
599254e4ee71SNavdeep Parhar 
599354e4ee71SNavdeep Parhar 	KASSERT(((uintptr_t)(*to) & 0xf) == 0,
599454e4ee71SNavdeep Parhar 	    ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
59957951040fSNavdeep Parhar 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
59967951040fSNavdeep Parhar 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
599754e4ee71SNavdeep Parhar 
59987951040fSNavdeep Parhar 	get_pkt_gl(m, gl);
59997951040fSNavdeep Parhar 	nsegs = gl->sg_nseg;
60007951040fSNavdeep Parhar 	MPASS(nsegs > 0);
60017951040fSNavdeep Parhar 
60027951040fSNavdeep Parhar 	nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
600354e4ee71SNavdeep Parhar 	flitp = (__be64 *)(*to);
60047951040fSNavdeep Parhar 	wrap = (__be64 *)(&eq->desc[eq->sidx]);
60057951040fSNavdeep Parhar 	seg = &gl->sg_segs[0];
600654e4ee71SNavdeep Parhar 	usgl = (void *)flitp;
600754e4ee71SNavdeep Parhar 
600854e4ee71SNavdeep Parhar 	/*
600954e4ee71SNavdeep Parhar 	 * We start at a 16 byte boundary somewhere inside the tx descriptor
601054e4ee71SNavdeep Parhar 	 * ring, so we're at least 16 bytes away from the status page.  There is
601154e4ee71SNavdeep Parhar 	 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
601254e4ee71SNavdeep Parhar 	 */
601354e4ee71SNavdeep Parhar 
601454e4ee71SNavdeep Parhar 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
60157951040fSNavdeep Parhar 	    V_ULPTX_NSGE(nsegs));
60167951040fSNavdeep Parhar 	usgl->len0 = htobe32(seg->ss_len);
60177951040fSNavdeep Parhar 	usgl->addr0 = htobe64(seg->ss_paddr);
601854e4ee71SNavdeep Parhar 	seg++;
601954e4ee71SNavdeep Parhar 
60207951040fSNavdeep Parhar 	if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) {
602154e4ee71SNavdeep Parhar 
602254e4ee71SNavdeep Parhar 		/* Won't wrap around at all */
602354e4ee71SNavdeep Parhar 
60247951040fSNavdeep Parhar 		for (i = 0; i < nsegs - 1; i++, seg++) {
60257951040fSNavdeep Parhar 			usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
60267951040fSNavdeep Parhar 			usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
602754e4ee71SNavdeep Parhar 		}
602854e4ee71SNavdeep Parhar 		if (i & 1)
602954e4ee71SNavdeep Parhar 			usgl->sge[i / 2].len[1] = htobe32(0);
60307951040fSNavdeep Parhar 		flitp += nflits;
603154e4ee71SNavdeep Parhar 	} else {
603254e4ee71SNavdeep Parhar 
603354e4ee71SNavdeep Parhar 		/* Will wrap somewhere in the rest of the SGL */
603454e4ee71SNavdeep Parhar 
603554e4ee71SNavdeep Parhar 		/* 2 flits already written, write the rest flit by flit */
603654e4ee71SNavdeep Parhar 		flitp = (void *)(usgl + 1);
60377951040fSNavdeep Parhar 		for (i = 0; i < nflits - 2; i++) {
60387951040fSNavdeep Parhar 			if (flitp == wrap)
603954e4ee71SNavdeep Parhar 				flitp = (void *)eq->desc;
60407951040fSNavdeep Parhar 			*flitp++ = get_flit(seg, nsegs - 1, i);
604154e4ee71SNavdeep Parhar 		}
604254e4ee71SNavdeep Parhar 	}
604354e4ee71SNavdeep Parhar 
60447951040fSNavdeep Parhar 	if (nflits & 1) {
60457951040fSNavdeep Parhar 		MPASS(((uintptr_t)flitp) & 0xf);
60467951040fSNavdeep Parhar 		*flitp++ = 0;
60477951040fSNavdeep Parhar 	}
604854e4ee71SNavdeep Parhar 
60497951040fSNavdeep Parhar 	MPASS((((uintptr_t)flitp) & 0xf) == 0);
60507951040fSNavdeep Parhar 	if (__predict_false(flitp == wrap))
605154e4ee71SNavdeep Parhar 		*to = (void *)eq->desc;
605254e4ee71SNavdeep Parhar 	else
60537951040fSNavdeep Parhar 		*to = (void *)flitp;
605454e4ee71SNavdeep Parhar }
605554e4ee71SNavdeep Parhar 
605654e4ee71SNavdeep Parhar static inline void
605754e4ee71SNavdeep Parhar copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
605854e4ee71SNavdeep Parhar {
60597951040fSNavdeep Parhar 
60607951040fSNavdeep Parhar 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
60617951040fSNavdeep Parhar 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
60627951040fSNavdeep Parhar 
60637951040fSNavdeep Parhar 	if (__predict_true((uintptr_t)(*to) + len <=
60647951040fSNavdeep Parhar 	    (uintptr_t)&eq->desc[eq->sidx])) {
606554e4ee71SNavdeep Parhar 		bcopy(from, *to, len);
606654e4ee71SNavdeep Parhar 		(*to) += len;
606754e4ee71SNavdeep Parhar 	} else {
60687951040fSNavdeep Parhar 		int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
606954e4ee71SNavdeep Parhar 
607054e4ee71SNavdeep Parhar 		bcopy(from, *to, portion);
607154e4ee71SNavdeep Parhar 		from += portion;
607254e4ee71SNavdeep Parhar 		portion = len - portion;	/* remaining */
607354e4ee71SNavdeep Parhar 		bcopy(from, (void *)eq->desc, portion);
607454e4ee71SNavdeep Parhar 		(*to) = (caddr_t)eq->desc + portion;
607554e4ee71SNavdeep Parhar 	}
607654e4ee71SNavdeep Parhar }
607754e4ee71SNavdeep Parhar 
607854e4ee71SNavdeep Parhar static inline void
60797951040fSNavdeep Parhar ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n)
608054e4ee71SNavdeep Parhar {
60817951040fSNavdeep Parhar 	u_int db;
60827951040fSNavdeep Parhar 
60837951040fSNavdeep Parhar 	MPASS(n > 0);
6084d14b0ac1SNavdeep Parhar 
6085d14b0ac1SNavdeep Parhar 	db = eq->doorbells;
60867951040fSNavdeep Parhar 	if (n > 1)
608777ad3c41SNavdeep Parhar 		clrbit(&db, DOORBELL_WCWR);
6088d14b0ac1SNavdeep Parhar 	wmb();
6089d14b0ac1SNavdeep Parhar 
6090d14b0ac1SNavdeep Parhar 	switch (ffs(db) - 1) {
6091d14b0ac1SNavdeep Parhar 	case DOORBELL_UDB:
60927951040fSNavdeep Parhar 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
60937951040fSNavdeep Parhar 		break;
6094d14b0ac1SNavdeep Parhar 
609577ad3c41SNavdeep Parhar 	case DOORBELL_WCWR: {
6096d14b0ac1SNavdeep Parhar 		volatile uint64_t *dst, *src;
6097d14b0ac1SNavdeep Parhar 		int i;
6098d14b0ac1SNavdeep Parhar 
6099d14b0ac1SNavdeep Parhar 		/*
6100d14b0ac1SNavdeep Parhar 		 * Queues whose 128B doorbell segment fits in the page do not
6101d14b0ac1SNavdeep Parhar 		 * use relative qid (udb_qid is always 0).  Only queues with
610277ad3c41SNavdeep Parhar 		 * doorbell segments can do WCWR.
6103d14b0ac1SNavdeep Parhar 		 */
61047951040fSNavdeep Parhar 		KASSERT(eq->udb_qid == 0 && n == 1,
6105d14b0ac1SNavdeep Parhar 		    ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
61067951040fSNavdeep Parhar 		    __func__, eq->doorbells, n, eq->dbidx, eq));
6107d14b0ac1SNavdeep Parhar 
6108d14b0ac1SNavdeep Parhar 		dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
6109d14b0ac1SNavdeep Parhar 		    UDBS_DB_OFFSET);
61107951040fSNavdeep Parhar 		i = eq->dbidx;
6111d14b0ac1SNavdeep Parhar 		src = (void *)&eq->desc[i];
6112d14b0ac1SNavdeep Parhar 		while (src != (void *)&eq->desc[i + 1])
6113d14b0ac1SNavdeep Parhar 			*dst++ = *src++;
6114d14b0ac1SNavdeep Parhar 		wmb();
61157951040fSNavdeep Parhar 		break;
6116d14b0ac1SNavdeep Parhar 	}
6117d14b0ac1SNavdeep Parhar 
6118d14b0ac1SNavdeep Parhar 	case DOORBELL_UDBWC:
61197951040fSNavdeep Parhar 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
6120d14b0ac1SNavdeep Parhar 		wmb();
61217951040fSNavdeep Parhar 		break;
6122d14b0ac1SNavdeep Parhar 
6123d14b0ac1SNavdeep Parhar 	case DOORBELL_KDB:
6124315048f2SJohn Baldwin 		t4_write_reg(sc, sc->sge_kdoorbell_reg,
61257951040fSNavdeep Parhar 		    V_QID(eq->cntxt_id) | V_PIDX(n));
61267951040fSNavdeep Parhar 		break;
612754e4ee71SNavdeep Parhar 	}
612854e4ee71SNavdeep Parhar 
61297951040fSNavdeep Parhar 	IDXINCR(eq->dbidx, n, eq->sidx);
61307951040fSNavdeep Parhar }
61317951040fSNavdeep Parhar 
61327951040fSNavdeep Parhar static inline u_int
61337951040fSNavdeep Parhar reclaimable_tx_desc(struct sge_eq *eq)
613454e4ee71SNavdeep Parhar {
61357951040fSNavdeep Parhar 	uint16_t hw_cidx;
613654e4ee71SNavdeep Parhar 
61377951040fSNavdeep Parhar 	hw_cidx = read_hw_cidx(eq);
61387951040fSNavdeep Parhar 	return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx));
61397951040fSNavdeep Parhar }
614054e4ee71SNavdeep Parhar 
61417951040fSNavdeep Parhar static inline u_int
61427951040fSNavdeep Parhar total_available_tx_desc(struct sge_eq *eq)
61437951040fSNavdeep Parhar {
61447951040fSNavdeep Parhar 	uint16_t hw_cidx, pidx;
61457951040fSNavdeep Parhar 
61467951040fSNavdeep Parhar 	hw_cidx = read_hw_cidx(eq);
61477951040fSNavdeep Parhar 	pidx = eq->pidx;
61487951040fSNavdeep Parhar 
61497951040fSNavdeep Parhar 	if (pidx == hw_cidx)
61507951040fSNavdeep Parhar 		return (eq->sidx - 1);
615154e4ee71SNavdeep Parhar 	else
61527951040fSNavdeep Parhar 		return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1);
61537951040fSNavdeep Parhar }
61547951040fSNavdeep Parhar 
61557951040fSNavdeep Parhar static inline uint16_t
61567951040fSNavdeep Parhar read_hw_cidx(struct sge_eq *eq)
61577951040fSNavdeep Parhar {
61587951040fSNavdeep Parhar 	struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
61597951040fSNavdeep Parhar 	uint16_t cidx = spg->cidx;	/* stable snapshot */
61607951040fSNavdeep Parhar 
61617951040fSNavdeep Parhar 	return (be16toh(cidx));
6162e874ff7aSNavdeep Parhar }
616354e4ee71SNavdeep Parhar 
6164e874ff7aSNavdeep Parhar /*
61657951040fSNavdeep Parhar  * Reclaim 'n' descriptors approximately.
6166e874ff7aSNavdeep Parhar  */
61677951040fSNavdeep Parhar static u_int
61687951040fSNavdeep Parhar reclaim_tx_descs(struct sge_txq *txq, u_int n)
6169e874ff7aSNavdeep Parhar {
6170e874ff7aSNavdeep Parhar 	struct tx_sdesc *txsd;
6171f7dfe243SNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
61727951040fSNavdeep Parhar 	u_int can_reclaim, reclaimed;
617354e4ee71SNavdeep Parhar 
6174733b9277SNavdeep Parhar 	TXQ_LOCK_ASSERT_OWNED(txq);
61757951040fSNavdeep Parhar 	MPASS(n > 0);
6176e874ff7aSNavdeep Parhar 
61777951040fSNavdeep Parhar 	reclaimed = 0;
61787951040fSNavdeep Parhar 	can_reclaim = reclaimable_tx_desc(eq);
61797951040fSNavdeep Parhar 	while (can_reclaim && reclaimed < n) {
618054e4ee71SNavdeep Parhar 		int ndesc;
61817951040fSNavdeep Parhar 		struct mbuf *m, *nextpkt;
618254e4ee71SNavdeep Parhar 
6183f7dfe243SNavdeep Parhar 		txsd = &txq->sdesc[eq->cidx];
618454e4ee71SNavdeep Parhar 		ndesc = txsd->desc_used;
618554e4ee71SNavdeep Parhar 
618654e4ee71SNavdeep Parhar 		/* Firmware doesn't return "partial" credits. */
618754e4ee71SNavdeep Parhar 		KASSERT(can_reclaim >= ndesc,
618854e4ee71SNavdeep Parhar 		    ("%s: unexpected number of credits: %d, %d",
618954e4ee71SNavdeep Parhar 		    __func__, can_reclaim, ndesc));
6190dcd50a20SJohn Baldwin 		KASSERT(ndesc != 0,
6191dcd50a20SJohn Baldwin 		    ("%s: descriptor with no credits: cidx %d",
6192dcd50a20SJohn Baldwin 		    __func__, eq->cidx));
619354e4ee71SNavdeep Parhar 
61947951040fSNavdeep Parhar 		for (m = txsd->m; m != NULL; m = nextpkt) {
61957951040fSNavdeep Parhar 			nextpkt = m->m_nextpkt;
61967951040fSNavdeep Parhar 			m->m_nextpkt = NULL;
61977951040fSNavdeep Parhar 			m_freem(m);
61987951040fSNavdeep Parhar 		}
619954e4ee71SNavdeep Parhar 		reclaimed += ndesc;
620054e4ee71SNavdeep Parhar 		can_reclaim -= ndesc;
62017951040fSNavdeep Parhar 		IDXINCR(eq->cidx, ndesc, eq->sidx);
620254e4ee71SNavdeep Parhar 	}
620354e4ee71SNavdeep Parhar 
620454e4ee71SNavdeep Parhar 	return (reclaimed);
620554e4ee71SNavdeep Parhar }
620654e4ee71SNavdeep Parhar 
620754e4ee71SNavdeep Parhar static void
62087951040fSNavdeep Parhar tx_reclaim(void *arg, int n)
620954e4ee71SNavdeep Parhar {
62107951040fSNavdeep Parhar 	struct sge_txq *txq = arg;
62117951040fSNavdeep Parhar 	struct sge_eq *eq = &txq->eq;
621254e4ee71SNavdeep Parhar 
62137951040fSNavdeep Parhar 	do {
62147951040fSNavdeep Parhar 		if (TXQ_TRYLOCK(txq) == 0)
62157951040fSNavdeep Parhar 			break;
62167951040fSNavdeep Parhar 		n = reclaim_tx_descs(txq, 32);
62177951040fSNavdeep Parhar 		if (eq->cidx == eq->pidx)
62187951040fSNavdeep Parhar 			eq->equeqidx = eq->pidx;
62197951040fSNavdeep Parhar 		TXQ_UNLOCK(txq);
62207951040fSNavdeep Parhar 	} while (n > 0);
622154e4ee71SNavdeep Parhar }
622254e4ee71SNavdeep Parhar 
622354e4ee71SNavdeep Parhar static __be64
62247951040fSNavdeep Parhar get_flit(struct sglist_seg *segs, int nsegs, int idx)
622554e4ee71SNavdeep Parhar {
622654e4ee71SNavdeep Parhar 	int i = (idx / 3) * 2;
622754e4ee71SNavdeep Parhar 
622854e4ee71SNavdeep Parhar 	switch (idx % 3) {
622954e4ee71SNavdeep Parhar 	case 0: {
6230f078ecf6SWojciech Macek 		uint64_t rc;
623154e4ee71SNavdeep Parhar 
6232f078ecf6SWojciech Macek 		rc = (uint64_t)segs[i].ss_len << 32;
623354e4ee71SNavdeep Parhar 		if (i + 1 < nsegs)
6234f078ecf6SWojciech Macek 			rc |= (uint64_t)(segs[i + 1].ss_len);
623554e4ee71SNavdeep Parhar 
6236f078ecf6SWojciech Macek 		return (htobe64(rc));
623754e4ee71SNavdeep Parhar 	}
623854e4ee71SNavdeep Parhar 	case 1:
62397951040fSNavdeep Parhar 		return (htobe64(segs[i].ss_paddr));
624054e4ee71SNavdeep Parhar 	case 2:
62417951040fSNavdeep Parhar 		return (htobe64(segs[i + 1].ss_paddr));
624254e4ee71SNavdeep Parhar 	}
624354e4ee71SNavdeep Parhar 
624454e4ee71SNavdeep Parhar 	return (0);
624554e4ee71SNavdeep Parhar }
624654e4ee71SNavdeep Parhar 
624746e1e307SNavdeep Parhar static int
624846e1e307SNavdeep Parhar find_refill_source(struct adapter *sc, int maxp, bool packing)
624954e4ee71SNavdeep Parhar {
625046e1e307SNavdeep Parhar 	int i, zidx = -1;
625146e1e307SNavdeep Parhar 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
625254e4ee71SNavdeep Parhar 
625346e1e307SNavdeep Parhar 	if (packing) {
625446e1e307SNavdeep Parhar 		for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
625546e1e307SNavdeep Parhar 			if (rxb->hwidx2 == -1)
625646e1e307SNavdeep Parhar 				continue;
625746e1e307SNavdeep Parhar 			if (rxb->size1 < PAGE_SIZE &&
625846e1e307SNavdeep Parhar 			    rxb->size1 < largest_rx_cluster)
625946e1e307SNavdeep Parhar 				continue;
626046e1e307SNavdeep Parhar 			if (rxb->size1 > largest_rx_cluster)
626138035ed6SNavdeep Parhar 				break;
626246e1e307SNavdeep Parhar 			MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE);
626346e1e307SNavdeep Parhar 			if (rxb->size2 >= maxp)
626446e1e307SNavdeep Parhar 				return (i);
626546e1e307SNavdeep Parhar 			zidx = i;
626638035ed6SNavdeep Parhar 		}
626738035ed6SNavdeep Parhar 	} else {
626846e1e307SNavdeep Parhar 		for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
626946e1e307SNavdeep Parhar 			if (rxb->hwidx1 == -1)
627046e1e307SNavdeep Parhar 				continue;
627146e1e307SNavdeep Parhar 			if (rxb->size1 > largest_rx_cluster)
627238035ed6SNavdeep Parhar 				break;
627346e1e307SNavdeep Parhar 			if (rxb->size1 >= maxp)
627446e1e307SNavdeep Parhar 				return (i);
627546e1e307SNavdeep Parhar 			zidx = i;
627638035ed6SNavdeep Parhar 		}
627738035ed6SNavdeep Parhar 	}
627838035ed6SNavdeep Parhar 
627946e1e307SNavdeep Parhar 	return (zidx);
628054e4ee71SNavdeep Parhar }
6281ecb79ca4SNavdeep Parhar 
6282733b9277SNavdeep Parhar static void
6283733b9277SNavdeep Parhar add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
6284ecb79ca4SNavdeep Parhar {
6285733b9277SNavdeep Parhar 	mtx_lock(&sc->sfl_lock);
6286733b9277SNavdeep Parhar 	FL_LOCK(fl);
6287733b9277SNavdeep Parhar 	if ((fl->flags & FL_DOOMED) == 0) {
6288733b9277SNavdeep Parhar 		fl->flags |= FL_STARVING;
6289733b9277SNavdeep Parhar 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
6290733b9277SNavdeep Parhar 		callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
6291733b9277SNavdeep Parhar 	}
6292733b9277SNavdeep Parhar 	FL_UNLOCK(fl);
6293733b9277SNavdeep Parhar 	mtx_unlock(&sc->sfl_lock);
6294733b9277SNavdeep Parhar }
6295ecb79ca4SNavdeep Parhar 
62967951040fSNavdeep Parhar static void
62977951040fSNavdeep Parhar handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
62987951040fSNavdeep Parhar {
62997951040fSNavdeep Parhar 	struct sge_wrq *wrq = (void *)eq;
63007951040fSNavdeep Parhar 
63017951040fSNavdeep Parhar 	atomic_readandclear_int(&eq->equiq);
63027951040fSNavdeep Parhar 	taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
63037951040fSNavdeep Parhar }
63047951040fSNavdeep Parhar 
63057951040fSNavdeep Parhar static void
63067951040fSNavdeep Parhar handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
63077951040fSNavdeep Parhar {
63087951040fSNavdeep Parhar 	struct sge_txq *txq = (void *)eq;
63097951040fSNavdeep Parhar 
631043bbae19SNavdeep Parhar 	MPASS(eq->type == EQ_ETH);
63117951040fSNavdeep Parhar 
63127951040fSNavdeep Parhar 	atomic_readandclear_int(&eq->equiq);
6313d735920dSNavdeep Parhar 	if (mp_ring_is_idle(txq->r))
63147951040fSNavdeep Parhar 		taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
6315d735920dSNavdeep Parhar 	else
6316d735920dSNavdeep Parhar 		mp_ring_check_drainage(txq->r, 64);
63177951040fSNavdeep Parhar }
63187951040fSNavdeep Parhar 
6319733b9277SNavdeep Parhar static int
6320733b9277SNavdeep Parhar handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
6321733b9277SNavdeep Parhar     struct mbuf *m)
6322733b9277SNavdeep Parhar {
6323733b9277SNavdeep Parhar 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
6324733b9277SNavdeep Parhar 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
6325733b9277SNavdeep Parhar 	struct adapter *sc = iq->adapter;
6326733b9277SNavdeep Parhar 	struct sge *s = &sc->sge;
6327733b9277SNavdeep Parhar 	struct sge_eq *eq;
63287951040fSNavdeep Parhar 	static void (*h[])(struct adapter *, struct sge_eq *) = {NULL,
63297951040fSNavdeep Parhar 		&handle_wrq_egr_update, &handle_eth_egr_update,
63307951040fSNavdeep Parhar 		&handle_wrq_egr_update};
6331733b9277SNavdeep Parhar 
6332733b9277SNavdeep Parhar 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6333733b9277SNavdeep Parhar 	    rss->opcode));
6334733b9277SNavdeep Parhar 
6335ec55567cSJohn Baldwin 	eq = s->eqmap[qid - s->eq_start - s->eq_base];
633643bbae19SNavdeep Parhar 	(*h[eq->type])(sc, eq);
6337ecb79ca4SNavdeep Parhar 
6338ecb79ca4SNavdeep Parhar 	return (0);
6339ecb79ca4SNavdeep Parhar }
6340f7dfe243SNavdeep Parhar 
63410abd31e2SNavdeep Parhar /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
63420abd31e2SNavdeep Parhar CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
63430abd31e2SNavdeep Parhar     offsetof(struct cpl_fw6_msg, data));
63440abd31e2SNavdeep Parhar 
6345733b9277SNavdeep Parhar static int
63461b4cc91fSNavdeep Parhar handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
634756599263SNavdeep Parhar {
63481b4cc91fSNavdeep Parhar 	struct adapter *sc = iq->adapter;
634956599263SNavdeep Parhar 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
635056599263SNavdeep Parhar 
6351733b9277SNavdeep Parhar 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6352733b9277SNavdeep Parhar 	    rss->opcode));
6353733b9277SNavdeep Parhar 
63540abd31e2SNavdeep Parhar 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
63550abd31e2SNavdeep Parhar 		const struct rss_header *rss2;
63560abd31e2SNavdeep Parhar 
63570abd31e2SNavdeep Parhar 		rss2 = (const struct rss_header *)&cpl->data[0];
6358671bf2b8SNavdeep Parhar 		return (t4_cpl_handler[rss2->opcode](iq, rss2, m));
63590abd31e2SNavdeep Parhar 	}
63600abd31e2SNavdeep Parhar 
6361671bf2b8SNavdeep Parhar 	return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0]));
6362f7dfe243SNavdeep Parhar }
6363af49c942SNavdeep Parhar 
6364069af0ebSJohn Baldwin /**
6365069af0ebSJohn Baldwin  *	t4_handle_wrerr_rpl - process a FW work request error message
6366069af0ebSJohn Baldwin  *	@adap: the adapter
6367069af0ebSJohn Baldwin  *	@rpl: start of the FW message
6368069af0ebSJohn Baldwin  */
6369069af0ebSJohn Baldwin static int
6370069af0ebSJohn Baldwin t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl)
6371069af0ebSJohn Baldwin {
6372069af0ebSJohn Baldwin 	u8 opcode = *(const u8 *)rpl;
6373069af0ebSJohn Baldwin 	const struct fw_error_cmd *e = (const void *)rpl;
6374069af0ebSJohn Baldwin 	unsigned int i;
6375069af0ebSJohn Baldwin 
6376069af0ebSJohn Baldwin 	if (opcode != FW_ERROR_CMD) {
6377069af0ebSJohn Baldwin 		log(LOG_ERR,
6378069af0ebSJohn Baldwin 		    "%s: Received WRERR_RPL message with opcode %#x\n",
6379069af0ebSJohn Baldwin 		    device_get_nameunit(adap->dev), opcode);
6380069af0ebSJohn Baldwin 		return (EINVAL);
6381069af0ebSJohn Baldwin 	}
6382069af0ebSJohn Baldwin 	log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev),
6383069af0ebSJohn Baldwin 	    G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" :
6384069af0ebSJohn Baldwin 	    "non-fatal");
6385069af0ebSJohn Baldwin 	switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) {
6386069af0ebSJohn Baldwin 	case FW_ERROR_TYPE_EXCEPTION:
6387069af0ebSJohn Baldwin 		log(LOG_ERR, "exception info:\n");
6388069af0ebSJohn Baldwin 		for (i = 0; i < nitems(e->u.exception.info); i++)
6389069af0ebSJohn Baldwin 			log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ",
6390069af0ebSJohn Baldwin 			    be32toh(e->u.exception.info[i]));
6391069af0ebSJohn Baldwin 		log(LOG_ERR, "\n");
6392069af0ebSJohn Baldwin 		break;
6393069af0ebSJohn Baldwin 	case FW_ERROR_TYPE_HWMODULE:
6394069af0ebSJohn Baldwin 		log(LOG_ERR, "HW module regaddr %08x regval %08x\n",
6395069af0ebSJohn Baldwin 		    be32toh(e->u.hwmodule.regaddr),
6396069af0ebSJohn Baldwin 		    be32toh(e->u.hwmodule.regval));
6397069af0ebSJohn Baldwin 		break;
6398069af0ebSJohn Baldwin 	case FW_ERROR_TYPE_WR:
6399069af0ebSJohn Baldwin 		log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n",
6400069af0ebSJohn Baldwin 		    be16toh(e->u.wr.cidx),
6401069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)),
6402069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)),
6403069af0ebSJohn Baldwin 		    be32toh(e->u.wr.eqid));
6404069af0ebSJohn Baldwin 		for (i = 0; i < nitems(e->u.wr.wrhdr); i++)
6405069af0ebSJohn Baldwin 			log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ",
6406069af0ebSJohn Baldwin 			    e->u.wr.wrhdr[i]);
6407069af0ebSJohn Baldwin 		log(LOG_ERR, "\n");
6408069af0ebSJohn Baldwin 		break;
6409069af0ebSJohn Baldwin 	case FW_ERROR_TYPE_ACL:
6410069af0ebSJohn Baldwin 		log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s",
6411069af0ebSJohn Baldwin 		    be16toh(e->u.acl.cidx),
6412069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)),
6413069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)),
6414069af0ebSJohn Baldwin 		    be32toh(e->u.acl.eqid),
6415069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" :
6416069af0ebSJohn Baldwin 		    "MAC");
6417069af0ebSJohn Baldwin 		for (i = 0; i < nitems(e->u.acl.val); i++)
6418069af0ebSJohn Baldwin 			log(LOG_ERR, " %02x", e->u.acl.val[i]);
6419069af0ebSJohn Baldwin 		log(LOG_ERR, "\n");
6420069af0ebSJohn Baldwin 		break;
6421069af0ebSJohn Baldwin 	default:
6422069af0ebSJohn Baldwin 		log(LOG_ERR, "type %#x\n",
6423069af0ebSJohn Baldwin 		    G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type)));
6424069af0ebSJohn Baldwin 		return (EINVAL);
6425069af0ebSJohn Baldwin 	}
6426069af0ebSJohn Baldwin 	return (0);
6427069af0ebSJohn Baldwin }
6428069af0ebSJohn Baldwin 
642946e1e307SNavdeep Parhar static inline bool
643046e1e307SNavdeep Parhar bufidx_used(struct adapter *sc, int idx)
643146e1e307SNavdeep Parhar {
643246e1e307SNavdeep Parhar 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
643346e1e307SNavdeep Parhar 	int i;
643446e1e307SNavdeep Parhar 
643546e1e307SNavdeep Parhar 	for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
643646e1e307SNavdeep Parhar 		if (rxb->size1 > largest_rx_cluster)
643746e1e307SNavdeep Parhar 			continue;
643846e1e307SNavdeep Parhar 		if (rxb->hwidx1 == idx || rxb->hwidx2 == idx)
643946e1e307SNavdeep Parhar 			return (true);
644046e1e307SNavdeep Parhar 	}
644146e1e307SNavdeep Parhar 
644246e1e307SNavdeep Parhar 	return (false);
644346e1e307SNavdeep Parhar }
644446e1e307SNavdeep Parhar 
644538035ed6SNavdeep Parhar static int
644638035ed6SNavdeep Parhar sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
644738035ed6SNavdeep Parhar {
644846e1e307SNavdeep Parhar 	struct adapter *sc = arg1;
644946e1e307SNavdeep Parhar 	struct sge_params *sp = &sc->params.sge;
645038035ed6SNavdeep Parhar 	int i, rc;
645138035ed6SNavdeep Parhar 	struct sbuf sb;
645238035ed6SNavdeep Parhar 	char c;
645338035ed6SNavdeep Parhar 
645446e1e307SNavdeep Parhar 	sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND);
645546e1e307SNavdeep Parhar 	for (i = 0; i < SGE_FLBUF_SIZES; i++) {
645646e1e307SNavdeep Parhar 		if (bufidx_used(sc, i))
645738035ed6SNavdeep Parhar 			c = '*';
645838035ed6SNavdeep Parhar 		else
645938035ed6SNavdeep Parhar 			c = '\0';
646038035ed6SNavdeep Parhar 
646146e1e307SNavdeep Parhar 		sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c);
646238035ed6SNavdeep Parhar 	}
646338035ed6SNavdeep Parhar 	sbuf_trim(&sb);
646438035ed6SNavdeep Parhar 	sbuf_finish(&sb);
646538035ed6SNavdeep Parhar 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
646638035ed6SNavdeep Parhar 	sbuf_delete(&sb);
646738035ed6SNavdeep Parhar 	return (rc);
646838035ed6SNavdeep Parhar }
646902f972e8SNavdeep Parhar 
6470786099deSNavdeep Parhar #ifdef RATELIMIT
6471ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6)
6472786099deSNavdeep Parhar /*
6473786099deSNavdeep Parhar  * len16 for a txpkt WR with a GL.  Includes the firmware work request header.
6474786099deSNavdeep Parhar  */
6475786099deSNavdeep Parhar static inline u_int
6476786099deSNavdeep Parhar txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso)
6477786099deSNavdeep Parhar {
6478786099deSNavdeep Parhar 	u_int n;
6479786099deSNavdeep Parhar 
6480786099deSNavdeep Parhar 	MPASS(immhdrs > 0);
6481786099deSNavdeep Parhar 
6482786099deSNavdeep Parhar 	n = roundup2(sizeof(struct fw_eth_tx_eo_wr) +
6483786099deSNavdeep Parhar 	    sizeof(struct cpl_tx_pkt_core) + immhdrs, 16);
6484786099deSNavdeep Parhar 	if (__predict_false(nsegs == 0))
6485786099deSNavdeep Parhar 		goto done;
6486786099deSNavdeep Parhar 
6487786099deSNavdeep Parhar 	nsegs--; /* first segment is part of ulptx_sgl */
6488786099deSNavdeep Parhar 	n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
6489786099deSNavdeep Parhar 	if (tso)
6490786099deSNavdeep Parhar 		n += sizeof(struct cpl_tx_pkt_lso_core);
6491786099deSNavdeep Parhar 
6492786099deSNavdeep Parhar done:
6493786099deSNavdeep Parhar 	return (howmany(n, 16));
6494786099deSNavdeep Parhar }
6495ffbb373cSNavdeep Parhar #endif
6496786099deSNavdeep Parhar 
6497786099deSNavdeep Parhar #define ETID_FLOWC_NPARAMS 6
6498786099deSNavdeep Parhar #define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \
6499786099deSNavdeep Parhar     ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16))
6500786099deSNavdeep Parhar #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16))
6501786099deSNavdeep Parhar 
6502786099deSNavdeep Parhar static int
6503e38a50e8SJohn Baldwin send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi,
6504786099deSNavdeep Parhar     struct vi_info *vi)
6505786099deSNavdeep Parhar {
6506786099deSNavdeep Parhar 	struct wrq_cookie cookie;
6507edb518f4SNavdeep Parhar 	u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN;
6508786099deSNavdeep Parhar 	struct fw_flowc_wr *flowc;
6509786099deSNavdeep Parhar 
6510786099deSNavdeep Parhar 	mtx_assert(&cst->lock, MA_OWNED);
6511786099deSNavdeep Parhar 	MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) ==
6512786099deSNavdeep Parhar 	    EO_FLOWC_PENDING);
6513786099deSNavdeep Parhar 
6514077ba6a8SJohn Baldwin 	flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie);
6515786099deSNavdeep Parhar 	if (__predict_false(flowc == NULL))
6516786099deSNavdeep Parhar 		return (ENOMEM);
6517786099deSNavdeep Parhar 
6518786099deSNavdeep Parhar 	bzero(flowc, ETID_FLOWC_LEN);
6519786099deSNavdeep Parhar 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6520786099deSNavdeep Parhar 	    V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0));
6521786099deSNavdeep Parhar 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) |
6522786099deSNavdeep Parhar 	    V_FW_WR_FLOWID(cst->etid));
6523786099deSNavdeep Parhar 	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
6524786099deSNavdeep Parhar 	flowc->mnemval[0].val = htobe32(pfvf);
6525786099deSNavdeep Parhar 	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
6526786099deSNavdeep Parhar 	flowc->mnemval[1].val = htobe32(pi->tx_chan);
6527786099deSNavdeep Parhar 	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
6528786099deSNavdeep Parhar 	flowc->mnemval[2].val = htobe32(pi->tx_chan);
6529786099deSNavdeep Parhar 	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
6530786099deSNavdeep Parhar 	flowc->mnemval[3].val = htobe32(cst->iqid);
6531786099deSNavdeep Parhar 	flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE;
6532786099deSNavdeep Parhar 	flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);
6533786099deSNavdeep Parhar 	flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
6534786099deSNavdeep Parhar 	flowc->mnemval[5].val = htobe32(cst->schedcl);
6535786099deSNavdeep Parhar 
6536077ba6a8SJohn Baldwin 	commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6537786099deSNavdeep Parhar 
6538786099deSNavdeep Parhar 	cst->flags &= ~EO_FLOWC_PENDING;
6539786099deSNavdeep Parhar 	cst->flags |= EO_FLOWC_RPL_PENDING;
6540786099deSNavdeep Parhar 	MPASS(cst->tx_credits >= ETID_FLOWC_LEN16);	/* flowc is first WR. */
6541786099deSNavdeep Parhar 	cst->tx_credits -= ETID_FLOWC_LEN16;
6542786099deSNavdeep Parhar 
6543786099deSNavdeep Parhar 	return (0);
6544786099deSNavdeep Parhar }
6545786099deSNavdeep Parhar 
6546786099deSNavdeep Parhar #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))
6547786099deSNavdeep Parhar 
6548786099deSNavdeep Parhar void
6549e38a50e8SJohn Baldwin send_etid_flush_wr(struct cxgbe_rate_tag *cst)
6550786099deSNavdeep Parhar {
6551786099deSNavdeep Parhar 	struct fw_flowc_wr *flowc;
6552786099deSNavdeep Parhar 	struct wrq_cookie cookie;
6553786099deSNavdeep Parhar 
6554786099deSNavdeep Parhar 	mtx_assert(&cst->lock, MA_OWNED);
6555786099deSNavdeep Parhar 
6556077ba6a8SJohn Baldwin 	flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie);
6557786099deSNavdeep Parhar 	if (__predict_false(flowc == NULL))
6558786099deSNavdeep Parhar 		CXGBE_UNIMPLEMENTED(__func__);
6559786099deSNavdeep Parhar 
6560786099deSNavdeep Parhar 	bzero(flowc, ETID_FLUSH_LEN16 * 16);
6561786099deSNavdeep Parhar 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6562786099deSNavdeep Parhar 	    V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL);
6563786099deSNavdeep Parhar 	flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) |
6564786099deSNavdeep Parhar 	    V_FW_WR_FLOWID(cst->etid));
6565786099deSNavdeep Parhar 
6566077ba6a8SJohn Baldwin 	commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6567786099deSNavdeep Parhar 
6568786099deSNavdeep Parhar 	cst->flags |= EO_FLUSH_RPL_PENDING;
6569786099deSNavdeep Parhar 	MPASS(cst->tx_credits >= ETID_FLUSH_LEN16);
6570786099deSNavdeep Parhar 	cst->tx_credits -= ETID_FLUSH_LEN16;
6571786099deSNavdeep Parhar 	cst->ncompl++;
6572786099deSNavdeep Parhar }
6573786099deSNavdeep Parhar 
6574786099deSNavdeep Parhar static void
6575e38a50e8SJohn Baldwin write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr,
6576786099deSNavdeep Parhar     struct mbuf *m0, int compl)
6577786099deSNavdeep Parhar {
6578786099deSNavdeep Parhar 	struct cpl_tx_pkt_core *cpl;
6579786099deSNavdeep Parhar 	uint64_t ctrl1;
6580786099deSNavdeep Parhar 	uint32_t ctrl;	/* used in many unrelated places */
6581786099deSNavdeep Parhar 	int len16, pktlen, nsegs, immhdrs;
6582786099deSNavdeep Parhar 	uintptr_t p;
6583786099deSNavdeep Parhar 	struct ulptx_sgl *usgl;
6584786099deSNavdeep Parhar 	struct sglist sg;
6585786099deSNavdeep Parhar 	struct sglist_seg segs[38];	/* XXX: find real limit.  XXX: get off the stack */
6586786099deSNavdeep Parhar 
6587786099deSNavdeep Parhar 	mtx_assert(&cst->lock, MA_OWNED);
6588786099deSNavdeep Parhar 	M_ASSERTPKTHDR(m0);
6589786099deSNavdeep Parhar 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
6590786099deSNavdeep Parhar 	    m0->m_pkthdr.l4hlen > 0,
6591786099deSNavdeep Parhar 	    ("%s: ethofld mbuf %p is missing header lengths", __func__, m0));
6592786099deSNavdeep Parhar 
6593786099deSNavdeep Parhar 	len16 = mbuf_eo_len16(m0);
6594786099deSNavdeep Parhar 	nsegs = mbuf_eo_nsegs(m0);
6595786099deSNavdeep Parhar 	pktlen = m0->m_pkthdr.len;
6596786099deSNavdeep Parhar 	ctrl = sizeof(struct cpl_tx_pkt_core);
6597786099deSNavdeep Parhar 	if (needs_tso(m0))
6598786099deSNavdeep Parhar 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
6599786099deSNavdeep Parhar 	immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen;
6600786099deSNavdeep Parhar 	ctrl += immhdrs;
6601786099deSNavdeep Parhar 
6602786099deSNavdeep Parhar 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) |
6603786099deSNavdeep Parhar 	    V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl));
6604786099deSNavdeep Parhar 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) |
6605786099deSNavdeep Parhar 	    V_FW_WR_FLOWID(cst->etid));
6606786099deSNavdeep Parhar 	wr->r3 = 0;
6607a4a4ad2dSNavdeep Parhar 	if (needs_outer_udp_csum(m0)) {
66086933902dSNavdeep Parhar 		wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG;
66096933902dSNavdeep Parhar 		wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen;
66106933902dSNavdeep Parhar 		wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
66116933902dSNavdeep Parhar 		wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen;
66126933902dSNavdeep Parhar 		wr->u.udpseg.rtplen = 0;
66136933902dSNavdeep Parhar 		wr->u.udpseg.r4 = 0;
66146933902dSNavdeep Parhar 		wr->u.udpseg.mss = htobe16(pktlen - immhdrs);
66156933902dSNavdeep Parhar 		wr->u.udpseg.schedpktsize = wr->u.udpseg.mss;
66166933902dSNavdeep Parhar 		wr->u.udpseg.plen = htobe32(pktlen - immhdrs);
66176933902dSNavdeep Parhar 		cpl = (void *)(wr + 1);
66186933902dSNavdeep Parhar 	} else {
6619a4a4ad2dSNavdeep Parhar 		MPASS(needs_outer_tcp_csum(m0));
6620786099deSNavdeep Parhar 		wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG;
6621786099deSNavdeep Parhar 		wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen;
6622786099deSNavdeep Parhar 		wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
6623786099deSNavdeep Parhar 		wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen;
6624786099deSNavdeep Parhar 		wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0);
6625786099deSNavdeep Parhar 		wr->u.tcpseg.r4 = 0;
6626786099deSNavdeep Parhar 		wr->u.tcpseg.r5 = 0;
6627786099deSNavdeep Parhar 		wr->u.tcpseg.plen = htobe32(pktlen - immhdrs);
6628786099deSNavdeep Parhar 
6629786099deSNavdeep Parhar 		if (needs_tso(m0)) {
6630786099deSNavdeep Parhar 			struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
6631786099deSNavdeep Parhar 
6632786099deSNavdeep Parhar 			wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz);
6633786099deSNavdeep Parhar 
66346933902dSNavdeep Parhar 			ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
66356933902dSNavdeep Parhar 			    F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
6636c0236bd9SNavdeep Parhar 			    V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen -
6637c0236bd9SNavdeep Parhar 				ETHER_HDR_LEN) >> 2) |
66386933902dSNavdeep Parhar 			    V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
66396933902dSNavdeep Parhar 			    V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
6640786099deSNavdeep Parhar 			if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
6641786099deSNavdeep Parhar 				ctrl |= F_LSO_IPV6;
6642786099deSNavdeep Parhar 			lso->lso_ctrl = htobe32(ctrl);
6643786099deSNavdeep Parhar 			lso->ipid_ofst = htobe16(0);
6644786099deSNavdeep Parhar 			lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
6645786099deSNavdeep Parhar 			lso->seqno_offset = htobe32(0);
6646786099deSNavdeep Parhar 			lso->len = htobe32(pktlen);
6647786099deSNavdeep Parhar 
6648786099deSNavdeep Parhar 			cpl = (void *)(lso + 1);
6649786099deSNavdeep Parhar 		} else {
6650786099deSNavdeep Parhar 			wr->u.tcpseg.mss = htobe16(0xffff);
6651786099deSNavdeep Parhar 			cpl = (void *)(wr + 1);
6652786099deSNavdeep Parhar 		}
66536933902dSNavdeep Parhar 	}
6654786099deSNavdeep Parhar 
6655786099deSNavdeep Parhar 	/* Checksum offload must be requested for ethofld. */
6656a4a4ad2dSNavdeep Parhar 	MPASS(needs_outer_l4_csum(m0));
6657c0236bd9SNavdeep Parhar 	ctrl1 = csum_to_ctrl(cst->adapter, m0);
6658786099deSNavdeep Parhar 
6659786099deSNavdeep Parhar 	/* VLAN tag insertion */
6660786099deSNavdeep Parhar 	if (needs_vlan_insertion(m0)) {
6661786099deSNavdeep Parhar 		ctrl1 |= F_TXPKT_VLAN_VLD |
6662786099deSNavdeep Parhar 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
6663786099deSNavdeep Parhar 	}
6664786099deSNavdeep Parhar 
6665786099deSNavdeep Parhar 	/* CPL header */
6666786099deSNavdeep Parhar 	cpl->ctrl0 = cst->ctrl0;
6667786099deSNavdeep Parhar 	cpl->pack = 0;
6668786099deSNavdeep Parhar 	cpl->len = htobe16(pktlen);
6669786099deSNavdeep Parhar 	cpl->ctrl1 = htobe64(ctrl1);
6670786099deSNavdeep Parhar 
66716933902dSNavdeep Parhar 	/* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */
6672786099deSNavdeep Parhar 	p = (uintptr_t)(cpl + 1);
6673786099deSNavdeep Parhar 	m_copydata(m0, 0, immhdrs, (void *)p);
6674786099deSNavdeep Parhar 
6675786099deSNavdeep Parhar 	/* SGL */
6676786099deSNavdeep Parhar 	if (nsegs > 0) {
6677786099deSNavdeep Parhar 		int i, pad;
6678786099deSNavdeep Parhar 
6679786099deSNavdeep Parhar 		/* zero-pad upto next 16Byte boundary, if not 16Byte aligned */
6680786099deSNavdeep Parhar 		p += immhdrs;
6681786099deSNavdeep Parhar 		pad = 16 - (immhdrs & 0xf);
6682786099deSNavdeep Parhar 		bzero((void *)p, pad);
6683786099deSNavdeep Parhar 
6684786099deSNavdeep Parhar 		usgl = (void *)(p + pad);
6685786099deSNavdeep Parhar 		usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
6686786099deSNavdeep Parhar 		    V_ULPTX_NSGE(nsegs));
6687786099deSNavdeep Parhar 
6688786099deSNavdeep Parhar 		sglist_init(&sg, nitems(segs), segs);
6689786099deSNavdeep Parhar 		for (; m0 != NULL; m0 = m0->m_next) {
6690786099deSNavdeep Parhar 			if (__predict_false(m0->m_len == 0))
6691786099deSNavdeep Parhar 				continue;
6692786099deSNavdeep Parhar 			if (immhdrs >= m0->m_len) {
6693786099deSNavdeep Parhar 				immhdrs -= m0->m_len;
6694786099deSNavdeep Parhar 				continue;
6695786099deSNavdeep Parhar 			}
66966edfd179SGleb Smirnoff 			if (m0->m_flags & M_EXTPG)
669749b6b60eSGleb Smirnoff 				sglist_append_mbuf_epg(&sg, m0,
669849b6b60eSGleb Smirnoff 				    mtod(m0, vm_offset_t), m0->m_len);
669949b6b60eSGleb Smirnoff                         else
6700786099deSNavdeep Parhar 				sglist_append(&sg, mtod(m0, char *) + immhdrs,
6701786099deSNavdeep Parhar 				    m0->m_len - immhdrs);
6702786099deSNavdeep Parhar 			immhdrs = 0;
6703786099deSNavdeep Parhar 		}
6704786099deSNavdeep Parhar 		MPASS(sg.sg_nseg == nsegs);
6705786099deSNavdeep Parhar 
6706786099deSNavdeep Parhar 		/*
6707786099deSNavdeep Parhar 		 * Zero pad last 8B in case the WR doesn't end on a 16B
6708786099deSNavdeep Parhar 		 * boundary.
6709786099deSNavdeep Parhar 		 */
6710786099deSNavdeep Parhar 		*(uint64_t *)((char *)wr + len16 * 16 - 8) = 0;
6711786099deSNavdeep Parhar 
6712786099deSNavdeep Parhar 		usgl->len0 = htobe32(segs[0].ss_len);
6713786099deSNavdeep Parhar 		usgl->addr0 = htobe64(segs[0].ss_paddr);
6714786099deSNavdeep Parhar 		for (i = 0; i < nsegs - 1; i++) {
6715786099deSNavdeep Parhar 			usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len);
6716786099deSNavdeep Parhar 			usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr);
6717786099deSNavdeep Parhar 		}
6718786099deSNavdeep Parhar 		if (i & 1)
6719786099deSNavdeep Parhar 			usgl->sge[i / 2].len[1] = htobe32(0);
6720786099deSNavdeep Parhar 	}
6721786099deSNavdeep Parhar 
6722786099deSNavdeep Parhar }
6723786099deSNavdeep Parhar 
6724786099deSNavdeep Parhar static void
6725e38a50e8SJohn Baldwin ethofld_tx(struct cxgbe_rate_tag *cst)
6726786099deSNavdeep Parhar {
6727786099deSNavdeep Parhar 	struct mbuf *m;
6728786099deSNavdeep Parhar 	struct wrq_cookie cookie;
6729786099deSNavdeep Parhar 	int next_credits, compl;
6730786099deSNavdeep Parhar 	struct fw_eth_tx_eo_wr *wr;
6731786099deSNavdeep Parhar 
6732786099deSNavdeep Parhar 	mtx_assert(&cst->lock, MA_OWNED);
6733786099deSNavdeep Parhar 
6734786099deSNavdeep Parhar 	while ((m = mbufq_first(&cst->pending_tx)) != NULL) {
6735786099deSNavdeep Parhar 		M_ASSERTPKTHDR(m);
6736786099deSNavdeep Parhar 
6737786099deSNavdeep Parhar 		/* How many len16 credits do we need to send this mbuf. */
6738786099deSNavdeep Parhar 		next_credits = mbuf_eo_len16(m);
6739786099deSNavdeep Parhar 		MPASS(next_credits > 0);
6740786099deSNavdeep Parhar 		if (next_credits > cst->tx_credits) {
6741786099deSNavdeep Parhar 			/*
6742786099deSNavdeep Parhar 			 * Tx will make progress eventually because there is at
6743786099deSNavdeep Parhar 			 * least one outstanding fw4_ack that will return
6744786099deSNavdeep Parhar 			 * credits and kick the tx.
6745786099deSNavdeep Parhar 			 */
6746786099deSNavdeep Parhar 			MPASS(cst->ncompl > 0);
6747786099deSNavdeep Parhar 			return;
6748786099deSNavdeep Parhar 		}
6749077ba6a8SJohn Baldwin 		wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie);
6750786099deSNavdeep Parhar 		if (__predict_false(wr == NULL)) {
6751786099deSNavdeep Parhar 			/* XXX: wishful thinking, not a real assertion. */
6752786099deSNavdeep Parhar 			MPASS(cst->ncompl > 0);
6753786099deSNavdeep Parhar 			return;
6754786099deSNavdeep Parhar 		}
6755786099deSNavdeep Parhar 		cst->tx_credits -= next_credits;
6756786099deSNavdeep Parhar 		cst->tx_nocompl += next_credits;
6757786099deSNavdeep Parhar 		compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2;
675856fb710fSJohn Baldwin 		ETHER_BPF_MTAP(cst->com.ifp, m);
6759786099deSNavdeep Parhar 		write_ethofld_wr(cst, wr, m, compl);
6760077ba6a8SJohn Baldwin 		commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie);
6761786099deSNavdeep Parhar 		if (compl) {
6762786099deSNavdeep Parhar 			cst->ncompl++;
6763786099deSNavdeep Parhar 			cst->tx_nocompl	= 0;
6764786099deSNavdeep Parhar 		}
6765786099deSNavdeep Parhar 		(void) mbufq_dequeue(&cst->pending_tx);
6766fb3bc596SJohn Baldwin 
6767fb3bc596SJohn Baldwin 		/*
6768fb3bc596SJohn Baldwin 		 * Drop the mbuf's reference on the tag now rather
6769fb3bc596SJohn Baldwin 		 * than waiting until m_freem().  This ensures that
6770e38a50e8SJohn Baldwin 		 * cxgbe_rate_tag_free gets called when the inp drops
6771fb3bc596SJohn Baldwin 		 * its reference on the tag and there are no more
6772fb3bc596SJohn Baldwin 		 * mbufs in the pending_tx queue and can flush any
6773fb3bc596SJohn Baldwin 		 * pending requests.  Otherwise if the last mbuf
6774fb3bc596SJohn Baldwin 		 * doesn't request a completion the etid will never be
6775fb3bc596SJohn Baldwin 		 * released.
6776fb3bc596SJohn Baldwin 		 */
6777fb3bc596SJohn Baldwin 		m->m_pkthdr.snd_tag = NULL;
6778fb3bc596SJohn Baldwin 		m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
677956fb710fSJohn Baldwin 		m_snd_tag_rele(&cst->com);
6780fb3bc596SJohn Baldwin 
6781786099deSNavdeep Parhar 		mbufq_enqueue(&cst->pending_fwack, m);
6782786099deSNavdeep Parhar 	}
6783786099deSNavdeep Parhar }
6784786099deSNavdeep Parhar 
6785786099deSNavdeep Parhar int
6786786099deSNavdeep Parhar ethofld_transmit(struct ifnet *ifp, struct mbuf *m0)
6787786099deSNavdeep Parhar {
6788e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst;
6789786099deSNavdeep Parhar 	int rc;
6790786099deSNavdeep Parhar 
6791786099deSNavdeep Parhar 	MPASS(m0->m_nextpkt == NULL);
6792fb3bc596SJohn Baldwin 	MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG);
6793786099deSNavdeep Parhar 	MPASS(m0->m_pkthdr.snd_tag != NULL);
6794e38a50e8SJohn Baldwin 	cst = mst_to_crt(m0->m_pkthdr.snd_tag);
6795786099deSNavdeep Parhar 
6796786099deSNavdeep Parhar 	mtx_lock(&cst->lock);
6797786099deSNavdeep Parhar 	MPASS(cst->flags & EO_SND_TAG_REF);
6798786099deSNavdeep Parhar 
6799786099deSNavdeep Parhar 	if (__predict_false(cst->flags & EO_FLOWC_PENDING)) {
6800786099deSNavdeep Parhar 		struct vi_info *vi = ifp->if_softc;
6801786099deSNavdeep Parhar 		struct port_info *pi = vi->pi;
6802786099deSNavdeep Parhar 		struct adapter *sc = pi->adapter;
6803786099deSNavdeep Parhar 		const uint32_t rss_mask = vi->rss_size - 1;
6804786099deSNavdeep Parhar 		uint32_t rss_hash;
6805786099deSNavdeep Parhar 
6806786099deSNavdeep Parhar 		cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];
6807786099deSNavdeep Parhar 		if (M_HASHTYPE_ISHASH(m0))
6808786099deSNavdeep Parhar 			rss_hash = m0->m_pkthdr.flowid;
6809786099deSNavdeep Parhar 		else
6810786099deSNavdeep Parhar 			rss_hash = arc4random();
6811786099deSNavdeep Parhar 		/* We assume RSS hashing */
6812786099deSNavdeep Parhar 		cst->iqid = vi->rss[rss_hash & rss_mask];
6813786099deSNavdeep Parhar 		cst->eo_txq += rss_hash % vi->nofldtxq;
6814786099deSNavdeep Parhar 		rc = send_etid_flowc_wr(cst, pi, vi);
6815786099deSNavdeep Parhar 		if (rc != 0)
6816786099deSNavdeep Parhar 			goto done;
6817786099deSNavdeep Parhar 	}
6818786099deSNavdeep Parhar 
6819786099deSNavdeep Parhar 	if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) {
6820786099deSNavdeep Parhar 		rc = ENOBUFS;
6821786099deSNavdeep Parhar 		goto done;
6822786099deSNavdeep Parhar 	}
6823786099deSNavdeep Parhar 
6824786099deSNavdeep Parhar 	mbufq_enqueue(&cst->pending_tx, m0);
6825786099deSNavdeep Parhar 	cst->plen += m0->m_pkthdr.len;
6826786099deSNavdeep Parhar 
6827fb3bc596SJohn Baldwin 	/*
6828fb3bc596SJohn Baldwin 	 * Hold an extra reference on the tag while generating work
6829fb3bc596SJohn Baldwin 	 * requests to ensure that we don't try to free the tag during
6830fb3bc596SJohn Baldwin 	 * ethofld_tx() in case we are sending the final mbuf after
6831fb3bc596SJohn Baldwin 	 * the inp was freed.
6832fb3bc596SJohn Baldwin 	 */
683356fb710fSJohn Baldwin 	m_snd_tag_ref(&cst->com);
6834786099deSNavdeep Parhar 	ethofld_tx(cst);
6835fb3bc596SJohn Baldwin 	mtx_unlock(&cst->lock);
683656fb710fSJohn Baldwin 	m_snd_tag_rele(&cst->com);
6837fb3bc596SJohn Baldwin 	return (0);
6838fb3bc596SJohn Baldwin 
6839786099deSNavdeep Parhar done:
6840786099deSNavdeep Parhar 	mtx_unlock(&cst->lock);
6841786099deSNavdeep Parhar 	if (__predict_false(rc != 0))
6842786099deSNavdeep Parhar 		m_freem(m0);
6843786099deSNavdeep Parhar 	return (rc);
6844786099deSNavdeep Parhar }
6845786099deSNavdeep Parhar 
6846786099deSNavdeep Parhar static int
6847786099deSNavdeep Parhar ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
6848786099deSNavdeep Parhar {
6849786099deSNavdeep Parhar 	struct adapter *sc = iq->adapter;
6850786099deSNavdeep Parhar 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
6851786099deSNavdeep Parhar 	struct mbuf *m;
6852786099deSNavdeep Parhar 	u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
6853e38a50e8SJohn Baldwin 	struct cxgbe_rate_tag *cst;
6854786099deSNavdeep Parhar 	uint8_t credits = cpl->credits;
6855786099deSNavdeep Parhar 
6856786099deSNavdeep Parhar 	cst = lookup_etid(sc, etid);
6857786099deSNavdeep Parhar 	mtx_lock(&cst->lock);
6858786099deSNavdeep Parhar 	if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) {
6859786099deSNavdeep Parhar 		MPASS(credits >= ETID_FLOWC_LEN16);
6860786099deSNavdeep Parhar 		credits -= ETID_FLOWC_LEN16;
6861786099deSNavdeep Parhar 		cst->flags &= ~EO_FLOWC_RPL_PENDING;
6862786099deSNavdeep Parhar 	}
6863786099deSNavdeep Parhar 
6864786099deSNavdeep Parhar 	KASSERT(cst->ncompl > 0,
6865786099deSNavdeep Parhar 	    ("%s: etid %u (%p) wasn't expecting completion.",
6866786099deSNavdeep Parhar 	    __func__, etid, cst));
6867786099deSNavdeep Parhar 	cst->ncompl--;
6868786099deSNavdeep Parhar 
6869786099deSNavdeep Parhar 	while (credits > 0) {
6870786099deSNavdeep Parhar 		m = mbufq_dequeue(&cst->pending_fwack);
6871786099deSNavdeep Parhar 		if (__predict_false(m == NULL)) {
6872786099deSNavdeep Parhar 			/*
6873786099deSNavdeep Parhar 			 * The remaining credits are for the final flush that
6874786099deSNavdeep Parhar 			 * was issued when the tag was freed by the kernel.
6875786099deSNavdeep Parhar 			 */
6876786099deSNavdeep Parhar 			MPASS((cst->flags &
6877786099deSNavdeep Parhar 			    (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) ==
6878786099deSNavdeep Parhar 			    EO_FLUSH_RPL_PENDING);
6879786099deSNavdeep Parhar 			MPASS(credits == ETID_FLUSH_LEN16);
6880786099deSNavdeep Parhar 			MPASS(cst->tx_credits + cpl->credits == cst->tx_total);
6881786099deSNavdeep Parhar 			MPASS(cst->ncompl == 0);
6882786099deSNavdeep Parhar 
6883786099deSNavdeep Parhar 			cst->flags &= ~EO_FLUSH_RPL_PENDING;
6884786099deSNavdeep Parhar 			cst->tx_credits += cpl->credits;
6885e38a50e8SJohn Baldwin 			cxgbe_rate_tag_free_locked(cst);
6886786099deSNavdeep Parhar 			return (0);	/* cst is gone. */
6887786099deSNavdeep Parhar 		}
6888786099deSNavdeep Parhar 		KASSERT(m != NULL,
6889786099deSNavdeep Parhar 		    ("%s: too many credits (%u, %u)", __func__, cpl->credits,
6890786099deSNavdeep Parhar 		    credits));
6891786099deSNavdeep Parhar 		KASSERT(credits >= mbuf_eo_len16(m),
6892786099deSNavdeep Parhar 		    ("%s: too few credits (%u, %u, %u)", __func__,
6893786099deSNavdeep Parhar 		    cpl->credits, credits, mbuf_eo_len16(m)));
6894786099deSNavdeep Parhar 		credits -= mbuf_eo_len16(m);
6895786099deSNavdeep Parhar 		cst->plen -= m->m_pkthdr.len;
6896786099deSNavdeep Parhar 		m_freem(m);
6897786099deSNavdeep Parhar 	}
6898786099deSNavdeep Parhar 
6899786099deSNavdeep Parhar 	cst->tx_credits += cpl->credits;
6900786099deSNavdeep Parhar 	MPASS(cst->tx_credits <= cst->tx_total);
6901786099deSNavdeep Parhar 
6902fb3bc596SJohn Baldwin 	if (cst->flags & EO_SND_TAG_REF) {
6903fb3bc596SJohn Baldwin 		/*
6904fb3bc596SJohn Baldwin 		 * As with ethofld_transmit(), hold an extra reference
6905fb3bc596SJohn Baldwin 		 * so that the tag is stable across ethold_tx().
6906fb3bc596SJohn Baldwin 		 */
690756fb710fSJohn Baldwin 		m_snd_tag_ref(&cst->com);
6908786099deSNavdeep Parhar 		m = mbufq_first(&cst->pending_tx);
6909786099deSNavdeep Parhar 		if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m))
6910786099deSNavdeep Parhar 			ethofld_tx(cst);
6911786099deSNavdeep Parhar 		mtx_unlock(&cst->lock);
691256fb710fSJohn Baldwin 		m_snd_tag_rele(&cst->com);
6913fb3bc596SJohn Baldwin 	} else {
6914fb3bc596SJohn Baldwin 		/*
6915fb3bc596SJohn Baldwin 		 * There shouldn't be any pending packets if the tag
6916fb3bc596SJohn Baldwin 		 * was freed by the kernel since any pending packet
6917fb3bc596SJohn Baldwin 		 * should hold a reference to the tag.
6918fb3bc596SJohn Baldwin 		 */
6919fb3bc596SJohn Baldwin 		MPASS(mbufq_first(&cst->pending_tx) == NULL);
6920fb3bc596SJohn Baldwin 		mtx_unlock(&cst->lock);
6921fb3bc596SJohn Baldwin 	}
6922786099deSNavdeep Parhar 
6923786099deSNavdeep Parhar 	return (0);
6924786099deSNavdeep Parhar }
6925786099deSNavdeep Parhar #endif
6926