154e4ee71SNavdeep Parhar /*- 2718cf2ccSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3718cf2ccSPedro F. Giffuni * 454e4ee71SNavdeep Parhar * Copyright (c) 2011 Chelsio Communications, Inc. 554e4ee71SNavdeep Parhar * All rights reserved. 654e4ee71SNavdeep Parhar * Written by: Navdeep Parhar <np@FreeBSD.org> 754e4ee71SNavdeep Parhar * 854e4ee71SNavdeep Parhar * Redistribution and use in source and binary forms, with or without 954e4ee71SNavdeep Parhar * modification, are permitted provided that the following conditions 1054e4ee71SNavdeep Parhar * are met: 1154e4ee71SNavdeep Parhar * 1. Redistributions of source code must retain the above copyright 1254e4ee71SNavdeep Parhar * notice, this list of conditions and the following disclaimer. 1354e4ee71SNavdeep Parhar * 2. Redistributions in binary form must reproduce the above copyright 1454e4ee71SNavdeep Parhar * notice, this list of conditions and the following disclaimer in the 1554e4ee71SNavdeep Parhar * documentation and/or other materials provided with the distribution. 1654e4ee71SNavdeep Parhar * 1754e4ee71SNavdeep Parhar * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1854e4ee71SNavdeep Parhar * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1954e4ee71SNavdeep Parhar * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2054e4ee71SNavdeep Parhar * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2154e4ee71SNavdeep Parhar * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2254e4ee71SNavdeep Parhar * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2354e4ee71SNavdeep Parhar * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2454e4ee71SNavdeep Parhar * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2554e4ee71SNavdeep Parhar * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2654e4ee71SNavdeep Parhar * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2754e4ee71SNavdeep Parhar * SUCH DAMAGE. 2854e4ee71SNavdeep Parhar */ 2954e4ee71SNavdeep Parhar 3054e4ee71SNavdeep Parhar #include <sys/cdefs.h> 3154e4ee71SNavdeep Parhar __FBSDID("$FreeBSD$"); 3254e4ee71SNavdeep Parhar 3354e4ee71SNavdeep Parhar #include "opt_inet.h" 34a1ea9a82SNavdeep Parhar #include "opt_inet6.h" 35bddf7343SJohn Baldwin #include "opt_kern_tls.h" 36eff62dbaSNavdeep Parhar #include "opt_ratelimit.h" 3754e4ee71SNavdeep Parhar 3854e4ee71SNavdeep Parhar #include <sys/types.h> 39c3322cb9SGleb Smirnoff #include <sys/eventhandler.h> 4054e4ee71SNavdeep Parhar #include <sys/mbuf.h> 4154e4ee71SNavdeep Parhar #include <sys/socket.h> 4254e4ee71SNavdeep Parhar #include <sys/kernel.h> 43bddf7343SJohn Baldwin #include <sys/ktls.h> 44ecb79ca4SNavdeep Parhar #include <sys/malloc.h> 4514a634dfSMark Johnston #include <sys/msan.h> 46ecb79ca4SNavdeep Parhar #include <sys/queue.h> 4738035ed6SNavdeep Parhar #include <sys/sbuf.h> 48ecb79ca4SNavdeep Parhar #include <sys/taskqueue.h> 49480e603cSNavdeep Parhar #include <sys/time.h> 507951040fSNavdeep Parhar #include <sys/sglist.h> 5154e4ee71SNavdeep Parhar #include <sys/sysctl.h> 52733b9277SNavdeep Parhar #include <sys/smp.h> 53bddf7343SJohn Baldwin #include <sys/socketvar.h> 5482eff304SNavdeep Parhar #include <sys/counter.h> 5554e4ee71SNavdeep Parhar #include <net/bpf.h> 5654e4ee71SNavdeep Parhar #include <net/ethernet.h> 5754e4ee71SNavdeep Parhar #include <net/if.h> 5854e4ee71SNavdeep Parhar #include <net/if_vlan_var.h> 59a4a4ad2dSNavdeep Parhar #include <net/if_vxlan.h> 6054e4ee71SNavdeep Parhar #include <netinet/in.h> 6154e4ee71SNavdeep Parhar #include <netinet/ip.h> 62a1ea9a82SNavdeep Parhar #include <netinet/ip6.h> 6354e4ee71SNavdeep Parhar #include <netinet/tcp.h> 64786099deSNavdeep Parhar #include <netinet/udp.h> 656af45170SJohn Baldwin #include <machine/in_cksum.h> 6664db8966SDimitry Andric #include <machine/md_var.h> 6738035ed6SNavdeep Parhar #include <vm/vm.h> 6838035ed6SNavdeep Parhar #include <vm/pmap.h> 69298d969cSNavdeep Parhar #ifdef DEV_NETMAP 70298d969cSNavdeep Parhar #include <machine/bus.h> 71298d969cSNavdeep Parhar #include <sys/selinfo.h> 72298d969cSNavdeep Parhar #include <net/if_var.h> 73298d969cSNavdeep Parhar #include <net/netmap.h> 74298d969cSNavdeep Parhar #include <dev/netmap/netmap_kern.h> 75298d969cSNavdeep Parhar #endif 7654e4ee71SNavdeep Parhar 7754e4ee71SNavdeep Parhar #include "common/common.h" 7854e4ee71SNavdeep Parhar #include "common/t4_regs.h" 7954e4ee71SNavdeep Parhar #include "common/t4_regs_values.h" 8054e4ee71SNavdeep Parhar #include "common/t4_msg.h" 81671bf2b8SNavdeep Parhar #include "t4_l2t.h" 827951040fSNavdeep Parhar #include "t4_mp_ring.h" 8354e4ee71SNavdeep Parhar 84d14b0ac1SNavdeep Parhar #ifdef T4_PKT_TIMESTAMP 85d14b0ac1SNavdeep Parhar #define RX_COPY_THRESHOLD (MINCLSIZE - 8) 86d14b0ac1SNavdeep Parhar #else 87d14b0ac1SNavdeep Parhar #define RX_COPY_THRESHOLD MINCLSIZE 88d14b0ac1SNavdeep Parhar #endif 89d14b0ac1SNavdeep Parhar 905cdaef71SJohn Baldwin /* Internal mbuf flags stored in PH_loc.eight[1]. */ 91d76bbe17SJohn Baldwin #define MC_NOMAP 0x01 925cdaef71SJohn Baldwin #define MC_RAW_WR 0x02 93bddf7343SJohn Baldwin #define MC_TLS 0x04 945cdaef71SJohn Baldwin 959fb8886bSNavdeep Parhar /* 969fb8886bSNavdeep Parhar * Ethernet frames are DMA'd at this byte offset into the freelist buffer. 979fb8886bSNavdeep Parhar * 0-7 are valid values. 989fb8886bSNavdeep Parhar */ 99518bca2cSNavdeep Parhar static int fl_pktshift = 0; 1002d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, 1012d714dbcSJohn Baldwin "payload DMA offset in rx buffer (bytes)"); 10254e4ee71SNavdeep Parhar 1039fb8886bSNavdeep Parhar /* 1049fb8886bSNavdeep Parhar * Pad ethernet payload up to this boundary. 1059fb8886bSNavdeep Parhar * -1: driver should figure out a good value. 1061458bff9SNavdeep Parhar * 0: disable padding. 1071458bff9SNavdeep Parhar * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. 1089fb8886bSNavdeep Parhar */ 109298d969cSNavdeep Parhar int fl_pad = -1; 1102d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0, 1112d714dbcSJohn Baldwin "payload pad boundary (bytes)"); 1129fb8886bSNavdeep Parhar 1139fb8886bSNavdeep Parhar /* 1149fb8886bSNavdeep Parhar * Status page length. 1159fb8886bSNavdeep Parhar * -1: driver should figure out a good value. 1169fb8886bSNavdeep Parhar * 64 or 128 are the only other valid values. 1179fb8886bSNavdeep Parhar */ 11829c229e9SJohn Baldwin static int spg_len = -1; 1192d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0, 1202d714dbcSJohn Baldwin "status page size (bytes)"); 1219fb8886bSNavdeep Parhar 1229fb8886bSNavdeep Parhar /* 1239fb8886bSNavdeep Parhar * Congestion drops. 1249fb8886bSNavdeep Parhar * -1: no congestion feedback (not recommended). 1259fb8886bSNavdeep Parhar * 0: backpressure the channel instead of dropping packets right away. 1269fb8886bSNavdeep Parhar * 1: no backpressure, drop packets for the congested queue immediately. 1279fb8886bSNavdeep Parhar */ 1289fb8886bSNavdeep Parhar static int cong_drop = 0; 1292d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0, 1302d714dbcSJohn Baldwin "Congestion control for RX queues (0 = backpressure, 1 = drop"); 13154e4ee71SNavdeep Parhar 1321458bff9SNavdeep Parhar /* 1331458bff9SNavdeep Parhar * Deliver multiple frames in the same free list buffer if they fit. 1341458bff9SNavdeep Parhar * -1: let the driver decide whether to enable buffer packing or not. 1351458bff9SNavdeep Parhar * 0: disable buffer packing. 1361458bff9SNavdeep Parhar * 1: enable buffer packing. 1371458bff9SNavdeep Parhar */ 1381458bff9SNavdeep Parhar static int buffer_packing = -1; 1392d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing, 1402d714dbcSJohn Baldwin 0, "Enable buffer packing"); 1411458bff9SNavdeep Parhar 1421458bff9SNavdeep Parhar /* 1431458bff9SNavdeep Parhar * Start next frame in a packed buffer at this boundary. 1441458bff9SNavdeep Parhar * -1: driver should figure out a good value. 145e3207e19SNavdeep Parhar * T4: driver will ignore this and use the same value as fl_pad above. 146e3207e19SNavdeep Parhar * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. 1471458bff9SNavdeep Parhar */ 1481458bff9SNavdeep Parhar static int fl_pack = -1; 1492d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0, 1502d714dbcSJohn Baldwin "payload pack boundary (bytes)"); 1511458bff9SNavdeep Parhar 15238035ed6SNavdeep Parhar /* 15338035ed6SNavdeep Parhar * Largest rx cluster size that the driver is allowed to allocate. 15438035ed6SNavdeep Parhar */ 15538035ed6SNavdeep Parhar static int largest_rx_cluster = MJUM16BYTES; 1562d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN, 1572d714dbcSJohn Baldwin &largest_rx_cluster, 0, "Largest rx cluster (bytes)"); 15838035ed6SNavdeep Parhar 15938035ed6SNavdeep Parhar /* 16038035ed6SNavdeep Parhar * Size of cluster allocation that's most likely to succeed. The driver will 16138035ed6SNavdeep Parhar * fall back to this size if it fails to allocate clusters larger than this. 16238035ed6SNavdeep Parhar */ 16338035ed6SNavdeep Parhar static int safest_rx_cluster = PAGE_SIZE; 1642d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN, 1652d714dbcSJohn Baldwin &safest_rx_cluster, 0, "Safe rx cluster (bytes)"); 16638035ed6SNavdeep Parhar 167786099deSNavdeep Parhar #ifdef RATELIMIT 168786099deSNavdeep Parhar /* 169786099deSNavdeep Parhar * Knob to control TCP timestamp rewriting, and the granularity of the tick used 170786099deSNavdeep Parhar * for rewriting. -1 and 0-3 are all valid values. 171786099deSNavdeep Parhar * -1: hardware should leave the TCP timestamps alone. 172786099deSNavdeep Parhar * 0: 1ms 173786099deSNavdeep Parhar * 1: 100us 174786099deSNavdeep Parhar * 2: 10us 175786099deSNavdeep Parhar * 3: 1us 176786099deSNavdeep Parhar */ 177786099deSNavdeep Parhar static int tsclk = -1; 1782d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0, 1792d714dbcSJohn Baldwin "Control TCP timestamp rewriting when using pacing"); 180786099deSNavdeep Parhar 181786099deSNavdeep Parhar static int eo_max_backlog = 1024 * 1024; 1822d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog, 1832d714dbcSJohn Baldwin 0, "Maximum backlog of ratelimited data per flow"); 184786099deSNavdeep Parhar #endif 185786099deSNavdeep Parhar 186d491f8caSNavdeep Parhar /* 187d491f8caSNavdeep Parhar * The interrupt holdoff timers are multiplied by this value on T6+. 188d491f8caSNavdeep Parhar * 1 and 3-17 (both inclusive) are legal values. 189d491f8caSNavdeep Parhar */ 190d491f8caSNavdeep Parhar static int tscale = 1; 1912d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0, 1922d714dbcSJohn Baldwin "Interrupt holdoff timer scale on T6+"); 193d491f8caSNavdeep Parhar 19446f48ee5SNavdeep Parhar /* 19546f48ee5SNavdeep Parhar * Number of LRO entries in the lro_ctrl structure per rx queue. 19646f48ee5SNavdeep Parhar */ 19746f48ee5SNavdeep Parhar static int lro_entries = TCP_LRO_ENTRIES; 1982d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0, 1992d714dbcSJohn Baldwin "Number of LRO entries per RX queue"); 20046f48ee5SNavdeep Parhar 20146f48ee5SNavdeep Parhar /* 20246f48ee5SNavdeep Parhar * This enables presorting of frames before they're fed into tcp_lro_rx. 20346f48ee5SNavdeep Parhar */ 20446f48ee5SNavdeep Parhar static int lro_mbufs = 0; 2052d714dbcSJohn Baldwin SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0, 2062d714dbcSJohn Baldwin "Enable presorting of LRO frames"); 20746f48ee5SNavdeep Parhar 2087054f6ecSNavdeep Parhar static counter_u64_t pullups; 2097054f6ecSNavdeep Parhar SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups, 2107054f6ecSNavdeep Parhar "Number of mbuf pullups performed"); 2117054f6ecSNavdeep Parhar 2127054f6ecSNavdeep Parhar static counter_u64_t defrags; 2137054f6ecSNavdeep Parhar SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, defrags, CTLFLAG_RD, &defrags, 2147054f6ecSNavdeep Parhar "Number of mbuf defrags performed"); 2157054f6ecSNavdeep Parhar 2163447df8bSNavdeep Parhar static int t4_tx_coalesce = 1; 2173447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce, CTLFLAG_RWTUN, &t4_tx_coalesce, 0, 2183447df8bSNavdeep Parhar "tx coalescing allowed"); 2193447df8bSNavdeep Parhar 2203447df8bSNavdeep Parhar /* 2213447df8bSNavdeep Parhar * The driver will make aggressive attempts at tx coalescing if it sees these 2223447df8bSNavdeep Parhar * many packets eligible for coalescing in quick succession, with no more than 2233447df8bSNavdeep Parhar * the specified gap in between the eth_tx calls that delivered the packets. 2243447df8bSNavdeep Parhar */ 2253447df8bSNavdeep Parhar static int t4_tx_coalesce_pkts = 32; 2263447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_pkts, CTLFLAG_RWTUN, 2273447df8bSNavdeep Parhar &t4_tx_coalesce_pkts, 0, 2283447df8bSNavdeep Parhar "# of consecutive packets (1 - 255) that will trigger tx coalescing"); 2293447df8bSNavdeep Parhar static int t4_tx_coalesce_gap = 5; 2303447df8bSNavdeep Parhar SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_gap, CTLFLAG_RWTUN, 2313447df8bSNavdeep Parhar &t4_tx_coalesce_gap, 0, "tx gap (in microseconds)"); 2327054f6ecSNavdeep Parhar 233733b9277SNavdeep Parhar static int service_iq(struct sge_iq *, int); 2343098bcfcSNavdeep Parhar static int service_iq_fl(struct sge_iq *, int); 2354d6db4e0SNavdeep Parhar static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 2361486d2deSNavdeep Parhar static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *, 2371486d2deSNavdeep Parhar u_int); 23843bbae19SNavdeep Parhar static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int, 23943bbae19SNavdeep Parhar int, int); 240e3207e19SNavdeep Parhar static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 24190e7434aSNavdeep Parhar static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, 24243bbae19SNavdeep Parhar struct sge_iq *, char *); 243fe2ebb76SJohn Baldwin static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, 24443bbae19SNavdeep Parhar struct sysctl_ctx_list *, struct sysctl_oid *); 24543bbae19SNavdeep Parhar static void free_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *); 246348694daSNavdeep Parhar static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 247348694daSNavdeep Parhar struct sge_iq *); 248aa93b99aSNavdeep Parhar static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, 249aa93b99aSNavdeep Parhar struct sysctl_oid *, struct sge_fl *); 25043bbae19SNavdeep Parhar static int alloc_iq_fl_hwq(struct vi_info *, struct sge_iq *, struct sge_fl *); 25143bbae19SNavdeep Parhar static int free_iq_fl_hwq(struct adapter *, struct sge_iq *, struct sge_fl *); 252733b9277SNavdeep Parhar static int alloc_fwq(struct adapter *); 25343bbae19SNavdeep Parhar static void free_fwq(struct adapter *); 25443bbae19SNavdeep Parhar static int alloc_ctrlq(struct adapter *, int); 25543bbae19SNavdeep Parhar static void free_ctrlq(struct adapter *, int); 25643bbae19SNavdeep Parhar static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, int); 25743bbae19SNavdeep Parhar static void free_rxq(struct vi_info *, struct sge_rxq *); 25843bbae19SNavdeep Parhar static void add_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 25943bbae19SNavdeep Parhar struct sge_rxq *); 26009fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 261fe2ebb76SJohn Baldwin static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, 26243bbae19SNavdeep Parhar int); 26343bbae19SNavdeep Parhar static void free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); 26443bbae19SNavdeep Parhar static void add_ofld_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 26543bbae19SNavdeep Parhar struct sge_ofld_rxq *); 266733b9277SNavdeep Parhar #endif 267733b9277SNavdeep Parhar static int ctrl_eq_alloc(struct adapter *, struct sge_eq *); 268fe2ebb76SJohn Baldwin static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 269eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 270fe2ebb76SJohn Baldwin static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 271733b9277SNavdeep Parhar #endif 27243bbae19SNavdeep Parhar static int alloc_eq(struct adapter *, struct sge_eq *, struct sysctl_ctx_list *, 27343bbae19SNavdeep Parhar struct sysctl_oid *); 27443bbae19SNavdeep Parhar static void free_eq(struct adapter *, struct sge_eq *); 27543bbae19SNavdeep Parhar static void add_eq_sysctls(struct adapter *, struct sysctl_ctx_list *, 27643bbae19SNavdeep Parhar struct sysctl_oid *, struct sge_eq *); 27743bbae19SNavdeep Parhar static int alloc_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *); 27843bbae19SNavdeep Parhar static int free_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *); 279fe2ebb76SJohn Baldwin static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, 28043bbae19SNavdeep Parhar struct sysctl_ctx_list *, struct sysctl_oid *); 28143bbae19SNavdeep Parhar static void free_wrq(struct adapter *, struct sge_wrq *); 28243bbae19SNavdeep Parhar static void add_wrq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 28343bbae19SNavdeep Parhar struct sge_wrq *); 28443bbae19SNavdeep Parhar static int alloc_txq(struct vi_info *, struct sge_txq *, int); 28543bbae19SNavdeep Parhar static void free_txq(struct vi_info *, struct sge_txq *); 28643bbae19SNavdeep Parhar static void add_txq_sysctls(struct vi_info *, struct sysctl_ctx_list *, 28743bbae19SNavdeep Parhar struct sysctl_oid *, struct sge_txq *); 288077ba6a8SJohn Baldwin #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 28943bbae19SNavdeep Parhar static int alloc_ofld_txq(struct vi_info *, struct sge_ofld_txq *, int); 29043bbae19SNavdeep Parhar static void free_ofld_txq(struct vi_info *, struct sge_ofld_txq *); 29143bbae19SNavdeep Parhar static void add_ofld_txq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 29243bbae19SNavdeep Parhar struct sge_ofld_txq *); 293077ba6a8SJohn Baldwin #endif 29454e4ee71SNavdeep Parhar static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); 29554e4ee71SNavdeep Parhar static inline void ring_fl_db(struct adapter *, struct sge_fl *); 296733b9277SNavdeep Parhar static int refill_fl(struct adapter *, struct sge_fl *, int); 297733b9277SNavdeep Parhar static void refill_sfl(void *); 29846e1e307SNavdeep Parhar static int find_refill_source(struct adapter *, int, bool); 299733b9277SNavdeep Parhar static void add_fl_to_sfl(struct adapter *, struct sge_fl *); 30054e4ee71SNavdeep Parhar 3017951040fSNavdeep Parhar static inline void get_pkt_gl(struct mbuf *, struct sglist *); 302a4a4ad2dSNavdeep Parhar static inline u_int txpkt_len16(u_int, const u_int); 303a4a4ad2dSNavdeep Parhar static inline u_int txpkt_vm_len16(u_int, const u_int); 30430e3f2b4SNavdeep Parhar static inline void calculate_mbuf_len16(struct mbuf *, bool); 3057951040fSNavdeep Parhar static inline u_int txpkts0_len16(u_int); 3067951040fSNavdeep Parhar static inline u_int txpkts1_len16(void); 3075cdaef71SJohn Baldwin static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int); 308d735920dSNavdeep Parhar static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *, 309d735920dSNavdeep Parhar u_int); 310472a6004SNavdeep Parhar static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, 311d735920dSNavdeep Parhar struct mbuf *); 312d735920dSNavdeep Parhar static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *, 313d735920dSNavdeep Parhar int, bool *); 314d735920dSNavdeep Parhar static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *, 315d735920dSNavdeep Parhar int, bool *); 316d735920dSNavdeep Parhar static u_int write_txpkts_wr(struct adapter *, struct sge_txq *); 317d735920dSNavdeep Parhar static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *); 3187951040fSNavdeep Parhar static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); 31954e4ee71SNavdeep Parhar static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); 3207951040fSNavdeep Parhar static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); 3217951040fSNavdeep Parhar static inline uint16_t read_hw_cidx(struct sge_eq *); 3227951040fSNavdeep Parhar static inline u_int reclaimable_tx_desc(struct sge_eq *); 3237951040fSNavdeep Parhar static inline u_int total_available_tx_desc(struct sge_eq *); 3247951040fSNavdeep Parhar static u_int reclaim_tx_descs(struct sge_txq *, u_int); 3257951040fSNavdeep Parhar static void tx_reclaim(void *, int); 3267951040fSNavdeep Parhar static __be64 get_flit(struct sglist_seg *, int, int); 327733b9277SNavdeep Parhar static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 328733b9277SNavdeep Parhar struct mbuf *); 3291b4cc91fSNavdeep Parhar static int handle_fw_msg(struct sge_iq *, const struct rss_header *, 330733b9277SNavdeep Parhar struct mbuf *); 331069af0ebSJohn Baldwin static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); 3327951040fSNavdeep Parhar static void wrq_tx_drain(void *, int); 3337951040fSNavdeep Parhar static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); 33454e4ee71SNavdeep Parhar 33538035ed6SNavdeep Parhar static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 336786099deSNavdeep Parhar #ifdef RATELIMIT 337ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 338786099deSNavdeep Parhar static inline u_int txpkt_eo_len16(u_int, u_int, u_int); 339ffbb373cSNavdeep Parhar #endif 340786099deSNavdeep Parhar static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *, 341786099deSNavdeep Parhar struct mbuf *); 342786099deSNavdeep Parhar #endif 343f7dfe243SNavdeep Parhar 34482eff304SNavdeep Parhar static counter_u64_t extfree_refs; 34582eff304SNavdeep Parhar static counter_u64_t extfree_rels; 34682eff304SNavdeep Parhar 347671bf2b8SNavdeep Parhar an_handler_t t4_an_handler; 348671bf2b8SNavdeep Parhar fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; 349671bf2b8SNavdeep Parhar cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; 3504535e804SNavdeep Parhar cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]; 3514535e804SNavdeep Parhar cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]; 352111638bfSNavdeep Parhar cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES]; 35389f651e7SNavdeep Parhar cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES]; 3549c707b32SNavdeep Parhar cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES]; 355671bf2b8SNavdeep Parhar 3564535e804SNavdeep Parhar void 357671bf2b8SNavdeep Parhar t4_register_an_handler(an_handler_t h) 358671bf2b8SNavdeep Parhar { 3594535e804SNavdeep Parhar uintptr_t *loc; 360671bf2b8SNavdeep Parhar 3614535e804SNavdeep Parhar MPASS(h == NULL || t4_an_handler == NULL); 3624535e804SNavdeep Parhar 363671bf2b8SNavdeep Parhar loc = (uintptr_t *)&t4_an_handler; 3644535e804SNavdeep Parhar atomic_store_rel_ptr(loc, (uintptr_t)h); 365671bf2b8SNavdeep Parhar } 366671bf2b8SNavdeep Parhar 3674535e804SNavdeep Parhar void 368671bf2b8SNavdeep Parhar t4_register_fw_msg_handler(int type, fw_msg_handler_t h) 369671bf2b8SNavdeep Parhar { 3704535e804SNavdeep Parhar uintptr_t *loc; 371671bf2b8SNavdeep Parhar 3724535e804SNavdeep Parhar MPASS(type < nitems(t4_fw_msg_handler)); 3734535e804SNavdeep Parhar MPASS(h == NULL || t4_fw_msg_handler[type] == NULL); 374671bf2b8SNavdeep Parhar /* 375671bf2b8SNavdeep Parhar * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 376671bf2b8SNavdeep Parhar * handler dispatch table. Reject any attempt to install a handler for 377671bf2b8SNavdeep Parhar * this subtype. 378671bf2b8SNavdeep Parhar */ 3794535e804SNavdeep Parhar MPASS(type != FW_TYPE_RSSCPL); 3804535e804SNavdeep Parhar MPASS(type != FW6_TYPE_RSSCPL); 381671bf2b8SNavdeep Parhar 382671bf2b8SNavdeep Parhar loc = (uintptr_t *)&t4_fw_msg_handler[type]; 3834535e804SNavdeep Parhar atomic_store_rel_ptr(loc, (uintptr_t)h); 3844535e804SNavdeep Parhar } 385671bf2b8SNavdeep Parhar 3864535e804SNavdeep Parhar void 3874535e804SNavdeep Parhar t4_register_cpl_handler(int opcode, cpl_handler_t h) 3884535e804SNavdeep Parhar { 3894535e804SNavdeep Parhar uintptr_t *loc; 3904535e804SNavdeep Parhar 3914535e804SNavdeep Parhar MPASS(opcode < nitems(t4_cpl_handler)); 3924535e804SNavdeep Parhar MPASS(h == NULL || t4_cpl_handler[opcode] == NULL); 3934535e804SNavdeep Parhar 3944535e804SNavdeep Parhar loc = (uintptr_t *)&t4_cpl_handler[opcode]; 3954535e804SNavdeep Parhar atomic_store_rel_ptr(loc, (uintptr_t)h); 396671bf2b8SNavdeep Parhar } 397671bf2b8SNavdeep Parhar 398671bf2b8SNavdeep Parhar static int 3994535e804SNavdeep Parhar set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 4004535e804SNavdeep Parhar struct mbuf *m) 401671bf2b8SNavdeep Parhar { 4024535e804SNavdeep Parhar const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 4034535e804SNavdeep Parhar u_int tid; 4044535e804SNavdeep Parhar int cookie; 405671bf2b8SNavdeep Parhar 4064535e804SNavdeep Parhar MPASS(m == NULL); 4074535e804SNavdeep Parhar 4084535e804SNavdeep Parhar tid = GET_TID(cpl); 4095fc0f72fSNavdeep Parhar if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) { 4104535e804SNavdeep Parhar /* 4114535e804SNavdeep Parhar * The return code for filter-write is put in the CPL cookie so 4124535e804SNavdeep Parhar * we have to rely on the hardware tid (is_ftid) to determine 4134535e804SNavdeep Parhar * that this is a response to a filter. 4144535e804SNavdeep Parhar */ 4154535e804SNavdeep Parhar cookie = CPL_COOKIE_FILTER; 4164535e804SNavdeep Parhar } else { 4174535e804SNavdeep Parhar cookie = G_COOKIE(cpl->cookie); 4184535e804SNavdeep Parhar } 4194535e804SNavdeep Parhar MPASS(cookie > CPL_COOKIE_RESERVED); 4204535e804SNavdeep Parhar MPASS(cookie < nitems(set_tcb_rpl_handlers)); 4214535e804SNavdeep Parhar 4224535e804SNavdeep Parhar return (set_tcb_rpl_handlers[cookie](iq, rss, m)); 423671bf2b8SNavdeep Parhar } 424671bf2b8SNavdeep Parhar 4254535e804SNavdeep Parhar static int 4264535e804SNavdeep Parhar l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 4274535e804SNavdeep Parhar struct mbuf *m) 428671bf2b8SNavdeep Parhar { 4294535e804SNavdeep Parhar const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); 4304535e804SNavdeep Parhar unsigned int cookie; 431671bf2b8SNavdeep Parhar 4324535e804SNavdeep Parhar MPASS(m == NULL); 433671bf2b8SNavdeep Parhar 4344535e804SNavdeep Parhar cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER; 4354535e804SNavdeep Parhar return (l2t_write_rpl_handlers[cookie](iq, rss, m)); 4364535e804SNavdeep Parhar } 437671bf2b8SNavdeep Parhar 438111638bfSNavdeep Parhar static int 439111638bfSNavdeep Parhar act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 440111638bfSNavdeep Parhar struct mbuf *m) 441111638bfSNavdeep Parhar { 442111638bfSNavdeep Parhar const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 443111638bfSNavdeep Parhar u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status))); 444111638bfSNavdeep Parhar 445111638bfSNavdeep Parhar MPASS(m == NULL); 446111638bfSNavdeep Parhar MPASS(cookie != CPL_COOKIE_RESERVED); 447111638bfSNavdeep Parhar 448111638bfSNavdeep Parhar return (act_open_rpl_handlers[cookie](iq, rss, m)); 449111638bfSNavdeep Parhar } 450111638bfSNavdeep Parhar 45189f651e7SNavdeep Parhar static int 45289f651e7SNavdeep Parhar abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, 45389f651e7SNavdeep Parhar struct mbuf *m) 45489f651e7SNavdeep Parhar { 45589f651e7SNavdeep Parhar struct adapter *sc = iq->adapter; 45689f651e7SNavdeep Parhar u_int cookie; 45789f651e7SNavdeep Parhar 45889f651e7SNavdeep Parhar MPASS(m == NULL); 45989f651e7SNavdeep Parhar if (is_hashfilter(sc)) 46089f651e7SNavdeep Parhar cookie = CPL_COOKIE_HASHFILTER; 46189f651e7SNavdeep Parhar else 46289f651e7SNavdeep Parhar cookie = CPL_COOKIE_TOM; 46389f651e7SNavdeep Parhar 46489f651e7SNavdeep Parhar return (abort_rpl_rss_handlers[cookie](iq, rss, m)); 46589f651e7SNavdeep Parhar } 46689f651e7SNavdeep Parhar 4679c707b32SNavdeep Parhar static int 4689c707b32SNavdeep Parhar fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4699c707b32SNavdeep Parhar { 4709c707b32SNavdeep Parhar struct adapter *sc = iq->adapter; 4719c707b32SNavdeep Parhar const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 4729c707b32SNavdeep Parhar unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 4739c707b32SNavdeep Parhar u_int cookie; 4749c707b32SNavdeep Parhar 4759c707b32SNavdeep Parhar MPASS(m == NULL); 4769c707b32SNavdeep Parhar if (is_etid(sc, tid)) 4779c707b32SNavdeep Parhar cookie = CPL_COOKIE_ETHOFLD; 4789c707b32SNavdeep Parhar else 4799c707b32SNavdeep Parhar cookie = CPL_COOKIE_TOM; 4809c707b32SNavdeep Parhar 4819c707b32SNavdeep Parhar return (fw4_ack_handlers[cookie](iq, rss, m)); 4829c707b32SNavdeep Parhar } 4839c707b32SNavdeep Parhar 4844535e804SNavdeep Parhar static void 4854535e804SNavdeep Parhar t4_init_shared_cpl_handlers(void) 4864535e804SNavdeep Parhar { 4874535e804SNavdeep Parhar 4884535e804SNavdeep Parhar t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler); 4894535e804SNavdeep Parhar t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler); 490111638bfSNavdeep Parhar t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler); 49189f651e7SNavdeep Parhar t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler); 4929c707b32SNavdeep Parhar t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler); 4934535e804SNavdeep Parhar } 4944535e804SNavdeep Parhar 4954535e804SNavdeep Parhar void 4964535e804SNavdeep Parhar t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie) 4974535e804SNavdeep Parhar { 4984535e804SNavdeep Parhar uintptr_t *loc; 4994535e804SNavdeep Parhar 5004535e804SNavdeep Parhar MPASS(opcode < nitems(t4_cpl_handler)); 5014535e804SNavdeep Parhar MPASS(cookie > CPL_COOKIE_RESERVED); 5024535e804SNavdeep Parhar MPASS(cookie < NUM_CPL_COOKIES); 5034535e804SNavdeep Parhar MPASS(t4_cpl_handler[opcode] != NULL); 5044535e804SNavdeep Parhar 5054535e804SNavdeep Parhar switch (opcode) { 5064535e804SNavdeep Parhar case CPL_SET_TCB_RPL: 5074535e804SNavdeep Parhar loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie]; 5084535e804SNavdeep Parhar break; 5094535e804SNavdeep Parhar case CPL_L2T_WRITE_RPL: 5104535e804SNavdeep Parhar loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie]; 5114535e804SNavdeep Parhar break; 512111638bfSNavdeep Parhar case CPL_ACT_OPEN_RPL: 513111638bfSNavdeep Parhar loc = (uintptr_t *)&act_open_rpl_handlers[cookie]; 514111638bfSNavdeep Parhar break; 51589f651e7SNavdeep Parhar case CPL_ABORT_RPL_RSS: 51689f651e7SNavdeep Parhar loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie]; 51789f651e7SNavdeep Parhar break; 5189c707b32SNavdeep Parhar case CPL_FW4_ACK: 5199c707b32SNavdeep Parhar loc = (uintptr_t *)&fw4_ack_handlers[cookie]; 5209c707b32SNavdeep Parhar break; 5214535e804SNavdeep Parhar default: 5224535e804SNavdeep Parhar MPASS(0); 5234535e804SNavdeep Parhar return; 5244535e804SNavdeep Parhar } 5254535e804SNavdeep Parhar MPASS(h == NULL || *loc == (uintptr_t)NULL); 5264535e804SNavdeep Parhar atomic_store_rel_ptr(loc, (uintptr_t)h); 527671bf2b8SNavdeep Parhar } 528671bf2b8SNavdeep Parhar 52994586193SNavdeep Parhar /* 5301458bff9SNavdeep Parhar * Called on MOD_LOAD. Validates and calculates the SGE tunables. 53194586193SNavdeep Parhar */ 53294586193SNavdeep Parhar void 53394586193SNavdeep Parhar t4_sge_modload(void) 53494586193SNavdeep Parhar { 5354defc81bSNavdeep Parhar 5369fb8886bSNavdeep Parhar if (fl_pktshift < 0 || fl_pktshift > 7) { 5379fb8886bSNavdeep Parhar printf("Invalid hw.cxgbe.fl_pktshift value (%d)," 538518bca2cSNavdeep Parhar " using 0 instead.\n", fl_pktshift); 539518bca2cSNavdeep Parhar fl_pktshift = 0; 5409fb8886bSNavdeep Parhar } 5419fb8886bSNavdeep Parhar 5429fb8886bSNavdeep Parhar if (spg_len != 64 && spg_len != 128) { 5439fb8886bSNavdeep Parhar int len; 5449fb8886bSNavdeep Parhar 5459fb8886bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__) 5469fb8886bSNavdeep Parhar len = cpu_clflush_line_size > 64 ? 128 : 64; 5479fb8886bSNavdeep Parhar #else 5489fb8886bSNavdeep Parhar len = 64; 5499fb8886bSNavdeep Parhar #endif 5509fb8886bSNavdeep Parhar if (spg_len != -1) { 5519fb8886bSNavdeep Parhar printf("Invalid hw.cxgbe.spg_len value (%d)," 5529fb8886bSNavdeep Parhar " using %d instead.\n", spg_len, len); 5539fb8886bSNavdeep Parhar } 5549fb8886bSNavdeep Parhar spg_len = len; 5559fb8886bSNavdeep Parhar } 5569fb8886bSNavdeep Parhar 5579fb8886bSNavdeep Parhar if (cong_drop < -1 || cong_drop > 1) { 5589fb8886bSNavdeep Parhar printf("Invalid hw.cxgbe.cong_drop value (%d)," 5599fb8886bSNavdeep Parhar " using 0 instead.\n", cong_drop); 5609fb8886bSNavdeep Parhar cong_drop = 0; 5619fb8886bSNavdeep Parhar } 56282eff304SNavdeep Parhar 563d491f8caSNavdeep Parhar if (tscale != 1 && (tscale < 3 || tscale > 17)) { 564d491f8caSNavdeep Parhar printf("Invalid hw.cxgbe.tscale value (%d)," 565d491f8caSNavdeep Parhar " using 1 instead.\n", tscale); 566d491f8caSNavdeep Parhar tscale = 1; 567d491f8caSNavdeep Parhar } 568d491f8caSNavdeep Parhar 5697676c62aSNavdeep Parhar if (largest_rx_cluster != MCLBYTES && 5707676c62aSNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES 5717676c62aSNavdeep Parhar largest_rx_cluster != MJUMPAGESIZE && 5727676c62aSNavdeep Parhar #endif 5737676c62aSNavdeep Parhar largest_rx_cluster != MJUM9BYTES && 5747676c62aSNavdeep Parhar largest_rx_cluster != MJUM16BYTES) { 5757676c62aSNavdeep Parhar printf("Invalid hw.cxgbe.largest_rx_cluster value (%d)," 5767676c62aSNavdeep Parhar " using %d instead.\n", largest_rx_cluster, MJUM16BYTES); 5777676c62aSNavdeep Parhar largest_rx_cluster = MJUM16BYTES; 5787676c62aSNavdeep Parhar } 5797676c62aSNavdeep Parhar 5807676c62aSNavdeep Parhar if (safest_rx_cluster != MCLBYTES && 5817676c62aSNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES 5827676c62aSNavdeep Parhar safest_rx_cluster != MJUMPAGESIZE && 5837676c62aSNavdeep Parhar #endif 5847676c62aSNavdeep Parhar safest_rx_cluster != MJUM9BYTES && 5857676c62aSNavdeep Parhar safest_rx_cluster != MJUM16BYTES) { 5867676c62aSNavdeep Parhar printf("Invalid hw.cxgbe.safest_rx_cluster value (%d)," 5877676c62aSNavdeep Parhar " using %d instead.\n", safest_rx_cluster, MJUMPAGESIZE); 5887676c62aSNavdeep Parhar safest_rx_cluster = MJUMPAGESIZE; 5897676c62aSNavdeep Parhar } 5907676c62aSNavdeep Parhar 59182eff304SNavdeep Parhar extfree_refs = counter_u64_alloc(M_WAITOK); 59282eff304SNavdeep Parhar extfree_rels = counter_u64_alloc(M_WAITOK); 5937054f6ecSNavdeep Parhar pullups = counter_u64_alloc(M_WAITOK); 5947054f6ecSNavdeep Parhar defrags = counter_u64_alloc(M_WAITOK); 59582eff304SNavdeep Parhar counter_u64_zero(extfree_refs); 59682eff304SNavdeep Parhar counter_u64_zero(extfree_rels); 5977054f6ecSNavdeep Parhar counter_u64_zero(pullups); 5987054f6ecSNavdeep Parhar counter_u64_zero(defrags); 599671bf2b8SNavdeep Parhar 6004535e804SNavdeep Parhar t4_init_shared_cpl_handlers(); 601671bf2b8SNavdeep Parhar t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); 602671bf2b8SNavdeep Parhar t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); 603671bf2b8SNavdeep Parhar t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 604786099deSNavdeep Parhar #ifdef RATELIMIT 605786099deSNavdeep Parhar t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack, 606786099deSNavdeep Parhar CPL_COOKIE_ETHOFLD); 607786099deSNavdeep Parhar #endif 608671bf2b8SNavdeep Parhar t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); 609069af0ebSJohn Baldwin t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); 61082eff304SNavdeep Parhar } 61182eff304SNavdeep Parhar 61282eff304SNavdeep Parhar void 61382eff304SNavdeep Parhar t4_sge_modunload(void) 61482eff304SNavdeep Parhar { 61582eff304SNavdeep Parhar 61682eff304SNavdeep Parhar counter_u64_free(extfree_refs); 61782eff304SNavdeep Parhar counter_u64_free(extfree_rels); 6187054f6ecSNavdeep Parhar counter_u64_free(pullups); 6197054f6ecSNavdeep Parhar counter_u64_free(defrags); 62082eff304SNavdeep Parhar } 62182eff304SNavdeep Parhar 62282eff304SNavdeep Parhar uint64_t 62382eff304SNavdeep Parhar t4_sge_extfree_refs(void) 62482eff304SNavdeep Parhar { 62582eff304SNavdeep Parhar uint64_t refs, rels; 62682eff304SNavdeep Parhar 62782eff304SNavdeep Parhar rels = counter_u64_fetch(extfree_rels); 62882eff304SNavdeep Parhar refs = counter_u64_fetch(extfree_refs); 62982eff304SNavdeep Parhar 63082eff304SNavdeep Parhar return (refs - rels); 63194586193SNavdeep Parhar } 63294586193SNavdeep Parhar 63344c6fea8SNavdeep Parhar /* max 4096 */ 63444c6fea8SNavdeep Parhar #define MAX_PACK_BOUNDARY 512 63544c6fea8SNavdeep Parhar 636e3207e19SNavdeep Parhar static inline void 637e3207e19SNavdeep Parhar setup_pad_and_pack_boundaries(struct adapter *sc) 638e3207e19SNavdeep Parhar { 639e3207e19SNavdeep Parhar uint32_t v, m; 6400dbc6cfdSNavdeep Parhar int pad, pack, pad_shift; 641e3207e19SNavdeep Parhar 6420dbc6cfdSNavdeep Parhar pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : 6430dbc6cfdSNavdeep Parhar X_INGPADBOUNDARY_SHIFT; 644e3207e19SNavdeep Parhar pad = fl_pad; 6450dbc6cfdSNavdeep Parhar if (fl_pad < (1 << pad_shift) || 6460dbc6cfdSNavdeep Parhar fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || 6470dbc6cfdSNavdeep Parhar !powerof2(fl_pad)) { 648e3207e19SNavdeep Parhar /* 649e3207e19SNavdeep Parhar * If there is any chance that we might use buffer packing and 650e3207e19SNavdeep Parhar * the chip is a T4, then pick 64 as the pad/pack boundary. Set 6510dbc6cfdSNavdeep Parhar * it to the minimum allowed in all other cases. 652e3207e19SNavdeep Parhar */ 6530dbc6cfdSNavdeep Parhar pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; 654e3207e19SNavdeep Parhar 655e3207e19SNavdeep Parhar /* 656e3207e19SNavdeep Parhar * For fl_pad = 0 we'll still write a reasonable value to the 657e3207e19SNavdeep Parhar * register but all the freelists will opt out of padding. 658e3207e19SNavdeep Parhar * We'll complain here only if the user tried to set it to a 659e3207e19SNavdeep Parhar * value greater than 0 that was invalid. 660e3207e19SNavdeep Parhar */ 661e3207e19SNavdeep Parhar if (fl_pad > 0) { 662e3207e19SNavdeep Parhar device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" 663e3207e19SNavdeep Parhar " (%d), using %d instead.\n", fl_pad, pad); 664e3207e19SNavdeep Parhar } 665e3207e19SNavdeep Parhar } 666e3207e19SNavdeep Parhar m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); 6670dbc6cfdSNavdeep Parhar v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); 668e3207e19SNavdeep Parhar t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 669e3207e19SNavdeep Parhar 670e3207e19SNavdeep Parhar if (is_t4(sc)) { 671e3207e19SNavdeep Parhar if (fl_pack != -1 && fl_pack != pad) { 672e3207e19SNavdeep Parhar /* Complain but carry on. */ 673e3207e19SNavdeep Parhar device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," 674e3207e19SNavdeep Parhar " using %d instead.\n", fl_pack, pad); 675e3207e19SNavdeep Parhar } 676e3207e19SNavdeep Parhar return; 677e3207e19SNavdeep Parhar } 678e3207e19SNavdeep Parhar 679e3207e19SNavdeep Parhar pack = fl_pack; 680e3207e19SNavdeep Parhar if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || 681e3207e19SNavdeep Parhar !powerof2(fl_pack)) { 68244c6fea8SNavdeep Parhar if (sc->params.pci.mps > MAX_PACK_BOUNDARY) 68344c6fea8SNavdeep Parhar pack = MAX_PACK_BOUNDARY; 68444c6fea8SNavdeep Parhar else 685e3207e19SNavdeep Parhar pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); 686e3207e19SNavdeep Parhar MPASS(powerof2(pack)); 687e3207e19SNavdeep Parhar if (pack < 16) 688e3207e19SNavdeep Parhar pack = 16; 689e3207e19SNavdeep Parhar if (pack == 32) 690e3207e19SNavdeep Parhar pack = 64; 691e3207e19SNavdeep Parhar if (pack > 4096) 692e3207e19SNavdeep Parhar pack = 4096; 693e3207e19SNavdeep Parhar if (fl_pack != -1) { 694e3207e19SNavdeep Parhar device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" 695e3207e19SNavdeep Parhar " (%d), using %d instead.\n", fl_pack, pack); 696e3207e19SNavdeep Parhar } 697e3207e19SNavdeep Parhar } 698e3207e19SNavdeep Parhar m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); 699e3207e19SNavdeep Parhar if (pack == 16) 700e3207e19SNavdeep Parhar v = V_INGPACKBOUNDARY(0); 701e3207e19SNavdeep Parhar else 702e3207e19SNavdeep Parhar v = V_INGPACKBOUNDARY(ilog2(pack) - 5); 703e3207e19SNavdeep Parhar 704e3207e19SNavdeep Parhar MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ 705e3207e19SNavdeep Parhar t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); 706e3207e19SNavdeep Parhar } 707e3207e19SNavdeep Parhar 708cf738022SNavdeep Parhar /* 709cf738022SNavdeep Parhar * adap->params.vpd.cclk must be set up before this is called. 710cf738022SNavdeep Parhar */ 711d14b0ac1SNavdeep Parhar void 712d14b0ac1SNavdeep Parhar t4_tweak_chip_settings(struct adapter *sc) 713d14b0ac1SNavdeep Parhar { 71446e1e307SNavdeep Parhar int i, reg; 715d14b0ac1SNavdeep Parhar uint32_t v, m; 716d14b0ac1SNavdeep Parhar int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; 717cf738022SNavdeep Parhar int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; 718d14b0ac1SNavdeep Parhar int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 719d14b0ac1SNavdeep Parhar uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 72046e1e307SNavdeep Parhar static int sw_buf_sizes[] = { 7211458bff9SNavdeep Parhar MCLBYTES, 7221458bff9SNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES 7231458bff9SNavdeep Parhar MJUMPAGESIZE, 7241458bff9SNavdeep Parhar #endif 7251458bff9SNavdeep Parhar MJUM9BYTES, 72646e1e307SNavdeep Parhar MJUM16BYTES 7271458bff9SNavdeep Parhar }; 728d14b0ac1SNavdeep Parhar 729d14b0ac1SNavdeep Parhar KASSERT(sc->flags & MASTER_PF, 730d14b0ac1SNavdeep Parhar ("%s: trying to change chip settings when not master.", __func__)); 731d14b0ac1SNavdeep Parhar 7321458bff9SNavdeep Parhar m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; 733d14b0ac1SNavdeep Parhar v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | 7344defc81bSNavdeep Parhar V_EGRSTATUSPAGESIZE(spg_len == 128); 735d14b0ac1SNavdeep Parhar t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 73654e4ee71SNavdeep Parhar 737e3207e19SNavdeep Parhar setup_pad_and_pack_boundaries(sc); 7381458bff9SNavdeep Parhar 739d14b0ac1SNavdeep Parhar v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | 740733b9277SNavdeep Parhar V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | 741733b9277SNavdeep Parhar V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | 742733b9277SNavdeep Parhar V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | 743733b9277SNavdeep Parhar V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | 744733b9277SNavdeep Parhar V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | 745733b9277SNavdeep Parhar V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | 746733b9277SNavdeep Parhar V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); 747d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); 748733b9277SNavdeep Parhar 7499b11a65dSNavdeep Parhar t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096); 7509b11a65dSNavdeep Parhar t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536); 75146e1e307SNavdeep Parhar reg = A_SGE_FL_BUFFER_SIZE2; 75246e1e307SNavdeep Parhar for (i = 0; i < nitems(sw_buf_sizes); i++) { 75346e1e307SNavdeep Parhar MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 75446e1e307SNavdeep Parhar t4_write_reg(sc, reg, sw_buf_sizes[i]); 75546e1e307SNavdeep Parhar reg += 4; 75646e1e307SNavdeep Parhar MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 75746e1e307SNavdeep Parhar t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE); 75846e1e307SNavdeep Parhar reg += 4; 75954e4ee71SNavdeep Parhar } 76054e4ee71SNavdeep Parhar 761d14b0ac1SNavdeep Parhar v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | 762d14b0ac1SNavdeep Parhar V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); 763d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); 76454e4ee71SNavdeep Parhar 765cf738022SNavdeep Parhar KASSERT(intr_timer[0] <= timer_max, 766cf738022SNavdeep Parhar ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], 767cf738022SNavdeep Parhar timer_max)); 768cf738022SNavdeep Parhar for (i = 1; i < nitems(intr_timer); i++) { 769cf738022SNavdeep Parhar KASSERT(intr_timer[i] >= intr_timer[i - 1], 770cf738022SNavdeep Parhar ("%s: timers not listed in increasing order (%d)", 771cf738022SNavdeep Parhar __func__, i)); 772cf738022SNavdeep Parhar 773cf738022SNavdeep Parhar while (intr_timer[i] > timer_max) { 774cf738022SNavdeep Parhar if (i == nitems(intr_timer) - 1) { 775cf738022SNavdeep Parhar intr_timer[i] = timer_max; 776cf738022SNavdeep Parhar break; 777cf738022SNavdeep Parhar } 778cf738022SNavdeep Parhar intr_timer[i] += intr_timer[i - 1]; 779cf738022SNavdeep Parhar intr_timer[i] /= 2; 780cf738022SNavdeep Parhar } 781cf738022SNavdeep Parhar } 782cf738022SNavdeep Parhar 783d14b0ac1SNavdeep Parhar v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | 784d14b0ac1SNavdeep Parhar V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); 785d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); 786d14b0ac1SNavdeep Parhar v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | 787d14b0ac1SNavdeep Parhar V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); 788d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); 789d14b0ac1SNavdeep Parhar v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | 790d14b0ac1SNavdeep Parhar V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); 791d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); 79286e02bf2SNavdeep Parhar 793d491f8caSNavdeep Parhar if (chip_id(sc) >= CHELSIO_T6) { 794d491f8caSNavdeep Parhar m = V_TSCALE(M_TSCALE); 795d491f8caSNavdeep Parhar if (tscale == 1) 796d491f8caSNavdeep Parhar v = 0; 797d491f8caSNavdeep Parhar else 798d491f8caSNavdeep Parhar v = V_TSCALE(tscale - 2); 799d491f8caSNavdeep Parhar t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); 8002f318252SNavdeep Parhar 8012f318252SNavdeep Parhar if (sc->debug_flags & DF_DISABLE_TCB_CACHE) { 8022f318252SNavdeep Parhar m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN | 8032f318252SNavdeep Parhar V_WRTHRTHRESH(M_WRTHRTHRESH); 8042f318252SNavdeep Parhar t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); 8052f318252SNavdeep Parhar v &= ~m; 8062f318252SNavdeep Parhar v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | 8072f318252SNavdeep Parhar V_WRTHRTHRESH(16); 8082f318252SNavdeep Parhar t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); 8092f318252SNavdeep Parhar } 810d491f8caSNavdeep Parhar } 811d491f8caSNavdeep Parhar 8127cba15b1SNavdeep Parhar /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */ 813d14b0ac1SNavdeep Parhar v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 814d14b0ac1SNavdeep Parhar t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); 815d14b0ac1SNavdeep Parhar 8167cba15b1SNavdeep Parhar /* 8177cba15b1SNavdeep Parhar * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been 8187cba15b1SNavdeep Parhar * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we 8197cba15b1SNavdeep Parhar * may have to deal with is MAXPHYS + 1 page. 8207cba15b1SNavdeep Parhar */ 8217cba15b1SNavdeep Parhar v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4); 8227cba15b1SNavdeep Parhar t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v); 8237cba15b1SNavdeep Parhar 8247cba15b1SNavdeep Parhar /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ 8257cba15b1SNavdeep Parhar m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; 826d14b0ac1SNavdeep Parhar t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); 827d14b0ac1SNavdeep Parhar 828d14b0ac1SNavdeep Parhar m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 829d14b0ac1SNavdeep Parhar F_RESETDDPOFFSET; 830d14b0ac1SNavdeep Parhar v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 831d14b0ac1SNavdeep Parhar t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); 832d14b0ac1SNavdeep Parhar } 833d14b0ac1SNavdeep Parhar 834d14b0ac1SNavdeep Parhar /* 83546e1e307SNavdeep Parhar * SGE wants the buffer to be at least 64B and then a multiple of 16. Its 83646e1e307SNavdeep Parhar * address mut be 16B aligned. If padding is in use the buffer's start and end 83746e1e307SNavdeep Parhar * need to be aligned to the pad boundary as well. We'll just make sure that 83846e1e307SNavdeep Parhar * the size is a multiple of the pad boundary here, it is up to the buffer 83946e1e307SNavdeep Parhar * allocation code to make sure the start of the buffer is aligned. 84038035ed6SNavdeep Parhar */ 84138035ed6SNavdeep Parhar static inline int 842e3207e19SNavdeep Parhar hwsz_ok(struct adapter *sc, int hwsz) 84338035ed6SNavdeep Parhar { 84490e7434aSNavdeep Parhar int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; 84538035ed6SNavdeep Parhar 846b741402cSNavdeep Parhar return (hwsz >= 64 && (hwsz & mask) == 0); 84738035ed6SNavdeep Parhar } 84838035ed6SNavdeep Parhar 84938035ed6SNavdeep Parhar /* 850fae028ddSNavdeep Parhar * Initialize the rx buffer sizes and figure out which zones the buffers will 851fae028ddSNavdeep Parhar * be allocated from. 852d14b0ac1SNavdeep Parhar */ 853fae028ddSNavdeep Parhar void 854fae028ddSNavdeep Parhar t4_init_rx_buf_info(struct adapter *sc) 855d14b0ac1SNavdeep Parhar { 856d14b0ac1SNavdeep Parhar struct sge *s = &sc->sge; 85790e7434aSNavdeep Parhar struct sge_params *sp = &sc->params.sge; 858fae028ddSNavdeep Parhar int i, j, n; 85938035ed6SNavdeep Parhar static int sw_buf_sizes[] = { /* Sorted by size */ 8601458bff9SNavdeep Parhar MCLBYTES, 8611458bff9SNavdeep Parhar #if MJUMPAGESIZE != MCLBYTES 8621458bff9SNavdeep Parhar MJUMPAGESIZE, 8631458bff9SNavdeep Parhar #endif 8641458bff9SNavdeep Parhar MJUM9BYTES, 8651458bff9SNavdeep Parhar MJUM16BYTES 8661458bff9SNavdeep Parhar }; 86746e1e307SNavdeep Parhar struct rx_buf_info *rxb; 868d14b0ac1SNavdeep Parhar 86946e1e307SNavdeep Parhar s->safe_zidx = -1; 87046e1e307SNavdeep Parhar rxb = &s->rx_buf_info[0]; 87146e1e307SNavdeep Parhar for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 87246e1e307SNavdeep Parhar rxb->size1 = sw_buf_sizes[i]; 87346e1e307SNavdeep Parhar rxb->zone = m_getzone(rxb->size1); 87446e1e307SNavdeep Parhar rxb->type = m_gettype(rxb->size1); 87546e1e307SNavdeep Parhar rxb->size2 = 0; 87646e1e307SNavdeep Parhar rxb->hwidx1 = -1; 87746e1e307SNavdeep Parhar rxb->hwidx2 = -1; 87846e1e307SNavdeep Parhar for (j = 0; j < SGE_FLBUF_SIZES; j++) { 87946e1e307SNavdeep Parhar int hwsize = sp->sge_fl_buffer_size[j]; 88038035ed6SNavdeep Parhar 88146e1e307SNavdeep Parhar if (!hwsz_ok(sc, hwsize)) 882e3207e19SNavdeep Parhar continue; 883e3207e19SNavdeep Parhar 88446e1e307SNavdeep Parhar /* hwidx for size1 */ 88546e1e307SNavdeep Parhar if (rxb->hwidx1 == -1 && rxb->size1 == hwsize) 88646e1e307SNavdeep Parhar rxb->hwidx1 = j; 88738035ed6SNavdeep Parhar 88846e1e307SNavdeep Parhar /* hwidx for size2 (buffer packing) */ 88946e1e307SNavdeep Parhar if (rxb->size1 - CL_METADATA_SIZE < hwsize) 8901458bff9SNavdeep Parhar continue; 89146e1e307SNavdeep Parhar n = rxb->size1 - hwsize - CL_METADATA_SIZE; 8921458bff9SNavdeep Parhar if (n == 0) { 89346e1e307SNavdeep Parhar rxb->hwidx2 = j; 89446e1e307SNavdeep Parhar rxb->size2 = hwsize; 89546e1e307SNavdeep Parhar break; /* stop looking */ 896733b9277SNavdeep Parhar } 89746e1e307SNavdeep Parhar if (rxb->hwidx2 != -1) { 89846e1e307SNavdeep Parhar if (n < sp->sge_fl_buffer_size[rxb->hwidx2] - 89946e1e307SNavdeep Parhar hwsize - CL_METADATA_SIZE) { 90046e1e307SNavdeep Parhar rxb->hwidx2 = j; 90146e1e307SNavdeep Parhar rxb->size2 = hwsize; 90246e1e307SNavdeep Parhar } 90346e1e307SNavdeep Parhar } else if (n <= 2 * CL_METADATA_SIZE) { 90446e1e307SNavdeep Parhar rxb->hwidx2 = j; 90546e1e307SNavdeep Parhar rxb->size2 = hwsize; 90638035ed6SNavdeep Parhar } 90738035ed6SNavdeep Parhar } 90846e1e307SNavdeep Parhar if (rxb->hwidx2 != -1) 90946e1e307SNavdeep Parhar sc->flags |= BUF_PACKING_OK; 91046e1e307SNavdeep Parhar if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster) 91146e1e307SNavdeep Parhar s->safe_zidx = i; 912e3207e19SNavdeep Parhar } 913fae028ddSNavdeep Parhar } 914fae028ddSNavdeep Parhar 915fae028ddSNavdeep Parhar /* 916fae028ddSNavdeep Parhar * Verify some basic SGE settings for the PF and VF driver, and other 917fae028ddSNavdeep Parhar * miscellaneous settings for the PF driver. 918fae028ddSNavdeep Parhar */ 919fae028ddSNavdeep Parhar int 920fae028ddSNavdeep Parhar t4_verify_chip_settings(struct adapter *sc) 921fae028ddSNavdeep Parhar { 922fae028ddSNavdeep Parhar struct sge_params *sp = &sc->params.sge; 923fae028ddSNavdeep Parhar uint32_t m, v, r; 924fae028ddSNavdeep Parhar int rc = 0; 925fae028ddSNavdeep Parhar const uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 926fae028ddSNavdeep Parhar 927fae028ddSNavdeep Parhar m = F_RXPKTCPLMODE; 928fae028ddSNavdeep Parhar v = F_RXPKTCPLMODE; 929fae028ddSNavdeep Parhar r = sp->sge_control; 930fae028ddSNavdeep Parhar if ((r & m) != v) { 931fae028ddSNavdeep Parhar device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); 932fae028ddSNavdeep Parhar rc = EINVAL; 933fae028ddSNavdeep Parhar } 934fae028ddSNavdeep Parhar 935fae028ddSNavdeep Parhar /* 936fae028ddSNavdeep Parhar * If this changes then every single use of PAGE_SHIFT in the driver 937fae028ddSNavdeep Parhar * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. 938fae028ddSNavdeep Parhar */ 939fae028ddSNavdeep Parhar if (sp->page_shift != PAGE_SHIFT) { 940fae028ddSNavdeep Parhar device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); 941fae028ddSNavdeep Parhar rc = EINVAL; 942fae028ddSNavdeep Parhar } 943733b9277SNavdeep Parhar 9446af45170SJohn Baldwin if (sc->flags & IS_VF) 9456af45170SJohn Baldwin return (0); 9466af45170SJohn Baldwin 947d14b0ac1SNavdeep Parhar v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 948d14b0ac1SNavdeep Parhar r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); 949d14b0ac1SNavdeep Parhar if (r != v) { 950d14b0ac1SNavdeep Parhar device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); 951fae028ddSNavdeep Parhar if (sc->vres.ddp.size != 0) 952d14b0ac1SNavdeep Parhar rc = EINVAL; 953d14b0ac1SNavdeep Parhar } 954733b9277SNavdeep Parhar 955d14b0ac1SNavdeep Parhar m = v = F_TDDPTAGTCB; 956d14b0ac1SNavdeep Parhar r = t4_read_reg(sc, A_ULP_RX_CTL); 957d14b0ac1SNavdeep Parhar if ((r & m) != v) { 958d14b0ac1SNavdeep Parhar device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); 959fae028ddSNavdeep Parhar if (sc->vres.ddp.size != 0) 960d14b0ac1SNavdeep Parhar rc = EINVAL; 961d14b0ac1SNavdeep Parhar } 962d14b0ac1SNavdeep Parhar 963d14b0ac1SNavdeep Parhar m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 964d14b0ac1SNavdeep Parhar F_RESETDDPOFFSET; 965d14b0ac1SNavdeep Parhar v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 966d14b0ac1SNavdeep Parhar r = t4_read_reg(sc, A_TP_PARA_REG5); 967d14b0ac1SNavdeep Parhar if ((r & m) != v) { 968d14b0ac1SNavdeep Parhar device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); 969fae028ddSNavdeep Parhar if (sc->vres.ddp.size != 0) 970d14b0ac1SNavdeep Parhar rc = EINVAL; 971d14b0ac1SNavdeep Parhar } 972d14b0ac1SNavdeep Parhar 973733b9277SNavdeep Parhar return (rc); 97454e4ee71SNavdeep Parhar } 97554e4ee71SNavdeep Parhar 97654e4ee71SNavdeep Parhar int 97754e4ee71SNavdeep Parhar t4_create_dma_tag(struct adapter *sc) 97854e4ee71SNavdeep Parhar { 97954e4ee71SNavdeep Parhar int rc; 98054e4ee71SNavdeep Parhar 98154e4ee71SNavdeep Parhar rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, 98254e4ee71SNavdeep Parhar BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, 98354e4ee71SNavdeep Parhar BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, 98454e4ee71SNavdeep Parhar NULL, &sc->dmat); 98554e4ee71SNavdeep Parhar if (rc != 0) { 98654e4ee71SNavdeep Parhar device_printf(sc->dev, 98754e4ee71SNavdeep Parhar "failed to create main DMA tag: %d\n", rc); 98854e4ee71SNavdeep Parhar } 98954e4ee71SNavdeep Parhar 99054e4ee71SNavdeep Parhar return (rc); 99154e4ee71SNavdeep Parhar } 99254e4ee71SNavdeep Parhar 9936e22f9f3SNavdeep Parhar void 9946e22f9f3SNavdeep Parhar t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 9956e22f9f3SNavdeep Parhar struct sysctl_oid_list *children) 9966e22f9f3SNavdeep Parhar { 99790e7434aSNavdeep Parhar struct sge_params *sp = &sc->params.sge; 9986e22f9f3SNavdeep Parhar 99938035ed6SNavdeep Parhar SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", 10008741306bSNavdeep Parhar CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 10017029da5cSPawel Biernacki sysctl_bufsizes, "A", "freelist buffer sizes"); 100238035ed6SNavdeep Parhar 10036e22f9f3SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, 100490e7434aSNavdeep Parhar NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); 10056e22f9f3SNavdeep Parhar 10066e22f9f3SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, 100790e7434aSNavdeep Parhar NULL, sp->pad_boundary, "payload pad boundary (bytes)"); 10086e22f9f3SNavdeep Parhar 10096e22f9f3SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, 101090e7434aSNavdeep Parhar NULL, sp->spg_len, "status page size (bytes)"); 10116e22f9f3SNavdeep Parhar 10126e22f9f3SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, 10136e22f9f3SNavdeep Parhar NULL, cong_drop, "congestion drop setting"); 10141458bff9SNavdeep Parhar 10151458bff9SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, 101690e7434aSNavdeep Parhar NULL, sp->pack_boundary, "payload pack boundary (bytes)"); 10176e22f9f3SNavdeep Parhar } 10186e22f9f3SNavdeep Parhar 101954e4ee71SNavdeep Parhar int 102054e4ee71SNavdeep Parhar t4_destroy_dma_tag(struct adapter *sc) 102154e4ee71SNavdeep Parhar { 102254e4ee71SNavdeep Parhar if (sc->dmat) 102354e4ee71SNavdeep Parhar bus_dma_tag_destroy(sc->dmat); 102454e4ee71SNavdeep Parhar 102554e4ee71SNavdeep Parhar return (0); 102654e4ee71SNavdeep Parhar } 102754e4ee71SNavdeep Parhar 102854e4ee71SNavdeep Parhar /* 102937310a98SNavdeep Parhar * Allocate and initialize the firmware event queue, control queues, and special 103037310a98SNavdeep Parhar * purpose rx queues owned by the adapter. 103154e4ee71SNavdeep Parhar * 103254e4ee71SNavdeep Parhar * Returns errno on failure. Resources allocated up to that point may still be 103354e4ee71SNavdeep Parhar * allocated. Caller is responsible for cleanup in case this function fails. 103454e4ee71SNavdeep Parhar */ 103554e4ee71SNavdeep Parhar int 1036f7dfe243SNavdeep Parhar t4_setup_adapter_queues(struct adapter *sc) 103754e4ee71SNavdeep Parhar { 103837310a98SNavdeep Parhar int rc, i; 103954e4ee71SNavdeep Parhar 104054e4ee71SNavdeep Parhar ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 104154e4ee71SNavdeep Parhar 104256599263SNavdeep Parhar /* 104356599263SNavdeep Parhar * Firmware event queue 104456599263SNavdeep Parhar */ 1045733b9277SNavdeep Parhar rc = alloc_fwq(sc); 1046aa95b653SNavdeep Parhar if (rc != 0) 1047f7dfe243SNavdeep Parhar return (rc); 1048f7dfe243SNavdeep Parhar 1049f7dfe243SNavdeep Parhar /* 105037310a98SNavdeep Parhar * That's all for the VF driver. 1051f7dfe243SNavdeep Parhar */ 105237310a98SNavdeep Parhar if (sc->flags & IS_VF) 105337310a98SNavdeep Parhar return (rc); 105437310a98SNavdeep Parhar 105537310a98SNavdeep Parhar /* 105637310a98SNavdeep Parhar * XXX: General purpose rx queues, one per port. 105737310a98SNavdeep Parhar */ 105837310a98SNavdeep Parhar 105937310a98SNavdeep Parhar /* 106037310a98SNavdeep Parhar * Control queues, one per port. 106137310a98SNavdeep Parhar */ 106237310a98SNavdeep Parhar for_each_port(sc, i) { 106343bbae19SNavdeep Parhar rc = alloc_ctrlq(sc, i); 106437310a98SNavdeep Parhar if (rc != 0) 106537310a98SNavdeep Parhar return (rc); 106637310a98SNavdeep Parhar } 106754e4ee71SNavdeep Parhar 106854e4ee71SNavdeep Parhar return (rc); 106954e4ee71SNavdeep Parhar } 107054e4ee71SNavdeep Parhar 107154e4ee71SNavdeep Parhar /* 107254e4ee71SNavdeep Parhar * Idempotent 107354e4ee71SNavdeep Parhar */ 107454e4ee71SNavdeep Parhar int 1075f7dfe243SNavdeep Parhar t4_teardown_adapter_queues(struct adapter *sc) 107654e4ee71SNavdeep Parhar { 107737310a98SNavdeep Parhar int i; 107854e4ee71SNavdeep Parhar 107954e4ee71SNavdeep Parhar ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 108054e4ee71SNavdeep Parhar 1081b99651c5SNavdeep Parhar if (sc->sge.ctrlq != NULL) { 1082b99651c5SNavdeep Parhar MPASS(!(sc->flags & IS_VF)); /* VFs don't allocate ctrlq. */ 108337310a98SNavdeep Parhar for_each_port(sc, i) 108443bbae19SNavdeep Parhar free_ctrlq(sc, i); 1085b8bfcb71SNavdeep Parhar } 1086733b9277SNavdeep Parhar free_fwq(sc); 108754e4ee71SNavdeep Parhar 108854e4ee71SNavdeep Parhar return (0); 108954e4ee71SNavdeep Parhar } 109054e4ee71SNavdeep Parhar 10916a59b994SNavdeep Parhar /* Maximum payload that could arrive with a single iq descriptor. */ 10928340ece5SNavdeep Parhar static inline int 10936a59b994SNavdeep Parhar max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld) 10948340ece5SNavdeep Parhar { 10956a59b994SNavdeep Parhar int maxp; 10968340ece5SNavdeep Parhar 109738035ed6SNavdeep Parhar /* large enough even when hw VLAN extraction is disabled */ 10986a59b994SNavdeep Parhar maxp = sc->params.sge.fl_pktshift + ETHER_HDR_LEN + 10996a59b994SNavdeep Parhar ETHER_VLAN_ENCAP_LEN + ifp->if_mtu; 11006a59b994SNavdeep Parhar if (ofld && sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS && 11016a59b994SNavdeep Parhar maxp < sc->params.tp.max_rx_pdu) 11026a59b994SNavdeep Parhar maxp = sc->params.tp.max_rx_pdu; 11036a59b994SNavdeep Parhar return (maxp); 110438035ed6SNavdeep Parhar } 11056eb3180fSNavdeep Parhar 1106733b9277SNavdeep Parhar int 1107fe2ebb76SJohn Baldwin t4_setup_vi_queues(struct vi_info *vi) 1108733b9277SNavdeep Parhar { 110943bbae19SNavdeep Parhar int rc = 0, i, intr_idx; 1110733b9277SNavdeep Parhar struct sge_rxq *rxq; 1111733b9277SNavdeep Parhar struct sge_txq *txq; 111209fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 1113733b9277SNavdeep Parhar struct sge_ofld_rxq *ofld_rxq; 1114eff62dbaSNavdeep Parhar #endif 1115eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1116077ba6a8SJohn Baldwin struct sge_ofld_txq *ofld_txq; 1117298d969cSNavdeep Parhar #endif 1118298d969cSNavdeep Parhar #ifdef DEV_NETMAP 111943bbae19SNavdeep Parhar int saved_idx, iqidx; 1120298d969cSNavdeep Parhar struct sge_nm_rxq *nm_rxq; 1121298d969cSNavdeep Parhar struct sge_nm_txq *nm_txq; 1122733b9277SNavdeep Parhar #endif 112343bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 1124fe2ebb76SJohn Baldwin struct ifnet *ifp = vi->ifp; 11256a59b994SNavdeep Parhar int maxp; 1126733b9277SNavdeep Parhar 1127733b9277SNavdeep Parhar /* Interrupt vector to start from (when using multiple vectors) */ 1128f549e352SNavdeep Parhar intr_idx = vi->first_intr; 1129fe2ebb76SJohn Baldwin 1130fe2ebb76SJohn Baldwin #ifdef DEV_NETMAP 113162291463SNavdeep Parhar saved_idx = intr_idx; 113262291463SNavdeep Parhar if (ifp->if_capabilities & IFCAP_NETMAP) { 113362291463SNavdeep Parhar 113462291463SNavdeep Parhar /* netmap is supported with direct interrupts only. */ 1135f549e352SNavdeep Parhar MPASS(!forwarding_intr_to_fwq(sc)); 113643bbae19SNavdeep Parhar MPASS(vi->first_intr >= 0); 113762291463SNavdeep Parhar 1138fe2ebb76SJohn Baldwin /* 1139fe2ebb76SJohn Baldwin * We don't have buffers to back the netmap rx queues 1140fe2ebb76SJohn Baldwin * right now so we create the queues in a way that 1141fe2ebb76SJohn Baldwin * doesn't set off any congestion signal in the chip. 1142fe2ebb76SJohn Baldwin */ 1143fe2ebb76SJohn Baldwin for_each_nm_rxq(vi, i, nm_rxq) { 114443bbae19SNavdeep Parhar rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i); 1145fe2ebb76SJohn Baldwin if (rc != 0) 1146fe2ebb76SJohn Baldwin goto done; 1147fe2ebb76SJohn Baldwin intr_idx++; 1148fe2ebb76SJohn Baldwin } 1149fe2ebb76SJohn Baldwin 1150fe2ebb76SJohn Baldwin for_each_nm_txq(vi, i, nm_txq) { 1151f549e352SNavdeep Parhar iqidx = vi->first_nm_rxq + (i % vi->nnmrxq); 115243bbae19SNavdeep Parhar rc = alloc_nm_txq(vi, nm_txq, iqidx, i); 1153fe2ebb76SJohn Baldwin if (rc != 0) 1154fe2ebb76SJohn Baldwin goto done; 1155fe2ebb76SJohn Baldwin } 1156fe2ebb76SJohn Baldwin } 115762291463SNavdeep Parhar 115862291463SNavdeep Parhar /* Normal rx queues and netmap rx queues share the same interrupts. */ 115962291463SNavdeep Parhar intr_idx = saved_idx; 1160fe2ebb76SJohn Baldwin #endif 1161733b9277SNavdeep Parhar 1162733b9277SNavdeep Parhar /* 1163f549e352SNavdeep Parhar * Allocate rx queues first because a default iqid is required when 1164f549e352SNavdeep Parhar * creating a tx queue. 1165733b9277SNavdeep Parhar */ 11666a59b994SNavdeep Parhar maxp = max_rx_payload(sc, ifp, false); 1167fe2ebb76SJohn Baldwin for_each_rxq(vi, i, rxq) { 116843bbae19SNavdeep Parhar rc = alloc_rxq(vi, rxq, i, intr_idx, maxp); 116954e4ee71SNavdeep Parhar if (rc != 0) 117054e4ee71SNavdeep Parhar goto done; 117143bbae19SNavdeep Parhar if (!forwarding_intr_to_fwq(sc)) 1172733b9277SNavdeep Parhar intr_idx++; 1173733b9277SNavdeep Parhar } 117462291463SNavdeep Parhar #ifdef DEV_NETMAP 117562291463SNavdeep Parhar if (ifp->if_capabilities & IFCAP_NETMAP) 117662291463SNavdeep Parhar intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); 117762291463SNavdeep Parhar #endif 117809fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 11796a59b994SNavdeep Parhar maxp = max_rx_payload(sc, ifp, true); 1180fe2ebb76SJohn Baldwin for_each_ofld_rxq(vi, i, ofld_rxq) { 118143bbae19SNavdeep Parhar rc = alloc_ofld_rxq(vi, ofld_rxq, i, intr_idx, maxp); 1182733b9277SNavdeep Parhar if (rc != 0) 1183733b9277SNavdeep Parhar goto done; 118443bbae19SNavdeep Parhar if (!forwarding_intr_to_fwq(sc)) 1185733b9277SNavdeep Parhar intr_idx++; 1186733b9277SNavdeep Parhar } 1187733b9277SNavdeep Parhar #endif 1188733b9277SNavdeep Parhar 1189733b9277SNavdeep Parhar /* 1190f549e352SNavdeep Parhar * Now the tx queues. 1191733b9277SNavdeep Parhar */ 1192fe2ebb76SJohn Baldwin for_each_txq(vi, i, txq) { 119343bbae19SNavdeep Parhar rc = alloc_txq(vi, txq, i); 119454e4ee71SNavdeep Parhar if (rc != 0) 119554e4ee71SNavdeep Parhar goto done; 119654e4ee71SNavdeep Parhar } 1197eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1198fe2ebb76SJohn Baldwin for_each_ofld_txq(vi, i, ofld_txq) { 119943bbae19SNavdeep Parhar rc = alloc_ofld_txq(vi, ofld_txq, i); 1200298d969cSNavdeep Parhar if (rc != 0) 1201298d969cSNavdeep Parhar goto done; 1202298d969cSNavdeep Parhar } 1203298d969cSNavdeep Parhar #endif 120454e4ee71SNavdeep Parhar done: 120554e4ee71SNavdeep Parhar if (rc) 1206fe2ebb76SJohn Baldwin t4_teardown_vi_queues(vi); 120754e4ee71SNavdeep Parhar 120854e4ee71SNavdeep Parhar return (rc); 120954e4ee71SNavdeep Parhar } 121054e4ee71SNavdeep Parhar 121154e4ee71SNavdeep Parhar /* 121254e4ee71SNavdeep Parhar * Idempotent 121354e4ee71SNavdeep Parhar */ 121454e4ee71SNavdeep Parhar int 1215fe2ebb76SJohn Baldwin t4_teardown_vi_queues(struct vi_info *vi) 121654e4ee71SNavdeep Parhar { 121754e4ee71SNavdeep Parhar int i; 121854e4ee71SNavdeep Parhar struct sge_rxq *rxq; 121954e4ee71SNavdeep Parhar struct sge_txq *txq; 122037310a98SNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1221077ba6a8SJohn Baldwin struct sge_ofld_txq *ofld_txq; 122237310a98SNavdeep Parhar #endif 122309fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 1224733b9277SNavdeep Parhar struct sge_ofld_rxq *ofld_rxq; 1225eff62dbaSNavdeep Parhar #endif 1226298d969cSNavdeep Parhar #ifdef DEV_NETMAP 1227298d969cSNavdeep Parhar struct sge_nm_rxq *nm_rxq; 1228298d969cSNavdeep Parhar struct sge_nm_txq *nm_txq; 1229298d969cSNavdeep Parhar #endif 123054e4ee71SNavdeep Parhar 1231fe2ebb76SJohn Baldwin #ifdef DEV_NETMAP 123262291463SNavdeep Parhar if (vi->ifp->if_capabilities & IFCAP_NETMAP) { 1233fe2ebb76SJohn Baldwin for_each_nm_txq(vi, i, nm_txq) { 1234fe2ebb76SJohn Baldwin free_nm_txq(vi, nm_txq); 1235fe2ebb76SJohn Baldwin } 1236fe2ebb76SJohn Baldwin 1237fe2ebb76SJohn Baldwin for_each_nm_rxq(vi, i, nm_rxq) { 1238fe2ebb76SJohn Baldwin free_nm_rxq(vi, nm_rxq); 1239fe2ebb76SJohn Baldwin } 1240fe2ebb76SJohn Baldwin } 1241fe2ebb76SJohn Baldwin #endif 1242fe2ebb76SJohn Baldwin 1243733b9277SNavdeep Parhar /* 1244733b9277SNavdeep Parhar * Take down all the tx queues first, as they reference the rx queues 1245733b9277SNavdeep Parhar * (for egress updates, etc.). 1246733b9277SNavdeep Parhar */ 1247733b9277SNavdeep Parhar 1248fe2ebb76SJohn Baldwin for_each_txq(vi, i, txq) { 1249fe2ebb76SJohn Baldwin free_txq(vi, txq); 125054e4ee71SNavdeep Parhar } 1251eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1252fe2ebb76SJohn Baldwin for_each_ofld_txq(vi, i, ofld_txq) { 1253077ba6a8SJohn Baldwin free_ofld_txq(vi, ofld_txq); 1254733b9277SNavdeep Parhar } 1255733b9277SNavdeep Parhar #endif 1256733b9277SNavdeep Parhar 1257733b9277SNavdeep Parhar /* 1258f549e352SNavdeep Parhar * Then take down the rx queues. 1259733b9277SNavdeep Parhar */ 1260733b9277SNavdeep Parhar 1261fe2ebb76SJohn Baldwin for_each_rxq(vi, i, rxq) { 1262fe2ebb76SJohn Baldwin free_rxq(vi, rxq); 126354e4ee71SNavdeep Parhar } 126409fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 1265fe2ebb76SJohn Baldwin for_each_ofld_rxq(vi, i, ofld_rxq) { 1266fe2ebb76SJohn Baldwin free_ofld_rxq(vi, ofld_rxq); 1267733b9277SNavdeep Parhar } 1268733b9277SNavdeep Parhar #endif 1269733b9277SNavdeep Parhar 127054e4ee71SNavdeep Parhar return (0); 127154e4ee71SNavdeep Parhar } 127254e4ee71SNavdeep Parhar 1273733b9277SNavdeep Parhar /* 12743098bcfcSNavdeep Parhar * Interrupt handler when the driver is using only 1 interrupt. This is a very 12753098bcfcSNavdeep Parhar * unusual scenario. 12763098bcfcSNavdeep Parhar * 12773098bcfcSNavdeep Parhar * a) Deals with errors, if any. 12783098bcfcSNavdeep Parhar * b) Services firmware event queue, which is taking interrupts for all other 12793098bcfcSNavdeep Parhar * queues. 1280733b9277SNavdeep Parhar */ 128154e4ee71SNavdeep Parhar void 128254e4ee71SNavdeep Parhar t4_intr_all(void *arg) 128354e4ee71SNavdeep Parhar { 128454e4ee71SNavdeep Parhar struct adapter *sc = arg; 1285733b9277SNavdeep Parhar struct sge_iq *fwq = &sc->sge.fwq; 128654e4ee71SNavdeep Parhar 12873098bcfcSNavdeep Parhar MPASS(sc->intr_count == 1); 12883098bcfcSNavdeep Parhar 12891dca7005SNavdeep Parhar if (sc->intr_type == INTR_INTX) 12901dca7005SNavdeep Parhar t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 12911dca7005SNavdeep Parhar 129254e4ee71SNavdeep Parhar t4_intr_err(arg); 12933098bcfcSNavdeep Parhar t4_intr_evt(fwq); 129454e4ee71SNavdeep Parhar } 129554e4ee71SNavdeep Parhar 12963098bcfcSNavdeep Parhar /* 12973098bcfcSNavdeep Parhar * Interrupt handler for errors (installed directly when multiple interrupts are 12983098bcfcSNavdeep Parhar * being used, or called by t4_intr_all). 12993098bcfcSNavdeep Parhar */ 130054e4ee71SNavdeep Parhar void 130154e4ee71SNavdeep Parhar t4_intr_err(void *arg) 130254e4ee71SNavdeep Parhar { 130354e4ee71SNavdeep Parhar struct adapter *sc = arg; 1304dd3b96ecSNavdeep Parhar uint32_t v; 1305cb7c3f12SNavdeep Parhar const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0; 130654e4ee71SNavdeep Parhar 1307cb7c3f12SNavdeep Parhar if (sc->flags & ADAP_ERR) 1308cb7c3f12SNavdeep Parhar return; 1309cb7c3f12SNavdeep Parhar 1310dd3b96ecSNavdeep Parhar v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE)); 1311dd3b96ecSNavdeep Parhar if (v & F_PFSW) { 1312dd3b96ecSNavdeep Parhar sc->swintr++; 1313dd3b96ecSNavdeep Parhar t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v); 1314dd3b96ecSNavdeep Parhar } 1315dd3b96ecSNavdeep Parhar 1316cb7c3f12SNavdeep Parhar t4_slow_intr_handler(sc, verbose); 131754e4ee71SNavdeep Parhar } 131854e4ee71SNavdeep Parhar 13193098bcfcSNavdeep Parhar /* 13203098bcfcSNavdeep Parhar * Interrupt handler for iq-only queues. The firmware event queue is the only 13213098bcfcSNavdeep Parhar * such queue right now. 13223098bcfcSNavdeep Parhar */ 132354e4ee71SNavdeep Parhar void 132454e4ee71SNavdeep Parhar t4_intr_evt(void *arg) 132554e4ee71SNavdeep Parhar { 132654e4ee71SNavdeep Parhar struct sge_iq *iq = arg; 13272be67d29SNavdeep Parhar 1328733b9277SNavdeep Parhar if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1329733b9277SNavdeep Parhar service_iq(iq, 0); 1330da6e3387SNavdeep Parhar (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 13312be67d29SNavdeep Parhar } 13322be67d29SNavdeep Parhar } 13332be67d29SNavdeep Parhar 13343098bcfcSNavdeep Parhar /* 13353098bcfcSNavdeep Parhar * Interrupt handler for iq+fl queues. 13363098bcfcSNavdeep Parhar */ 1337733b9277SNavdeep Parhar void 1338733b9277SNavdeep Parhar t4_intr(void *arg) 13392be67d29SNavdeep Parhar { 13402be67d29SNavdeep Parhar struct sge_iq *iq = arg; 1341733b9277SNavdeep Parhar 1342733b9277SNavdeep Parhar if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 13433098bcfcSNavdeep Parhar service_iq_fl(iq, 0); 1344da6e3387SNavdeep Parhar (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1345733b9277SNavdeep Parhar } 1346733b9277SNavdeep Parhar } 1347733b9277SNavdeep Parhar 13483098bcfcSNavdeep Parhar #ifdef DEV_NETMAP 13493098bcfcSNavdeep Parhar /* 13503098bcfcSNavdeep Parhar * Interrupt handler for netmap rx queues. 13513098bcfcSNavdeep Parhar */ 13523098bcfcSNavdeep Parhar void 13533098bcfcSNavdeep Parhar t4_nm_intr(void *arg) 13543098bcfcSNavdeep Parhar { 13553098bcfcSNavdeep Parhar struct sge_nm_rxq *nm_rxq = arg; 13563098bcfcSNavdeep Parhar 13573098bcfcSNavdeep Parhar if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) { 13583098bcfcSNavdeep Parhar service_nm_rxq(nm_rxq); 1359da6e3387SNavdeep Parhar (void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON); 13603098bcfcSNavdeep Parhar } 13613098bcfcSNavdeep Parhar } 13623098bcfcSNavdeep Parhar 13633098bcfcSNavdeep Parhar /* 13643098bcfcSNavdeep Parhar * Interrupt handler for vectors shared between NIC and netmap rx queues. 13653098bcfcSNavdeep Parhar */ 136662291463SNavdeep Parhar void 136762291463SNavdeep Parhar t4_vi_intr(void *arg) 136862291463SNavdeep Parhar { 136962291463SNavdeep Parhar struct irq *irq = arg; 137062291463SNavdeep Parhar 13713098bcfcSNavdeep Parhar MPASS(irq->nm_rxq != NULL); 137262291463SNavdeep Parhar t4_nm_intr(irq->nm_rxq); 13733098bcfcSNavdeep Parhar 13743098bcfcSNavdeep Parhar MPASS(irq->rxq != NULL); 137562291463SNavdeep Parhar t4_intr(irq->rxq); 137662291463SNavdeep Parhar } 13773098bcfcSNavdeep Parhar #endif 137846f48ee5SNavdeep Parhar 1379733b9277SNavdeep Parhar /* 13803098bcfcSNavdeep Parhar * Deals with interrupts on an iq-only (no freelist) queue. 1381733b9277SNavdeep Parhar */ 1382733b9277SNavdeep Parhar static int 1383733b9277SNavdeep Parhar service_iq(struct sge_iq *iq, int budget) 1384733b9277SNavdeep Parhar { 1385733b9277SNavdeep Parhar struct sge_iq *q; 138654e4ee71SNavdeep Parhar struct adapter *sc = iq->adapter; 1387b2daa9a9SNavdeep Parhar struct iq_desc *d = &iq->desc[iq->cidx]; 13884d6db4e0SNavdeep Parhar int ndescs = 0, limit; 13893098bcfcSNavdeep Parhar int rsp_type; 1390733b9277SNavdeep Parhar uint32_t lq; 1391733b9277SNavdeep Parhar STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 1392733b9277SNavdeep Parhar 1393733b9277SNavdeep Parhar KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 13943098bcfcSNavdeep Parhar KASSERT((iq->flags & IQ_HAS_FL) == 0, 13953098bcfcSNavdeep Parhar ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq, 13963098bcfcSNavdeep Parhar iq->flags)); 13973098bcfcSNavdeep Parhar MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 13983098bcfcSNavdeep Parhar MPASS((iq->flags & IQ_LRO_ENABLED) == 0); 1399733b9277SNavdeep Parhar 14004d6db4e0SNavdeep Parhar limit = budget ? budget : iq->qsize / 16; 14014d6db4e0SNavdeep Parhar 1402733b9277SNavdeep Parhar /* 1403733b9277SNavdeep Parhar * We always come back and check the descriptor ring for new indirect 1404733b9277SNavdeep Parhar * interrupts and other responses after running a single handler. 1405733b9277SNavdeep Parhar */ 1406733b9277SNavdeep Parhar for (;;) { 1407b2daa9a9SNavdeep Parhar while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 140854e4ee71SNavdeep Parhar 140954e4ee71SNavdeep Parhar rmb(); 141054e4ee71SNavdeep Parhar 1411b2daa9a9SNavdeep Parhar rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1412b2daa9a9SNavdeep Parhar lq = be32toh(d->rsp.pldbuflen_qid); 141354e4ee71SNavdeep Parhar 1414733b9277SNavdeep Parhar switch (rsp_type) { 1415733b9277SNavdeep Parhar case X_RSPD_TYPE_FLBUF: 14163098bcfcSNavdeep Parhar panic("%s: data for an iq (%p) with no freelist", 14173098bcfcSNavdeep Parhar __func__, iq); 141854e4ee71SNavdeep Parhar 14193098bcfcSNavdeep Parhar /* NOTREACHED */ 1420733b9277SNavdeep Parhar 1421733b9277SNavdeep Parhar case X_RSPD_TYPE_CPL: 1422b2daa9a9SNavdeep Parhar KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1423733b9277SNavdeep Parhar ("%s: bad opcode %02x.", __func__, 1424b2daa9a9SNavdeep Parhar d->rss.opcode)); 14253098bcfcSNavdeep Parhar t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL); 1426733b9277SNavdeep Parhar break; 1427733b9277SNavdeep Parhar 1428733b9277SNavdeep Parhar case X_RSPD_TYPE_INTR: 142998005176SNavdeep Parhar /* 143098005176SNavdeep Parhar * There are 1K interrupt-capable queues (qids 0 143198005176SNavdeep Parhar * through 1023). A response type indicating a 143298005176SNavdeep Parhar * forwarded interrupt with a qid >= 1K is an 143398005176SNavdeep Parhar * iWARP async notification. 143498005176SNavdeep Parhar */ 14353098bcfcSNavdeep Parhar if (__predict_true(lq >= 1024)) { 1436671bf2b8SNavdeep Parhar t4_an_handler(iq, &d->rsp); 143798005176SNavdeep Parhar break; 143898005176SNavdeep Parhar } 143998005176SNavdeep Parhar 1440ec55567cSJohn Baldwin q = sc->sge.iqmap[lq - sc->sge.iq_start - 1441ec55567cSJohn Baldwin sc->sge.iq_base]; 1442733b9277SNavdeep Parhar if (atomic_cmpset_int(&q->state, IQS_IDLE, 1443733b9277SNavdeep Parhar IQS_BUSY)) { 14443098bcfcSNavdeep Parhar if (service_iq_fl(q, q->qsize / 16) == 0) { 1445da6e3387SNavdeep Parhar (void) atomic_cmpset_int(&q->state, 1446733b9277SNavdeep Parhar IQS_BUSY, IQS_IDLE); 1447733b9277SNavdeep Parhar } else { 1448733b9277SNavdeep Parhar STAILQ_INSERT_TAIL(&iql, q, 1449733b9277SNavdeep Parhar link); 1450733b9277SNavdeep Parhar } 1451733b9277SNavdeep Parhar } 1452733b9277SNavdeep Parhar break; 1453733b9277SNavdeep Parhar 1454733b9277SNavdeep Parhar default: 145598005176SNavdeep Parhar KASSERT(0, 145698005176SNavdeep Parhar ("%s: illegal response type %d on iq %p", 145798005176SNavdeep Parhar __func__, rsp_type, iq)); 145898005176SNavdeep Parhar log(LOG_ERR, 145998005176SNavdeep Parhar "%s: illegal response type %d on iq %p", 146098005176SNavdeep Parhar device_get_nameunit(sc->dev), rsp_type, iq); 146109fe6320SNavdeep Parhar break; 146254e4ee71SNavdeep Parhar } 146356599263SNavdeep Parhar 1464b2daa9a9SNavdeep Parhar d++; 1465b2daa9a9SNavdeep Parhar if (__predict_false(++iq->cidx == iq->sidx)) { 1466b2daa9a9SNavdeep Parhar iq->cidx = 0; 1467b2daa9a9SNavdeep Parhar iq->gen ^= F_RSPD_GEN; 1468b2daa9a9SNavdeep Parhar d = &iq->desc[0]; 1469b2daa9a9SNavdeep Parhar } 1470b2daa9a9SNavdeep Parhar if (__predict_false(++ndescs == limit)) { 1471315048f2SJohn Baldwin t4_write_reg(sc, sc->sge_gts_reg, 1472733b9277SNavdeep Parhar V_CIDXINC(ndescs) | 1473733b9277SNavdeep Parhar V_INGRESSQID(iq->cntxt_id) | 1474733b9277SNavdeep Parhar V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1475733b9277SNavdeep Parhar ndescs = 0; 1476733b9277SNavdeep Parhar 14773098bcfcSNavdeep Parhar if (budget) { 14783098bcfcSNavdeep Parhar return (EINPROGRESS); 14793098bcfcSNavdeep Parhar } 14803098bcfcSNavdeep Parhar } 14813098bcfcSNavdeep Parhar } 14823098bcfcSNavdeep Parhar 14833098bcfcSNavdeep Parhar if (STAILQ_EMPTY(&iql)) 14843098bcfcSNavdeep Parhar break; 14853098bcfcSNavdeep Parhar 14863098bcfcSNavdeep Parhar /* 14873098bcfcSNavdeep Parhar * Process the head only, and send it to the back of the list if 14883098bcfcSNavdeep Parhar * it's still not done. 14893098bcfcSNavdeep Parhar */ 14903098bcfcSNavdeep Parhar q = STAILQ_FIRST(&iql); 14913098bcfcSNavdeep Parhar STAILQ_REMOVE_HEAD(&iql, link); 14923098bcfcSNavdeep Parhar if (service_iq_fl(q, q->qsize / 8) == 0) 1493da6e3387SNavdeep Parhar (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); 14943098bcfcSNavdeep Parhar else 14953098bcfcSNavdeep Parhar STAILQ_INSERT_TAIL(&iql, q, link); 14963098bcfcSNavdeep Parhar } 14973098bcfcSNavdeep Parhar 14983098bcfcSNavdeep Parhar t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 14993098bcfcSNavdeep Parhar V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 15003098bcfcSNavdeep Parhar 15013098bcfcSNavdeep Parhar return (0); 15023098bcfcSNavdeep Parhar } 15033098bcfcSNavdeep Parhar 1504ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 15053098bcfcSNavdeep Parhar static inline int 15063098bcfcSNavdeep Parhar sort_before_lro(struct lro_ctrl *lro) 15073098bcfcSNavdeep Parhar { 15083098bcfcSNavdeep Parhar 15093098bcfcSNavdeep Parhar return (lro->lro_mbuf_max != 0); 15103098bcfcSNavdeep Parhar } 1511ffbb373cSNavdeep Parhar #endif 15123098bcfcSNavdeep Parhar 1513e7e08444SNavdeep Parhar static inline uint64_t 1514e7e08444SNavdeep Parhar last_flit_to_ns(struct adapter *sc, uint64_t lf) 1515e7e08444SNavdeep Parhar { 1516e7e08444SNavdeep Parhar uint64_t n = be64toh(lf) & 0xfffffffffffffff; /* 60b, not 64b. */ 1517e7e08444SNavdeep Parhar 1518e7e08444SNavdeep Parhar if (n > UINT64_MAX / 1000000) 1519e7e08444SNavdeep Parhar return (n / sc->params.vpd.cclk * 1000000); 1520e7e08444SNavdeep Parhar else 1521e7e08444SNavdeep Parhar return (n * 1000000 / sc->params.vpd.cclk); 1522e7e08444SNavdeep Parhar } 1523e7e08444SNavdeep Parhar 152446e1e307SNavdeep Parhar static inline void 152546e1e307SNavdeep Parhar move_to_next_rxbuf(struct sge_fl *fl) 152646e1e307SNavdeep Parhar { 152746e1e307SNavdeep Parhar 152846e1e307SNavdeep Parhar fl->rx_offset = 0; 152946e1e307SNavdeep Parhar if (__predict_false((++fl->cidx & 7) == 0)) { 153046e1e307SNavdeep Parhar uint16_t cidx = fl->cidx >> 3; 153146e1e307SNavdeep Parhar 153246e1e307SNavdeep Parhar if (__predict_false(cidx == fl->sidx)) 153346e1e307SNavdeep Parhar fl->cidx = cidx = 0; 153446e1e307SNavdeep Parhar fl->hw_cidx = cidx; 153546e1e307SNavdeep Parhar } 153646e1e307SNavdeep Parhar } 153746e1e307SNavdeep Parhar 15383098bcfcSNavdeep Parhar /* 15393098bcfcSNavdeep Parhar * Deals with interrupts on an iq+fl queue. 15403098bcfcSNavdeep Parhar */ 15413098bcfcSNavdeep Parhar static int 15423098bcfcSNavdeep Parhar service_iq_fl(struct sge_iq *iq, int budget) 15433098bcfcSNavdeep Parhar { 15443098bcfcSNavdeep Parhar struct sge_rxq *rxq = iq_to_rxq(iq); 15453098bcfcSNavdeep Parhar struct sge_fl *fl; 15463098bcfcSNavdeep Parhar struct adapter *sc = iq->adapter; 15473098bcfcSNavdeep Parhar struct iq_desc *d = &iq->desc[iq->cidx]; 154846e1e307SNavdeep Parhar int ndescs, limit; 154946e1e307SNavdeep Parhar int rsp_type, starved; 15503098bcfcSNavdeep Parhar uint32_t lq; 15513098bcfcSNavdeep Parhar uint16_t fl_hw_cidx; 15523098bcfcSNavdeep Parhar struct mbuf *m0; 15533098bcfcSNavdeep Parhar #if defined(INET) || defined(INET6) 15543098bcfcSNavdeep Parhar const struct timeval lro_timeout = {0, sc->lro_timeout}; 15553098bcfcSNavdeep Parhar struct lro_ctrl *lro = &rxq->lro; 15563098bcfcSNavdeep Parhar #endif 15573098bcfcSNavdeep Parhar 15583098bcfcSNavdeep Parhar KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 15593098bcfcSNavdeep Parhar MPASS(iq->flags & IQ_HAS_FL); 15603098bcfcSNavdeep Parhar 156146e1e307SNavdeep Parhar ndescs = 0; 15623098bcfcSNavdeep Parhar #if defined(INET) || defined(INET6) 15633098bcfcSNavdeep Parhar if (iq->flags & IQ_ADJ_CREDIT) { 15643098bcfcSNavdeep Parhar MPASS(sort_before_lro(lro)); 15653098bcfcSNavdeep Parhar iq->flags &= ~IQ_ADJ_CREDIT; 15663098bcfcSNavdeep Parhar if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { 15673098bcfcSNavdeep Parhar tcp_lro_flush_all(lro); 15683098bcfcSNavdeep Parhar t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) | 15693098bcfcSNavdeep Parhar V_INGRESSQID((u32)iq->cntxt_id) | 15703098bcfcSNavdeep Parhar V_SEINTARM(iq->intr_params)); 15713098bcfcSNavdeep Parhar return (0); 15723098bcfcSNavdeep Parhar } 15733098bcfcSNavdeep Parhar ndescs = 1; 15743098bcfcSNavdeep Parhar } 15753098bcfcSNavdeep Parhar #else 15763098bcfcSNavdeep Parhar MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 15773098bcfcSNavdeep Parhar #endif 15783098bcfcSNavdeep Parhar 157946e1e307SNavdeep Parhar limit = budget ? budget : iq->qsize / 16; 158046e1e307SNavdeep Parhar fl = &rxq->fl; 158146e1e307SNavdeep Parhar fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ 15823098bcfcSNavdeep Parhar while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 15833098bcfcSNavdeep Parhar 15843098bcfcSNavdeep Parhar rmb(); 15853098bcfcSNavdeep Parhar 15863098bcfcSNavdeep Parhar m0 = NULL; 15873098bcfcSNavdeep Parhar rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 15883098bcfcSNavdeep Parhar lq = be32toh(d->rsp.pldbuflen_qid); 15893098bcfcSNavdeep Parhar 15903098bcfcSNavdeep Parhar switch (rsp_type) { 15913098bcfcSNavdeep Parhar case X_RSPD_TYPE_FLBUF: 159246e1e307SNavdeep Parhar if (lq & F_RSPD_NEWBUF) { 159346e1e307SNavdeep Parhar if (fl->rx_offset > 0) 159446e1e307SNavdeep Parhar move_to_next_rxbuf(fl); 159546e1e307SNavdeep Parhar lq = G_RSPD_LEN(lq); 159646e1e307SNavdeep Parhar } 159746e1e307SNavdeep Parhar if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) { 159846e1e307SNavdeep Parhar FL_LOCK(fl); 159946e1e307SNavdeep Parhar refill_fl(sc, fl, 64); 160046e1e307SNavdeep Parhar FL_UNLOCK(fl); 160146e1e307SNavdeep Parhar fl_hw_cidx = fl->hw_cidx; 160246e1e307SNavdeep Parhar } 16033098bcfcSNavdeep Parhar 16041486d2deSNavdeep Parhar if (d->rss.opcode == CPL_RX_PKT) { 16051486d2deSNavdeep Parhar if (__predict_true(eth_rx(sc, rxq, d, lq) == 0)) 16061486d2deSNavdeep Parhar break; 16071486d2deSNavdeep Parhar goto out; 16081486d2deSNavdeep Parhar } 16093098bcfcSNavdeep Parhar m0 = get_fl_payload(sc, fl, lq); 16103098bcfcSNavdeep Parhar if (__predict_false(m0 == NULL)) 16113098bcfcSNavdeep Parhar goto out; 1612e7e08444SNavdeep Parhar 16133098bcfcSNavdeep Parhar /* fall through */ 16143098bcfcSNavdeep Parhar 16153098bcfcSNavdeep Parhar case X_RSPD_TYPE_CPL: 16163098bcfcSNavdeep Parhar KASSERT(d->rss.opcode < NUM_CPL_CMDS, 16173098bcfcSNavdeep Parhar ("%s: bad opcode %02x.", __func__, d->rss.opcode)); 16183098bcfcSNavdeep Parhar t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); 16193098bcfcSNavdeep Parhar break; 16203098bcfcSNavdeep Parhar 16213098bcfcSNavdeep Parhar case X_RSPD_TYPE_INTR: 16223098bcfcSNavdeep Parhar 16233098bcfcSNavdeep Parhar /* 16243098bcfcSNavdeep Parhar * There are 1K interrupt-capable queues (qids 0 16253098bcfcSNavdeep Parhar * through 1023). A response type indicating a 16263098bcfcSNavdeep Parhar * forwarded interrupt with a qid >= 1K is an 16273098bcfcSNavdeep Parhar * iWARP async notification. That is the only 16283098bcfcSNavdeep Parhar * acceptable indirect interrupt on this queue. 16293098bcfcSNavdeep Parhar */ 16303098bcfcSNavdeep Parhar if (__predict_false(lq < 1024)) { 16313098bcfcSNavdeep Parhar panic("%s: indirect interrupt on iq_fl %p " 16323098bcfcSNavdeep Parhar "with qid %u", __func__, iq, lq); 16333098bcfcSNavdeep Parhar } 16343098bcfcSNavdeep Parhar 16353098bcfcSNavdeep Parhar t4_an_handler(iq, &d->rsp); 16363098bcfcSNavdeep Parhar break; 16373098bcfcSNavdeep Parhar 16383098bcfcSNavdeep Parhar default: 16393098bcfcSNavdeep Parhar KASSERT(0, ("%s: illegal response type %d on iq %p", 16403098bcfcSNavdeep Parhar __func__, rsp_type, iq)); 16413098bcfcSNavdeep Parhar log(LOG_ERR, "%s: illegal response type %d on iq %p", 16423098bcfcSNavdeep Parhar device_get_nameunit(sc->dev), rsp_type, iq); 16433098bcfcSNavdeep Parhar break; 16443098bcfcSNavdeep Parhar } 16453098bcfcSNavdeep Parhar 16463098bcfcSNavdeep Parhar d++; 16473098bcfcSNavdeep Parhar if (__predict_false(++iq->cidx == iq->sidx)) { 16483098bcfcSNavdeep Parhar iq->cidx = 0; 16493098bcfcSNavdeep Parhar iq->gen ^= F_RSPD_GEN; 16503098bcfcSNavdeep Parhar d = &iq->desc[0]; 16513098bcfcSNavdeep Parhar } 16523098bcfcSNavdeep Parhar if (__predict_false(++ndescs == limit)) { 16533098bcfcSNavdeep Parhar t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 16543098bcfcSNavdeep Parhar V_INGRESSQID(iq->cntxt_id) | 16553098bcfcSNavdeep Parhar V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 16563098bcfcSNavdeep Parhar 1657480e603cSNavdeep Parhar #if defined(INET) || defined(INET6) 1658480e603cSNavdeep Parhar if (iq->flags & IQ_LRO_ENABLED && 165946f48ee5SNavdeep Parhar !sort_before_lro(lro) && 1660480e603cSNavdeep Parhar sc->lro_timeout != 0) { 16613098bcfcSNavdeep Parhar tcp_lro_flush_inactive(lro, &lro_timeout); 1662480e603cSNavdeep Parhar } 1663480e603cSNavdeep Parhar #endif 166446e1e307SNavdeep Parhar if (budget) 1665733b9277SNavdeep Parhar return (EINPROGRESS); 166646e1e307SNavdeep Parhar ndescs = 0; 16674d6db4e0SNavdeep Parhar } 1668861e42b2SNavdeep Parhar } 16693098bcfcSNavdeep Parhar out: 1670a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6) 1671733b9277SNavdeep Parhar if (iq->flags & IQ_LRO_ENABLED) { 167246f48ee5SNavdeep Parhar if (ndescs > 0 && lro->lro_mbuf_count > 8) { 167346f48ee5SNavdeep Parhar MPASS(sort_before_lro(lro)); 167446f48ee5SNavdeep Parhar /* hold back one credit and don't flush LRO state */ 167546f48ee5SNavdeep Parhar iq->flags |= IQ_ADJ_CREDIT; 167646f48ee5SNavdeep Parhar ndescs--; 167746f48ee5SNavdeep Parhar } else { 16786dd38b87SSepherosa Ziehau tcp_lro_flush_all(lro); 1679733b9277SNavdeep Parhar } 168046f48ee5SNavdeep Parhar } 1681733b9277SNavdeep Parhar #endif 1682733b9277SNavdeep Parhar 1683315048f2SJohn Baldwin t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1684733b9277SNavdeep Parhar V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1685733b9277SNavdeep Parhar 1686733b9277SNavdeep Parhar FL_LOCK(fl); 168738035ed6SNavdeep Parhar starved = refill_fl(sc, fl, 64); 1688733b9277SNavdeep Parhar FL_UNLOCK(fl); 1689733b9277SNavdeep Parhar if (__predict_false(starved != 0)) 1690733b9277SNavdeep Parhar add_fl_to_sfl(sc, fl); 1691733b9277SNavdeep Parhar 1692733b9277SNavdeep Parhar return (0); 1693733b9277SNavdeep Parhar } 1694733b9277SNavdeep Parhar 169538035ed6SNavdeep Parhar static inline struct cluster_metadata * 169646e1e307SNavdeep Parhar cl_metadata(struct fl_sdesc *sd) 16971458bff9SNavdeep Parhar { 16981458bff9SNavdeep Parhar 169946e1e307SNavdeep Parhar return ((void *)(sd->cl + sd->moff)); 17001458bff9SNavdeep Parhar } 17011458bff9SNavdeep Parhar 170215c28f87SGleb Smirnoff static void 1703e8fd18f3SGleb Smirnoff rxb_free(struct mbuf *m) 17041458bff9SNavdeep Parhar { 1705d6f79b27SNavdeep Parhar struct cluster_metadata *clm = m->m_ext.ext_arg1; 17061458bff9SNavdeep Parhar 1707d6f79b27SNavdeep Parhar uma_zfree(clm->zone, clm->cl); 170882eff304SNavdeep Parhar counter_u64_add(extfree_rels, 1); 17091458bff9SNavdeep Parhar } 17101458bff9SNavdeep Parhar 171138035ed6SNavdeep Parhar /* 171246e1e307SNavdeep Parhar * The mbuf returned comes from zone_muf and carries the payload in one of these 171346e1e307SNavdeep Parhar * ways 171446e1e307SNavdeep Parhar * a) complete frame inside the mbuf 171546e1e307SNavdeep Parhar * b) m_cljset (for clusters without metadata) 171646e1e307SNavdeep Parhar * d) m_extaddref (cluster with metadata) 171738035ed6SNavdeep Parhar */ 17181458bff9SNavdeep Parhar static struct mbuf * 1719b741402cSNavdeep Parhar get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1720b741402cSNavdeep Parhar int remaining) 172138035ed6SNavdeep Parhar { 172238035ed6SNavdeep Parhar struct mbuf *m; 172338035ed6SNavdeep Parhar struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 172446e1e307SNavdeep Parhar struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 172546e1e307SNavdeep Parhar struct cluster_metadata *clm; 1726b741402cSNavdeep Parhar int len, blen; 172738035ed6SNavdeep Parhar caddr_t payload; 172838035ed6SNavdeep Parhar 1729e3207e19SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 173046e1e307SNavdeep Parhar u_int l, pad; 1731b741402cSNavdeep Parhar 173246e1e307SNavdeep Parhar blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 173346e1e307SNavdeep Parhar len = min(remaining, blen); 173446e1e307SNavdeep Parhar payload = sd->cl + fl->rx_offset; 173546e1e307SNavdeep Parhar 173646e1e307SNavdeep Parhar l = fr_offset + len; 173746e1e307SNavdeep Parhar pad = roundup2(l, fl->buf_boundary) - l; 173846e1e307SNavdeep Parhar if (fl->rx_offset + len + pad < rxb->size2) 1739b741402cSNavdeep Parhar blen = len + pad; 174046e1e307SNavdeep Parhar MPASS(fl->rx_offset + blen <= rxb->size2); 1741e3207e19SNavdeep Parhar } else { 1742e3207e19SNavdeep Parhar MPASS(fl->rx_offset == 0); /* not packing */ 174346e1e307SNavdeep Parhar blen = rxb->size1; 174446e1e307SNavdeep Parhar len = min(remaining, blen); 174546e1e307SNavdeep Parhar payload = sd->cl; 1746e3207e19SNavdeep Parhar } 174738035ed6SNavdeep Parhar 174846e1e307SNavdeep Parhar if (fr_offset == 0) { 174946e1e307SNavdeep Parhar m = m_gethdr(M_NOWAIT, MT_DATA); 175046e1e307SNavdeep Parhar if (__predict_false(m == NULL)) 175146e1e307SNavdeep Parhar return (NULL); 175246e1e307SNavdeep Parhar m->m_pkthdr.len = remaining; 175346e1e307SNavdeep Parhar } else { 175446e1e307SNavdeep Parhar m = m_get(M_NOWAIT, MT_DATA); 175546e1e307SNavdeep Parhar if (__predict_false(m == NULL)) 175646e1e307SNavdeep Parhar return (NULL); 175746e1e307SNavdeep Parhar } 175846e1e307SNavdeep Parhar m->m_len = len; 175914a634dfSMark Johnston kmsan_mark(payload, len, KMSAN_STATE_INITED); 1760b741402cSNavdeep Parhar 176138035ed6SNavdeep Parhar if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { 176238035ed6SNavdeep Parhar /* copy data to mbuf */ 176338035ed6SNavdeep Parhar bcopy(payload, mtod(m, caddr_t), len); 176446e1e307SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 176546e1e307SNavdeep Parhar fl->rx_offset += blen; 176646e1e307SNavdeep Parhar MPASS(fl->rx_offset <= rxb->size2); 176746e1e307SNavdeep Parhar if (fl->rx_offset < rxb->size2) 176846e1e307SNavdeep Parhar return (m); /* without advancing the cidx */ 176946e1e307SNavdeep Parhar } 177046e1e307SNavdeep Parhar } else if (fl->flags & FL_BUF_PACKING) { 177146e1e307SNavdeep Parhar clm = cl_metadata(sd); 1772a9c4062aSNavdeep Parhar if (sd->nmbuf++ == 0) { 1773a9c4062aSNavdeep Parhar clm->refcount = 1; 177446e1e307SNavdeep Parhar clm->zone = rxb->zone; 1775d6f79b27SNavdeep Parhar clm->cl = sd->cl; 1776a9c4062aSNavdeep Parhar counter_u64_add(extfree_refs, 1); 1777a9c4062aSNavdeep Parhar } 1778d6f79b27SNavdeep Parhar m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm, 1779d6f79b27SNavdeep Parhar NULL); 178038035ed6SNavdeep Parhar 178146e1e307SNavdeep Parhar fl->rx_offset += blen; 178246e1e307SNavdeep Parhar MPASS(fl->rx_offset <= rxb->size2); 178346e1e307SNavdeep Parhar if (fl->rx_offset < rxb->size2) 178446e1e307SNavdeep Parhar return (m); /* without advancing the cidx */ 1785ccc69b2fSNavdeep Parhar } else { 178646e1e307SNavdeep Parhar m_cljset(m, sd->cl, rxb->type); 178738035ed6SNavdeep Parhar sd->cl = NULL; /* consumed, not a recycle candidate */ 178838035ed6SNavdeep Parhar } 178938035ed6SNavdeep Parhar 179046e1e307SNavdeep Parhar move_to_next_rxbuf(fl); 179138035ed6SNavdeep Parhar 179238035ed6SNavdeep Parhar return (m); 179338035ed6SNavdeep Parhar } 179438035ed6SNavdeep Parhar 179538035ed6SNavdeep Parhar static struct mbuf * 179646e1e307SNavdeep Parhar get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen) 17971458bff9SNavdeep Parhar { 179838035ed6SNavdeep Parhar struct mbuf *m0, *m, **pnext; 1799b741402cSNavdeep Parhar u_int remaining; 18001458bff9SNavdeep Parhar 18014d6db4e0SNavdeep Parhar if (__predict_false(fl->flags & FL_BUF_RESUME)) { 1802368541baSNavdeep Parhar M_ASSERTPKTHDR(fl->m0); 180346e1e307SNavdeep Parhar MPASS(fl->m0->m_pkthdr.len == plen); 180446e1e307SNavdeep Parhar MPASS(fl->remaining < plen); 18051458bff9SNavdeep Parhar 180638035ed6SNavdeep Parhar m0 = fl->m0; 180738035ed6SNavdeep Parhar pnext = fl->pnext; 1808b741402cSNavdeep Parhar remaining = fl->remaining; 18094d6db4e0SNavdeep Parhar fl->flags &= ~FL_BUF_RESUME; 181038035ed6SNavdeep Parhar goto get_segment; 18111458bff9SNavdeep Parhar } 18121458bff9SNavdeep Parhar 18131458bff9SNavdeep Parhar /* 181438035ed6SNavdeep Parhar * Payload starts at rx_offset in the current hw buffer. Its length is 181538035ed6SNavdeep Parhar * 'len' and it may span multiple hw buffers. 18161458bff9SNavdeep Parhar */ 18171458bff9SNavdeep Parhar 181846e1e307SNavdeep Parhar m0 = get_scatter_segment(sc, fl, 0, plen); 1819368541baSNavdeep Parhar if (m0 == NULL) 18204d6db4e0SNavdeep Parhar return (NULL); 182146e1e307SNavdeep Parhar remaining = plen - m0->m_len; 182238035ed6SNavdeep Parhar pnext = &m0->m_next; 1823b741402cSNavdeep Parhar while (remaining > 0) { 182438035ed6SNavdeep Parhar get_segment: 182538035ed6SNavdeep Parhar MPASS(fl->rx_offset == 0); 182646e1e307SNavdeep Parhar m = get_scatter_segment(sc, fl, plen - remaining, remaining); 18274d6db4e0SNavdeep Parhar if (__predict_false(m == NULL)) { 182838035ed6SNavdeep Parhar fl->m0 = m0; 182938035ed6SNavdeep Parhar fl->pnext = pnext; 1830b741402cSNavdeep Parhar fl->remaining = remaining; 18314d6db4e0SNavdeep Parhar fl->flags |= FL_BUF_RESUME; 18324d6db4e0SNavdeep Parhar return (NULL); 18331458bff9SNavdeep Parhar } 183438035ed6SNavdeep Parhar *pnext = m; 183538035ed6SNavdeep Parhar pnext = &m->m_next; 1836b741402cSNavdeep Parhar remaining -= m->m_len; 1837733b9277SNavdeep Parhar } 183838035ed6SNavdeep Parhar *pnext = NULL; 18394d6db4e0SNavdeep Parhar 1840dbbf46c4SNavdeep Parhar M_ASSERTPKTHDR(m0); 1841733b9277SNavdeep Parhar return (m0); 1842733b9277SNavdeep Parhar } 1843733b9277SNavdeep Parhar 1844733b9277SNavdeep Parhar static int 184587bbb333SNavdeep Parhar skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 184687bbb333SNavdeep Parhar int remaining) 184787bbb333SNavdeep Parhar { 184887bbb333SNavdeep Parhar struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 184987bbb333SNavdeep Parhar struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 185087bbb333SNavdeep Parhar int len, blen; 185187bbb333SNavdeep Parhar 185287bbb333SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 185387bbb333SNavdeep Parhar u_int l, pad; 185487bbb333SNavdeep Parhar 185587bbb333SNavdeep Parhar blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 185687bbb333SNavdeep Parhar len = min(remaining, blen); 185787bbb333SNavdeep Parhar 185887bbb333SNavdeep Parhar l = fr_offset + len; 185987bbb333SNavdeep Parhar pad = roundup2(l, fl->buf_boundary) - l; 186087bbb333SNavdeep Parhar if (fl->rx_offset + len + pad < rxb->size2) 186187bbb333SNavdeep Parhar blen = len + pad; 186287bbb333SNavdeep Parhar fl->rx_offset += blen; 186387bbb333SNavdeep Parhar MPASS(fl->rx_offset <= rxb->size2); 186487bbb333SNavdeep Parhar if (fl->rx_offset < rxb->size2) 186587bbb333SNavdeep Parhar return (len); /* without advancing the cidx */ 186687bbb333SNavdeep Parhar } else { 186787bbb333SNavdeep Parhar MPASS(fl->rx_offset == 0); /* not packing */ 186887bbb333SNavdeep Parhar blen = rxb->size1; 186987bbb333SNavdeep Parhar len = min(remaining, blen); 187087bbb333SNavdeep Parhar } 187187bbb333SNavdeep Parhar move_to_next_rxbuf(fl); 187287bbb333SNavdeep Parhar return (len); 187387bbb333SNavdeep Parhar } 187487bbb333SNavdeep Parhar 187587bbb333SNavdeep Parhar static inline void 187687bbb333SNavdeep Parhar skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen) 187787bbb333SNavdeep Parhar { 187887bbb333SNavdeep Parhar int remaining, fr_offset, len; 187987bbb333SNavdeep Parhar 188087bbb333SNavdeep Parhar fr_offset = 0; 188187bbb333SNavdeep Parhar remaining = plen; 188287bbb333SNavdeep Parhar while (remaining > 0) { 188387bbb333SNavdeep Parhar len = skip_scatter_segment(sc, fl, fr_offset, remaining); 188487bbb333SNavdeep Parhar fr_offset += len; 188587bbb333SNavdeep Parhar remaining -= len; 188687bbb333SNavdeep Parhar } 188787bbb333SNavdeep Parhar } 188887bbb333SNavdeep Parhar 188987bbb333SNavdeep Parhar static inline int 189087bbb333SNavdeep Parhar get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen) 189187bbb333SNavdeep Parhar { 189287bbb333SNavdeep Parhar int len; 189387bbb333SNavdeep Parhar struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 189487bbb333SNavdeep Parhar struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 189587bbb333SNavdeep Parhar 189687bbb333SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) 189787bbb333SNavdeep Parhar len = rxb->size2 - fl->rx_offset; 189887bbb333SNavdeep Parhar else 189987bbb333SNavdeep Parhar len = rxb->size1; 190087bbb333SNavdeep Parhar 190187bbb333SNavdeep Parhar return (min(plen, len)); 190287bbb333SNavdeep Parhar } 190387bbb333SNavdeep Parhar 190487bbb333SNavdeep Parhar static int 19051486d2deSNavdeep Parhar eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d, 19061486d2deSNavdeep Parhar u_int plen) 1907733b9277SNavdeep Parhar { 19081486d2deSNavdeep Parhar struct mbuf *m0; 1909733b9277SNavdeep Parhar struct ifnet *ifp = rxq->ifp; 19101486d2deSNavdeep Parhar struct sge_fl *fl = &rxq->fl; 191187bbb333SNavdeep Parhar struct vi_info *vi = ifp->if_softc; 19121486d2deSNavdeep Parhar const struct cpl_rx_pkt *cpl; 1913a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6) 1914733b9277SNavdeep Parhar struct lro_ctrl *lro = &rxq->lro; 1915733b9277SNavdeep Parhar #endif 1916a4a4ad2dSNavdeep Parhar uint16_t err_vec, tnl_type, tnlhdr_len; 191770ca6229SNavdeep Parhar static const int sw_hashtype[4][2] = { 191870ca6229SNavdeep Parhar {M_HASHTYPE_NONE, M_HASHTYPE_NONE}, 191970ca6229SNavdeep Parhar {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6}, 192070ca6229SNavdeep Parhar {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6}, 192170ca6229SNavdeep Parhar {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6}, 192270ca6229SNavdeep Parhar }; 1923a4a4ad2dSNavdeep Parhar static const int sw_csum_flags[2][2] = { 1924a4a4ad2dSNavdeep Parhar { 1925a4a4ad2dSNavdeep Parhar /* IP, inner IP */ 1926a4a4ad2dSNavdeep Parhar CSUM_ENCAP_VXLAN | 1927a4a4ad2dSNavdeep Parhar CSUM_L3_CALC | CSUM_L3_VALID | 1928a4a4ad2dSNavdeep Parhar CSUM_L4_CALC | CSUM_L4_VALID | 1929a4a4ad2dSNavdeep Parhar CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 1930a4a4ad2dSNavdeep Parhar CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1931a4a4ad2dSNavdeep Parhar 1932a4a4ad2dSNavdeep Parhar /* IP, inner IP6 */ 1933a4a4ad2dSNavdeep Parhar CSUM_ENCAP_VXLAN | 1934a4a4ad2dSNavdeep Parhar CSUM_L3_CALC | CSUM_L3_VALID | 1935a4a4ad2dSNavdeep Parhar CSUM_L4_CALC | CSUM_L4_VALID | 1936a4a4ad2dSNavdeep Parhar CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1937a4a4ad2dSNavdeep Parhar }, 1938a4a4ad2dSNavdeep Parhar { 1939a4a4ad2dSNavdeep Parhar /* IP6, inner IP */ 1940a4a4ad2dSNavdeep Parhar CSUM_ENCAP_VXLAN | 1941a4a4ad2dSNavdeep Parhar CSUM_L4_CALC | CSUM_L4_VALID | 1942a4a4ad2dSNavdeep Parhar CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 1943a4a4ad2dSNavdeep Parhar CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1944a4a4ad2dSNavdeep Parhar 1945a4a4ad2dSNavdeep Parhar /* IP6, inner IP6 */ 1946a4a4ad2dSNavdeep Parhar CSUM_ENCAP_VXLAN | 1947a4a4ad2dSNavdeep Parhar CSUM_L4_CALC | CSUM_L4_VALID | 1948a4a4ad2dSNavdeep Parhar CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1949a4a4ad2dSNavdeep Parhar }, 1950a4a4ad2dSNavdeep Parhar }; 1951733b9277SNavdeep Parhar 19521486d2deSNavdeep Parhar MPASS(plen > sc->params.sge.fl_pktshift); 195387bbb333SNavdeep Parhar if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) && 195487bbb333SNavdeep Parhar __predict_true((fl->flags & FL_BUF_RESUME) == 0)) { 195587bbb333SNavdeep Parhar struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 195687bbb333SNavdeep Parhar caddr_t frame; 195787bbb333SNavdeep Parhar int rc, slen; 195887bbb333SNavdeep Parhar 195987bbb333SNavdeep Parhar slen = get_segment_len(sc, fl, plen) - 196087bbb333SNavdeep Parhar sc->params.sge.fl_pktshift; 196187bbb333SNavdeep Parhar frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift; 196287bbb333SNavdeep Parhar CURVNET_SET_QUIET(ifp->if_vnet); 196387bbb333SNavdeep Parhar rc = pfil_run_hooks(vi->pfil, frame, ifp, 196487bbb333SNavdeep Parhar slen | PFIL_MEMPTR | PFIL_IN, NULL); 196587bbb333SNavdeep Parhar CURVNET_RESTORE(); 196687bbb333SNavdeep Parhar if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) { 196787bbb333SNavdeep Parhar skip_fl_payload(sc, fl, plen); 196887bbb333SNavdeep Parhar return (0); 196987bbb333SNavdeep Parhar } 197087bbb333SNavdeep Parhar if (rc == PFIL_REALLOCED) { 197187bbb333SNavdeep Parhar skip_fl_payload(sc, fl, plen); 197287bbb333SNavdeep Parhar m0 = pfil_mem2mbuf(frame); 197387bbb333SNavdeep Parhar goto have_mbuf; 197487bbb333SNavdeep Parhar } 197587bbb333SNavdeep Parhar } 197687bbb333SNavdeep Parhar 19771486d2deSNavdeep Parhar m0 = get_fl_payload(sc, fl, plen); 19781486d2deSNavdeep Parhar if (__predict_false(m0 == NULL)) 19791486d2deSNavdeep Parhar return (ENOMEM); 1980733b9277SNavdeep Parhar 198190e7434aSNavdeep Parhar m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; 198290e7434aSNavdeep Parhar m0->m_len -= sc->params.sge.fl_pktshift; 198390e7434aSNavdeep Parhar m0->m_data += sc->params.sge.fl_pktshift; 198454e4ee71SNavdeep Parhar 198587bbb333SNavdeep Parhar have_mbuf: 198654e4ee71SNavdeep Parhar m0->m_pkthdr.rcvif = ifp; 19871486d2deSNavdeep Parhar M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]); 19881486d2deSNavdeep Parhar m0->m_pkthdr.flowid = be32toh(d->rss.hash_val); 198954e4ee71SNavdeep Parhar 19901486d2deSNavdeep Parhar cpl = (const void *)(&d->rss + 1); 1991a4a4ad2dSNavdeep Parhar if (sc->params.tp.rx_pkt_encap) { 1992a4a4ad2dSNavdeep Parhar const uint16_t ev = be16toh(cpl->err_vec); 19939600bf00SNavdeep Parhar 1994a4a4ad2dSNavdeep Parhar err_vec = G_T6_COMPR_RXERR_VEC(ev); 1995a4a4ad2dSNavdeep Parhar tnl_type = G_T6_RX_TNL_TYPE(ev); 1996a4a4ad2dSNavdeep Parhar tnlhdr_len = G_T6_RX_TNLHDR_LEN(ev); 1997a4a4ad2dSNavdeep Parhar } else { 1998a4a4ad2dSNavdeep Parhar err_vec = be16toh(cpl->err_vec); 1999a4a4ad2dSNavdeep Parhar tnl_type = 0; 2000a4a4ad2dSNavdeep Parhar tnlhdr_len = 0; 2001a4a4ad2dSNavdeep Parhar } 2002a4a4ad2dSNavdeep Parhar if (cpl->csum_calc && err_vec == 0) { 2003a4a4ad2dSNavdeep Parhar int ipv6 = !!(cpl->l2info & htobe32(F_RXF_IP6)); 2004a4a4ad2dSNavdeep Parhar 2005a4a4ad2dSNavdeep Parhar /* checksum(s) calculated and found to be correct. */ 2006a4a4ad2dSNavdeep Parhar 2007a4a4ad2dSNavdeep Parhar MPASS((cpl->l2info & htobe32(F_RXF_IP)) ^ 2008a4a4ad2dSNavdeep Parhar (cpl->l2info & htobe32(F_RXF_IP6))); 200954e4ee71SNavdeep Parhar m0->m_pkthdr.csum_data = be16toh(cpl->csum); 2010a4a4ad2dSNavdeep Parhar if (tnl_type == 0) { 2011a4a4ad2dSNavdeep Parhar if (!ipv6 && ifp->if_capenable & IFCAP_RXCSUM) { 2012a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags = CSUM_L3_CALC | 2013a4a4ad2dSNavdeep Parhar CSUM_L3_VALID | CSUM_L4_CALC | 2014a4a4ad2dSNavdeep Parhar CSUM_L4_VALID; 2015a4a4ad2dSNavdeep Parhar } else if (ipv6 && ifp->if_capenable & IFCAP_RXCSUM_IPV6) { 2016a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags = CSUM_L4_CALC | 2017a4a4ad2dSNavdeep Parhar CSUM_L4_VALID; 2018a4a4ad2dSNavdeep Parhar } 2019a4a4ad2dSNavdeep Parhar rxq->rxcsum++; 2020a4a4ad2dSNavdeep Parhar } else { 2021a4a4ad2dSNavdeep Parhar MPASS(tnl_type == RX_PKT_TNL_TYPE_VXLAN); 2022d107ee06SNavdeep Parhar 2023d107ee06SNavdeep Parhar M_HASHTYPE_SETINNER(m0); 2024a4a4ad2dSNavdeep Parhar if (__predict_false(cpl->ip_frag)) { 2025a4a4ad2dSNavdeep Parhar /* 2026a4a4ad2dSNavdeep Parhar * csum_data is for the inner frame (which is an 2027a4a4ad2dSNavdeep Parhar * IP fragment) and is not 0xffff. There is no 2028a4a4ad2dSNavdeep Parhar * way to pass the inner csum_data to the stack. 2029a4a4ad2dSNavdeep Parhar * We don't want the stack to use the inner 2030a4a4ad2dSNavdeep Parhar * csum_data to validate the outer frame or it 2031a4a4ad2dSNavdeep Parhar * will get rejected. So we fix csum_data here 2032a4a4ad2dSNavdeep Parhar * and let sw do the checksum of inner IP 2033a4a4ad2dSNavdeep Parhar * fragments. 2034a4a4ad2dSNavdeep Parhar * 2035a4a4ad2dSNavdeep Parhar * XXX: Need 32b for csum_data2 in an rx mbuf. 2036a4a4ad2dSNavdeep Parhar * Maybe stuff it into rcv_tstmp? 2037a4a4ad2dSNavdeep Parhar */ 203854e4ee71SNavdeep Parhar m0->m_pkthdr.csum_data = 0xffff; 2039a4a4ad2dSNavdeep Parhar if (ipv6) { 2040a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags = CSUM_L4_CALC | 2041a4a4ad2dSNavdeep Parhar CSUM_L4_VALID; 2042a4a4ad2dSNavdeep Parhar } else { 2043a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags = CSUM_L3_CALC | 2044a4a4ad2dSNavdeep Parhar CSUM_L3_VALID | CSUM_L4_CALC | 2045a4a4ad2dSNavdeep Parhar CSUM_L4_VALID; 2046a4a4ad2dSNavdeep Parhar } 2047a4a4ad2dSNavdeep Parhar } else { 2048a4a4ad2dSNavdeep Parhar int outer_ipv6; 2049a4a4ad2dSNavdeep Parhar 2050a4a4ad2dSNavdeep Parhar MPASS(m0->m_pkthdr.csum_data == 0xffff); 2051a4a4ad2dSNavdeep Parhar 2052a4a4ad2dSNavdeep Parhar outer_ipv6 = tnlhdr_len >= 2053a4a4ad2dSNavdeep Parhar sizeof(struct ether_header) + 2054a4a4ad2dSNavdeep Parhar sizeof(struct ip6_hdr); 2055a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags = 2056a4a4ad2dSNavdeep Parhar sw_csum_flags[outer_ipv6][ipv6]; 2057a4a4ad2dSNavdeep Parhar } 2058a4a4ad2dSNavdeep Parhar rxq->vxlan_rxcsum++; 2059a4a4ad2dSNavdeep Parhar } 206054e4ee71SNavdeep Parhar } 206154e4ee71SNavdeep Parhar 206254e4ee71SNavdeep Parhar if (cpl->vlan_ex) { 206354e4ee71SNavdeep Parhar m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); 206454e4ee71SNavdeep Parhar m0->m_flags |= M_VLANTAG; 206554e4ee71SNavdeep Parhar rxq->vlan_extraction++; 206654e4ee71SNavdeep Parhar } 206754e4ee71SNavdeep Parhar 20681486d2deSNavdeep Parhar if (rxq->iq.flags & IQ_RX_TIMESTAMP) { 20691486d2deSNavdeep Parhar /* 20701486d2deSNavdeep Parhar * Fill up rcv_tstmp but do not set M_TSTMP. 20711486d2deSNavdeep Parhar * rcv_tstmp is not in the format that the 20721486d2deSNavdeep Parhar * kernel expects and we don't want to mislead 20731486d2deSNavdeep Parhar * it. For now this is only for custom code 20741486d2deSNavdeep Parhar * that knows how to interpret cxgbe's stamp. 20751486d2deSNavdeep Parhar */ 20761486d2deSNavdeep Parhar m0->m_pkthdr.rcv_tstmp = 20771486d2deSNavdeep Parhar last_flit_to_ns(sc, d->rsp.u.last_flit); 20781486d2deSNavdeep Parhar #ifdef notyet 20791486d2deSNavdeep Parhar m0->m_flags |= M_TSTMP; 20801486d2deSNavdeep Parhar #endif 20811486d2deSNavdeep Parhar } 20821486d2deSNavdeep Parhar 208350575ce1SAndrew Gallatin #ifdef NUMA 208450575ce1SAndrew Gallatin m0->m_pkthdr.numa_domain = ifp->if_numa_domain; 208550575ce1SAndrew Gallatin #endif 2086a1ea9a82SNavdeep Parhar #if defined(INET) || defined(INET6) 2087a4a4ad2dSNavdeep Parhar if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 && 20889087a3dfSNavdeep Parhar (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 || 20899087a3dfSNavdeep Parhar M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) { 209046f48ee5SNavdeep Parhar if (sort_before_lro(lro)) { 209146f48ee5SNavdeep Parhar tcp_lro_queue_mbuf(lro, m0); 209246f48ee5SNavdeep Parhar return (0); /* queued for sort, then LRO */ 209346f48ee5SNavdeep Parhar } 209446f48ee5SNavdeep Parhar if (tcp_lro_rx(lro, m0, 0) == 0) 209546f48ee5SNavdeep Parhar return (0); /* queued for LRO */ 209646f48ee5SNavdeep Parhar } 209754e4ee71SNavdeep Parhar #endif 20987d29df59SNavdeep Parhar ifp->if_input(ifp, m0); 209954e4ee71SNavdeep Parhar 2100733b9277SNavdeep Parhar return (0); 210154e4ee71SNavdeep Parhar } 210254e4ee71SNavdeep Parhar 2103733b9277SNavdeep Parhar /* 21047951040fSNavdeep Parhar * Must drain the wrq or make sure that someone else will. 21057951040fSNavdeep Parhar */ 21067951040fSNavdeep Parhar static void 21077951040fSNavdeep Parhar wrq_tx_drain(void *arg, int n) 21087951040fSNavdeep Parhar { 21097951040fSNavdeep Parhar struct sge_wrq *wrq = arg; 21107951040fSNavdeep Parhar struct sge_eq *eq = &wrq->eq; 21117951040fSNavdeep Parhar 21127951040fSNavdeep Parhar EQ_LOCK(eq); 21137951040fSNavdeep Parhar if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 21147951040fSNavdeep Parhar drain_wrq_wr_list(wrq->adapter, wrq); 21157951040fSNavdeep Parhar EQ_UNLOCK(eq); 21167951040fSNavdeep Parhar } 21177951040fSNavdeep Parhar 21187951040fSNavdeep Parhar static void 21197951040fSNavdeep Parhar drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) 21207951040fSNavdeep Parhar { 21217951040fSNavdeep Parhar struct sge_eq *eq = &wrq->eq; 21227951040fSNavdeep Parhar u_int available, dbdiff; /* # of hardware descriptors */ 21237951040fSNavdeep Parhar u_int n; 21247951040fSNavdeep Parhar struct wrqe *wr; 21257951040fSNavdeep Parhar struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 21267951040fSNavdeep Parhar 21277951040fSNavdeep Parhar EQ_LOCK_ASSERT_OWNED(eq); 21287951040fSNavdeep Parhar MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 21297951040fSNavdeep Parhar wr = STAILQ_FIRST(&wrq->wr_list); 21307951040fSNavdeep Parhar MPASS(wr != NULL); /* Must be called with something useful to do */ 2131cda2ab0eSNavdeep Parhar MPASS(eq->pidx == eq->dbidx); 2132cda2ab0eSNavdeep Parhar dbdiff = 0; 21337951040fSNavdeep Parhar 21347951040fSNavdeep Parhar do { 21357951040fSNavdeep Parhar eq->cidx = read_hw_cidx(eq); 21367951040fSNavdeep Parhar if (eq->pidx == eq->cidx) 21377951040fSNavdeep Parhar available = eq->sidx - 1; 21387951040fSNavdeep Parhar else 21397951040fSNavdeep Parhar available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 21407951040fSNavdeep Parhar 21417951040fSNavdeep Parhar MPASS(wr->wrq == wrq); 21427951040fSNavdeep Parhar n = howmany(wr->wr_len, EQ_ESIZE); 21437951040fSNavdeep Parhar if (available < n) 2144cda2ab0eSNavdeep Parhar break; 21457951040fSNavdeep Parhar 21467951040fSNavdeep Parhar dst = (void *)&eq->desc[eq->pidx]; 21477951040fSNavdeep Parhar if (__predict_true(eq->sidx - eq->pidx > n)) { 21487951040fSNavdeep Parhar /* Won't wrap, won't end exactly at the status page. */ 21497951040fSNavdeep Parhar bcopy(&wr->wr[0], dst, wr->wr_len); 21507951040fSNavdeep Parhar eq->pidx += n; 21517951040fSNavdeep Parhar } else { 21527951040fSNavdeep Parhar int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; 21537951040fSNavdeep Parhar 21547951040fSNavdeep Parhar bcopy(&wr->wr[0], dst, first_portion); 21557951040fSNavdeep Parhar if (wr->wr_len > first_portion) { 21567951040fSNavdeep Parhar bcopy(&wr->wr[first_portion], &eq->desc[0], 21577951040fSNavdeep Parhar wr->wr_len - first_portion); 21587951040fSNavdeep Parhar } 21597951040fSNavdeep Parhar eq->pidx = n - (eq->sidx - eq->pidx); 21607951040fSNavdeep Parhar } 21610459a175SNavdeep Parhar wrq->tx_wrs_copied++; 21627951040fSNavdeep Parhar 21637951040fSNavdeep Parhar if (available < eq->sidx / 4 && 21647951040fSNavdeep Parhar atomic_cmpset_int(&eq->equiq, 0, 1)) { 2165ddf09ad6SNavdeep Parhar /* 2166ddf09ad6SNavdeep Parhar * XXX: This is not 100% reliable with some 2167ddf09ad6SNavdeep Parhar * types of WRs. But this is a very unusual 2168ddf09ad6SNavdeep Parhar * situation for an ofld/ctrl queue anyway. 2169ddf09ad6SNavdeep Parhar */ 21707951040fSNavdeep Parhar dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 21717951040fSNavdeep Parhar F_FW_WR_EQUEQ); 21727951040fSNavdeep Parhar } 21737951040fSNavdeep Parhar 21747951040fSNavdeep Parhar dbdiff += n; 21757951040fSNavdeep Parhar if (dbdiff >= 16) { 21767951040fSNavdeep Parhar ring_eq_db(sc, eq, dbdiff); 21777951040fSNavdeep Parhar dbdiff = 0; 21787951040fSNavdeep Parhar } 21797951040fSNavdeep Parhar 21807951040fSNavdeep Parhar STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 21817951040fSNavdeep Parhar free_wrqe(wr); 21827951040fSNavdeep Parhar MPASS(wrq->nwr_pending > 0); 21837951040fSNavdeep Parhar wrq->nwr_pending--; 21847951040fSNavdeep Parhar MPASS(wrq->ndesc_needed >= n); 21857951040fSNavdeep Parhar wrq->ndesc_needed -= n; 21867951040fSNavdeep Parhar } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); 21877951040fSNavdeep Parhar 21887951040fSNavdeep Parhar if (dbdiff) 21897951040fSNavdeep Parhar ring_eq_db(sc, eq, dbdiff); 21907951040fSNavdeep Parhar } 21917951040fSNavdeep Parhar 21927951040fSNavdeep Parhar /* 2193733b9277SNavdeep Parhar * Doesn't fail. Holds on to work requests it can't send right away. 2194733b9277SNavdeep Parhar */ 219509fe6320SNavdeep Parhar void 219609fe6320SNavdeep Parhar t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 2197733b9277SNavdeep Parhar { 2198733b9277SNavdeep Parhar #ifdef INVARIANTS 21997951040fSNavdeep Parhar struct sge_eq *eq = &wrq->eq; 2200733b9277SNavdeep Parhar #endif 2201733b9277SNavdeep Parhar 22027951040fSNavdeep Parhar EQ_LOCK_ASSERT_OWNED(eq); 22037951040fSNavdeep Parhar MPASS(wr != NULL); 22047951040fSNavdeep Parhar MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); 22057951040fSNavdeep Parhar MPASS((wr->wr_len & 0x7) == 0); 2206733b9277SNavdeep Parhar 22077951040fSNavdeep Parhar STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); 22087951040fSNavdeep Parhar wrq->nwr_pending++; 22097951040fSNavdeep Parhar wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); 2210733b9277SNavdeep Parhar 22117951040fSNavdeep Parhar if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) 22127951040fSNavdeep Parhar return; /* commit_wrq_wr will drain wr_list as well. */ 2213733b9277SNavdeep Parhar 22147951040fSNavdeep Parhar drain_wrq_wr_list(sc, wrq); 2215733b9277SNavdeep Parhar 22167951040fSNavdeep Parhar /* Doorbell must have caught up to the pidx. */ 22177951040fSNavdeep Parhar MPASS(eq->pidx == eq->dbidx); 221854e4ee71SNavdeep Parhar } 221954e4ee71SNavdeep Parhar 222054e4ee71SNavdeep Parhar void 222154e4ee71SNavdeep Parhar t4_update_fl_bufsize(struct ifnet *ifp) 222254e4ee71SNavdeep Parhar { 2223fe2ebb76SJohn Baldwin struct vi_info *vi = ifp->if_softc; 22247c228be3SNavdeep Parhar struct adapter *sc = vi->adapter; 222554e4ee71SNavdeep Parhar struct sge_rxq *rxq; 22266eb3180fSNavdeep Parhar #ifdef TCP_OFFLOAD 22276eb3180fSNavdeep Parhar struct sge_ofld_rxq *ofld_rxq; 22286eb3180fSNavdeep Parhar #endif 222954e4ee71SNavdeep Parhar struct sge_fl *fl; 22306a59b994SNavdeep Parhar int i, maxp; 223154e4ee71SNavdeep Parhar 22326a59b994SNavdeep Parhar maxp = max_rx_payload(sc, ifp, false); 2233fe2ebb76SJohn Baldwin for_each_rxq(vi, i, rxq) { 223454e4ee71SNavdeep Parhar fl = &rxq->fl; 223554e4ee71SNavdeep Parhar 223654e4ee71SNavdeep Parhar FL_LOCK(fl); 223746e1e307SNavdeep Parhar fl->zidx = find_refill_source(sc, maxp, 223846e1e307SNavdeep Parhar fl->flags & FL_BUF_PACKING); 223954e4ee71SNavdeep Parhar FL_UNLOCK(fl); 224054e4ee71SNavdeep Parhar } 22416eb3180fSNavdeep Parhar #ifdef TCP_OFFLOAD 22426a59b994SNavdeep Parhar maxp = max_rx_payload(sc, ifp, true); 2243fe2ebb76SJohn Baldwin for_each_ofld_rxq(vi, i, ofld_rxq) { 22446eb3180fSNavdeep Parhar fl = &ofld_rxq->fl; 22456eb3180fSNavdeep Parhar 22466eb3180fSNavdeep Parhar FL_LOCK(fl); 224746e1e307SNavdeep Parhar fl->zidx = find_refill_source(sc, maxp, 224846e1e307SNavdeep Parhar fl->flags & FL_BUF_PACKING); 22496eb3180fSNavdeep Parhar FL_UNLOCK(fl); 22506eb3180fSNavdeep Parhar } 22516eb3180fSNavdeep Parhar #endif 225254e4ee71SNavdeep Parhar } 225354e4ee71SNavdeep Parhar 22547951040fSNavdeep Parhar static inline int 22557951040fSNavdeep Parhar mbuf_nsegs(struct mbuf *m) 2256733b9277SNavdeep Parhar { 22570835ddc7SNavdeep Parhar 22587951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 2259a4a4ad2dSNavdeep Parhar KASSERT(m->m_pkthdr.inner_l5hlen > 0, 22607951040fSNavdeep Parhar ("%s: mbuf %p missing information on # of segments.", __func__, m)); 22617951040fSNavdeep Parhar 2262a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.inner_l5hlen); 22637951040fSNavdeep Parhar } 22647951040fSNavdeep Parhar 22657951040fSNavdeep Parhar static inline void 22667951040fSNavdeep Parhar set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) 22677951040fSNavdeep Parhar { 22687951040fSNavdeep Parhar 22697951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 2270a4a4ad2dSNavdeep Parhar m->m_pkthdr.inner_l5hlen = nsegs; 22717951040fSNavdeep Parhar } 22727951040fSNavdeep Parhar 22737951040fSNavdeep Parhar static inline int 22745cdaef71SJohn Baldwin mbuf_cflags(struct mbuf *m) 22755cdaef71SJohn Baldwin { 22765cdaef71SJohn Baldwin 22775cdaef71SJohn Baldwin M_ASSERTPKTHDR(m); 22785cdaef71SJohn Baldwin return (m->m_pkthdr.PH_loc.eight[4]); 22795cdaef71SJohn Baldwin } 22805cdaef71SJohn Baldwin 22815cdaef71SJohn Baldwin static inline void 22825cdaef71SJohn Baldwin set_mbuf_cflags(struct mbuf *m, uint8_t flags) 22835cdaef71SJohn Baldwin { 22845cdaef71SJohn Baldwin 22855cdaef71SJohn Baldwin M_ASSERTPKTHDR(m); 22865cdaef71SJohn Baldwin m->m_pkthdr.PH_loc.eight[4] = flags; 22875cdaef71SJohn Baldwin } 22885cdaef71SJohn Baldwin 22895cdaef71SJohn Baldwin static inline int 22907951040fSNavdeep Parhar mbuf_len16(struct mbuf *m) 22917951040fSNavdeep Parhar { 22927951040fSNavdeep Parhar int n; 22937951040fSNavdeep Parhar 22947951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 22957951040fSNavdeep Parhar n = m->m_pkthdr.PH_loc.eight[0]; 2296bddf7343SJohn Baldwin if (!(mbuf_cflags(m) & MC_TLS)) 22977951040fSNavdeep Parhar MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 22987951040fSNavdeep Parhar 22997951040fSNavdeep Parhar return (n); 23007951040fSNavdeep Parhar } 23017951040fSNavdeep Parhar 23027951040fSNavdeep Parhar static inline void 23037951040fSNavdeep Parhar set_mbuf_len16(struct mbuf *m, uint8_t len16) 23047951040fSNavdeep Parhar { 23057951040fSNavdeep Parhar 23067951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 230730e3f2b4SNavdeep Parhar if (!(mbuf_cflags(m) & MC_TLS)) 230830e3f2b4SNavdeep Parhar MPASS(len16 > 0 && len16 <= SGE_MAX_WR_LEN / 16); 23097951040fSNavdeep Parhar m->m_pkthdr.PH_loc.eight[0] = len16; 23107951040fSNavdeep Parhar } 23117951040fSNavdeep Parhar 2312786099deSNavdeep Parhar #ifdef RATELIMIT 2313786099deSNavdeep Parhar static inline int 2314786099deSNavdeep Parhar mbuf_eo_nsegs(struct mbuf *m) 2315786099deSNavdeep Parhar { 2316786099deSNavdeep Parhar 2317786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2318786099deSNavdeep Parhar return (m->m_pkthdr.PH_loc.eight[1]); 2319786099deSNavdeep Parhar } 2320786099deSNavdeep Parhar 2321ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 2322786099deSNavdeep Parhar static inline void 2323786099deSNavdeep Parhar set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs) 2324786099deSNavdeep Parhar { 2325786099deSNavdeep Parhar 2326786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2327786099deSNavdeep Parhar m->m_pkthdr.PH_loc.eight[1] = nsegs; 2328786099deSNavdeep Parhar } 2329ffbb373cSNavdeep Parhar #endif 2330786099deSNavdeep Parhar 2331786099deSNavdeep Parhar static inline int 2332786099deSNavdeep Parhar mbuf_eo_len16(struct mbuf *m) 2333786099deSNavdeep Parhar { 2334786099deSNavdeep Parhar int n; 2335786099deSNavdeep Parhar 2336786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2337786099deSNavdeep Parhar n = m->m_pkthdr.PH_loc.eight[2]; 2338786099deSNavdeep Parhar MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2339786099deSNavdeep Parhar 2340786099deSNavdeep Parhar return (n); 2341786099deSNavdeep Parhar } 2342786099deSNavdeep Parhar 2343ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 2344786099deSNavdeep Parhar static inline void 2345786099deSNavdeep Parhar set_mbuf_eo_len16(struct mbuf *m, uint8_t len16) 2346786099deSNavdeep Parhar { 2347786099deSNavdeep Parhar 2348786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2349786099deSNavdeep Parhar m->m_pkthdr.PH_loc.eight[2] = len16; 2350786099deSNavdeep Parhar } 2351ffbb373cSNavdeep Parhar #endif 2352786099deSNavdeep Parhar 2353786099deSNavdeep Parhar static inline int 2354786099deSNavdeep Parhar mbuf_eo_tsclk_tsoff(struct mbuf *m) 2355786099deSNavdeep Parhar { 2356786099deSNavdeep Parhar 2357786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2358786099deSNavdeep Parhar return (m->m_pkthdr.PH_loc.eight[3]); 2359786099deSNavdeep Parhar } 2360786099deSNavdeep Parhar 2361ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 2362786099deSNavdeep Parhar static inline void 2363786099deSNavdeep Parhar set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff) 2364786099deSNavdeep Parhar { 2365786099deSNavdeep Parhar 2366786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2367786099deSNavdeep Parhar m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff; 2368786099deSNavdeep Parhar } 2369ffbb373cSNavdeep Parhar #endif 2370786099deSNavdeep Parhar 2371786099deSNavdeep Parhar static inline int 237256fb710fSJohn Baldwin needs_eo(struct m_snd_tag *mst) 2373786099deSNavdeep Parhar { 2374786099deSNavdeep Parhar 2375c782ea8bSJohn Baldwin return (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT); 2376786099deSNavdeep Parhar } 2377786099deSNavdeep Parhar #endif 2378786099deSNavdeep Parhar 23795cdaef71SJohn Baldwin /* 23805cdaef71SJohn Baldwin * Try to allocate an mbuf to contain a raw work request. To make it 23815cdaef71SJohn Baldwin * easy to construct the work request, don't allocate a chain but a 23825cdaef71SJohn Baldwin * single mbuf. 23835cdaef71SJohn Baldwin */ 23845cdaef71SJohn Baldwin struct mbuf * 23855cdaef71SJohn Baldwin alloc_wr_mbuf(int len, int how) 23865cdaef71SJohn Baldwin { 23875cdaef71SJohn Baldwin struct mbuf *m; 23885cdaef71SJohn Baldwin 23895cdaef71SJohn Baldwin if (len <= MHLEN) 23905cdaef71SJohn Baldwin m = m_gethdr(how, MT_DATA); 23915cdaef71SJohn Baldwin else if (len <= MCLBYTES) 23925cdaef71SJohn Baldwin m = m_getcl(how, MT_DATA, M_PKTHDR); 23935cdaef71SJohn Baldwin else 23945cdaef71SJohn Baldwin m = NULL; 23955cdaef71SJohn Baldwin if (m == NULL) 23965cdaef71SJohn Baldwin return (NULL); 23975cdaef71SJohn Baldwin m->m_pkthdr.len = len; 23985cdaef71SJohn Baldwin m->m_len = len; 23995cdaef71SJohn Baldwin set_mbuf_cflags(m, MC_RAW_WR); 24005cdaef71SJohn Baldwin set_mbuf_len16(m, howmany(len, 16)); 24015cdaef71SJohn Baldwin return (m); 24025cdaef71SJohn Baldwin } 24035cdaef71SJohn Baldwin 2404a4a4ad2dSNavdeep Parhar static inline bool 2405c0236bd9SNavdeep Parhar needs_hwcsum(struct mbuf *m) 2406c0236bd9SNavdeep Parhar { 2407a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | 2408a4a4ad2dSNavdeep Parhar CSUM_IP_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP | 2409a4a4ad2dSNavdeep Parhar CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_IP6_UDP | 2410a4a4ad2dSNavdeep Parhar CSUM_IP6_TCP | CSUM_IP6_TSO | CSUM_INNER_IP6_UDP | 2411a4a4ad2dSNavdeep Parhar CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO; 2412c0236bd9SNavdeep Parhar 2413c0236bd9SNavdeep Parhar M_ASSERTPKTHDR(m); 2414c0236bd9SNavdeep Parhar 2415a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 2416c0236bd9SNavdeep Parhar } 2417c0236bd9SNavdeep Parhar 2418a4a4ad2dSNavdeep Parhar static inline bool 24197951040fSNavdeep Parhar needs_tso(struct mbuf *m) 24207951040fSNavdeep Parhar { 2421a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP_TSO | CSUM_IP6_TSO | 2422a4a4ad2dSNavdeep Parhar CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO; 24237951040fSNavdeep Parhar 24247951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 24257951040fSNavdeep Parhar 2426a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 24277951040fSNavdeep Parhar } 24287951040fSNavdeep Parhar 2429a4a4ad2dSNavdeep Parhar static inline bool 2430a4a4ad2dSNavdeep Parhar needs_vxlan_csum(struct mbuf *m) 2431a4a4ad2dSNavdeep Parhar { 2432a4a4ad2dSNavdeep Parhar 2433a4a4ad2dSNavdeep Parhar M_ASSERTPKTHDR(m); 2434a4a4ad2dSNavdeep Parhar 2435a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN); 2436a4a4ad2dSNavdeep Parhar } 2437a4a4ad2dSNavdeep Parhar 2438a4a4ad2dSNavdeep Parhar static inline bool 2439a4a4ad2dSNavdeep Parhar needs_vxlan_tso(struct mbuf *m) 2440a4a4ad2dSNavdeep Parhar { 2441a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_ENCAP_VXLAN | CSUM_INNER_IP_TSO | 2442a4a4ad2dSNavdeep Parhar CSUM_INNER_IP6_TSO; 2443a4a4ad2dSNavdeep Parhar 2444a4a4ad2dSNavdeep Parhar M_ASSERTPKTHDR(m); 2445a4a4ad2dSNavdeep Parhar 2446a4a4ad2dSNavdeep Parhar return ((m->m_pkthdr.csum_flags & csum_flags) != 0 && 2447a4a4ad2dSNavdeep Parhar (m->m_pkthdr.csum_flags & csum_flags) != CSUM_ENCAP_VXLAN); 2448a4a4ad2dSNavdeep Parhar } 2449a4a4ad2dSNavdeep Parhar 2450ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 2451a4a4ad2dSNavdeep Parhar static inline bool 2452a4a4ad2dSNavdeep Parhar needs_inner_tcp_csum(struct mbuf *m) 2453a4a4ad2dSNavdeep Parhar { 2454a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO; 2455a4a4ad2dSNavdeep Parhar 2456a4a4ad2dSNavdeep Parhar M_ASSERTPKTHDR(m); 2457a4a4ad2dSNavdeep Parhar 2458a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 2459a4a4ad2dSNavdeep Parhar } 2460ffbb373cSNavdeep Parhar #endif 2461a4a4ad2dSNavdeep Parhar 2462a4a4ad2dSNavdeep Parhar static inline bool 24637951040fSNavdeep Parhar needs_l3_csum(struct mbuf *m) 24647951040fSNavdeep Parhar { 2465a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP | CSUM_IP_TSO | CSUM_INNER_IP | 2466a4a4ad2dSNavdeep Parhar CSUM_INNER_IP_TSO; 24677951040fSNavdeep Parhar 24687951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 24697951040fSNavdeep Parhar 2470a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 24717951040fSNavdeep Parhar } 24727951040fSNavdeep Parhar 2473a4a4ad2dSNavdeep Parhar static inline bool 2474a4a4ad2dSNavdeep Parhar needs_outer_tcp_csum(struct mbuf *m) 2475c0236bd9SNavdeep Parhar { 2476a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP_TCP | CSUM_IP_TSO | CSUM_IP6_TCP | 2477a4a4ad2dSNavdeep Parhar CSUM_IP6_TSO; 2478c0236bd9SNavdeep Parhar 2479c0236bd9SNavdeep Parhar M_ASSERTPKTHDR(m); 2480a4a4ad2dSNavdeep Parhar 2481a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 2482c0236bd9SNavdeep Parhar } 2483c0236bd9SNavdeep Parhar 2484c0236bd9SNavdeep Parhar #ifdef RATELIMIT 2485a4a4ad2dSNavdeep Parhar static inline bool 2486a4a4ad2dSNavdeep Parhar needs_outer_l4_csum(struct mbuf *m) 24877951040fSNavdeep Parhar { 2488a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_TSO | 2489a4a4ad2dSNavdeep Parhar CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_TSO; 24907951040fSNavdeep Parhar 24917951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 24927951040fSNavdeep Parhar 2493a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 24947951040fSNavdeep Parhar } 24957951040fSNavdeep Parhar 2496a4a4ad2dSNavdeep Parhar static inline bool 2497a4a4ad2dSNavdeep Parhar needs_outer_udp_csum(struct mbuf *m) 2498786099deSNavdeep Parhar { 2499a4a4ad2dSNavdeep Parhar const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP6_UDP; 2500786099deSNavdeep Parhar 2501786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2502a4a4ad2dSNavdeep Parhar 2503a4a4ad2dSNavdeep Parhar return (m->m_pkthdr.csum_flags & csum_flags); 2504786099deSNavdeep Parhar } 2505c3fce948SNavdeep Parhar #endif 2506786099deSNavdeep Parhar 2507a4a4ad2dSNavdeep Parhar static inline bool 25087951040fSNavdeep Parhar needs_vlan_insertion(struct mbuf *m) 25097951040fSNavdeep Parhar { 25107951040fSNavdeep Parhar 25117951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 25127951040fSNavdeep Parhar 2513a6a8ff35SNavdeep Parhar return (m->m_flags & M_VLANTAG); 25147951040fSNavdeep Parhar } 25157951040fSNavdeep Parhar 2516*94e6b3feSNavdeep Parhar #if defined(INET) || defined(INET6) 25177951040fSNavdeep Parhar static void * 25187951040fSNavdeep Parhar m_advance(struct mbuf **pm, int *poffset, int len) 25197951040fSNavdeep Parhar { 25207951040fSNavdeep Parhar struct mbuf *m = *pm; 25217951040fSNavdeep Parhar int offset = *poffset; 25227951040fSNavdeep Parhar uintptr_t p = 0; 25237951040fSNavdeep Parhar 25247951040fSNavdeep Parhar MPASS(len > 0); 25257951040fSNavdeep Parhar 2526e06ab612SJohn Baldwin for (;;) { 25277951040fSNavdeep Parhar if (offset + len < m->m_len) { 25287951040fSNavdeep Parhar offset += len; 25297951040fSNavdeep Parhar p = mtod(m, uintptr_t) + offset; 25307951040fSNavdeep Parhar break; 25317951040fSNavdeep Parhar } 25327951040fSNavdeep Parhar len -= m->m_len - offset; 25337951040fSNavdeep Parhar m = m->m_next; 25347951040fSNavdeep Parhar offset = 0; 25357951040fSNavdeep Parhar MPASS(m != NULL); 25367951040fSNavdeep Parhar } 25377951040fSNavdeep Parhar *poffset = offset; 25387951040fSNavdeep Parhar *pm = m; 25397951040fSNavdeep Parhar return ((void *)p); 25407951040fSNavdeep Parhar } 2541*94e6b3feSNavdeep Parhar #endif 25427951040fSNavdeep Parhar 2543d76bbe17SJohn Baldwin static inline int 2544d76bbe17SJohn Baldwin count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) 2545d76bbe17SJohn Baldwin { 2546d76bbe17SJohn Baldwin vm_paddr_t paddr; 2547d76bbe17SJohn Baldwin int i, len, off, pglen, pgoff, seglen, segoff; 2548d76bbe17SJohn Baldwin int nsegs = 0; 2549d76bbe17SJohn Baldwin 2550365e8da4SGleb Smirnoff M_ASSERTEXTPG(m); 2551d76bbe17SJohn Baldwin off = mtod(m, vm_offset_t); 2552d76bbe17SJohn Baldwin len = m->m_len; 2553d76bbe17SJohn Baldwin off += skip; 2554d76bbe17SJohn Baldwin len -= skip; 2555d76bbe17SJohn Baldwin 25567b6c99d0SGleb Smirnoff if (m->m_epg_hdrlen != 0) { 25577b6c99d0SGleb Smirnoff if (off >= m->m_epg_hdrlen) { 25587b6c99d0SGleb Smirnoff off -= m->m_epg_hdrlen; 2559d76bbe17SJohn Baldwin } else { 25607b6c99d0SGleb Smirnoff seglen = m->m_epg_hdrlen - off; 2561d76bbe17SJohn Baldwin segoff = off; 2562d76bbe17SJohn Baldwin seglen = min(seglen, len); 2563d76bbe17SJohn Baldwin off = 0; 2564d76bbe17SJohn Baldwin len -= seglen; 2565d76bbe17SJohn Baldwin paddr = pmap_kextract( 25660c103266SGleb Smirnoff (vm_offset_t)&m->m_epg_hdr[segoff]); 2567d76bbe17SJohn Baldwin if (*nextaddr != paddr) 2568d76bbe17SJohn Baldwin nsegs++; 2569d76bbe17SJohn Baldwin *nextaddr = paddr + seglen; 2570d76bbe17SJohn Baldwin } 2571d76bbe17SJohn Baldwin } 25727b6c99d0SGleb Smirnoff pgoff = m->m_epg_1st_off; 25737b6c99d0SGleb Smirnoff for (i = 0; i < m->m_epg_npgs && len > 0; i++) { 2574c4ee38f8SGleb Smirnoff pglen = m_epg_pagelen(m, i, pgoff); 2575d76bbe17SJohn Baldwin if (off >= pglen) { 2576d76bbe17SJohn Baldwin off -= pglen; 2577d76bbe17SJohn Baldwin pgoff = 0; 2578d76bbe17SJohn Baldwin continue; 2579d76bbe17SJohn Baldwin } 2580d76bbe17SJohn Baldwin seglen = pglen - off; 2581d76bbe17SJohn Baldwin segoff = pgoff + off; 2582d76bbe17SJohn Baldwin off = 0; 2583d76bbe17SJohn Baldwin seglen = min(seglen, len); 2584d76bbe17SJohn Baldwin len -= seglen; 25850c103266SGleb Smirnoff paddr = m->m_epg_pa[i] + segoff; 2586d76bbe17SJohn Baldwin if (*nextaddr != paddr) 2587d76bbe17SJohn Baldwin nsegs++; 2588d76bbe17SJohn Baldwin *nextaddr = paddr + seglen; 2589d76bbe17SJohn Baldwin pgoff = 0; 2590d76bbe17SJohn Baldwin }; 2591d76bbe17SJohn Baldwin if (len != 0) { 25927b6c99d0SGleb Smirnoff seglen = min(len, m->m_epg_trllen - off); 2593d76bbe17SJohn Baldwin len -= seglen; 25940c103266SGleb Smirnoff paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]); 2595d76bbe17SJohn Baldwin if (*nextaddr != paddr) 2596d76bbe17SJohn Baldwin nsegs++; 2597d76bbe17SJohn Baldwin *nextaddr = paddr + seglen; 2598d76bbe17SJohn Baldwin } 2599d76bbe17SJohn Baldwin 2600d76bbe17SJohn Baldwin return (nsegs); 2601d76bbe17SJohn Baldwin } 2602d76bbe17SJohn Baldwin 2603d76bbe17SJohn Baldwin 26047951040fSNavdeep Parhar /* 26057951040fSNavdeep Parhar * Can deal with empty mbufs in the chain that have m_len = 0, but the chain 2606786099deSNavdeep Parhar * must have at least one mbuf that's not empty. It is possible for this 2607786099deSNavdeep Parhar * routine to return 0 if skip accounts for all the contents of the mbuf chain. 26087951040fSNavdeep Parhar */ 26097951040fSNavdeep Parhar static inline int 2610d76bbe17SJohn Baldwin count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags) 26117951040fSNavdeep Parhar { 2612d76bbe17SJohn Baldwin vm_paddr_t nextaddr, paddr; 261377e9044cSNavdeep Parhar vm_offset_t va; 26147951040fSNavdeep Parhar int len, nsegs; 26157951040fSNavdeep Parhar 2616786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 2617786099deSNavdeep Parhar MPASS(m->m_pkthdr.len > 0); 2618786099deSNavdeep Parhar MPASS(m->m_pkthdr.len >= skip); 26197951040fSNavdeep Parhar 26207951040fSNavdeep Parhar nsegs = 0; 2621d76bbe17SJohn Baldwin nextaddr = 0; 26227951040fSNavdeep Parhar for (; m; m = m->m_next) { 26237951040fSNavdeep Parhar len = m->m_len; 26247951040fSNavdeep Parhar if (__predict_false(len == 0)) 26257951040fSNavdeep Parhar continue; 2626786099deSNavdeep Parhar if (skip >= len) { 2627786099deSNavdeep Parhar skip -= len; 2628786099deSNavdeep Parhar continue; 2629786099deSNavdeep Parhar } 26306edfd179SGleb Smirnoff if ((m->m_flags & M_EXTPG) != 0) { 2631d76bbe17SJohn Baldwin *cflags |= MC_NOMAP; 2632d76bbe17SJohn Baldwin nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr); 2633d76bbe17SJohn Baldwin skip = 0; 2634d76bbe17SJohn Baldwin continue; 2635d76bbe17SJohn Baldwin } 2636786099deSNavdeep Parhar va = mtod(m, vm_offset_t) + skip; 2637786099deSNavdeep Parhar len -= skip; 2638786099deSNavdeep Parhar skip = 0; 2639d76bbe17SJohn Baldwin paddr = pmap_kextract(va); 2640786099deSNavdeep Parhar nsegs += sglist_count((void *)(uintptr_t)va, len); 2641d76bbe17SJohn Baldwin if (paddr == nextaddr) 26427951040fSNavdeep Parhar nsegs--; 2643d76bbe17SJohn Baldwin nextaddr = pmap_kextract(va + len - 1) + 1; 26447951040fSNavdeep Parhar } 26457951040fSNavdeep Parhar 26467951040fSNavdeep Parhar return (nsegs); 26477951040fSNavdeep Parhar } 26487951040fSNavdeep Parhar 26497951040fSNavdeep Parhar /* 2650a4a4ad2dSNavdeep Parhar * The maximum number of segments that can fit in a WR. 2651a4a4ad2dSNavdeep Parhar */ 2652a4a4ad2dSNavdeep Parhar static int 265330e3f2b4SNavdeep Parhar max_nsegs_allowed(struct mbuf *m, bool vm_wr) 2654a4a4ad2dSNavdeep Parhar { 2655a4a4ad2dSNavdeep Parhar 265630e3f2b4SNavdeep Parhar if (vm_wr) { 265730e3f2b4SNavdeep Parhar if (needs_tso(m)) 265830e3f2b4SNavdeep Parhar return (TX_SGL_SEGS_VM_TSO); 265930e3f2b4SNavdeep Parhar return (TX_SGL_SEGS_VM); 266030e3f2b4SNavdeep Parhar } 266130e3f2b4SNavdeep Parhar 2662a4a4ad2dSNavdeep Parhar if (needs_tso(m)) { 2663a4a4ad2dSNavdeep Parhar if (needs_vxlan_tso(m)) 2664a4a4ad2dSNavdeep Parhar return (TX_SGL_SEGS_VXLAN_TSO); 2665a4a4ad2dSNavdeep Parhar else 2666a4a4ad2dSNavdeep Parhar return (TX_SGL_SEGS_TSO); 2667a4a4ad2dSNavdeep Parhar } 2668a4a4ad2dSNavdeep Parhar 2669a4a4ad2dSNavdeep Parhar return (TX_SGL_SEGS); 2670a4a4ad2dSNavdeep Parhar } 2671a4a4ad2dSNavdeep Parhar 2672b9820bcaSNavdeep Parhar static struct timeval txerr_ratecheck = {0}; 2673b9820bcaSNavdeep Parhar static const struct timeval txerr_interval = {3, 0}; 2674b9820bcaSNavdeep Parhar 2675a4a4ad2dSNavdeep Parhar /* 26767951040fSNavdeep Parhar * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: 26777951040fSNavdeep Parhar * a) caller can assume it's been freed if this function returns with an error. 26787951040fSNavdeep Parhar * b) it may get defragged up if the gather list is too long for the hardware. 26797951040fSNavdeep Parhar */ 26807951040fSNavdeep Parhar int 268130e3f2b4SNavdeep Parhar parse_pkt(struct mbuf **mp, bool vm_wr) 26827951040fSNavdeep Parhar { 26837951040fSNavdeep Parhar struct mbuf *m0 = *mp, *m; 268439d5cbdcSNavdeep Parhar int rc, nsegs, defragged = 0; 26857951040fSNavdeep Parhar struct ether_header *eh; 268639d5cbdcSNavdeep Parhar #ifdef INET 26877951040fSNavdeep Parhar void *l3hdr; 268839d5cbdcSNavdeep Parhar #endif 26897951040fSNavdeep Parhar #if defined(INET) || defined(INET6) 269039d5cbdcSNavdeep Parhar int offset; 26917951040fSNavdeep Parhar struct tcphdr *tcp; 26927951040fSNavdeep Parhar #endif 2693bddf7343SJohn Baldwin #if defined(KERN_TLS) || defined(RATELIMIT) 269456fb710fSJohn Baldwin struct m_snd_tag *mst; 2695e38a50e8SJohn Baldwin #endif 26967951040fSNavdeep Parhar uint16_t eh_type; 2697d76bbe17SJohn Baldwin uint8_t cflags; 26987951040fSNavdeep Parhar 2699d76bbe17SJohn Baldwin cflags = 0; 27007951040fSNavdeep Parhar M_ASSERTPKTHDR(m0); 27017951040fSNavdeep Parhar if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { 27027951040fSNavdeep Parhar rc = EINVAL; 27037951040fSNavdeep Parhar fail: 27047951040fSNavdeep Parhar m_freem(m0); 27057951040fSNavdeep Parhar *mp = NULL; 27067951040fSNavdeep Parhar return (rc); 27077951040fSNavdeep Parhar } 27087951040fSNavdeep Parhar restart: 27097951040fSNavdeep Parhar /* 27107951040fSNavdeep Parhar * First count the number of gather list segments in the payload. 27117951040fSNavdeep Parhar * Defrag the mbuf if nsegs exceeds the hardware limit. 27127951040fSNavdeep Parhar */ 27137951040fSNavdeep Parhar M_ASSERTPKTHDR(m0); 27147951040fSNavdeep Parhar MPASS(m0->m_pkthdr.len > 0); 2715d76bbe17SJohn Baldwin nsegs = count_mbuf_nsegs(m0, 0, &cflags); 2716bddf7343SJohn Baldwin #if defined(KERN_TLS) || defined(RATELIMIT) 2717e38a50e8SJohn Baldwin if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) 271856fb710fSJohn Baldwin mst = m0->m_pkthdr.snd_tag; 2719e38a50e8SJohn Baldwin else 272056fb710fSJohn Baldwin mst = NULL; 2721e38a50e8SJohn Baldwin #endif 2722bddf7343SJohn Baldwin #ifdef KERN_TLS 2723c782ea8bSJohn Baldwin if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) { 2724bddf7343SJohn Baldwin int len16; 2725bddf7343SJohn Baldwin 2726bddf7343SJohn Baldwin cflags |= MC_TLS; 2727bddf7343SJohn Baldwin set_mbuf_cflags(m0, cflags); 2728bddf7343SJohn Baldwin rc = t6_ktls_parse_pkt(m0, &nsegs, &len16); 2729bddf7343SJohn Baldwin if (rc != 0) 2730bddf7343SJohn Baldwin goto fail; 2731bddf7343SJohn Baldwin set_mbuf_nsegs(m0, nsegs); 2732bddf7343SJohn Baldwin set_mbuf_len16(m0, len16); 2733bddf7343SJohn Baldwin return (0); 2734bddf7343SJohn Baldwin } 2735bddf7343SJohn Baldwin #endif 273630e3f2b4SNavdeep Parhar if (nsegs > max_nsegs_allowed(m0, vm_wr)) { 27377054f6ecSNavdeep Parhar if (defragged++ > 0) { 27387951040fSNavdeep Parhar rc = EFBIG; 27397951040fSNavdeep Parhar goto fail; 27407951040fSNavdeep Parhar } 27417054f6ecSNavdeep Parhar counter_u64_add(defrags, 1); 27427054f6ecSNavdeep Parhar if ((m = m_defrag(m0, M_NOWAIT)) == NULL) { 27437054f6ecSNavdeep Parhar rc = ENOMEM; 27447054f6ecSNavdeep Parhar goto fail; 27457054f6ecSNavdeep Parhar } 27467951040fSNavdeep Parhar *mp = m0 = m; /* update caller's copy after defrag */ 27477951040fSNavdeep Parhar goto restart; 27487951040fSNavdeep Parhar } 27497951040fSNavdeep Parhar 2750d76bbe17SJohn Baldwin if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN && 2751d76bbe17SJohn Baldwin !(cflags & MC_NOMAP))) { 27527054f6ecSNavdeep Parhar counter_u64_add(pullups, 1); 27537951040fSNavdeep Parhar m0 = m_pullup(m0, m0->m_pkthdr.len); 27547951040fSNavdeep Parhar if (m0 == NULL) { 27557951040fSNavdeep Parhar /* Should have left well enough alone. */ 27567951040fSNavdeep Parhar rc = EFBIG; 27577951040fSNavdeep Parhar goto fail; 27587951040fSNavdeep Parhar } 27597951040fSNavdeep Parhar *mp = m0; /* update caller's copy after pullup */ 27607951040fSNavdeep Parhar goto restart; 27617951040fSNavdeep Parhar } 27627951040fSNavdeep Parhar set_mbuf_nsegs(m0, nsegs); 2763d76bbe17SJohn Baldwin set_mbuf_cflags(m0, cflags); 276430e3f2b4SNavdeep Parhar calculate_mbuf_len16(m0, vm_wr); 27657951040fSNavdeep Parhar 2766786099deSNavdeep Parhar #ifdef RATELIMIT 2767786099deSNavdeep Parhar /* 2768786099deSNavdeep Parhar * Ethofld is limited to TCP and UDP for now, and only when L4 hw 2769a4a4ad2dSNavdeep Parhar * checksumming is enabled. needs_outer_l4_csum happens to check for 2770a4a4ad2dSNavdeep Parhar * all the right things. 2771786099deSNavdeep Parhar */ 277256fb710fSJohn Baldwin if (__predict_false(needs_eo(mst) && !needs_outer_l4_csum(m0))) { 2773fb3bc596SJohn Baldwin m_snd_tag_rele(m0->m_pkthdr.snd_tag); 2774786099deSNavdeep Parhar m0->m_pkthdr.snd_tag = NULL; 2775fb3bc596SJohn Baldwin m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 277656fb710fSJohn Baldwin mst = NULL; 2777fb3bc596SJohn Baldwin } 2778786099deSNavdeep Parhar #endif 2779786099deSNavdeep Parhar 2780c0236bd9SNavdeep Parhar if (!needs_hwcsum(m0) 2781786099deSNavdeep Parhar #ifdef RATELIMIT 278256fb710fSJohn Baldwin && !needs_eo(mst) 2783786099deSNavdeep Parhar #endif 2784c0236bd9SNavdeep Parhar ) 27857951040fSNavdeep Parhar return (0); 27867951040fSNavdeep Parhar 27877951040fSNavdeep Parhar m = m0; 27887951040fSNavdeep Parhar eh = mtod(m, struct ether_header *); 27897951040fSNavdeep Parhar eh_type = ntohs(eh->ether_type); 27907951040fSNavdeep Parhar if (eh_type == ETHERTYPE_VLAN) { 27917951040fSNavdeep Parhar struct ether_vlan_header *evh = (void *)eh; 27927951040fSNavdeep Parhar 27937951040fSNavdeep Parhar eh_type = ntohs(evh->evl_proto); 27947951040fSNavdeep Parhar m0->m_pkthdr.l2hlen = sizeof(*evh); 27957951040fSNavdeep Parhar } else 27967951040fSNavdeep Parhar m0->m_pkthdr.l2hlen = sizeof(*eh); 27977951040fSNavdeep Parhar 279839d5cbdcSNavdeep Parhar #if defined(INET) || defined(INET6) 27997951040fSNavdeep Parhar offset = 0; 280039d5cbdcSNavdeep Parhar #ifdef INET 28017951040fSNavdeep Parhar l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); 280239d5cbdcSNavdeep Parhar #else 280339d5cbdcSNavdeep Parhar m_advance(&m, &offset, m0->m_pkthdr.l2hlen); 280439d5cbdcSNavdeep Parhar #endif 280539d5cbdcSNavdeep Parhar #endif 28067951040fSNavdeep Parhar 28077951040fSNavdeep Parhar switch (eh_type) { 28087951040fSNavdeep Parhar #ifdef INET6 28097951040fSNavdeep Parhar case ETHERTYPE_IPV6: 2810a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); 28117951040fSNavdeep Parhar break; 28127951040fSNavdeep Parhar #endif 28137951040fSNavdeep Parhar #ifdef INET 28147951040fSNavdeep Parhar case ETHERTYPE_IP: 28157951040fSNavdeep Parhar { 28167951040fSNavdeep Parhar struct ip *ip = l3hdr; 28177951040fSNavdeep Parhar 2818a4a4ad2dSNavdeep Parhar if (needs_vxlan_csum(m0)) { 2819a4a4ad2dSNavdeep Parhar /* Driver will do the outer IP hdr checksum. */ 2820a4a4ad2dSNavdeep Parhar ip->ip_sum = 0; 2821a4a4ad2dSNavdeep Parhar if (needs_vxlan_tso(m0)) { 2822a4a4ad2dSNavdeep Parhar const uint16_t ipl = ip->ip_len; 2823a4a4ad2dSNavdeep Parhar 2824a4a4ad2dSNavdeep Parhar ip->ip_len = 0; 2825a4a4ad2dSNavdeep Parhar ip->ip_sum = ~in_cksum_hdr(ip); 2826a4a4ad2dSNavdeep Parhar ip->ip_len = ipl; 2827a4a4ad2dSNavdeep Parhar } else 2828a4a4ad2dSNavdeep Parhar ip->ip_sum = in_cksum_hdr(ip); 2829a4a4ad2dSNavdeep Parhar } 2830a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l3hlen = ip->ip_hl << 2; 28317951040fSNavdeep Parhar break; 28327951040fSNavdeep Parhar } 28337951040fSNavdeep Parhar #endif 28347951040fSNavdeep Parhar default: 2835b9820bcaSNavdeep Parhar if (ratecheck(&txerr_ratecheck, &txerr_interval)) { 2836b9820bcaSNavdeep Parhar log(LOG_ERR, "%s: ethertype 0x%04x unknown. " 2837b9820bcaSNavdeep Parhar "if_cxgbe must be compiled with the same " 2838b9820bcaSNavdeep Parhar "INET/INET6 options as the kernel.\n", __func__, 2839b9820bcaSNavdeep Parhar eh_type); 2840b9820bcaSNavdeep Parhar } 2841b9820bcaSNavdeep Parhar rc = EINVAL; 2842b9820bcaSNavdeep Parhar goto fail; 28437951040fSNavdeep Parhar } 28447951040fSNavdeep Parhar 284539d5cbdcSNavdeep Parhar #if defined(INET) || defined(INET6) 2846a4a4ad2dSNavdeep Parhar if (needs_vxlan_csum(m0)) { 2847a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l4hlen = sizeof(struct udphdr); 2848a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l5hlen = sizeof(struct vxlan_header); 2849a4a4ad2dSNavdeep Parhar 2850a4a4ad2dSNavdeep Parhar /* Inner headers. */ 2851a4a4ad2dSNavdeep Parhar eh = m_advance(&m, &offset, m0->m_pkthdr.l3hlen + 2852a4a4ad2dSNavdeep Parhar sizeof(struct udphdr) + sizeof(struct vxlan_header)); 2853a4a4ad2dSNavdeep Parhar eh_type = ntohs(eh->ether_type); 2854a4a4ad2dSNavdeep Parhar if (eh_type == ETHERTYPE_VLAN) { 2855a4a4ad2dSNavdeep Parhar struct ether_vlan_header *evh = (void *)eh; 2856a4a4ad2dSNavdeep Parhar 2857a4a4ad2dSNavdeep Parhar eh_type = ntohs(evh->evl_proto); 2858a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l2hlen = sizeof(*evh); 2859a4a4ad2dSNavdeep Parhar } else 2860a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l2hlen = sizeof(*eh); 286139d5cbdcSNavdeep Parhar #ifdef INET 2862a4a4ad2dSNavdeep Parhar l3hdr = m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen); 286339d5cbdcSNavdeep Parhar #else 286439d5cbdcSNavdeep Parhar m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen); 286539d5cbdcSNavdeep Parhar #endif 2866a4a4ad2dSNavdeep Parhar 2867a4a4ad2dSNavdeep Parhar switch (eh_type) { 2868a4a4ad2dSNavdeep Parhar #ifdef INET6 2869a4a4ad2dSNavdeep Parhar case ETHERTYPE_IPV6: 2870a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l3hlen = sizeof(struct ip6_hdr); 2871a4a4ad2dSNavdeep Parhar break; 2872a4a4ad2dSNavdeep Parhar #endif 2873a4a4ad2dSNavdeep Parhar #ifdef INET 2874a4a4ad2dSNavdeep Parhar case ETHERTYPE_IP: 2875a4a4ad2dSNavdeep Parhar { 2876a4a4ad2dSNavdeep Parhar struct ip *ip = l3hdr; 2877a4a4ad2dSNavdeep Parhar 2878a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l3hlen = ip->ip_hl << 2; 2879a4a4ad2dSNavdeep Parhar break; 2880a4a4ad2dSNavdeep Parhar } 2881a4a4ad2dSNavdeep Parhar #endif 2882a4a4ad2dSNavdeep Parhar default: 2883b9820bcaSNavdeep Parhar if (ratecheck(&txerr_ratecheck, &txerr_interval)) { 2884b9820bcaSNavdeep Parhar log(LOG_ERR, "%s: VXLAN hw offload requested" 2885b9820bcaSNavdeep Parhar "with unknown ethertype 0x%04x. if_cxgbe " 2886b9820bcaSNavdeep Parhar "must be compiled with the same INET/INET6 " 2887b9820bcaSNavdeep Parhar "options as the kernel.\n", __func__, 2888b9820bcaSNavdeep Parhar eh_type); 2889b9820bcaSNavdeep Parhar } 2890b9820bcaSNavdeep Parhar rc = EINVAL; 2891b9820bcaSNavdeep Parhar goto fail; 2892a4a4ad2dSNavdeep Parhar } 2893a4a4ad2dSNavdeep Parhar if (needs_inner_tcp_csum(m0)) { 2894a4a4ad2dSNavdeep Parhar tcp = m_advance(&m, &offset, m0->m_pkthdr.inner_l3hlen); 2895a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l4hlen = tcp->th_off * 4; 2896a4a4ad2dSNavdeep Parhar } 2897a4a4ad2dSNavdeep Parhar MPASS((m0->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); 2898a4a4ad2dSNavdeep Parhar m0->m_pkthdr.csum_flags &= CSUM_INNER_IP6_UDP | 2899a4a4ad2dSNavdeep Parhar CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP | 2900a4a4ad2dSNavdeep Parhar CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | 2901a4a4ad2dSNavdeep Parhar CSUM_ENCAP_VXLAN; 2902a4a4ad2dSNavdeep Parhar } 2903a4a4ad2dSNavdeep Parhar 2904a4a4ad2dSNavdeep Parhar if (needs_outer_tcp_csum(m0)) { 29057951040fSNavdeep Parhar tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); 29067951040fSNavdeep Parhar m0->m_pkthdr.l4hlen = tcp->th_off * 4; 2907786099deSNavdeep Parhar #ifdef RATELIMIT 2908786099deSNavdeep Parhar if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) { 2909786099deSNavdeep Parhar set_mbuf_eo_tsclk_tsoff(m0, 2910786099deSNavdeep Parhar V_FW_ETH_TX_EO_WR_TSCLK(tsclk) | 2911786099deSNavdeep Parhar V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1)); 2912786099deSNavdeep Parhar } else 2913786099deSNavdeep Parhar set_mbuf_eo_tsclk_tsoff(m0, 0); 2914a4a4ad2dSNavdeep Parhar } else if (needs_outer_udp_csum(m0)) { 2915786099deSNavdeep Parhar m0->m_pkthdr.l4hlen = sizeof(struct udphdr); 2916786099deSNavdeep Parhar #endif 29176af45170SJohn Baldwin } 2918786099deSNavdeep Parhar #ifdef RATELIMIT 291956fb710fSJohn Baldwin if (needs_eo(mst)) { 2920786099deSNavdeep Parhar u_int immhdrs; 2921786099deSNavdeep Parhar 2922786099deSNavdeep Parhar /* EO WRs have the headers in the WR and not the GL. */ 2923786099deSNavdeep Parhar immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + 2924786099deSNavdeep Parhar m0->m_pkthdr.l4hlen; 2925d76bbe17SJohn Baldwin cflags = 0; 2926d76bbe17SJohn Baldwin nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags); 2927d76bbe17SJohn Baldwin MPASS(cflags == mbuf_cflags(m0)); 2928786099deSNavdeep Parhar set_mbuf_eo_nsegs(m0, nsegs); 2929786099deSNavdeep Parhar set_mbuf_eo_len16(m0, 2930786099deSNavdeep Parhar txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0))); 2931786099deSNavdeep Parhar } 2932786099deSNavdeep Parhar #endif 29337951040fSNavdeep Parhar #endif 29347951040fSNavdeep Parhar MPASS(m0 == *mp); 29357951040fSNavdeep Parhar return (0); 29367951040fSNavdeep Parhar } 29377951040fSNavdeep Parhar 29387951040fSNavdeep Parhar void * 29397951040fSNavdeep Parhar start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) 29407951040fSNavdeep Parhar { 29417951040fSNavdeep Parhar struct sge_eq *eq = &wrq->eq; 29427951040fSNavdeep Parhar struct adapter *sc = wrq->adapter; 29437951040fSNavdeep Parhar int ndesc, available; 29447951040fSNavdeep Parhar struct wrqe *wr; 29457951040fSNavdeep Parhar void *w; 29467951040fSNavdeep Parhar 29477951040fSNavdeep Parhar MPASS(len16 > 0); 29480cadedfcSNavdeep Parhar ndesc = tx_len16_to_desc(len16); 29497951040fSNavdeep Parhar MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); 29507951040fSNavdeep Parhar 29517951040fSNavdeep Parhar EQ_LOCK(eq); 29527951040fSNavdeep Parhar 29538d6ae10aSNavdeep Parhar if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 29547951040fSNavdeep Parhar drain_wrq_wr_list(sc, wrq); 29557951040fSNavdeep Parhar 29567951040fSNavdeep Parhar if (!STAILQ_EMPTY(&wrq->wr_list)) { 29577951040fSNavdeep Parhar slowpath: 29587951040fSNavdeep Parhar EQ_UNLOCK(eq); 29597951040fSNavdeep Parhar wr = alloc_wrqe(len16 * 16, wrq); 29607951040fSNavdeep Parhar if (__predict_false(wr == NULL)) 29617951040fSNavdeep Parhar return (NULL); 29627951040fSNavdeep Parhar cookie->pidx = -1; 29637951040fSNavdeep Parhar cookie->ndesc = ndesc; 29647951040fSNavdeep Parhar return (&wr->wr); 29657951040fSNavdeep Parhar } 29667951040fSNavdeep Parhar 29677951040fSNavdeep Parhar eq->cidx = read_hw_cidx(eq); 29687951040fSNavdeep Parhar if (eq->pidx == eq->cidx) 29697951040fSNavdeep Parhar available = eq->sidx - 1; 29707951040fSNavdeep Parhar else 29717951040fSNavdeep Parhar available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 29727951040fSNavdeep Parhar if (available < ndesc) 29737951040fSNavdeep Parhar goto slowpath; 29747951040fSNavdeep Parhar 29757951040fSNavdeep Parhar cookie->pidx = eq->pidx; 29767951040fSNavdeep Parhar cookie->ndesc = ndesc; 29777951040fSNavdeep Parhar TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); 29787951040fSNavdeep Parhar 29797951040fSNavdeep Parhar w = &eq->desc[eq->pidx]; 29807951040fSNavdeep Parhar IDXINCR(eq->pidx, ndesc, eq->sidx); 2981f50c49ccSNavdeep Parhar if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { 29827951040fSNavdeep Parhar w = &wrq->ss[0]; 29837951040fSNavdeep Parhar wrq->ss_pidx = cookie->pidx; 29847951040fSNavdeep Parhar wrq->ss_len = len16 * 16; 29857951040fSNavdeep Parhar } 29867951040fSNavdeep Parhar 29877951040fSNavdeep Parhar EQ_UNLOCK(eq); 29887951040fSNavdeep Parhar 29897951040fSNavdeep Parhar return (w); 29907951040fSNavdeep Parhar } 29917951040fSNavdeep Parhar 29927951040fSNavdeep Parhar void 29937951040fSNavdeep Parhar commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) 29947951040fSNavdeep Parhar { 29957951040fSNavdeep Parhar struct sge_eq *eq = &wrq->eq; 29967951040fSNavdeep Parhar struct adapter *sc = wrq->adapter; 29977951040fSNavdeep Parhar int ndesc, pidx; 29987951040fSNavdeep Parhar struct wrq_cookie *prev, *next; 29997951040fSNavdeep Parhar 30007951040fSNavdeep Parhar if (cookie->pidx == -1) { 30017951040fSNavdeep Parhar struct wrqe *wr = __containerof(w, struct wrqe, wr); 30027951040fSNavdeep Parhar 30037951040fSNavdeep Parhar t4_wrq_tx(sc, wr); 30047951040fSNavdeep Parhar return; 30057951040fSNavdeep Parhar } 30067951040fSNavdeep Parhar 30077951040fSNavdeep Parhar if (__predict_false(w == &wrq->ss[0])) { 30087951040fSNavdeep Parhar int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; 30097951040fSNavdeep Parhar 30107951040fSNavdeep Parhar MPASS(wrq->ss_len > n); /* WR had better wrap around. */ 30117951040fSNavdeep Parhar bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); 30127951040fSNavdeep Parhar bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); 30137951040fSNavdeep Parhar wrq->tx_wrs_ss++; 30147951040fSNavdeep Parhar } else 30157951040fSNavdeep Parhar wrq->tx_wrs_direct++; 30167951040fSNavdeep Parhar 30177951040fSNavdeep Parhar EQ_LOCK(eq); 30188d6ae10aSNavdeep Parhar ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ 30198d6ae10aSNavdeep Parhar pidx = cookie->pidx; 30208d6ae10aSNavdeep Parhar MPASS(pidx >= 0 && pidx < eq->sidx); 30217951040fSNavdeep Parhar prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); 30227951040fSNavdeep Parhar next = TAILQ_NEXT(cookie, link); 30237951040fSNavdeep Parhar if (prev == NULL) { 30247951040fSNavdeep Parhar MPASS(pidx == eq->dbidx); 30252e09fe91SNavdeep Parhar if (next == NULL || ndesc >= 16) { 30262e09fe91SNavdeep Parhar int available; 30272e09fe91SNavdeep Parhar struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 30282e09fe91SNavdeep Parhar 30292e09fe91SNavdeep Parhar /* 30302e09fe91SNavdeep Parhar * Note that the WR via which we'll request tx updates 30312e09fe91SNavdeep Parhar * is at pidx and not eq->pidx, which has moved on 30322e09fe91SNavdeep Parhar * already. 30332e09fe91SNavdeep Parhar */ 30342e09fe91SNavdeep Parhar dst = (void *)&eq->desc[pidx]; 30352e09fe91SNavdeep Parhar available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 30362e09fe91SNavdeep Parhar if (available < eq->sidx / 4 && 30372e09fe91SNavdeep Parhar atomic_cmpset_int(&eq->equiq, 0, 1)) { 3038ddf09ad6SNavdeep Parhar /* 3039ddf09ad6SNavdeep Parhar * XXX: This is not 100% reliable with some 3040ddf09ad6SNavdeep Parhar * types of WRs. But this is a very unusual 3041ddf09ad6SNavdeep Parhar * situation for an ofld/ctrl queue anyway. 3042ddf09ad6SNavdeep Parhar */ 30432e09fe91SNavdeep Parhar dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 30442e09fe91SNavdeep Parhar F_FW_WR_EQUEQ); 30452e09fe91SNavdeep Parhar } 30462e09fe91SNavdeep Parhar 30477951040fSNavdeep Parhar ring_eq_db(wrq->adapter, eq, ndesc); 30482e09fe91SNavdeep Parhar } else { 30497951040fSNavdeep Parhar MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); 30507951040fSNavdeep Parhar next->pidx = pidx; 30517951040fSNavdeep Parhar next->ndesc += ndesc; 30527951040fSNavdeep Parhar } 30537951040fSNavdeep Parhar } else { 30547951040fSNavdeep Parhar MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); 30557951040fSNavdeep Parhar prev->ndesc += ndesc; 30567951040fSNavdeep Parhar } 30577951040fSNavdeep Parhar TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); 30587951040fSNavdeep Parhar 30597951040fSNavdeep Parhar if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 30607951040fSNavdeep Parhar drain_wrq_wr_list(sc, wrq); 30617951040fSNavdeep Parhar 30627951040fSNavdeep Parhar #ifdef INVARIANTS 30637951040fSNavdeep Parhar if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { 30647951040fSNavdeep Parhar /* Doorbell must have caught up to the pidx. */ 30657951040fSNavdeep Parhar MPASS(wrq->eq.pidx == wrq->eq.dbidx); 30667951040fSNavdeep Parhar } 30677951040fSNavdeep Parhar #endif 30687951040fSNavdeep Parhar EQ_UNLOCK(eq); 30697951040fSNavdeep Parhar } 30707951040fSNavdeep Parhar 30717951040fSNavdeep Parhar static u_int 30727951040fSNavdeep Parhar can_resume_eth_tx(struct mp_ring *r) 30737951040fSNavdeep Parhar { 30747951040fSNavdeep Parhar struct sge_eq *eq = r->cookie; 30757951040fSNavdeep Parhar 30767951040fSNavdeep Parhar return (total_available_tx_desc(eq) > eq->sidx / 8); 30777951040fSNavdeep Parhar } 30787951040fSNavdeep Parhar 3079d735920dSNavdeep Parhar static inline bool 30807951040fSNavdeep Parhar cannot_use_txpkts(struct mbuf *m) 30817951040fSNavdeep Parhar { 30827951040fSNavdeep Parhar /* maybe put a GL limit too, to avoid silliness? */ 30837951040fSNavdeep Parhar 3084bddf7343SJohn Baldwin return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0); 30857951040fSNavdeep Parhar } 30867951040fSNavdeep Parhar 30871404daa7SNavdeep Parhar static inline int 30881404daa7SNavdeep Parhar discard_tx(struct sge_eq *eq) 30891404daa7SNavdeep Parhar { 30901404daa7SNavdeep Parhar 30911404daa7SNavdeep Parhar return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); 30921404daa7SNavdeep Parhar } 30931404daa7SNavdeep Parhar 30945cdaef71SJohn Baldwin static inline int 3095d735920dSNavdeep Parhar wr_can_update_eq(void *p) 30965cdaef71SJohn Baldwin { 3097d735920dSNavdeep Parhar struct fw_eth_tx_pkts_wr *wr = p; 30985cdaef71SJohn Baldwin 30995cdaef71SJohn Baldwin switch (G_FW_WR_OP(be32toh(wr->op_pkd))) { 31005cdaef71SJohn Baldwin case FW_ULPTX_WR: 31015cdaef71SJohn Baldwin case FW_ETH_TX_PKT_WR: 31025cdaef71SJohn Baldwin case FW_ETH_TX_PKTS_WR: 3103693a9dfcSNavdeep Parhar case FW_ETH_TX_PKTS2_WR: 31045cdaef71SJohn Baldwin case FW_ETH_TX_PKT_VM_WR: 3105d735920dSNavdeep Parhar case FW_ETH_TX_PKTS_VM_WR: 31065cdaef71SJohn Baldwin return (1); 31075cdaef71SJohn Baldwin default: 31085cdaef71SJohn Baldwin return (0); 31095cdaef71SJohn Baldwin } 31105cdaef71SJohn Baldwin } 31115cdaef71SJohn Baldwin 3112d735920dSNavdeep Parhar static inline void 3113d735920dSNavdeep Parhar set_txupdate_flags(struct sge_txq *txq, u_int avail, 3114d735920dSNavdeep Parhar struct fw_eth_tx_pkt_wr *wr) 3115d735920dSNavdeep Parhar { 3116d735920dSNavdeep Parhar struct sge_eq *eq = &txq->eq; 3117d735920dSNavdeep Parhar struct txpkts *txp = &txq->txp; 3118d735920dSNavdeep Parhar 3119d735920dSNavdeep Parhar if ((txp->npkt > 0 || avail < eq->sidx / 2) && 3120d735920dSNavdeep Parhar atomic_cmpset_int(&eq->equiq, 0, 1)) { 3121d735920dSNavdeep Parhar wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3122d735920dSNavdeep Parhar eq->equeqidx = eq->pidx; 3123d735920dSNavdeep Parhar } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 3124d735920dSNavdeep Parhar wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 3125d735920dSNavdeep Parhar eq->equeqidx = eq->pidx; 3126d735920dSNavdeep Parhar } 3127d735920dSNavdeep Parhar } 3128d735920dSNavdeep Parhar 31293447df8bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__) 31303447df8bSNavdeep Parhar extern uint64_t tsc_freq; 31313447df8bSNavdeep Parhar #endif 31323447df8bSNavdeep Parhar 31333447df8bSNavdeep Parhar static inline bool 31343447df8bSNavdeep Parhar record_eth_tx_time(struct sge_txq *txq) 31353447df8bSNavdeep Parhar { 31363447df8bSNavdeep Parhar const uint64_t cycles = get_cyclecount(); 31373447df8bSNavdeep Parhar const uint64_t last_tx = txq->last_tx; 31383447df8bSNavdeep Parhar #if defined(__i386__) || defined(__amd64__) 31393447df8bSNavdeep Parhar const uint64_t itg = tsc_freq * t4_tx_coalesce_gap / 1000000; 31403447df8bSNavdeep Parhar #else 31413447df8bSNavdeep Parhar const uint64_t itg = 0; 31423447df8bSNavdeep Parhar #endif 31433447df8bSNavdeep Parhar 31443447df8bSNavdeep Parhar MPASS(cycles >= last_tx); 31453447df8bSNavdeep Parhar txq->last_tx = cycles; 31463447df8bSNavdeep Parhar return (cycles - last_tx < itg); 31473447df8bSNavdeep Parhar } 31483447df8bSNavdeep Parhar 31497951040fSNavdeep Parhar /* 31507951040fSNavdeep Parhar * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to 31517951040fSNavdeep Parhar * be consumed. Return the actual number consumed. 0 indicates a stall. 31527951040fSNavdeep Parhar */ 31537951040fSNavdeep Parhar static u_int 3154d735920dSNavdeep Parhar eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing) 31557951040fSNavdeep Parhar { 31567951040fSNavdeep Parhar struct sge_txq *txq = r->cookie; 31577951040fSNavdeep Parhar struct ifnet *ifp = txq->ifp; 3158d735920dSNavdeep Parhar struct sge_eq *eq = &txq->eq; 3159d735920dSNavdeep Parhar struct txpkts *txp = &txq->txp; 3160fe2ebb76SJohn Baldwin struct vi_info *vi = ifp->if_softc; 31617c228be3SNavdeep Parhar struct adapter *sc = vi->adapter; 31627951040fSNavdeep Parhar u_int total, remaining; /* # of packets */ 3163d735920dSNavdeep Parhar u_int n, avail, dbdiff; /* # of hardware descriptors */ 3164d735920dSNavdeep Parhar int i, rc; 3165d735920dSNavdeep Parhar struct mbuf *m0; 31663447df8bSNavdeep Parhar bool snd, recent_tx; 3167d735920dSNavdeep Parhar void *wr; /* start of the last WR written to the ring */ 3168d735920dSNavdeep Parhar 3169d735920dSNavdeep Parhar TXQ_LOCK_ASSERT_OWNED(txq); 31703447df8bSNavdeep Parhar recent_tx = record_eth_tx_time(txq); 31717951040fSNavdeep Parhar 31727951040fSNavdeep Parhar remaining = IDXDIFF(pidx, cidx, r->size); 31731404daa7SNavdeep Parhar if (__predict_false(discard_tx(eq))) { 3174d735920dSNavdeep Parhar for (i = 0; i < txp->npkt; i++) 3175d735920dSNavdeep Parhar m_freem(txp->mb[i]); 3176d735920dSNavdeep Parhar txp->npkt = 0; 31777951040fSNavdeep Parhar while (cidx != pidx) { 31787951040fSNavdeep Parhar m0 = r->items[cidx]; 31797951040fSNavdeep Parhar m_freem(m0); 31807951040fSNavdeep Parhar if (++cidx == r->size) 31817951040fSNavdeep Parhar cidx = 0; 31827951040fSNavdeep Parhar } 3183d735920dSNavdeep Parhar reclaim_tx_descs(txq, eq->sidx); 3184d735920dSNavdeep Parhar *coalescing = false; 3185d735920dSNavdeep Parhar return (remaining); /* emptied */ 31867951040fSNavdeep Parhar } 31877951040fSNavdeep Parhar 31887951040fSNavdeep Parhar /* How many hardware descriptors do we have readily available. */ 31893447df8bSNavdeep Parhar if (eq->pidx == eq->cidx) 3190d735920dSNavdeep Parhar avail = eq->sidx - 1; 31913447df8bSNavdeep Parhar else 3192d735920dSNavdeep Parhar avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 31937951040fSNavdeep Parhar 3194d735920dSNavdeep Parhar total = 0; 3195d735920dSNavdeep Parhar if (remaining == 0) { 31963447df8bSNavdeep Parhar txp->score = 0; 31973447df8bSNavdeep Parhar txq->txpkts_flush++; 3198d735920dSNavdeep Parhar goto send_txpkts; 3199d735920dSNavdeep Parhar } 3200d735920dSNavdeep Parhar 3201d735920dSNavdeep Parhar dbdiff = 0; 3202d735920dSNavdeep Parhar MPASS(remaining > 0); 32037951040fSNavdeep Parhar while (remaining > 0) { 32047951040fSNavdeep Parhar m0 = r->items[cidx]; 32057951040fSNavdeep Parhar M_ASSERTPKTHDR(m0); 32067951040fSNavdeep Parhar MPASS(m0->m_nextpkt == NULL); 32077951040fSNavdeep Parhar 3208d735920dSNavdeep Parhar if (avail < 2 * SGE_MAX_WR_NDESC) 3209d735920dSNavdeep Parhar avail += reclaim_tx_descs(txq, 64); 3210d735920dSNavdeep Parhar 32113447df8bSNavdeep Parhar if (t4_tx_coalesce == 0 && txp->npkt == 0) 32123447df8bSNavdeep Parhar goto skip_coalescing; 32133447df8bSNavdeep Parhar if (cannot_use_txpkts(m0)) 32143447df8bSNavdeep Parhar txp->score = 0; 32153447df8bSNavdeep Parhar else if (recent_tx) { 32163447df8bSNavdeep Parhar if (++txp->score == 0) 32173447df8bSNavdeep Parhar txp->score = UINT8_MAX; 32183447df8bSNavdeep Parhar } else 32193447df8bSNavdeep Parhar txp->score = 1; 32203447df8bSNavdeep Parhar if (txp->npkt > 0 || remaining > 1 || 32213447df8bSNavdeep Parhar txp->score >= t4_tx_coalesce_pkts || 3222d735920dSNavdeep Parhar atomic_load_int(&txq->eq.equiq) != 0) { 322330e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3224d735920dSNavdeep Parhar rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd); 3225d735920dSNavdeep Parhar else 3226d735920dSNavdeep Parhar rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd); 3227d735920dSNavdeep Parhar } else { 3228d735920dSNavdeep Parhar snd = false; 3229d735920dSNavdeep Parhar rc = EINVAL; 3230d735920dSNavdeep Parhar } 3231d735920dSNavdeep Parhar if (snd) { 3232d735920dSNavdeep Parhar MPASS(txp->npkt > 0); 3233d735920dSNavdeep Parhar for (i = 0; i < txp->npkt; i++) 3234d735920dSNavdeep Parhar ETHER_BPF_MTAP(ifp, txp->mb[i]); 3235d735920dSNavdeep Parhar if (txp->npkt > 1) { 3236d735920dSNavdeep Parhar MPASS(avail >= tx_len16_to_desc(txp->len16)); 323730e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3238d735920dSNavdeep Parhar n = write_txpkts_vm_wr(sc, txq); 3239d735920dSNavdeep Parhar else 3240d735920dSNavdeep Parhar n = write_txpkts_wr(sc, txq); 3241d735920dSNavdeep Parhar } else { 3242d735920dSNavdeep Parhar MPASS(avail >= 3243d735920dSNavdeep Parhar tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 324430e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3245d735920dSNavdeep Parhar n = write_txpkt_vm_wr(sc, txq, 3246d735920dSNavdeep Parhar txp->mb[0]); 3247d735920dSNavdeep Parhar else 3248d735920dSNavdeep Parhar n = write_txpkt_wr(sc, txq, txp->mb[0], 3249d735920dSNavdeep Parhar avail); 3250d735920dSNavdeep Parhar } 3251d735920dSNavdeep Parhar MPASS(n <= SGE_MAX_WR_NDESC); 3252d735920dSNavdeep Parhar avail -= n; 3253d735920dSNavdeep Parhar dbdiff += n; 3254d735920dSNavdeep Parhar wr = &eq->desc[eq->pidx]; 3255d735920dSNavdeep Parhar IDXINCR(eq->pidx, n, eq->sidx); 3256d735920dSNavdeep Parhar txp->npkt = 0; /* emptied */ 3257d735920dSNavdeep Parhar } 3258d735920dSNavdeep Parhar if (rc == 0) { 3259d735920dSNavdeep Parhar /* m0 was coalesced into txq->txpkts. */ 3260d735920dSNavdeep Parhar goto next_mbuf; 3261d735920dSNavdeep Parhar } 3262d735920dSNavdeep Parhar if (rc == EAGAIN) { 3263d735920dSNavdeep Parhar /* 3264d735920dSNavdeep Parhar * m0 is suitable for tx coalescing but could not be 3265d735920dSNavdeep Parhar * combined with the existing txq->txpkts, which has now 3266d735920dSNavdeep Parhar * been transmitted. Start a new txpkts with m0. 3267d735920dSNavdeep Parhar */ 3268d735920dSNavdeep Parhar MPASS(snd); 3269d735920dSNavdeep Parhar MPASS(txp->npkt == 0); 3270d735920dSNavdeep Parhar continue; 32717951040fSNavdeep Parhar } 32727951040fSNavdeep Parhar 3273d735920dSNavdeep Parhar MPASS(rc != 0 && rc != EAGAIN); 3274d735920dSNavdeep Parhar MPASS(txp->npkt == 0); 32753447df8bSNavdeep Parhar skip_coalescing: 3276565b8fceSNavdeep Parhar n = tx_len16_to_desc(mbuf_len16(m0)); 3277565b8fceSNavdeep Parhar if (__predict_false(avail < n)) { 3278565b8fceSNavdeep Parhar avail += reclaim_tx_descs(txq, min(n, 32)); 3279565b8fceSNavdeep Parhar if (avail < n) 3280565b8fceSNavdeep Parhar break; /* out of descriptors */ 3281565b8fceSNavdeep Parhar } 3282565b8fceSNavdeep Parhar 3283d735920dSNavdeep Parhar wr = &eq->desc[eq->pidx]; 3284bddf7343SJohn Baldwin if (mbuf_cflags(m0) & MC_RAW_WR) { 3285d735920dSNavdeep Parhar n = write_raw_wr(txq, wr, m0, avail); 3286bddf7343SJohn Baldwin #ifdef KERN_TLS 3287bddf7343SJohn Baldwin } else if (mbuf_cflags(m0) & MC_TLS) { 3288bddf7343SJohn Baldwin ETHER_BPF_MTAP(ifp, m0); 3289d735920dSNavdeep Parhar n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0), 3290d735920dSNavdeep Parhar avail); 3291bddf7343SJohn Baldwin #endif 32927951040fSNavdeep Parhar } else { 32933bbb68f0SNavdeep Parhar ETHER_BPF_MTAP(ifp, m0); 329430e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3295d735920dSNavdeep Parhar n = write_txpkt_vm_wr(sc, txq, m0); 3296d735920dSNavdeep Parhar else 3297d735920dSNavdeep Parhar n = write_txpkt_wr(sc, txq, m0, avail); 3298d735920dSNavdeep Parhar } 3299d735920dSNavdeep Parhar MPASS(n >= 1 && n <= avail); 3300bddf7343SJohn Baldwin if (!(mbuf_cflags(m0) & MC_TLS)) 3301bddf7343SJohn Baldwin MPASS(n <= SGE_MAX_WR_NDESC); 33027951040fSNavdeep Parhar 3303d735920dSNavdeep Parhar avail -= n; 33047951040fSNavdeep Parhar dbdiff += n; 33057951040fSNavdeep Parhar IDXINCR(eq->pidx, n, eq->sidx); 33067951040fSNavdeep Parhar 3307d735920dSNavdeep Parhar if (dbdiff >= 512 / EQ_ESIZE) { /* X_FETCHBURSTMAX_512B */ 3308d735920dSNavdeep Parhar if (wr_can_update_eq(wr)) 3309d735920dSNavdeep Parhar set_txupdate_flags(txq, avail, wr); 33107951040fSNavdeep Parhar ring_eq_db(sc, eq, dbdiff); 3311d735920dSNavdeep Parhar avail += reclaim_tx_descs(txq, 32); 33127951040fSNavdeep Parhar dbdiff = 0; 33137951040fSNavdeep Parhar } 3314d735920dSNavdeep Parhar next_mbuf: 3315d735920dSNavdeep Parhar total++; 3316d735920dSNavdeep Parhar remaining--; 3317d735920dSNavdeep Parhar if (__predict_false(++cidx == r->size)) 3318d735920dSNavdeep Parhar cidx = 0; 33197951040fSNavdeep Parhar } 33207951040fSNavdeep Parhar if (dbdiff != 0) { 3321d735920dSNavdeep Parhar if (wr_can_update_eq(wr)) 3322d735920dSNavdeep Parhar set_txupdate_flags(txq, avail, wr); 33237951040fSNavdeep Parhar ring_eq_db(sc, eq, dbdiff); 33247951040fSNavdeep Parhar reclaim_tx_descs(txq, 32); 3325d735920dSNavdeep Parhar } else if (eq->pidx == eq->cidx && txp->npkt > 0 && 3326d735920dSNavdeep Parhar atomic_load_int(&txq->eq.equiq) == 0) { 3327d735920dSNavdeep Parhar /* 3328d735920dSNavdeep Parhar * If nothing was submitted to the chip for tx (it was coalesced 3329d735920dSNavdeep Parhar * into txpkts instead) and there is no tx update outstanding 3330d735920dSNavdeep Parhar * then we need to send txpkts now. 3331d735920dSNavdeep Parhar */ 3332d735920dSNavdeep Parhar send_txpkts: 3333d735920dSNavdeep Parhar MPASS(txp->npkt > 0); 3334d735920dSNavdeep Parhar for (i = 0; i < txp->npkt; i++) 3335d735920dSNavdeep Parhar ETHER_BPF_MTAP(ifp, txp->mb[i]); 3336d735920dSNavdeep Parhar if (txp->npkt > 1) { 3337d735920dSNavdeep Parhar MPASS(avail >= tx_len16_to_desc(txp->len16)); 333830e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3339d735920dSNavdeep Parhar n = write_txpkts_vm_wr(sc, txq); 3340d735920dSNavdeep Parhar else 3341d735920dSNavdeep Parhar n = write_txpkts_wr(sc, txq); 3342d735920dSNavdeep Parhar } else { 3343d735920dSNavdeep Parhar MPASS(avail >= 3344d735920dSNavdeep Parhar tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 334530e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 3346d735920dSNavdeep Parhar n = write_txpkt_vm_wr(sc, txq, txp->mb[0]); 3347d735920dSNavdeep Parhar else 3348d735920dSNavdeep Parhar n = write_txpkt_wr(sc, txq, txp->mb[0], avail); 33497951040fSNavdeep Parhar } 3350d735920dSNavdeep Parhar MPASS(n <= SGE_MAX_WR_NDESC); 3351d735920dSNavdeep Parhar wr = &eq->desc[eq->pidx]; 3352d735920dSNavdeep Parhar IDXINCR(eq->pidx, n, eq->sidx); 3353d735920dSNavdeep Parhar txp->npkt = 0; /* emptied */ 3354d735920dSNavdeep Parhar 3355d735920dSNavdeep Parhar MPASS(wr_can_update_eq(wr)); 3356d735920dSNavdeep Parhar set_txupdate_flags(txq, avail - n, wr); 3357d735920dSNavdeep Parhar ring_eq_db(sc, eq, n); 3358d735920dSNavdeep Parhar reclaim_tx_descs(txq, 32); 3359d735920dSNavdeep Parhar } 3360d735920dSNavdeep Parhar *coalescing = txp->npkt > 0; 33617951040fSNavdeep Parhar 33627951040fSNavdeep Parhar return (total); 3363733b9277SNavdeep Parhar } 3364733b9277SNavdeep Parhar 336554e4ee71SNavdeep Parhar static inline void 336654e4ee71SNavdeep Parhar init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 336743bbae19SNavdeep Parhar int qsize, int intr_idx, int cong) 336854e4ee71SNavdeep Parhar { 3369b2daa9a9SNavdeep Parhar 337054e4ee71SNavdeep Parhar KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 337154e4ee71SNavdeep Parhar ("%s: bad tmr_idx %d", __func__, tmr_idx)); 337254e4ee71SNavdeep Parhar KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ 337354e4ee71SNavdeep Parhar ("%s: bad pktc_idx %d", __func__, pktc_idx)); 337443bbae19SNavdeep Parhar KASSERT(intr_idx >= -1 && intr_idx < sc->intr_count, 337543bbae19SNavdeep Parhar ("%s: bad intr_idx %d", __func__, intr_idx)); 337654e4ee71SNavdeep Parhar 337754e4ee71SNavdeep Parhar iq->flags = 0; 337843bbae19SNavdeep Parhar iq->state = IQS_DISABLED; 337954e4ee71SNavdeep Parhar iq->adapter = sc; 33807a32954cSNavdeep Parhar iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 33817a32954cSNavdeep Parhar iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 33827a32954cSNavdeep Parhar if (pktc_idx >= 0) { 33837a32954cSNavdeep Parhar iq->intr_params |= F_QINTR_CNT_EN; 338454e4ee71SNavdeep Parhar iq->intr_pktc_idx = pktc_idx; 33857a32954cSNavdeep Parhar } 3386d14b0ac1SNavdeep Parhar iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ 338790e7434aSNavdeep Parhar iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; 338843bbae19SNavdeep Parhar iq->intr_idx = intr_idx; 338943bbae19SNavdeep Parhar iq->cong = cong; 339054e4ee71SNavdeep Parhar } 339154e4ee71SNavdeep Parhar 339254e4ee71SNavdeep Parhar static inline void 3393e3207e19SNavdeep Parhar init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) 339454e4ee71SNavdeep Parhar { 339543bbae19SNavdeep Parhar struct sge_params *sp = &sc->params.sge; 33961458bff9SNavdeep Parhar 339754e4ee71SNavdeep Parhar fl->qsize = qsize; 339890e7434aSNavdeep Parhar fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 339954e4ee71SNavdeep Parhar strlcpy(fl->lockname, name, sizeof(fl->lockname)); 340043bbae19SNavdeep Parhar mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); 3401e3207e19SNavdeep Parhar if (sc->flags & BUF_PACKING_OK && 3402e3207e19SNavdeep Parhar ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ 3403e3207e19SNavdeep Parhar (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ 34041458bff9SNavdeep Parhar fl->flags |= FL_BUF_PACKING; 340546e1e307SNavdeep Parhar fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING); 340646e1e307SNavdeep Parhar fl->safe_zidx = sc->sge.safe_zidx; 340743bbae19SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 340843bbae19SNavdeep Parhar fl->lowat = roundup2(sp->fl_starve_threshold2, 8); 340943bbae19SNavdeep Parhar fl->buf_boundary = sp->pack_boundary; 341043bbae19SNavdeep Parhar } else { 341143bbae19SNavdeep Parhar fl->lowat = roundup2(sp->fl_starve_threshold, 8); 341243bbae19SNavdeep Parhar fl->buf_boundary = 16; 341343bbae19SNavdeep Parhar } 341443bbae19SNavdeep Parhar if (fl_pad && fl->buf_boundary < sp->pad_boundary) 341543bbae19SNavdeep Parhar fl->buf_boundary = sp->pad_boundary; 341654e4ee71SNavdeep Parhar } 341754e4ee71SNavdeep Parhar 341854e4ee71SNavdeep Parhar static inline void 341990e7434aSNavdeep Parhar init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, 342043bbae19SNavdeep Parhar uint8_t tx_chan, struct sge_iq *iq, char *name) 342154e4ee71SNavdeep Parhar { 342243bbae19SNavdeep Parhar KASSERT(eqtype >= EQ_CTRL && eqtype <= EQ_OFLD, 342343bbae19SNavdeep Parhar ("%s: bad qtype %d", __func__, eqtype)); 3424733b9277SNavdeep Parhar 342543bbae19SNavdeep Parhar eq->type = eqtype; 3426733b9277SNavdeep Parhar eq->tx_chan = tx_chan; 342743bbae19SNavdeep Parhar eq->iq = iq; 342890e7434aSNavdeep Parhar eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 3429f7dfe243SNavdeep Parhar strlcpy(eq->lockname, name, sizeof(eq->lockname)); 343043bbae19SNavdeep Parhar mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 343154e4ee71SNavdeep Parhar } 343254e4ee71SNavdeep Parhar 34338eba75edSNavdeep Parhar int 343454e4ee71SNavdeep Parhar alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 343554e4ee71SNavdeep Parhar bus_dmamap_t *map, bus_addr_t *pa, void **va) 343654e4ee71SNavdeep Parhar { 343754e4ee71SNavdeep Parhar int rc; 343854e4ee71SNavdeep Parhar 343954e4ee71SNavdeep Parhar rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, 344054e4ee71SNavdeep Parhar BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); 344154e4ee71SNavdeep Parhar if (rc != 0) { 344243bbae19SNavdeep Parhar CH_ERR(sc, "cannot allocate DMA tag: %d\n", rc); 344354e4ee71SNavdeep Parhar goto done; 344454e4ee71SNavdeep Parhar } 344554e4ee71SNavdeep Parhar 344654e4ee71SNavdeep Parhar rc = bus_dmamem_alloc(*tag, va, 344754e4ee71SNavdeep Parhar BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); 344854e4ee71SNavdeep Parhar if (rc != 0) { 344943bbae19SNavdeep Parhar CH_ERR(sc, "cannot allocate DMA memory: %d\n", rc); 345054e4ee71SNavdeep Parhar goto done; 345154e4ee71SNavdeep Parhar } 345254e4ee71SNavdeep Parhar 345354e4ee71SNavdeep Parhar rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); 345454e4ee71SNavdeep Parhar if (rc != 0) { 345543bbae19SNavdeep Parhar CH_ERR(sc, "cannot load DMA map: %d\n", rc); 345654e4ee71SNavdeep Parhar goto done; 345754e4ee71SNavdeep Parhar } 345854e4ee71SNavdeep Parhar done: 345954e4ee71SNavdeep Parhar if (rc) 346054e4ee71SNavdeep Parhar free_ring(sc, *tag, *map, *pa, *va); 346154e4ee71SNavdeep Parhar 346254e4ee71SNavdeep Parhar return (rc); 346354e4ee71SNavdeep Parhar } 346454e4ee71SNavdeep Parhar 34658eba75edSNavdeep Parhar int 346654e4ee71SNavdeep Parhar free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, 346754e4ee71SNavdeep Parhar bus_addr_t pa, void *va) 346854e4ee71SNavdeep Parhar { 346954e4ee71SNavdeep Parhar if (pa) 347054e4ee71SNavdeep Parhar bus_dmamap_unload(tag, map); 347154e4ee71SNavdeep Parhar if (va) 347254e4ee71SNavdeep Parhar bus_dmamem_free(tag, va, map); 347354e4ee71SNavdeep Parhar if (tag) 347454e4ee71SNavdeep Parhar bus_dma_tag_destroy(tag); 347554e4ee71SNavdeep Parhar 347654e4ee71SNavdeep Parhar return (0); 347754e4ee71SNavdeep Parhar } 347854e4ee71SNavdeep Parhar 347954e4ee71SNavdeep Parhar /* 348043bbae19SNavdeep Parhar * Allocates the software resources (mainly memory and sysctl nodes) for an 348143bbae19SNavdeep Parhar * ingress queue and an optional freelist. 348254e4ee71SNavdeep Parhar * 348343bbae19SNavdeep Parhar * Sets IQ_SW_ALLOCATED and returns 0 on success. 348454e4ee71SNavdeep Parhar */ 348554e4ee71SNavdeep Parhar static int 3486fe2ebb76SJohn Baldwin alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, 348743bbae19SNavdeep Parhar struct sysctl_ctx_list *ctx, struct sysctl_oid *oid) 348854e4ee71SNavdeep Parhar { 348943bbae19SNavdeep Parhar int rc; 349054e4ee71SNavdeep Parhar size_t len; 349143bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 349243bbae19SNavdeep Parhar 349343bbae19SNavdeep Parhar MPASS(!(iq->flags & IQ_SW_ALLOCATED)); 349454e4ee71SNavdeep Parhar 3495b2daa9a9SNavdeep Parhar len = iq->qsize * IQ_ESIZE; 349654e4ee71SNavdeep Parhar rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, 349754e4ee71SNavdeep Parhar (void **)&iq->desc); 349854e4ee71SNavdeep Parhar if (rc != 0) 349954e4ee71SNavdeep Parhar return (rc); 350054e4ee71SNavdeep Parhar 350143bbae19SNavdeep Parhar if (fl) { 350243bbae19SNavdeep Parhar len = fl->qsize * EQ_ESIZE; 350343bbae19SNavdeep Parhar rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, 350443bbae19SNavdeep Parhar &fl->ba, (void **)&fl->desc); 350543bbae19SNavdeep Parhar if (rc) { 350643bbae19SNavdeep Parhar free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, 350743bbae19SNavdeep Parhar iq->desc); 350843bbae19SNavdeep Parhar return (rc); 350943bbae19SNavdeep Parhar } 351043bbae19SNavdeep Parhar 351143bbae19SNavdeep Parhar /* Allocate space for one software descriptor per buffer. */ 351243bbae19SNavdeep Parhar fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), 351343bbae19SNavdeep Parhar M_CXGBE, M_ZERO | M_WAITOK); 351443bbae19SNavdeep Parhar 351543bbae19SNavdeep Parhar add_fl_sysctls(sc, ctx, oid, fl); 351643bbae19SNavdeep Parhar iq->flags |= IQ_HAS_FL; 351743bbae19SNavdeep Parhar } 351843bbae19SNavdeep Parhar add_iq_sysctls(ctx, oid, iq); 351943bbae19SNavdeep Parhar iq->flags |= IQ_SW_ALLOCATED; 352043bbae19SNavdeep Parhar 352143bbae19SNavdeep Parhar return (0); 352243bbae19SNavdeep Parhar } 352343bbae19SNavdeep Parhar 352443bbae19SNavdeep Parhar /* 352543bbae19SNavdeep Parhar * Frees all software resources (memory and locks) associated with an ingress 352643bbae19SNavdeep Parhar * queue and an optional freelist. 352743bbae19SNavdeep Parhar */ 352843bbae19SNavdeep Parhar static void 352943bbae19SNavdeep Parhar free_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) 353043bbae19SNavdeep Parhar { 353143bbae19SNavdeep Parhar MPASS(iq->flags & IQ_SW_ALLOCATED); 353243bbae19SNavdeep Parhar 353343bbae19SNavdeep Parhar if (fl) { 353443bbae19SNavdeep Parhar MPASS(iq->flags & IQ_HAS_FL); 353543bbae19SNavdeep Parhar free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc); 353643bbae19SNavdeep Parhar free_fl_buffers(sc, fl); 353743bbae19SNavdeep Parhar free(fl->sdesc, M_CXGBE); 353843bbae19SNavdeep Parhar mtx_destroy(&fl->fl_lock); 353943bbae19SNavdeep Parhar bzero(fl, sizeof(*fl)); 354043bbae19SNavdeep Parhar } 354143bbae19SNavdeep Parhar free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); 354243bbae19SNavdeep Parhar bzero(iq, sizeof(*iq)); 354343bbae19SNavdeep Parhar } 354443bbae19SNavdeep Parhar 354543bbae19SNavdeep Parhar /* 354643bbae19SNavdeep Parhar * Allocates a hardware ingress queue and an optional freelist that will be 354743bbae19SNavdeep Parhar * associated with it. 354843bbae19SNavdeep Parhar * 354943bbae19SNavdeep Parhar * Returns errno on failure. Resources allocated up to that point may still be 355043bbae19SNavdeep Parhar * allocated. Caller is responsible for cleanup in case this function fails. 355143bbae19SNavdeep Parhar */ 355243bbae19SNavdeep Parhar static int 355343bbae19SNavdeep Parhar alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) 355443bbae19SNavdeep Parhar { 355543bbae19SNavdeep Parhar int rc, i, cntxt_id; 355643bbae19SNavdeep Parhar struct fw_iq_cmd c; 355743bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 355843bbae19SNavdeep Parhar __be32 v = 0; 355943bbae19SNavdeep Parhar 356043bbae19SNavdeep Parhar MPASS (!(iq->flags & IQ_HW_ALLOCATED)); 356143bbae19SNavdeep Parhar 356254e4ee71SNavdeep Parhar bzero(&c, sizeof(c)); 356354e4ee71SNavdeep Parhar c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 356454e4ee71SNavdeep Parhar F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 356554e4ee71SNavdeep Parhar V_FW_IQ_CMD_VFN(0)); 356654e4ee71SNavdeep Parhar 356754e4ee71SNavdeep Parhar c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 356854e4ee71SNavdeep Parhar FW_LEN16(c)); 356954e4ee71SNavdeep Parhar 357054e4ee71SNavdeep Parhar /* Special handling for firmware event queue */ 357154e4ee71SNavdeep Parhar if (iq == &sc->sge.fwq) 357254e4ee71SNavdeep Parhar v |= F_FW_IQ_CMD_IQASYNCH; 357354e4ee71SNavdeep Parhar 357443bbae19SNavdeep Parhar if (iq->intr_idx < 0) { 3575f549e352SNavdeep Parhar /* Forwarded interrupts, all headed to fwq */ 3576f549e352SNavdeep Parhar v |= F_FW_IQ_CMD_IQANDST; 3577f549e352SNavdeep Parhar v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id); 3578f549e352SNavdeep Parhar } else { 357943bbae19SNavdeep Parhar KASSERT(iq->intr_idx < sc->intr_count, 358043bbae19SNavdeep Parhar ("%s: invalid direct intr_idx %d", __func__, iq->intr_idx)); 358143bbae19SNavdeep Parhar v |= V_FW_IQ_CMD_IQANDSTINDEX(iq->intr_idx); 3582f549e352SNavdeep Parhar } 358354e4ee71SNavdeep Parhar 358443bbae19SNavdeep Parhar bzero(iq->desc, iq->qsize * IQ_ESIZE); 358554e4ee71SNavdeep Parhar c.type_to_iqandstindex = htobe32(v | 358654e4ee71SNavdeep Parhar V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 3587fe2ebb76SJohn Baldwin V_FW_IQ_CMD_VIID(vi->viid) | 358854e4ee71SNavdeep Parhar V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 358943bbae19SNavdeep Parhar c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(vi->pi->tx_chan) | 359054e4ee71SNavdeep Parhar F_FW_IQ_CMD_IQGTSMODE | 359154e4ee71SNavdeep Parhar V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 3592b2daa9a9SNavdeep Parhar V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); 359354e4ee71SNavdeep Parhar c.iqsize = htobe16(iq->qsize); 359454e4ee71SNavdeep Parhar c.iqaddr = htobe64(iq->ba); 359543bbae19SNavdeep Parhar if (iq->cong >= 0) 3596bc14b14dSNavdeep Parhar c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); 359754e4ee71SNavdeep Parhar 359854e4ee71SNavdeep Parhar if (fl) { 359943bbae19SNavdeep Parhar bzero(fl->desc, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len); 3600214c3582SNavdeep Parhar c.iqns_to_fl0congen |= 3601bc14b14dSNavdeep Parhar htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 3602bc14b14dSNavdeep Parhar F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | 36031458bff9SNavdeep Parhar (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | 36041458bff9SNavdeep Parhar (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 36051458bff9SNavdeep Parhar 0)); 360643bbae19SNavdeep Parhar if (iq->cong >= 0) { 3607bc14b14dSNavdeep Parhar c.iqns_to_fl0congen |= 360843bbae19SNavdeep Parhar htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(iq->cong) | 3609bc14b14dSNavdeep Parhar F_FW_IQ_CMD_FL0CONGCIF | 3610bc14b14dSNavdeep Parhar F_FW_IQ_CMD_FL0CONGEN); 3611bc14b14dSNavdeep Parhar } 361254e4ee71SNavdeep Parhar c.fl0dcaen_to_fl0cidxfthresh = 3613ed7e5640SNavdeep Parhar htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3614adb0cd84SNavdeep Parhar X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) | 3615ed7e5640SNavdeep Parhar V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? 3616ed7e5640SNavdeep Parhar X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 361754e4ee71SNavdeep Parhar c.fl0size = htobe16(fl->qsize); 361854e4ee71SNavdeep Parhar c.fl0addr = htobe64(fl->ba); 361954e4ee71SNavdeep Parhar } 362054e4ee71SNavdeep Parhar 362154e4ee71SNavdeep Parhar rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 362254e4ee71SNavdeep Parhar if (rc != 0) { 362343bbae19SNavdeep Parhar CH_ERR(sc, "failed to create hw ingress queue: %d\n", rc); 362454e4ee71SNavdeep Parhar return (rc); 362554e4ee71SNavdeep Parhar } 362654e4ee71SNavdeep Parhar 362754e4ee71SNavdeep Parhar iq->cidx = 0; 3628b2daa9a9SNavdeep Parhar iq->gen = F_RSPD_GEN; 362954e4ee71SNavdeep Parhar iq->cntxt_id = be16toh(c.iqid); 363054e4ee71SNavdeep Parhar iq->abs_id = be16toh(c.physiqid); 363154e4ee71SNavdeep Parhar 363254e4ee71SNavdeep Parhar cntxt_id = iq->cntxt_id - sc->sge.iq_start; 3633b20b25e7SNavdeep Parhar if (cntxt_id >= sc->sge.iqmap_sz) { 3634733b9277SNavdeep Parhar panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 3635b20b25e7SNavdeep Parhar cntxt_id, sc->sge.iqmap_sz - 1); 3636733b9277SNavdeep Parhar } 363754e4ee71SNavdeep Parhar sc->sge.iqmap[cntxt_id] = iq; 363854e4ee71SNavdeep Parhar 363954e4ee71SNavdeep Parhar if (fl) { 36404d6db4e0SNavdeep Parhar u_int qid; 364143bbae19SNavdeep Parhar #ifdef INVARIANTS 364243bbae19SNavdeep Parhar MPASS(!(fl->flags & FL_BUF_RESUME)); 364343bbae19SNavdeep Parhar for (i = 0; i < fl->sidx * 8; i++) 364443bbae19SNavdeep Parhar MPASS(fl->sdesc[i].cl == NULL); 364543bbae19SNavdeep Parhar #endif 364654e4ee71SNavdeep Parhar fl->cntxt_id = be16toh(c.fl0id); 364743bbae19SNavdeep Parhar fl->pidx = fl->cidx = fl->hw_cidx = fl->dbidx = 0; 364843bbae19SNavdeep Parhar fl->rx_offset = 0; 364943bbae19SNavdeep Parhar fl->flags &= ~(FL_STARVING | FL_DOOMED); 365054e4ee71SNavdeep Parhar 36519f1f7ec9SNavdeep Parhar cntxt_id = fl->cntxt_id - sc->sge.eq_start; 3652b20b25e7SNavdeep Parhar if (cntxt_id >= sc->sge.eqmap_sz) { 3653733b9277SNavdeep Parhar panic("%s: fl->cntxt_id (%d) more than the max (%d)", 3654b20b25e7SNavdeep Parhar __func__, cntxt_id, sc->sge.eqmap_sz - 1); 3655733b9277SNavdeep Parhar } 365654e4ee71SNavdeep Parhar sc->sge.eqmap[cntxt_id] = (void *)fl; 365754e4ee71SNavdeep Parhar 36584d6db4e0SNavdeep Parhar qid = fl->cntxt_id; 36594d6db4e0SNavdeep Parhar if (isset(&sc->doorbells, DOORBELL_UDB)) { 366090e7434aSNavdeep Parhar uint32_t s_qpp = sc->params.sge.eq_s_qpp; 36614d6db4e0SNavdeep Parhar uint32_t mask = (1 << s_qpp) - 1; 36624d6db4e0SNavdeep Parhar volatile uint8_t *udb; 36634d6db4e0SNavdeep Parhar 36644d6db4e0SNavdeep Parhar udb = sc->udbs_base + UDBS_DB_OFFSET; 36654d6db4e0SNavdeep Parhar udb += (qid >> s_qpp) << PAGE_SHIFT; 36664d6db4e0SNavdeep Parhar qid &= mask; 36674d6db4e0SNavdeep Parhar if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { 36684d6db4e0SNavdeep Parhar udb += qid << UDBS_SEG_SHIFT; 36694d6db4e0SNavdeep Parhar qid = 0; 36704d6db4e0SNavdeep Parhar } 36714d6db4e0SNavdeep Parhar fl->udb = (volatile void *)udb; 36724d6db4e0SNavdeep Parhar } 3673d1205d09SNavdeep Parhar fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; 36744d6db4e0SNavdeep Parhar 367554e4ee71SNavdeep Parhar FL_LOCK(fl); 3676733b9277SNavdeep Parhar /* Enough to make sure the SGE doesn't think it's starved */ 3677733b9277SNavdeep Parhar refill_fl(sc, fl, fl->lowat); 367854e4ee71SNavdeep Parhar FL_UNLOCK(fl); 367954e4ee71SNavdeep Parhar } 368054e4ee71SNavdeep Parhar 368143bbae19SNavdeep Parhar if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && iq->cong >= 0) { 3682ba41ec48SNavdeep Parhar uint32_t param, val; 3683ba41ec48SNavdeep Parhar 3684ba41ec48SNavdeep Parhar param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 3685ba41ec48SNavdeep Parhar V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 3686ba41ec48SNavdeep Parhar V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 368743bbae19SNavdeep Parhar if (iq->cong == 0) 368873cd9220SNavdeep Parhar val = 1 << 19; 368973cd9220SNavdeep Parhar else { 369073cd9220SNavdeep Parhar val = 2 << 19; 369173cd9220SNavdeep Parhar for (i = 0; i < 4; i++) { 369243bbae19SNavdeep Parhar if (iq->cong & (1 << i)) 369373cd9220SNavdeep Parhar val |= 1 << (i << 2); 369473cd9220SNavdeep Parhar } 369573cd9220SNavdeep Parhar } 369673cd9220SNavdeep Parhar 3697ba41ec48SNavdeep Parhar rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3698ba41ec48SNavdeep Parhar if (rc != 0) { 3699ba41ec48SNavdeep Parhar /* report error but carry on */ 370043bbae19SNavdeep Parhar CH_ERR(sc, "failed to set congestion manager context " 370143bbae19SNavdeep Parhar "for ingress queue %d: %d\n", iq->cntxt_id, rc); 3702ba41ec48SNavdeep Parhar } 3703ba41ec48SNavdeep Parhar } 3704ba41ec48SNavdeep Parhar 370554e4ee71SNavdeep Parhar /* Enable IQ interrupts */ 3706733b9277SNavdeep Parhar atomic_store_rel_int(&iq->state, IQS_IDLE); 3707315048f2SJohn Baldwin t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | 370854e4ee71SNavdeep Parhar V_INGRESSQID(iq->cntxt_id)); 370954e4ee71SNavdeep Parhar 371043bbae19SNavdeep Parhar iq->flags |= IQ_HW_ALLOCATED; 371143bbae19SNavdeep Parhar 371254e4ee71SNavdeep Parhar return (0); 371354e4ee71SNavdeep Parhar } 371454e4ee71SNavdeep Parhar 371554e4ee71SNavdeep Parhar static int 371643bbae19SNavdeep Parhar free_iq_fl_hwq(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) 371754e4ee71SNavdeep Parhar { 371838035ed6SNavdeep Parhar int rc; 371954e4ee71SNavdeep Parhar 372043bbae19SNavdeep Parhar MPASS(iq->flags & IQ_HW_ALLOCATED); 372143bbae19SNavdeep Parhar rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP, 372243bbae19SNavdeep Parhar iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); 372354e4ee71SNavdeep Parhar if (rc != 0) { 372443bbae19SNavdeep Parhar CH_ERR(sc, "failed to free iq %p: %d\n", iq, rc); 372554e4ee71SNavdeep Parhar return (rc); 372654e4ee71SNavdeep Parhar } 372743bbae19SNavdeep Parhar iq->flags &= ~IQ_HW_ALLOCATED; 372854e4ee71SNavdeep Parhar 372954e4ee71SNavdeep Parhar return (0); 373054e4ee71SNavdeep Parhar } 373154e4ee71SNavdeep Parhar 373238035ed6SNavdeep Parhar static void 3733348694daSNavdeep Parhar add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 3734348694daSNavdeep Parhar struct sge_iq *iq) 3735348694daSNavdeep Parhar { 373643bbae19SNavdeep Parhar struct sysctl_oid_list *children; 3737348694daSNavdeep Parhar 373843bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 373943bbae19SNavdeep Parhar return; 374043bbae19SNavdeep Parhar 374143bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 3742348694daSNavdeep Parhar SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba, 3743348694daSNavdeep Parhar "bus address of descriptor ring"); 3744348694daSNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3745348694daSNavdeep Parhar iq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); 3746473f6163SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, 3747473f6163SNavdeep Parhar &iq->abs_id, 0, "absolute id of the queue"); 3748473f6163SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3749473f6163SNavdeep Parhar &iq->cntxt_id, 0, "SGE context id of the queue"); 3750473f6163SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &iq->cidx, 3751473f6163SNavdeep Parhar 0, "consumer index"); 3752348694daSNavdeep Parhar } 3753348694daSNavdeep Parhar 3754348694daSNavdeep Parhar static void 3755aa93b99aSNavdeep Parhar add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 3756aa93b99aSNavdeep Parhar struct sysctl_oid *oid, struct sge_fl *fl) 375738035ed6SNavdeep Parhar { 375843bbae19SNavdeep Parhar struct sysctl_oid_list *children; 375938035ed6SNavdeep Parhar 376043bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 376143bbae19SNavdeep Parhar return; 376243bbae19SNavdeep Parhar 376343bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 37647029da5cSPawel Biernacki oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", 37657029da5cSPawel Biernacki CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist"); 376638035ed6SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 376738035ed6SNavdeep Parhar 3768aa93b99aSNavdeep Parhar SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3769aa93b99aSNavdeep Parhar &fl->ba, "bus address of descriptor ring"); 3770aa93b99aSNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3771aa93b99aSNavdeep Parhar fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, 3772aa93b99aSNavdeep Parhar "desc ring size in bytes"); 3773473f6163SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 3774473f6163SNavdeep Parhar &fl->cntxt_id, 0, "SGE context id of the freelist"); 3775e3207e19SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, 3776e3207e19SNavdeep Parhar fl_pad ? 1 : 0, "padding enabled"); 3777e3207e19SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, 3778e3207e19SNavdeep Parhar fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); 377938035ed6SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 378038035ed6SNavdeep Parhar 0, "consumer index"); 378138035ed6SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 378238035ed6SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", 378338035ed6SNavdeep Parhar CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); 378438035ed6SNavdeep Parhar } 378538035ed6SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 378638035ed6SNavdeep Parhar 0, "producer index"); 378738035ed6SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", 378838035ed6SNavdeep Parhar CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); 378938035ed6SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", 379038035ed6SNavdeep Parhar CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); 379138035ed6SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", 379238035ed6SNavdeep Parhar CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); 379338035ed6SNavdeep Parhar } 379438035ed6SNavdeep Parhar 379543bbae19SNavdeep Parhar /* 379643bbae19SNavdeep Parhar * Idempotent. 379743bbae19SNavdeep Parhar */ 379854e4ee71SNavdeep Parhar static int 3799733b9277SNavdeep Parhar alloc_fwq(struct adapter *sc) 380054e4ee71SNavdeep Parhar { 3801733b9277SNavdeep Parhar int rc, intr_idx; 380256599263SNavdeep Parhar struct sge_iq *fwq = &sc->sge.fwq; 380343bbae19SNavdeep Parhar struct vi_info *vi = &sc->port[0]->vi[0]; 380456599263SNavdeep Parhar 380543bbae19SNavdeep Parhar if (!(fwq->flags & IQ_SW_ALLOCATED)) { 380643bbae19SNavdeep Parhar MPASS(!(fwq->flags & IQ_HW_ALLOCATED)); 380743bbae19SNavdeep Parhar 38086af45170SJohn Baldwin if (sc->flags & IS_VF) 38096af45170SJohn Baldwin intr_idx = 0; 38104535e804SNavdeep Parhar else 3811733b9277SNavdeep Parhar intr_idx = sc->intr_count > 1 ? 1 : 0; 381243bbae19SNavdeep Parhar init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, intr_idx, -1); 381343bbae19SNavdeep Parhar rc = alloc_iq_fl(vi, fwq, NULL, &sc->ctx, sc->fwq_oid); 3814733b9277SNavdeep Parhar if (rc != 0) { 381543bbae19SNavdeep Parhar CH_ERR(sc, "failed to allocate fwq: %d\n", rc); 381656599263SNavdeep Parhar return (rc); 3817733b9277SNavdeep Parhar } 381843bbae19SNavdeep Parhar MPASS(fwq->flags & IQ_SW_ALLOCATED); 381943bbae19SNavdeep Parhar } 382056599263SNavdeep Parhar 382143bbae19SNavdeep Parhar if (!(fwq->flags & IQ_HW_ALLOCATED)) { 382243bbae19SNavdeep Parhar MPASS(fwq->flags & IQ_SW_ALLOCATED); 382343bbae19SNavdeep Parhar 382443bbae19SNavdeep Parhar rc = alloc_iq_fl_hwq(vi, fwq, NULL); 382543bbae19SNavdeep Parhar if (rc != 0) { 382643bbae19SNavdeep Parhar CH_ERR(sc, "failed to create hw fwq: %d\n", rc); 382743bbae19SNavdeep Parhar return (rc); 382843bbae19SNavdeep Parhar } 382943bbae19SNavdeep Parhar MPASS(fwq->flags & IQ_HW_ALLOCATED); 383043bbae19SNavdeep Parhar } 383156599263SNavdeep Parhar 3832733b9277SNavdeep Parhar return (0); 3833733b9277SNavdeep Parhar } 3834733b9277SNavdeep Parhar 383543bbae19SNavdeep Parhar /* 383643bbae19SNavdeep Parhar * Idempotent. 383743bbae19SNavdeep Parhar */ 383843bbae19SNavdeep Parhar static void 3839733b9277SNavdeep Parhar free_fwq(struct adapter *sc) 3840733b9277SNavdeep Parhar { 384143bbae19SNavdeep Parhar struct sge_iq *fwq = &sc->sge.fwq; 384243bbae19SNavdeep Parhar 384343bbae19SNavdeep Parhar if (fwq->flags & IQ_HW_ALLOCATED) { 384443bbae19SNavdeep Parhar MPASS(fwq->flags & IQ_SW_ALLOCATED); 384543bbae19SNavdeep Parhar free_iq_fl_hwq(sc, fwq, NULL); 384643bbae19SNavdeep Parhar MPASS(!(fwq->flags & IQ_HW_ALLOCATED)); 3847733b9277SNavdeep Parhar } 3848733b9277SNavdeep Parhar 384943bbae19SNavdeep Parhar if (fwq->flags & IQ_SW_ALLOCATED) { 385043bbae19SNavdeep Parhar MPASS(!(fwq->flags & IQ_HW_ALLOCATED)); 385143bbae19SNavdeep Parhar free_iq_fl(sc, fwq, NULL); 385243bbae19SNavdeep Parhar MPASS(!(fwq->flags & IQ_SW_ALLOCATED)); 385343bbae19SNavdeep Parhar } 385443bbae19SNavdeep Parhar } 385543bbae19SNavdeep Parhar 385643bbae19SNavdeep Parhar /* 385743bbae19SNavdeep Parhar * Idempotent. 385843bbae19SNavdeep Parhar */ 3859733b9277SNavdeep Parhar static int 386043bbae19SNavdeep Parhar alloc_ctrlq(struct adapter *sc, int idx) 3861733b9277SNavdeep Parhar { 3862733b9277SNavdeep Parhar int rc; 3863733b9277SNavdeep Parhar char name[16]; 386443bbae19SNavdeep Parhar struct sysctl_oid *oid; 386543bbae19SNavdeep Parhar struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx]; 3866733b9277SNavdeep Parhar 386743bbae19SNavdeep Parhar MPASS(idx < sc->params.nports); 386837310a98SNavdeep Parhar 386943bbae19SNavdeep Parhar if (!(ctrlq->eq.flags & EQ_SW_ALLOCATED)) { 387043bbae19SNavdeep Parhar MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED)); 387143bbae19SNavdeep Parhar 387237310a98SNavdeep Parhar snprintf(name, sizeof(name), "%d", idx); 387343bbae19SNavdeep Parhar oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(sc->ctrlq_oid), 387443bbae19SNavdeep Parhar OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 387543bbae19SNavdeep Parhar "ctrl queue"); 387637310a98SNavdeep Parhar 387743bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%s ctrlq%d", 387843bbae19SNavdeep Parhar device_get_nameunit(sc->dev), idx); 387943bbae19SNavdeep Parhar init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, 388043bbae19SNavdeep Parhar sc->port[idx]->tx_chan, &sc->sge.fwq, name); 388143bbae19SNavdeep Parhar rc = alloc_wrq(sc, NULL, ctrlq, &sc->ctx, oid); 388243bbae19SNavdeep Parhar if (rc != 0) { 388343bbae19SNavdeep Parhar CH_ERR(sc, "failed to allocate ctrlq%d: %d\n", idx, rc); 388443bbae19SNavdeep Parhar sysctl_remove_oid(oid, 1, 1); 388556599263SNavdeep Parhar return (rc); 388656599263SNavdeep Parhar } 388743bbae19SNavdeep Parhar MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED); 388843bbae19SNavdeep Parhar } 388943bbae19SNavdeep Parhar 389043bbae19SNavdeep Parhar if (!(ctrlq->eq.flags & EQ_HW_ALLOCATED)) { 389143bbae19SNavdeep Parhar MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED); 389243bbae19SNavdeep Parhar 389343bbae19SNavdeep Parhar rc = alloc_eq_hwq(sc, NULL, &ctrlq->eq); 389443bbae19SNavdeep Parhar if (rc != 0) { 389543bbae19SNavdeep Parhar CH_ERR(sc, "failed to create hw ctrlq%d: %d\n", idx, rc); 389643bbae19SNavdeep Parhar return (rc); 389743bbae19SNavdeep Parhar } 389843bbae19SNavdeep Parhar MPASS(ctrlq->eq.flags & EQ_HW_ALLOCATED); 389943bbae19SNavdeep Parhar } 390043bbae19SNavdeep Parhar 390143bbae19SNavdeep Parhar return (0); 390243bbae19SNavdeep Parhar } 390343bbae19SNavdeep Parhar 390443bbae19SNavdeep Parhar /* 390543bbae19SNavdeep Parhar * Idempotent. 390643bbae19SNavdeep Parhar */ 390743bbae19SNavdeep Parhar static void 390843bbae19SNavdeep Parhar free_ctrlq(struct adapter *sc, int idx) 390943bbae19SNavdeep Parhar { 391043bbae19SNavdeep Parhar struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx]; 391143bbae19SNavdeep Parhar 391243bbae19SNavdeep Parhar if (ctrlq->eq.flags & EQ_HW_ALLOCATED) { 391343bbae19SNavdeep Parhar MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED); 391443bbae19SNavdeep Parhar free_eq_hwq(sc, NULL, &ctrlq->eq); 391543bbae19SNavdeep Parhar MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED)); 391643bbae19SNavdeep Parhar } 391743bbae19SNavdeep Parhar 391843bbae19SNavdeep Parhar if (ctrlq->eq.flags & EQ_SW_ALLOCATED) { 391943bbae19SNavdeep Parhar MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED)); 392043bbae19SNavdeep Parhar free_wrq(sc, ctrlq); 392143bbae19SNavdeep Parhar MPASS(!(ctrlq->eq.flags & EQ_SW_ALLOCATED)); 392243bbae19SNavdeep Parhar } 392343bbae19SNavdeep Parhar } 392456599263SNavdeep Parhar 39251605bac6SNavdeep Parhar int 39269af71ab3SNavdeep Parhar tnl_cong(struct port_info *pi, int drop) 39279fb8886bSNavdeep Parhar { 39289fb8886bSNavdeep Parhar 39299af71ab3SNavdeep Parhar if (drop == -1) 39309fb8886bSNavdeep Parhar return (-1); 39319af71ab3SNavdeep Parhar else if (drop == 1) 39329fb8886bSNavdeep Parhar return (0); 39339fb8886bSNavdeep Parhar else 39345bcae8ddSNavdeep Parhar return (pi->rx_e_chan_map); 39359fb8886bSNavdeep Parhar } 39369fb8886bSNavdeep Parhar 393743bbae19SNavdeep Parhar /* 393843bbae19SNavdeep Parhar * Idempotent. 393943bbae19SNavdeep Parhar */ 3940733b9277SNavdeep Parhar static int 394143bbae19SNavdeep Parhar alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int idx, int intr_idx, 394243bbae19SNavdeep Parhar int maxp) 394354e4ee71SNavdeep Parhar { 394454e4ee71SNavdeep Parhar int rc; 39457c228be3SNavdeep Parhar struct adapter *sc = vi->adapter; 394643bbae19SNavdeep Parhar struct ifnet *ifp = vi->ifp; 394743bbae19SNavdeep Parhar struct sysctl_oid *oid; 394854e4ee71SNavdeep Parhar char name[16]; 394954e4ee71SNavdeep Parhar 395043bbae19SNavdeep Parhar if (!(rxq->iq.flags & IQ_SW_ALLOCATED)) { 395143bbae19SNavdeep Parhar MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); 395243bbae19SNavdeep Parhar #if defined(INET) || defined(INET6) 395343bbae19SNavdeep Parhar rc = tcp_lro_init_args(&rxq->lro, ifp, lro_entries, lro_mbufs); 395454e4ee71SNavdeep Parhar if (rc != 0) 395554e4ee71SNavdeep Parhar return (rc); 395643bbae19SNavdeep Parhar MPASS(rxq->lro.ifp == ifp); /* also indicates LRO init'ed */ 395743bbae19SNavdeep Parhar #endif 395843bbae19SNavdeep Parhar rxq->ifp = ifp; 395943bbae19SNavdeep Parhar 396043bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%d", idx); 396143bbae19SNavdeep Parhar oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->rxq_oid), 396243bbae19SNavdeep Parhar OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 396343bbae19SNavdeep Parhar "rx queue"); 396443bbae19SNavdeep Parhar 396543bbae19SNavdeep Parhar init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq, 396643bbae19SNavdeep Parhar intr_idx, tnl_cong(vi->pi, cong_drop)); 3967df8437a9SAndrew Gallatin #if defined(INET) || defined(INET6) 3968df8437a9SAndrew Gallatin if (ifp->if_capenable & IFCAP_LRO) 3969df8437a9SAndrew Gallatin rxq->iq.flags |= IQ_LRO_ENABLED; 3970df8437a9SAndrew Gallatin #endif 3971df8437a9SAndrew Gallatin if (ifp->if_capenable & IFCAP_HWRXTSTMP) 3972df8437a9SAndrew Gallatin rxq->iq.flags |= IQ_RX_TIMESTAMP; 397343bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%s rxq%d-fl", 397443bbae19SNavdeep Parhar device_get_nameunit(vi->dev), idx); 397543bbae19SNavdeep Parhar init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); 397643bbae19SNavdeep Parhar rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid); 397743bbae19SNavdeep Parhar if (rc != 0) { 397843bbae19SNavdeep Parhar CH_ERR(vi, "failed to allocate rxq%d: %d\n", idx, rc); 397943bbae19SNavdeep Parhar sysctl_remove_oid(oid, 1, 1); 398043bbae19SNavdeep Parhar #if defined(INET) || defined(INET6) 398143bbae19SNavdeep Parhar tcp_lro_free(&rxq->lro); 398243bbae19SNavdeep Parhar rxq->lro.ifp = NULL; 398343bbae19SNavdeep Parhar #endif 398443bbae19SNavdeep Parhar return (rc); 398543bbae19SNavdeep Parhar } 398643bbae19SNavdeep Parhar MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); 398743bbae19SNavdeep Parhar add_rxq_sysctls(&vi->ctx, oid, rxq); 398843bbae19SNavdeep Parhar } 398943bbae19SNavdeep Parhar 399043bbae19SNavdeep Parhar if (!(rxq->iq.flags & IQ_HW_ALLOCATED)) { 399143bbae19SNavdeep Parhar MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); 399243bbae19SNavdeep Parhar rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl); 399343bbae19SNavdeep Parhar if (rc != 0) { 399443bbae19SNavdeep Parhar CH_ERR(vi, "failed to create hw rxq%d: %d\n", idx, rc); 399543bbae19SNavdeep Parhar return (rc); 399643bbae19SNavdeep Parhar } 399743bbae19SNavdeep Parhar MPASS(rxq->iq.flags & IQ_HW_ALLOCATED); 399854e4ee71SNavdeep Parhar 3999ec55567cSJohn Baldwin if (idx == 0) 4000ec55567cSJohn Baldwin sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; 4001ec55567cSJohn Baldwin else 4002ec55567cSJohn Baldwin KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, 4003ec55567cSJohn Baldwin ("iq_base mismatch")); 4004ec55567cSJohn Baldwin KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, 4005ec55567cSJohn Baldwin ("PF with non-zero iq_base")); 4006ec55567cSJohn Baldwin 40074d6db4e0SNavdeep Parhar /* 400843bbae19SNavdeep Parhar * The freelist is just barely above the starvation threshold 400943bbae19SNavdeep Parhar * right now, fill it up a bit more. 40104d6db4e0SNavdeep Parhar */ 40119b4d7b4eSNavdeep Parhar FL_LOCK(&rxq->fl); 4012ec55567cSJohn Baldwin refill_fl(sc, &rxq->fl, 128); 40139b4d7b4eSNavdeep Parhar FL_UNLOCK(&rxq->fl); 401454e4ee71SNavdeep Parhar } 401554e4ee71SNavdeep Parhar 401643bbae19SNavdeep Parhar return (0); 401743bbae19SNavdeep Parhar } 401843bbae19SNavdeep Parhar 401943bbae19SNavdeep Parhar /* 402043bbae19SNavdeep Parhar * Idempotent. 402143bbae19SNavdeep Parhar */ 402243bbae19SNavdeep Parhar static void 4023fe2ebb76SJohn Baldwin free_rxq(struct vi_info *vi, struct sge_rxq *rxq) 402454e4ee71SNavdeep Parhar { 402543bbae19SNavdeep Parhar if (rxq->iq.flags & IQ_HW_ALLOCATED) { 402643bbae19SNavdeep Parhar MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); 402743bbae19SNavdeep Parhar free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl); 402843bbae19SNavdeep Parhar MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); 402954e4ee71SNavdeep Parhar } 403043bbae19SNavdeep Parhar 403143bbae19SNavdeep Parhar if (rxq->iq.flags & IQ_SW_ALLOCATED) { 403243bbae19SNavdeep Parhar MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); 403343bbae19SNavdeep Parhar #if defined(INET) || defined(INET6) 403443bbae19SNavdeep Parhar tcp_lro_free(&rxq->lro); 403554e4ee71SNavdeep Parhar #endif 403643bbae19SNavdeep Parhar free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl); 403743bbae19SNavdeep Parhar MPASS(!(rxq->iq.flags & IQ_SW_ALLOCATED)); 403854e4ee71SNavdeep Parhar bzero(rxq, sizeof(*rxq)); 403943bbae19SNavdeep Parhar } 404043bbae19SNavdeep Parhar } 404154e4ee71SNavdeep Parhar 404243bbae19SNavdeep Parhar static void 404343bbae19SNavdeep Parhar add_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 404443bbae19SNavdeep Parhar struct sge_rxq *rxq) 404543bbae19SNavdeep Parhar { 404643bbae19SNavdeep Parhar struct sysctl_oid_list *children; 404743bbae19SNavdeep Parhar 404843bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 404943bbae19SNavdeep Parhar return; 405043bbae19SNavdeep Parhar 405143bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 405243bbae19SNavdeep Parhar #if defined(INET) || defined(INET6) 405343bbae19SNavdeep Parhar SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, 405443bbae19SNavdeep Parhar &rxq->lro.lro_queued, 0, NULL); 405543bbae19SNavdeep Parhar SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, 405643bbae19SNavdeep Parhar &rxq->lro.lro_flushed, 0, NULL); 405743bbae19SNavdeep Parhar #endif 405843bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, 405943bbae19SNavdeep Parhar &rxq->rxcsum, "# of times hardware assisted with checksum"); 406043bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD, 406143bbae19SNavdeep Parhar &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag"); 406243bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_rxcsum", CTLFLAG_RD, 406343bbae19SNavdeep Parhar &rxq->vxlan_rxcsum, 406443bbae19SNavdeep Parhar "# of times hardware assisted with inner checksum (VXLAN)"); 406554e4ee71SNavdeep Parhar } 406654e4ee71SNavdeep Parhar 406709fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD 406843bbae19SNavdeep Parhar /* 406943bbae19SNavdeep Parhar * Idempotent. 407043bbae19SNavdeep Parhar */ 407154e4ee71SNavdeep Parhar static int 407243bbae19SNavdeep Parhar alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx, 407343bbae19SNavdeep Parhar int intr_idx, int maxp) 4074f7dfe243SNavdeep Parhar { 4075733b9277SNavdeep Parhar int rc; 407643bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 407743bbae19SNavdeep Parhar struct sysctl_oid *oid; 4078733b9277SNavdeep Parhar char name[16]; 4079f7dfe243SNavdeep Parhar 408043bbae19SNavdeep Parhar if (!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)) { 408143bbae19SNavdeep Parhar MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); 4082733b9277SNavdeep Parhar 4083733b9277SNavdeep Parhar snprintf(name, sizeof(name), "%d", idx); 408443bbae19SNavdeep Parhar oid = SYSCTL_ADD_NODE(&vi->ctx, 408543bbae19SNavdeep Parhar SYSCTL_CHILDREN(vi->ofld_rxq_oid), OID_AUTO, name, 408643bbae19SNavdeep Parhar CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload rx queue"); 4087733b9277SNavdeep Parhar 408843bbae19SNavdeep Parhar init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx, 408943bbae19SNavdeep Parhar vi->qsize_rxq, intr_idx, 0); 409043bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", 409143bbae19SNavdeep Parhar device_get_nameunit(vi->dev), idx); 409243bbae19SNavdeep Parhar init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); 409343bbae19SNavdeep Parhar rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx, 409443bbae19SNavdeep Parhar oid); 409543bbae19SNavdeep Parhar if (rc != 0) { 409643bbae19SNavdeep Parhar CH_ERR(vi, "failed to allocate ofld_rxq%d: %d\n", idx, 409743bbae19SNavdeep Parhar rc); 409843bbae19SNavdeep Parhar sysctl_remove_oid(oid, 1, 1); 409943bbae19SNavdeep Parhar return (rc); 410043bbae19SNavdeep Parhar } 410143bbae19SNavdeep Parhar MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); 4102a9f0cf48SJohn Baldwin ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK); 4103a9f0cf48SJohn Baldwin ofld_rxq->rx_iscsi_ddp_setup_error = 4104a9f0cf48SJohn Baldwin counter_u64_alloc(M_WAITOK); 410543bbae19SNavdeep Parhar add_ofld_rxq_sysctls(&vi->ctx, oid, ofld_rxq); 410643bbae19SNavdeep Parhar } 4107fe496dc0SJohn Baldwin 410843bbae19SNavdeep Parhar if (!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)) { 410943bbae19SNavdeep Parhar MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); 411043bbae19SNavdeep Parhar rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl); 411143bbae19SNavdeep Parhar if (rc != 0) { 411243bbae19SNavdeep Parhar CH_ERR(vi, "failed to create hw ofld_rxq%d: %d\n", idx, 411343bbae19SNavdeep Parhar rc); 411443bbae19SNavdeep Parhar return (rc); 411543bbae19SNavdeep Parhar } 411643bbae19SNavdeep Parhar MPASS(ofld_rxq->iq.flags & IQ_HW_ALLOCATED); 411743bbae19SNavdeep Parhar } 4118733b9277SNavdeep Parhar return (rc); 4119733b9277SNavdeep Parhar } 4120733b9277SNavdeep Parhar 412143bbae19SNavdeep Parhar /* 412243bbae19SNavdeep Parhar * Idempotent. 412343bbae19SNavdeep Parhar */ 412443bbae19SNavdeep Parhar static void 4125fe2ebb76SJohn Baldwin free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) 4126733b9277SNavdeep Parhar { 412743bbae19SNavdeep Parhar if (ofld_rxq->iq.flags & IQ_HW_ALLOCATED) { 412843bbae19SNavdeep Parhar MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); 412943bbae19SNavdeep Parhar free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl); 413043bbae19SNavdeep Parhar MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); 413143bbae19SNavdeep Parhar } 4132733b9277SNavdeep Parhar 413343bbae19SNavdeep Parhar if (ofld_rxq->iq.flags & IQ_SW_ALLOCATED) { 413443bbae19SNavdeep Parhar MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); 413543bbae19SNavdeep Parhar free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl); 413643bbae19SNavdeep Parhar MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)); 4137a9f0cf48SJohn Baldwin counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok); 4138a9f0cf48SJohn Baldwin counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error); 4139733b9277SNavdeep Parhar bzero(ofld_rxq, sizeof(*ofld_rxq)); 414043bbae19SNavdeep Parhar } 414143bbae19SNavdeep Parhar } 4142733b9277SNavdeep Parhar 414343bbae19SNavdeep Parhar static void 414443bbae19SNavdeep Parhar add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 414543bbae19SNavdeep Parhar struct sge_ofld_rxq *ofld_rxq) 414643bbae19SNavdeep Parhar { 414743bbae19SNavdeep Parhar struct sysctl_oid_list *children; 414843bbae19SNavdeep Parhar 414943bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 415043bbae19SNavdeep Parhar return; 415143bbae19SNavdeep Parhar 415243bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 41534b6ed075SJohn Baldwin SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, 415443bbae19SNavdeep Parhar "rx_toe_tls_records", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_records, 415543bbae19SNavdeep Parhar "# of TOE TLS records received"); 41564b6ed075SJohn Baldwin SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, 415743bbae19SNavdeep Parhar "rx_toe_tls_octets", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_octets, 415843bbae19SNavdeep Parhar "# of payload octets in received TOE TLS records"); 41594b6ed075SJohn Baldwin 41604b6ed075SJohn Baldwin oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "iscsi", 41614b6ed075SJohn Baldwin CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE iSCSI statistics"); 41624b6ed075SJohn Baldwin children = SYSCTL_CHILDREN(oid); 41634b6ed075SJohn Baldwin 41644b6ed075SJohn Baldwin SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok", 41654b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_ok, 41664b6ed075SJohn Baldwin "# of times DDP buffer was setup successfully."); 41674b6ed075SJohn Baldwin SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error", 41684b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_error, 41694b6ed075SJohn Baldwin "# of times DDP buffer setup failed."); 41704b6ed075SJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_octets", 41714b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_octets, 0, 41724b6ed075SJohn Baldwin "# of octets placed directly"); 41734b6ed075SJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_pdus", 41744b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_pdus, 0, 41754b6ed075SJohn Baldwin "# of PDUs with data placed directly."); 41764b6ed075SJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_octets", 41774b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_octets, 0, 41784b6ed075SJohn Baldwin "# of data octets delivered in freelist"); 41794b6ed075SJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_pdus", 41804b6ed075SJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_pdus, 0, 41814b6ed075SJohn Baldwin "# of PDUs with data delivered in freelist"); 41824d4cf62eSJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "padding_errors", 41834d4cf62eSJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_padding_errors, 0, 41844d4cf62eSJohn Baldwin "# of PDUs with invalid padding"); 41854d4cf62eSJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "header_digest_errors", 41864d4cf62eSJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_header_digest_errors, 0, 41874d4cf62eSJohn Baldwin "# of PDUs with invalid header digests"); 41884d4cf62eSJohn Baldwin SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors", 41894d4cf62eSJohn Baldwin CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0, 41904d4cf62eSJohn Baldwin "# of PDUs with invalid data digests"); 4191733b9277SNavdeep Parhar } 4192733b9277SNavdeep Parhar #endif 4193733b9277SNavdeep Parhar 4194ddf09ad6SNavdeep Parhar /* 4195ddf09ad6SNavdeep Parhar * Returns a reasonable automatic cidx flush threshold for a given queue size. 4196ddf09ad6SNavdeep Parhar */ 4197ddf09ad6SNavdeep Parhar static u_int 4198ddf09ad6SNavdeep Parhar qsize_to_fthresh(int qsize) 4199ddf09ad6SNavdeep Parhar { 4200ddf09ad6SNavdeep Parhar u_int fthresh; 4201ddf09ad6SNavdeep Parhar 4202ddf09ad6SNavdeep Parhar while (!powerof2(qsize)) 4203ddf09ad6SNavdeep Parhar qsize++; 4204ddf09ad6SNavdeep Parhar fthresh = ilog2(qsize); 4205ddf09ad6SNavdeep Parhar if (fthresh > X_CIDXFLUSHTHRESH_128) 4206ddf09ad6SNavdeep Parhar fthresh = X_CIDXFLUSHTHRESH_128; 4207ddf09ad6SNavdeep Parhar 4208ddf09ad6SNavdeep Parhar return (fthresh); 4209ddf09ad6SNavdeep Parhar } 4210ddf09ad6SNavdeep Parhar 4211733b9277SNavdeep Parhar static int 4212733b9277SNavdeep Parhar ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 4213733b9277SNavdeep Parhar { 4214733b9277SNavdeep Parhar int rc, cntxt_id; 4215733b9277SNavdeep Parhar struct fw_eq_ctrl_cmd c; 421690e7434aSNavdeep Parhar int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4217f7dfe243SNavdeep Parhar 4218f7dfe243SNavdeep Parhar bzero(&c, sizeof(c)); 4219f7dfe243SNavdeep Parhar 4220f7dfe243SNavdeep Parhar c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 4221f7dfe243SNavdeep Parhar F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 4222f7dfe243SNavdeep Parhar V_FW_EQ_CTRL_CMD_VFN(0)); 4223f7dfe243SNavdeep Parhar c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 4224f7dfe243SNavdeep Parhar F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 42257951040fSNavdeep Parhar c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); 4226f7dfe243SNavdeep Parhar c.physeqid_pkd = htobe32(0); 4227f7dfe243SNavdeep Parhar c.fetchszm_to_iqid = 422887b027baSNavdeep Parhar htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 4229733b9277SNavdeep Parhar V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 423056599263SNavdeep Parhar F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 4231f7dfe243SNavdeep Parhar c.dcaen_to_eqsize = 4232adb0cd84SNavdeep Parhar htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4233adb0cd84SNavdeep Parhar X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 4234f7dfe243SNavdeep Parhar V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 4235ddf09ad6SNavdeep Parhar V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 42367951040fSNavdeep Parhar V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); 4237f7dfe243SNavdeep Parhar c.eqaddr = htobe64(eq->ba); 4238f7dfe243SNavdeep Parhar 4239f7dfe243SNavdeep Parhar rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 4240f7dfe243SNavdeep Parhar if (rc != 0) { 424143bbae19SNavdeep Parhar CH_ERR(sc, "failed to create hw ctrlq for tx_chan %d: %d\n", 424243bbae19SNavdeep Parhar eq->tx_chan, rc); 4243f7dfe243SNavdeep Parhar return (rc); 4244f7dfe243SNavdeep Parhar } 4245f7dfe243SNavdeep Parhar 4246f7dfe243SNavdeep Parhar eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); 424776c89022SNavdeep Parhar eq->abs_id = G_FW_EQ_CTRL_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 4248f7dfe243SNavdeep Parhar cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4249b20b25e7SNavdeep Parhar if (cntxt_id >= sc->sge.eqmap_sz) 4250733b9277SNavdeep Parhar panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4251b20b25e7SNavdeep Parhar cntxt_id, sc->sge.eqmap_sz - 1); 4252f7dfe243SNavdeep Parhar sc->sge.eqmap[cntxt_id] = eq; 4253f7dfe243SNavdeep Parhar 4254f7dfe243SNavdeep Parhar return (rc); 4255f7dfe243SNavdeep Parhar } 4256f7dfe243SNavdeep Parhar 4257f7dfe243SNavdeep Parhar static int 4258fe2ebb76SJohn Baldwin eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 425954e4ee71SNavdeep Parhar { 426054e4ee71SNavdeep Parhar int rc, cntxt_id; 426154e4ee71SNavdeep Parhar struct fw_eq_eth_cmd c; 426290e7434aSNavdeep Parhar int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 426354e4ee71SNavdeep Parhar 426454e4ee71SNavdeep Parhar bzero(&c, sizeof(c)); 426554e4ee71SNavdeep Parhar 426654e4ee71SNavdeep Parhar c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 426754e4ee71SNavdeep Parhar F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 426854e4ee71SNavdeep Parhar V_FW_EQ_ETH_CMD_VFN(0)); 426954e4ee71SNavdeep Parhar c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 427054e4ee71SNavdeep Parhar F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 42717951040fSNavdeep Parhar c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 4272fe2ebb76SJohn Baldwin F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); 427354e4ee71SNavdeep Parhar c.fetchszm_to_iqid = 42747951040fSNavdeep Parhar htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | 4275733b9277SNavdeep Parhar V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 4276aa2457e1SNavdeep Parhar V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 4277adb0cd84SNavdeep Parhar c.dcaen_to_eqsize = 4278adb0cd84SNavdeep Parhar htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4279adb0cd84SNavdeep Parhar X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 428054e4ee71SNavdeep Parhar V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 42817951040fSNavdeep Parhar V_FW_EQ_ETH_CMD_EQSIZE(qsize)); 428254e4ee71SNavdeep Parhar c.eqaddr = htobe64(eq->ba); 428354e4ee71SNavdeep Parhar 428454e4ee71SNavdeep Parhar rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 428554e4ee71SNavdeep Parhar if (rc != 0) { 4286fe2ebb76SJohn Baldwin device_printf(vi->dev, 4287733b9277SNavdeep Parhar "failed to create Ethernet egress queue: %d\n", rc); 4288733b9277SNavdeep Parhar return (rc); 4289733b9277SNavdeep Parhar } 4290733b9277SNavdeep Parhar 4291733b9277SNavdeep Parhar eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); 4292ec55567cSJohn Baldwin eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 4293733b9277SNavdeep Parhar cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4294b20b25e7SNavdeep Parhar if (cntxt_id >= sc->sge.eqmap_sz) 4295733b9277SNavdeep Parhar panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4296b20b25e7SNavdeep Parhar cntxt_id, sc->sge.eqmap_sz - 1); 4297733b9277SNavdeep Parhar sc->sge.eqmap[cntxt_id] = eq; 4298733b9277SNavdeep Parhar 429954e4ee71SNavdeep Parhar return (rc); 430054e4ee71SNavdeep Parhar } 430154e4ee71SNavdeep Parhar 4302eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4303733b9277SNavdeep Parhar static int 4304fe2ebb76SJohn Baldwin ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 4305733b9277SNavdeep Parhar { 4306733b9277SNavdeep Parhar int rc, cntxt_id; 4307733b9277SNavdeep Parhar struct fw_eq_ofld_cmd c; 430890e7434aSNavdeep Parhar int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 430954e4ee71SNavdeep Parhar 4310733b9277SNavdeep Parhar bzero(&c, sizeof(c)); 4311733b9277SNavdeep Parhar 4312733b9277SNavdeep Parhar c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 4313733b9277SNavdeep Parhar F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 4314733b9277SNavdeep Parhar V_FW_EQ_OFLD_CMD_VFN(0)); 4315733b9277SNavdeep Parhar c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 4316733b9277SNavdeep Parhar F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 4317733b9277SNavdeep Parhar c.fetchszm_to_iqid = 4318ddf09ad6SNavdeep Parhar htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 4319733b9277SNavdeep Parhar V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 4320733b9277SNavdeep Parhar F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 4321733b9277SNavdeep Parhar c.dcaen_to_eqsize = 4322adb0cd84SNavdeep Parhar htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4323adb0cd84SNavdeep Parhar X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 4324733b9277SNavdeep Parhar V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 4325ddf09ad6SNavdeep Parhar V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 43267951040fSNavdeep Parhar V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); 4327733b9277SNavdeep Parhar c.eqaddr = htobe64(eq->ba); 4328733b9277SNavdeep Parhar 4329733b9277SNavdeep Parhar rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 4330733b9277SNavdeep Parhar if (rc != 0) { 4331fe2ebb76SJohn Baldwin device_printf(vi->dev, 4332733b9277SNavdeep Parhar "failed to create egress queue for TCP offload: %d\n", rc); 4333733b9277SNavdeep Parhar return (rc); 4334733b9277SNavdeep Parhar } 4335733b9277SNavdeep Parhar 4336733b9277SNavdeep Parhar eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); 433776c89022SNavdeep Parhar eq->abs_id = G_FW_EQ_OFLD_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 433854e4ee71SNavdeep Parhar cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4339b20b25e7SNavdeep Parhar if (cntxt_id >= sc->sge.eqmap_sz) 4340733b9277SNavdeep Parhar panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4341b20b25e7SNavdeep Parhar cntxt_id, sc->sge.eqmap_sz - 1); 434254e4ee71SNavdeep Parhar sc->sge.eqmap[cntxt_id] = eq; 434354e4ee71SNavdeep Parhar 4344733b9277SNavdeep Parhar return (rc); 4345733b9277SNavdeep Parhar } 4346733b9277SNavdeep Parhar #endif 4347733b9277SNavdeep Parhar 434843bbae19SNavdeep Parhar /* SW only */ 4349733b9277SNavdeep Parhar static int 435043bbae19SNavdeep Parhar alloc_eq(struct adapter *sc, struct sge_eq *eq, struct sysctl_ctx_list *ctx, 435143bbae19SNavdeep Parhar struct sysctl_oid *oid) 4352733b9277SNavdeep Parhar { 43537951040fSNavdeep Parhar int rc, qsize; 4354733b9277SNavdeep Parhar size_t len; 4355733b9277SNavdeep Parhar 435643bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_SW_ALLOCATED)); 4357733b9277SNavdeep Parhar 435890e7434aSNavdeep Parhar qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 43597951040fSNavdeep Parhar len = qsize * EQ_ESIZE; 436043bbae19SNavdeep Parhar rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba, 436143bbae19SNavdeep Parhar (void **)&eq->desc); 4362733b9277SNavdeep Parhar if (rc) 4363733b9277SNavdeep Parhar return (rc); 436443bbae19SNavdeep Parhar if (ctx != NULL && oid != NULL) 436543bbae19SNavdeep Parhar add_eq_sysctls(sc, ctx, oid, eq); 436643bbae19SNavdeep Parhar eq->flags |= EQ_SW_ALLOCATED; 4367733b9277SNavdeep Parhar 436843bbae19SNavdeep Parhar return (0); 436943bbae19SNavdeep Parhar } 437043bbae19SNavdeep Parhar 437143bbae19SNavdeep Parhar /* SW only */ 437243bbae19SNavdeep Parhar static void 437343bbae19SNavdeep Parhar free_eq(struct adapter *sc, struct sge_eq *eq) 437443bbae19SNavdeep Parhar { 437543bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 43765ef87bf8SNavdeep Parhar if (eq->type == EQ_ETH) 437743bbae19SNavdeep Parhar MPASS(eq->pidx == eq->cidx); 437843bbae19SNavdeep Parhar 437943bbae19SNavdeep Parhar free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); 438043bbae19SNavdeep Parhar mtx_destroy(&eq->eq_lock); 438143bbae19SNavdeep Parhar bzero(eq, sizeof(*eq)); 438243bbae19SNavdeep Parhar } 438343bbae19SNavdeep Parhar 438443bbae19SNavdeep Parhar static void 438543bbae19SNavdeep Parhar add_eq_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 438643bbae19SNavdeep Parhar struct sysctl_oid *oid, struct sge_eq *eq) 438743bbae19SNavdeep Parhar { 438843bbae19SNavdeep Parhar struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 438943bbae19SNavdeep Parhar 439043bbae19SNavdeep Parhar SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba, 439143bbae19SNavdeep Parhar "bus address of descriptor ring"); 439243bbae19SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 439343bbae19SNavdeep Parhar eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, 439443bbae19SNavdeep Parhar "desc ring size in bytes"); 439543bbae19SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, 439643bbae19SNavdeep Parhar &eq->abs_id, 0, "absolute id of the queue"); 439743bbae19SNavdeep Parhar SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 439843bbae19SNavdeep Parhar &eq->cntxt_id, 0, "SGE context id of the queue"); 439943bbae19SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &eq->cidx, 440043bbae19SNavdeep Parhar 0, "consumer index"); 440143bbae19SNavdeep Parhar SYSCTL_ADD_U16(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &eq->pidx, 440243bbae19SNavdeep Parhar 0, "producer index"); 440343bbae19SNavdeep Parhar SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 440443bbae19SNavdeep Parhar eq->sidx, "status page index"); 440543bbae19SNavdeep Parhar } 440643bbae19SNavdeep Parhar 440743bbae19SNavdeep Parhar static int 440843bbae19SNavdeep Parhar alloc_eq_hwq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 440943bbae19SNavdeep Parhar { 441043bbae19SNavdeep Parhar int rc; 441143bbae19SNavdeep Parhar 441243bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 441343bbae19SNavdeep Parhar 441443bbae19SNavdeep Parhar eq->iqid = eq->iq->cntxt_id; 4415ddf09ad6SNavdeep Parhar eq->pidx = eq->cidx = eq->dbidx = 0; 4416ddf09ad6SNavdeep Parhar /* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */ 4417ddf09ad6SNavdeep Parhar eq->equeqidx = 0; 4418d14b0ac1SNavdeep Parhar eq->doorbells = sc->doorbells; 441943bbae19SNavdeep Parhar bzero(eq->desc, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len); 4420733b9277SNavdeep Parhar 442143bbae19SNavdeep Parhar switch (eq->type) { 4422733b9277SNavdeep Parhar case EQ_CTRL: 4423733b9277SNavdeep Parhar rc = ctrl_eq_alloc(sc, eq); 4424733b9277SNavdeep Parhar break; 4425733b9277SNavdeep Parhar 4426733b9277SNavdeep Parhar case EQ_ETH: 4427fe2ebb76SJohn Baldwin rc = eth_eq_alloc(sc, vi, eq); 4428733b9277SNavdeep Parhar break; 4429733b9277SNavdeep Parhar 4430eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4431733b9277SNavdeep Parhar case EQ_OFLD: 4432fe2ebb76SJohn Baldwin rc = ofld_eq_alloc(sc, vi, eq); 4433733b9277SNavdeep Parhar break; 4434733b9277SNavdeep Parhar #endif 4435733b9277SNavdeep Parhar 4436733b9277SNavdeep Parhar default: 443743bbae19SNavdeep Parhar panic("%s: invalid eq type %d.", __func__, eq->type); 4438733b9277SNavdeep Parhar } 4439733b9277SNavdeep Parhar if (rc != 0) { 444043bbae19SNavdeep Parhar CH_ERR(sc, "failed to allocate egress queue(%d): %d\n", 444143bbae19SNavdeep Parhar eq->type, rc); 444243bbae19SNavdeep Parhar return (rc); 4443733b9277SNavdeep Parhar } 4444733b9277SNavdeep Parhar 4445d14b0ac1SNavdeep Parhar if (isset(&eq->doorbells, DOORBELL_UDB) || 4446d14b0ac1SNavdeep Parhar isset(&eq->doorbells, DOORBELL_UDBWC) || 444777ad3c41SNavdeep Parhar isset(&eq->doorbells, DOORBELL_WCWR)) { 444890e7434aSNavdeep Parhar uint32_t s_qpp = sc->params.sge.eq_s_qpp; 4449d14b0ac1SNavdeep Parhar uint32_t mask = (1 << s_qpp) - 1; 4450d14b0ac1SNavdeep Parhar volatile uint8_t *udb; 4451d14b0ac1SNavdeep Parhar 4452d14b0ac1SNavdeep Parhar udb = sc->udbs_base + UDBS_DB_OFFSET; 4453d14b0ac1SNavdeep Parhar udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 4454d14b0ac1SNavdeep Parhar eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 4455f10405b3SNavdeep Parhar if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) 445677ad3c41SNavdeep Parhar clrbit(&eq->doorbells, DOORBELL_WCWR); 4457d14b0ac1SNavdeep Parhar else { 4458d14b0ac1SNavdeep Parhar udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 4459d14b0ac1SNavdeep Parhar eq->udb_qid = 0; 4460d14b0ac1SNavdeep Parhar } 4461d14b0ac1SNavdeep Parhar eq->udb = (volatile void *)udb; 4462d14b0ac1SNavdeep Parhar } 4463d14b0ac1SNavdeep Parhar 446443bbae19SNavdeep Parhar eq->flags |= EQ_HW_ALLOCATED; 446543bbae19SNavdeep Parhar return (0); 4466733b9277SNavdeep Parhar } 4467733b9277SNavdeep Parhar 4468733b9277SNavdeep Parhar static int 446943bbae19SNavdeep Parhar free_eq_hwq(struct adapter *sc, struct vi_info *vi __unused, struct sge_eq *eq) 4470733b9277SNavdeep Parhar { 4471733b9277SNavdeep Parhar int rc; 4472733b9277SNavdeep Parhar 447343bbae19SNavdeep Parhar MPASS(eq->flags & EQ_HW_ALLOCATED); 447443bbae19SNavdeep Parhar 447543bbae19SNavdeep Parhar switch (eq->type) { 4476733b9277SNavdeep Parhar case EQ_CTRL: 447743bbae19SNavdeep Parhar rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); 4478733b9277SNavdeep Parhar break; 4479733b9277SNavdeep Parhar case EQ_ETH: 448043bbae19SNavdeep Parhar rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); 4481733b9277SNavdeep Parhar break; 4482eff62dbaSNavdeep Parhar #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4483733b9277SNavdeep Parhar case EQ_OFLD: 448443bbae19SNavdeep Parhar rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); 4485733b9277SNavdeep Parhar break; 4486733b9277SNavdeep Parhar #endif 4487733b9277SNavdeep Parhar default: 448843bbae19SNavdeep Parhar panic("%s: invalid eq type %d.", __func__, eq->type); 4489733b9277SNavdeep Parhar } 4490733b9277SNavdeep Parhar if (rc != 0) { 449143bbae19SNavdeep Parhar CH_ERR(sc, "failed to free eq (type %d): %d\n", eq->type, rc); 4492733b9277SNavdeep Parhar return (rc); 4493733b9277SNavdeep Parhar } 449443bbae19SNavdeep Parhar eq->flags &= ~EQ_HW_ALLOCATED; 4495733b9277SNavdeep Parhar 4496733b9277SNavdeep Parhar return (0); 4497733b9277SNavdeep Parhar } 4498733b9277SNavdeep Parhar 4499733b9277SNavdeep Parhar static int 4500fe2ebb76SJohn Baldwin alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, 450143bbae19SNavdeep Parhar struct sysctl_ctx_list *ctx, struct sysctl_oid *oid) 4502733b9277SNavdeep Parhar { 450343bbae19SNavdeep Parhar struct sge_eq *eq = &wrq->eq; 4504733b9277SNavdeep Parhar int rc; 4505733b9277SNavdeep Parhar 450643bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_SW_ALLOCATED)); 450743bbae19SNavdeep Parhar 450843bbae19SNavdeep Parhar rc = alloc_eq(sc, eq, ctx, oid); 4509733b9277SNavdeep Parhar if (rc) 4510733b9277SNavdeep Parhar return (rc); 451143bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 451243bbae19SNavdeep Parhar /* Can't fail after this. */ 4513733b9277SNavdeep Parhar 4514733b9277SNavdeep Parhar wrq->adapter = sc; 45157951040fSNavdeep Parhar TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); 45167951040fSNavdeep Parhar TAILQ_INIT(&wrq->incomplete_wrs); 451709fe6320SNavdeep Parhar STAILQ_INIT(&wrq->wr_list); 45187951040fSNavdeep Parhar wrq->nwr_pending = 0; 45197951040fSNavdeep Parhar wrq->ndesc_needed = 0; 452043bbae19SNavdeep Parhar add_wrq_sysctls(ctx, oid, wrq); 4521733b9277SNavdeep Parhar 452243bbae19SNavdeep Parhar return (0); 452343bbae19SNavdeep Parhar } 452443bbae19SNavdeep Parhar 452543bbae19SNavdeep Parhar static void 452643bbae19SNavdeep Parhar free_wrq(struct adapter *sc, struct sge_wrq *wrq) 452743bbae19SNavdeep Parhar { 452843bbae19SNavdeep Parhar free_eq(sc, &wrq->eq); 452943bbae19SNavdeep Parhar MPASS(wrq->nwr_pending == 0); 45305ef87bf8SNavdeep Parhar MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 45315ef87bf8SNavdeep Parhar MPASS(STAILQ_EMPTY(&wrq->wr_list)); 453243bbae19SNavdeep Parhar bzero(wrq, sizeof(*wrq)); 453343bbae19SNavdeep Parhar } 453443bbae19SNavdeep Parhar 453543bbae19SNavdeep Parhar static void 453643bbae19SNavdeep Parhar add_wrq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 453743bbae19SNavdeep Parhar struct sge_wrq *wrq) 453843bbae19SNavdeep Parhar { 453943bbae19SNavdeep Parhar struct sysctl_oid_list *children; 454043bbae19SNavdeep Parhar 454143bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 454243bbae19SNavdeep Parhar return; 454343bbae19SNavdeep Parhar 454443bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 45457951040fSNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, 45467951040fSNavdeep Parhar &wrq->tx_wrs_direct, "# of work requests (direct)"); 45477951040fSNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, 45487951040fSNavdeep Parhar &wrq->tx_wrs_copied, "# of work requests (copied)"); 45490459a175SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, 45500459a175SNavdeep Parhar &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); 4551733b9277SNavdeep Parhar } 4552733b9277SNavdeep Parhar 455343bbae19SNavdeep Parhar /* 455443bbae19SNavdeep Parhar * Idempotent. 455543bbae19SNavdeep Parhar */ 4556733b9277SNavdeep Parhar static int 455743bbae19SNavdeep Parhar alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx) 4558733b9277SNavdeep Parhar { 455943bbae19SNavdeep Parhar int rc, iqidx; 4560fe2ebb76SJohn Baldwin struct port_info *pi = vi->pi; 456143bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 4562733b9277SNavdeep Parhar struct sge_eq *eq = &txq->eq; 4563d735920dSNavdeep Parhar struct txpkts *txp; 4564733b9277SNavdeep Parhar char name[16]; 456543bbae19SNavdeep Parhar struct sysctl_oid *oid; 4566733b9277SNavdeep Parhar 456743bbae19SNavdeep Parhar if (!(eq->flags & EQ_SW_ALLOCATED)) { 456843bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 456943bbae19SNavdeep Parhar 457043bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%d", idx); 457143bbae19SNavdeep Parhar oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->txq_oid), 457243bbae19SNavdeep Parhar OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 457343bbae19SNavdeep Parhar "tx queue"); 457443bbae19SNavdeep Parhar 457543bbae19SNavdeep Parhar iqidx = vi->first_rxq + (idx % vi->nrxq); 457643bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%s txq%d", 457743bbae19SNavdeep Parhar device_get_nameunit(vi->dev), idx); 457843bbae19SNavdeep Parhar init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, 457943bbae19SNavdeep Parhar &sc->sge.rxq[iqidx].iq, name); 458043bbae19SNavdeep Parhar 458143bbae19SNavdeep Parhar rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, 458243bbae19SNavdeep Parhar can_resume_eth_tx, M_CXGBE, &eq->eq_lock, M_WAITOK); 45837951040fSNavdeep Parhar if (rc != 0) { 458443bbae19SNavdeep Parhar CH_ERR(vi, "failed to allocate mp_ring for txq%d: %d\n", 458543bbae19SNavdeep Parhar idx, rc); 458643bbae19SNavdeep Parhar failed: 458743bbae19SNavdeep Parhar sysctl_remove_oid(oid, 1, 1); 45887951040fSNavdeep Parhar return (rc); 45897951040fSNavdeep Parhar } 45907951040fSNavdeep Parhar 459143bbae19SNavdeep Parhar rc = alloc_eq(sc, eq, &vi->ctx, oid); 459243bbae19SNavdeep Parhar if (rc) { 459343bbae19SNavdeep Parhar CH_ERR(vi, "failed to allocate txq%d: %d\n", idx, rc); 45947951040fSNavdeep Parhar mp_ring_free(txq->r); 459543bbae19SNavdeep Parhar goto failed; 459643bbae19SNavdeep Parhar } 459743bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 459843bbae19SNavdeep Parhar /* Can't fail after this point. */ 459943bbae19SNavdeep Parhar 460043bbae19SNavdeep Parhar TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); 460143bbae19SNavdeep Parhar txq->ifp = vi->ifp; 460243bbae19SNavdeep Parhar txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); 460343bbae19SNavdeep Parhar txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, 460443bbae19SNavdeep Parhar M_ZERO | M_WAITOK); 460543bbae19SNavdeep Parhar 460643bbae19SNavdeep Parhar add_txq_sysctls(vi, &vi->ctx, oid, txq); 46077951040fSNavdeep Parhar } 4608733b9277SNavdeep Parhar 460943bbae19SNavdeep Parhar if (!(eq->flags & EQ_HW_ALLOCATED)) { 461043bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 461143bbae19SNavdeep Parhar rc = alloc_eq_hwq(sc, vi, eq); 461243bbae19SNavdeep Parhar if (rc != 0) { 461343bbae19SNavdeep Parhar CH_ERR(vi, "failed to create hw txq%d: %d\n", idx, rc); 461443bbae19SNavdeep Parhar return (rc); 461543bbae19SNavdeep Parhar } 461643bbae19SNavdeep Parhar MPASS(eq->flags & EQ_HW_ALLOCATED); 46177951040fSNavdeep Parhar /* Can't fail after this point. */ 46187951040fSNavdeep Parhar 4619ec55567cSJohn Baldwin if (idx == 0) 4620ec55567cSJohn Baldwin sc->sge.eq_base = eq->abs_id - eq->cntxt_id; 4621ec55567cSJohn Baldwin else 4622ec55567cSJohn Baldwin KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, 4623ec55567cSJohn Baldwin ("eq_base mismatch")); 4624ec55567cSJohn Baldwin KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, 4625ec55567cSJohn Baldwin ("PF with non-zero eq_base")); 4626ec55567cSJohn Baldwin 4627d735920dSNavdeep Parhar txp = &txq->txp; 4628d735920dSNavdeep Parhar MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr); 4629d735920dSNavdeep Parhar txq->txp.max_npkt = min(nitems(txp->mb), 4630d735920dSNavdeep Parhar sc->params.max_pkts_per_eth_tx_pkts_wr); 463130e3f2b4SNavdeep Parhar if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF)) 463230e3f2b4SNavdeep Parhar txq->txp.max_npkt--; 4633d735920dSNavdeep Parhar 463443bbae19SNavdeep Parhar if (vi->flags & TX_USES_VM_WR) 463543bbae19SNavdeep Parhar txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 463643bbae19SNavdeep Parhar V_TXPKT_INTF(pi->tx_chan)); 463743bbae19SNavdeep Parhar else 463843bbae19SNavdeep Parhar txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 463943bbae19SNavdeep Parhar V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 464043bbae19SNavdeep Parhar V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 464143bbae19SNavdeep Parhar 464243bbae19SNavdeep Parhar txq->tc_idx = -1; 464343bbae19SNavdeep Parhar } 464443bbae19SNavdeep Parhar 464543bbae19SNavdeep Parhar return (0); 464643bbae19SNavdeep Parhar } 464743bbae19SNavdeep Parhar 464843bbae19SNavdeep Parhar /* 464943bbae19SNavdeep Parhar * Idempotent. 465043bbae19SNavdeep Parhar */ 465143bbae19SNavdeep Parhar static void 465243bbae19SNavdeep Parhar free_txq(struct vi_info *vi, struct sge_txq *txq) 465343bbae19SNavdeep Parhar { 465443bbae19SNavdeep Parhar struct adapter *sc = vi->adapter; 465543bbae19SNavdeep Parhar struct sge_eq *eq = &txq->eq; 465643bbae19SNavdeep Parhar 465743bbae19SNavdeep Parhar if (eq->flags & EQ_HW_ALLOCATED) { 465843bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 465943bbae19SNavdeep Parhar free_eq_hwq(sc, NULL, eq); 466043bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 466143bbae19SNavdeep Parhar } 466243bbae19SNavdeep Parhar 466343bbae19SNavdeep Parhar if (eq->flags & EQ_SW_ALLOCATED) { 466443bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 466543bbae19SNavdeep Parhar sglist_free(txq->gl); 466643bbae19SNavdeep Parhar free(txq->sdesc, M_CXGBE); 466743bbae19SNavdeep Parhar mp_ring_free(txq->r); 466843bbae19SNavdeep Parhar free_eq(sc, eq); 466943bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_SW_ALLOCATED)); 467043bbae19SNavdeep Parhar bzero(txq, sizeof(*txq)); 467143bbae19SNavdeep Parhar } 467243bbae19SNavdeep Parhar } 467343bbae19SNavdeep Parhar 467443bbae19SNavdeep Parhar static void 467543bbae19SNavdeep Parhar add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx, 467643bbae19SNavdeep Parhar struct sysctl_oid *oid, struct sge_txq *txq) 467743bbae19SNavdeep Parhar { 467843bbae19SNavdeep Parhar struct adapter *sc; 467943bbae19SNavdeep Parhar struct sysctl_oid_list *children; 468043bbae19SNavdeep Parhar 468143bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 468243bbae19SNavdeep Parhar return; 468343bbae19SNavdeep Parhar 468443bbae19SNavdeep Parhar sc = vi->adapter; 468554e4ee71SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 468654e4ee71SNavdeep Parhar 468743bbae19SNavdeep Parhar mp_ring_sysctls(txq->r, ctx, children); 468859bc8ce0SNavdeep Parhar 468943bbae19SNavdeep Parhar SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tc", 469043bbae19SNavdeep Parhar CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, txq - sc->sge.txq, 469143bbae19SNavdeep Parhar sysctl_tc, "I", "traffic class (-1 means none)"); 469202f972e8SNavdeep Parhar 469343bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, 469454e4ee71SNavdeep Parhar &txq->txcsum, "# of times hardware assisted with checksum"); 469543bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD, 469643bbae19SNavdeep Parhar &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag"); 469743bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 4698a1ea9a82SNavdeep Parhar &txq->tso_wrs, "# of TSO work requests"); 469943bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 470054e4ee71SNavdeep Parhar &txq->imm_wrs, "# of work requests with immediate data"); 470143bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 470254e4ee71SNavdeep Parhar &txq->sgl_wrs, "# of work requests with direct SGL"); 470343bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 470454e4ee71SNavdeep Parhar &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); 470543bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD, 470643bbae19SNavdeep Parhar &txq->txpkts0_wrs, "# of txpkts (type 0) work requests"); 470743bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD, 470843bbae19SNavdeep Parhar &txq->txpkts1_wrs, "# of txpkts (type 1) work requests"); 470943bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD, 471043bbae19SNavdeep Parhar &txq->txpkts0_pkts, 47117951040fSNavdeep Parhar "# of frames tx'd using type0 txpkts work requests"); 471243bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD, 471343bbae19SNavdeep Parhar &txq->txpkts1_pkts, 47147951040fSNavdeep Parhar "# of frames tx'd using type1 txpkts work requests"); 471543bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts_flush", CTLFLAG_RD, 471643bbae19SNavdeep Parhar &txq->txpkts_flush, 47173447df8bSNavdeep Parhar "# of times txpkts had to be flushed out by an egress-update"); 471843bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD, 47195cdaef71SJohn Baldwin &txq->raw_wrs, "# of raw work requests (non-packets)"); 472043bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_tso_wrs", CTLFLAG_RD, 472143bbae19SNavdeep Parhar &txq->vxlan_tso_wrs, "# of VXLAN TSO work requests"); 472243bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_txcsum", CTLFLAG_RD, 472343bbae19SNavdeep Parhar &txq->vxlan_txcsum, 4724a4a4ad2dSNavdeep Parhar "# of times hardware assisted with inner checksums (VXLAN)"); 4725bddf7343SJohn Baldwin 4726bddf7343SJohn Baldwin #ifdef KERN_TLS 472715f33555SNavdeep Parhar if (is_ktls(sc)) { 472843bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_records", 472943bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_records, 4730bddf7343SJohn Baldwin "# of NIC TLS records transmitted"); 473143bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_short", 473243bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_short, 4733bddf7343SJohn Baldwin "# of short NIC TLS records transmitted"); 473443bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_partial", 473543bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_partial, 4736bddf7343SJohn Baldwin "# of partial NIC TLS records transmitted"); 473743bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_full", 473843bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_full, 4739bddf7343SJohn Baldwin "# of full NIC TLS records transmitted"); 474043bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_octets", 474143bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_octets, 4742bddf7343SJohn Baldwin "# of payload octets in transmitted NIC TLS records"); 474343bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_waste", 474443bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_waste, 4745bddf7343SJohn Baldwin "# of octets DMAd but not transmitted in NIC TLS records"); 474643bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_options", 474743bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_options, 4748bddf7343SJohn Baldwin "# of NIC TLS options-only packets transmitted"); 474943bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_header", 475043bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_header, 4751bddf7343SJohn Baldwin "# of NIC TLS header-only packets transmitted"); 475243bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin", 475343bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_fin, 4754bddf7343SJohn Baldwin "# of NIC TLS FIN-only packets transmitted"); 475543bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin_short", 475643bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_fin_short, 4757bddf7343SJohn Baldwin "# of NIC TLS padded FIN packets on short TLS records"); 475843bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_cbc", 475943bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_cbc, 4760bddf7343SJohn Baldwin "# of NIC TLS sessions using AES-CBC"); 476143bbae19SNavdeep Parhar SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_gcm", 476243bbae19SNavdeep Parhar CTLFLAG_RD, &txq->kern_tls_gcm, 4763bddf7343SJohn Baldwin "# of NIC TLS sessions using AES-GCM"); 4764bddf7343SJohn Baldwin } 4765bddf7343SJohn Baldwin #endif 476654e4ee71SNavdeep Parhar } 476754e4ee71SNavdeep Parhar 4768077ba6a8SJohn Baldwin #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 476943bbae19SNavdeep Parhar /* 477043bbae19SNavdeep Parhar * Idempotent. 477143bbae19SNavdeep Parhar */ 4772077ba6a8SJohn Baldwin static int 477343bbae19SNavdeep Parhar alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx) 4774077ba6a8SJohn Baldwin { 477543bbae19SNavdeep Parhar struct sysctl_oid *oid; 477643bbae19SNavdeep Parhar struct port_info *pi = vi->pi; 4777077ba6a8SJohn Baldwin struct adapter *sc = vi->adapter; 477843bbae19SNavdeep Parhar struct sge_eq *eq = &ofld_txq->wrq.eq; 477943bbae19SNavdeep Parhar int rc, iqidx; 4780077ba6a8SJohn Baldwin char name[16]; 4781077ba6a8SJohn Baldwin 478243bbae19SNavdeep Parhar MPASS(idx >= 0); 478343bbae19SNavdeep Parhar MPASS(idx < vi->nofldtxq); 4784077ba6a8SJohn Baldwin 478543bbae19SNavdeep Parhar if (!(eq->flags & EQ_SW_ALLOCATED)) { 4786077ba6a8SJohn Baldwin snprintf(name, sizeof(name), "%d", idx); 478743bbae19SNavdeep Parhar oid = SYSCTL_ADD_NODE(&vi->ctx, 478843bbae19SNavdeep Parhar SYSCTL_CHILDREN(vi->ofld_txq_oid), OID_AUTO, name, 4789077ba6a8SJohn Baldwin CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue"); 4790077ba6a8SJohn Baldwin 479143bbae19SNavdeep Parhar snprintf(name, sizeof(name), "%s ofld_txq%d", 479243bbae19SNavdeep Parhar device_get_nameunit(vi->dev), idx); 479343bbae19SNavdeep Parhar if (vi->nofldrxq > 0) { 479443bbae19SNavdeep Parhar iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq); 479543bbae19SNavdeep Parhar init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan, 479643bbae19SNavdeep Parhar &sc->sge.ofld_rxq[iqidx].iq, name); 479743bbae19SNavdeep Parhar } else { 479843bbae19SNavdeep Parhar iqidx = vi->first_rxq + (idx % vi->nrxq); 479943bbae19SNavdeep Parhar init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan, 480043bbae19SNavdeep Parhar &sc->sge.rxq[iqidx].iq, name); 480143bbae19SNavdeep Parhar } 480243bbae19SNavdeep Parhar 480343bbae19SNavdeep Parhar rc = alloc_wrq(sc, vi, &ofld_txq->wrq, &vi->ctx, oid); 480443bbae19SNavdeep Parhar if (rc != 0) { 480543bbae19SNavdeep Parhar CH_ERR(vi, "failed to allocate ofld_txq%d: %d\n", idx, 480643bbae19SNavdeep Parhar rc); 480743bbae19SNavdeep Parhar sysctl_remove_oid(oid, 1, 1); 4808077ba6a8SJohn Baldwin return (rc); 480943bbae19SNavdeep Parhar } 481043bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 481143bbae19SNavdeep Parhar /* Can't fail after this point. */ 4812077ba6a8SJohn Baldwin 4813568e69e4SJohn Baldwin ofld_txq->tx_iscsi_pdus = counter_u64_alloc(M_WAITOK); 4814568e69e4SJohn Baldwin ofld_txq->tx_iscsi_octets = counter_u64_alloc(M_WAITOK); 48155b27e4b2SJohn Baldwin ofld_txq->tx_iscsi_iso_wrs = counter_u64_alloc(M_WAITOK); 4816fe496dc0SJohn Baldwin ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK); 4817fe496dc0SJohn Baldwin ofld_txq->tx_toe_tls_octets = counter_u64_alloc(M_WAITOK); 481843bbae19SNavdeep Parhar add_ofld_txq_sysctls(&vi->ctx, oid, ofld_txq); 4819077ba6a8SJohn Baldwin } 4820077ba6a8SJohn Baldwin 482143bbae19SNavdeep Parhar if (!(eq->flags & EQ_HW_ALLOCATED)) { 482243bbae19SNavdeep Parhar rc = alloc_eq_hwq(sc, vi, eq); 482343bbae19SNavdeep Parhar if (rc != 0) { 482443bbae19SNavdeep Parhar CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx, 482543bbae19SNavdeep Parhar rc); 482643bbae19SNavdeep Parhar return (rc); 482743bbae19SNavdeep Parhar } 482843bbae19SNavdeep Parhar MPASS(eq->flags & EQ_HW_ALLOCATED); 482943bbae19SNavdeep Parhar } 483043bbae19SNavdeep Parhar 483143bbae19SNavdeep Parhar return (0); 483243bbae19SNavdeep Parhar } 483343bbae19SNavdeep Parhar 483443bbae19SNavdeep Parhar /* 483543bbae19SNavdeep Parhar * Idempotent. 483643bbae19SNavdeep Parhar */ 483743bbae19SNavdeep Parhar static void 4838077ba6a8SJohn Baldwin free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq) 4839077ba6a8SJohn Baldwin { 4840077ba6a8SJohn Baldwin struct adapter *sc = vi->adapter; 484143bbae19SNavdeep Parhar struct sge_eq *eq = &ofld_txq->wrq.eq; 4842077ba6a8SJohn Baldwin 484343bbae19SNavdeep Parhar if (eq->flags & EQ_HW_ALLOCATED) { 484443bbae19SNavdeep Parhar MPASS(eq->flags & EQ_SW_ALLOCATED); 484543bbae19SNavdeep Parhar free_eq_hwq(sc, NULL, eq); 484643bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 484743bbae19SNavdeep Parhar } 4848077ba6a8SJohn Baldwin 484943bbae19SNavdeep Parhar if (eq->flags & EQ_SW_ALLOCATED) { 485043bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_HW_ALLOCATED)); 4851568e69e4SJohn Baldwin counter_u64_free(ofld_txq->tx_iscsi_pdus); 4852568e69e4SJohn Baldwin counter_u64_free(ofld_txq->tx_iscsi_octets); 48535b27e4b2SJohn Baldwin counter_u64_free(ofld_txq->tx_iscsi_iso_wrs); 4854fe496dc0SJohn Baldwin counter_u64_free(ofld_txq->tx_toe_tls_records); 4855fe496dc0SJohn Baldwin counter_u64_free(ofld_txq->tx_toe_tls_octets); 485643bbae19SNavdeep Parhar free_wrq(sc, &ofld_txq->wrq); 485743bbae19SNavdeep Parhar MPASS(!(eq->flags & EQ_SW_ALLOCATED)); 4858077ba6a8SJohn Baldwin bzero(ofld_txq, sizeof(*ofld_txq)); 485943bbae19SNavdeep Parhar } 486043bbae19SNavdeep Parhar } 486143bbae19SNavdeep Parhar 486243bbae19SNavdeep Parhar static void 486343bbae19SNavdeep Parhar add_ofld_txq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 486443bbae19SNavdeep Parhar struct sge_ofld_txq *ofld_txq) 486543bbae19SNavdeep Parhar { 486643bbae19SNavdeep Parhar struct sysctl_oid_list *children; 486743bbae19SNavdeep Parhar 486843bbae19SNavdeep Parhar if (ctx == NULL || oid == NULL) 486943bbae19SNavdeep Parhar return; 487043bbae19SNavdeep Parhar 487143bbae19SNavdeep Parhar children = SYSCTL_CHILDREN(oid); 487243bbae19SNavdeep Parhar SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_pdus", 487343bbae19SNavdeep Parhar CTLFLAG_RD, &ofld_txq->tx_iscsi_pdus, 487443bbae19SNavdeep Parhar "# of iSCSI PDUs transmitted"); 487543bbae19SNavdeep Parhar SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_octets", 487643bbae19SNavdeep Parhar CTLFLAG_RD, &ofld_txq->tx_iscsi_octets, 487743bbae19SNavdeep Parhar "# of payload octets in transmitted iSCSI PDUs"); 48785b27e4b2SJohn Baldwin SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_iso_wrs", 48795b27e4b2SJohn Baldwin CTLFLAG_RD, &ofld_txq->tx_iscsi_iso_wrs, 48805b27e4b2SJohn Baldwin "# of iSCSI segmentation offload work requests"); 488143bbae19SNavdeep Parhar SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_records", 488243bbae19SNavdeep Parhar CTLFLAG_RD, &ofld_txq->tx_toe_tls_records, 488343bbae19SNavdeep Parhar "# of TOE TLS records transmitted"); 488443bbae19SNavdeep Parhar SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_octets", 488543bbae19SNavdeep Parhar CTLFLAG_RD, &ofld_txq->tx_toe_tls_octets, 488643bbae19SNavdeep Parhar "# of payload octets in transmitted TOE TLS records"); 4887077ba6a8SJohn Baldwin } 4888077ba6a8SJohn Baldwin #endif 4889077ba6a8SJohn Baldwin 489054e4ee71SNavdeep Parhar static void 489154e4ee71SNavdeep Parhar oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 489254e4ee71SNavdeep Parhar { 489354e4ee71SNavdeep Parhar bus_addr_t *ba = arg; 489454e4ee71SNavdeep Parhar 489554e4ee71SNavdeep Parhar KASSERT(nseg == 1, 489654e4ee71SNavdeep Parhar ("%s meant for single segment mappings only.", __func__)); 489754e4ee71SNavdeep Parhar 489854e4ee71SNavdeep Parhar *ba = error ? 0 : segs->ds_addr; 489954e4ee71SNavdeep Parhar } 490054e4ee71SNavdeep Parhar 490154e4ee71SNavdeep Parhar static inline void 490254e4ee71SNavdeep Parhar ring_fl_db(struct adapter *sc, struct sge_fl *fl) 490354e4ee71SNavdeep Parhar { 49044d6db4e0SNavdeep Parhar uint32_t n, v; 490554e4ee71SNavdeep Parhar 490646e1e307SNavdeep Parhar n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx); 49074d6db4e0SNavdeep Parhar MPASS(n > 0); 4908d14b0ac1SNavdeep Parhar 490954e4ee71SNavdeep Parhar wmb(); 49104d6db4e0SNavdeep Parhar v = fl->dbval | V_PIDX(n); 49114d6db4e0SNavdeep Parhar if (fl->udb) 49124d6db4e0SNavdeep Parhar *fl->udb = htole32(v); 49134d6db4e0SNavdeep Parhar else 4914315048f2SJohn Baldwin t4_write_reg(sc, sc->sge_kdoorbell_reg, v); 49154d6db4e0SNavdeep Parhar IDXINCR(fl->dbidx, n, fl->sidx); 491654e4ee71SNavdeep Parhar } 491754e4ee71SNavdeep Parhar 4918fb12416cSNavdeep Parhar /* 49194d6db4e0SNavdeep Parhar * Fills up the freelist by allocating up to 'n' buffers. Buffers that are 49204d6db4e0SNavdeep Parhar * recycled do not count towards this allocation budget. 4921733b9277SNavdeep Parhar * 49224d6db4e0SNavdeep Parhar * Returns non-zero to indicate that this freelist should be added to the list 49234d6db4e0SNavdeep Parhar * of starving freelists. 4924fb12416cSNavdeep Parhar */ 4925733b9277SNavdeep Parhar static int 49264d6db4e0SNavdeep Parhar refill_fl(struct adapter *sc, struct sge_fl *fl, int n) 492754e4ee71SNavdeep Parhar { 49284d6db4e0SNavdeep Parhar __be64 *d; 49294d6db4e0SNavdeep Parhar struct fl_sdesc *sd; 493038035ed6SNavdeep Parhar uintptr_t pa; 493154e4ee71SNavdeep Parhar caddr_t cl; 493246e1e307SNavdeep Parhar struct rx_buf_info *rxb; 493338035ed6SNavdeep Parhar struct cluster_metadata *clm; 4934294e62beSAlexander Motin uint16_t max_pidx, zidx = fl->zidx; 49354d6db4e0SNavdeep Parhar uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ 493654e4ee71SNavdeep Parhar 493754e4ee71SNavdeep Parhar FL_LOCK_ASSERT_OWNED(fl); 493854e4ee71SNavdeep Parhar 49394d6db4e0SNavdeep Parhar /* 4940453130d9SPedro F. Giffuni * We always stop at the beginning of the hardware descriptor that's just 49414d6db4e0SNavdeep Parhar * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, 49424d6db4e0SNavdeep Parhar * which would mean an empty freelist to the chip. 49434d6db4e0SNavdeep Parhar */ 49444d6db4e0SNavdeep Parhar max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; 49454d6db4e0SNavdeep Parhar if (fl->pidx == max_pidx * 8) 49464d6db4e0SNavdeep Parhar return (0); 494754e4ee71SNavdeep Parhar 49484d6db4e0SNavdeep Parhar d = &fl->desc[fl->pidx]; 49494d6db4e0SNavdeep Parhar sd = &fl->sdesc[fl->pidx]; 4950294e62beSAlexander Motin rxb = &sc->sge.rx_buf_info[zidx]; 49514d6db4e0SNavdeep Parhar 49524d6db4e0SNavdeep Parhar while (n > 0) { 495354e4ee71SNavdeep Parhar 495454e4ee71SNavdeep Parhar if (sd->cl != NULL) { 495554e4ee71SNavdeep Parhar 4956c3fb7725SNavdeep Parhar if (sd->nmbuf == 0) { 495738035ed6SNavdeep Parhar /* 495838035ed6SNavdeep Parhar * Fast recycle without involving any atomics on 495938035ed6SNavdeep Parhar * the cluster's metadata (if the cluster has 496038035ed6SNavdeep Parhar * metadata). This happens when all frames 496138035ed6SNavdeep Parhar * received in the cluster were small enough to 496238035ed6SNavdeep Parhar * fit within a single mbuf each. 496338035ed6SNavdeep Parhar */ 496438035ed6SNavdeep Parhar fl->cl_fast_recycled++; 4965a9c4062aSNavdeep Parhar goto recycled; 496638035ed6SNavdeep Parhar } 496754e4ee71SNavdeep Parhar 496838035ed6SNavdeep Parhar /* 496938035ed6SNavdeep Parhar * Cluster is guaranteed to have metadata. Clusters 497038035ed6SNavdeep Parhar * without metadata always take the fast recycle path 497138035ed6SNavdeep Parhar * when they're recycled. 497238035ed6SNavdeep Parhar */ 497346e1e307SNavdeep Parhar clm = cl_metadata(sd); 497438035ed6SNavdeep Parhar MPASS(clm != NULL); 49751458bff9SNavdeep Parhar 497638035ed6SNavdeep Parhar if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 497738035ed6SNavdeep Parhar fl->cl_recycled++; 497882eff304SNavdeep Parhar counter_u64_add(extfree_rels, 1); 497954e4ee71SNavdeep Parhar goto recycled; 498054e4ee71SNavdeep Parhar } 49811458bff9SNavdeep Parhar sd->cl = NULL; /* gave up my reference */ 49821458bff9SNavdeep Parhar } 498338035ed6SNavdeep Parhar MPASS(sd->cl == NULL); 498446e1e307SNavdeep Parhar cl = uma_zalloc(rxb->zone, M_NOWAIT); 49852b9010f0SNavdeep Parhar if (__predict_false(cl == NULL)) { 4986294e62beSAlexander Motin if (zidx != fl->safe_zidx) { 4987294e62beSAlexander Motin zidx = fl->safe_zidx; 4988294e62beSAlexander Motin rxb = &sc->sge.rx_buf_info[zidx]; 498946e1e307SNavdeep Parhar cl = uma_zalloc(rxb->zone, M_NOWAIT); 49902b9010f0SNavdeep Parhar } 49912b9010f0SNavdeep Parhar if (cl == NULL) 499254e4ee71SNavdeep Parhar break; 499354e4ee71SNavdeep Parhar } 499438035ed6SNavdeep Parhar fl->cl_allocated++; 49954d6db4e0SNavdeep Parhar n--; 499654e4ee71SNavdeep Parhar 499738035ed6SNavdeep Parhar pa = pmap_kextract((vm_offset_t)cl); 499854e4ee71SNavdeep Parhar sd->cl = cl; 4999294e62beSAlexander Motin sd->zidx = zidx; 500046e1e307SNavdeep Parhar 500146e1e307SNavdeep Parhar if (fl->flags & FL_BUF_PACKING) { 500246e1e307SNavdeep Parhar *d = htobe64(pa | rxb->hwidx2); 500346e1e307SNavdeep Parhar sd->moff = rxb->size2; 500446e1e307SNavdeep Parhar } else { 500546e1e307SNavdeep Parhar *d = htobe64(pa | rxb->hwidx1); 500646e1e307SNavdeep Parhar sd->moff = 0; 500746e1e307SNavdeep Parhar } 50087d29df59SNavdeep Parhar recycled: 5009c3fb7725SNavdeep Parhar sd->nmbuf = 0; 501038035ed6SNavdeep Parhar d++; 501154e4ee71SNavdeep Parhar sd++; 501246e1e307SNavdeep Parhar if (__predict_false((++fl->pidx & 7) == 0)) { 501346e1e307SNavdeep Parhar uint16_t pidx = fl->pidx >> 3; 50144d6db4e0SNavdeep Parhar 50154d6db4e0SNavdeep Parhar if (__predict_false(pidx == fl->sidx)) { 501654e4ee71SNavdeep Parhar fl->pidx = 0; 50174d6db4e0SNavdeep Parhar pidx = 0; 501854e4ee71SNavdeep Parhar sd = fl->sdesc; 501954e4ee71SNavdeep Parhar d = fl->desc; 502054e4ee71SNavdeep Parhar } 502146e1e307SNavdeep Parhar if (n < 8 || pidx == max_pidx) 50224d6db4e0SNavdeep Parhar break; 50234d6db4e0SNavdeep Parhar 50244d6db4e0SNavdeep Parhar if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) 50254d6db4e0SNavdeep Parhar ring_fl_db(sc, fl); 50264d6db4e0SNavdeep Parhar } 502754e4ee71SNavdeep Parhar } 5028fb12416cSNavdeep Parhar 502946e1e307SNavdeep Parhar if ((fl->pidx >> 3) != fl->dbidx) 5030fb12416cSNavdeep Parhar ring_fl_db(sc, fl); 5031733b9277SNavdeep Parhar 5032733b9277SNavdeep Parhar return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 5033733b9277SNavdeep Parhar } 5034733b9277SNavdeep Parhar 5035733b9277SNavdeep Parhar /* 5036733b9277SNavdeep Parhar * Attempt to refill all starving freelists. 5037733b9277SNavdeep Parhar */ 5038733b9277SNavdeep Parhar static void 5039733b9277SNavdeep Parhar refill_sfl(void *arg) 5040733b9277SNavdeep Parhar { 5041733b9277SNavdeep Parhar struct adapter *sc = arg; 5042733b9277SNavdeep Parhar struct sge_fl *fl, *fl_temp; 5043733b9277SNavdeep Parhar 5044fe2ebb76SJohn Baldwin mtx_assert(&sc->sfl_lock, MA_OWNED); 5045733b9277SNavdeep Parhar TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 5046733b9277SNavdeep Parhar FL_LOCK(fl); 5047733b9277SNavdeep Parhar refill_fl(sc, fl, 64); 5048733b9277SNavdeep Parhar if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 5049733b9277SNavdeep Parhar TAILQ_REMOVE(&sc->sfl, fl, link); 5050733b9277SNavdeep Parhar fl->flags &= ~FL_STARVING; 5051733b9277SNavdeep Parhar } 5052733b9277SNavdeep Parhar FL_UNLOCK(fl); 5053733b9277SNavdeep Parhar } 5054733b9277SNavdeep Parhar 5055733b9277SNavdeep Parhar if (!TAILQ_EMPTY(&sc->sfl)) 5056733b9277SNavdeep Parhar callout_schedule(&sc->sfl_callout, hz / 5); 505754e4ee71SNavdeep Parhar } 505854e4ee71SNavdeep Parhar 505943bbae19SNavdeep Parhar /* 506043bbae19SNavdeep Parhar * Release the driver's reference on all buffers in the given freelist. Buffers 506143bbae19SNavdeep Parhar * with kernel references cannot be freed and will prevent the driver from being 506243bbae19SNavdeep Parhar * unloaded safely. 506343bbae19SNavdeep Parhar */ 506443bbae19SNavdeep Parhar void 506543bbae19SNavdeep Parhar free_fl_buffers(struct adapter *sc, struct sge_fl *fl) 506654e4ee71SNavdeep Parhar { 506754e4ee71SNavdeep Parhar struct fl_sdesc *sd; 506838035ed6SNavdeep Parhar struct cluster_metadata *clm; 506954e4ee71SNavdeep Parhar int i; 507054e4ee71SNavdeep Parhar 507154e4ee71SNavdeep Parhar sd = fl->sdesc; 50724d6db4e0SNavdeep Parhar for (i = 0; i < fl->sidx * 8; i++, sd++) { 507338035ed6SNavdeep Parhar if (sd->cl == NULL) 507438035ed6SNavdeep Parhar continue; 507554e4ee71SNavdeep Parhar 507682eff304SNavdeep Parhar if (sd->nmbuf == 0) 507746e1e307SNavdeep Parhar uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl); 507846e1e307SNavdeep Parhar else if (fl->flags & FL_BUF_PACKING) { 507946e1e307SNavdeep Parhar clm = cl_metadata(sd); 508046e1e307SNavdeep Parhar if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 508146e1e307SNavdeep Parhar uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, 508246e1e307SNavdeep Parhar sd->cl); 508382eff304SNavdeep Parhar counter_u64_add(extfree_rels, 1); 508454e4ee71SNavdeep Parhar } 508546e1e307SNavdeep Parhar } 508638035ed6SNavdeep Parhar sd->cl = NULL; 508754e4ee71SNavdeep Parhar } 508854e4ee71SNavdeep Parhar 508943bbae19SNavdeep Parhar if (fl->flags & FL_BUF_RESUME) { 509043bbae19SNavdeep Parhar m_freem(fl->m0); 509143bbae19SNavdeep Parhar fl->flags &= ~FL_BUF_RESUME; 509243bbae19SNavdeep Parhar } 509354e4ee71SNavdeep Parhar } 509454e4ee71SNavdeep Parhar 50957951040fSNavdeep Parhar static inline void 50967951040fSNavdeep Parhar get_pkt_gl(struct mbuf *m, struct sglist *gl) 509754e4ee71SNavdeep Parhar { 50987951040fSNavdeep Parhar int rc; 509954e4ee71SNavdeep Parhar 51007951040fSNavdeep Parhar M_ASSERTPKTHDR(m); 510154e4ee71SNavdeep Parhar 51027951040fSNavdeep Parhar sglist_reset(gl); 51037951040fSNavdeep Parhar rc = sglist_append_mbuf(gl, m); 51047951040fSNavdeep Parhar if (__predict_false(rc != 0)) { 51057951040fSNavdeep Parhar panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " 51067951040fSNavdeep Parhar "with %d.", __func__, m, mbuf_nsegs(m), rc); 510754e4ee71SNavdeep Parhar } 510854e4ee71SNavdeep Parhar 51097951040fSNavdeep Parhar KASSERT(gl->sg_nseg == mbuf_nsegs(m), 51107951040fSNavdeep Parhar ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, 51117951040fSNavdeep Parhar mbuf_nsegs(m), gl->sg_nseg)); 511230e3f2b4SNavdeep Parhar #if 0 /* vm_wr not readily available here. */ 511330e3f2b4SNavdeep Parhar KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= max_nsegs_allowed(m, vm_wr), 51147951040fSNavdeep Parhar ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, 511530e3f2b4SNavdeep Parhar gl->sg_nseg, max_nsegs_allowed(m, vm_wr))); 511630e3f2b4SNavdeep Parhar #endif 511754e4ee71SNavdeep Parhar } 511854e4ee71SNavdeep Parhar 511954e4ee71SNavdeep Parhar /* 51207951040fSNavdeep Parhar * len16 for a txpkt WR with a GL. Includes the firmware work request header. 512154e4ee71SNavdeep Parhar */ 51227951040fSNavdeep Parhar static inline u_int 5123a4a4ad2dSNavdeep Parhar txpkt_len16(u_int nsegs, const u_int extra) 51247951040fSNavdeep Parhar { 51257951040fSNavdeep Parhar u_int n; 51267951040fSNavdeep Parhar 51277951040fSNavdeep Parhar MPASS(nsegs > 0); 51287951040fSNavdeep Parhar 51297951040fSNavdeep Parhar nsegs--; /* first segment is part of ulptx_sgl */ 5130a4a4ad2dSNavdeep Parhar n = extra + sizeof(struct fw_eth_tx_pkt_wr) + 5131a4a4ad2dSNavdeep Parhar sizeof(struct cpl_tx_pkt_core) + 51327951040fSNavdeep Parhar sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 51337951040fSNavdeep Parhar 51347951040fSNavdeep Parhar return (howmany(n, 16)); 51357951040fSNavdeep Parhar } 513654e4ee71SNavdeep Parhar 513754e4ee71SNavdeep Parhar /* 51386af45170SJohn Baldwin * len16 for a txpkt_vm WR with a GL. Includes the firmware work 51396af45170SJohn Baldwin * request header. 51406af45170SJohn Baldwin */ 51416af45170SJohn Baldwin static inline u_int 5142a4a4ad2dSNavdeep Parhar txpkt_vm_len16(u_int nsegs, const u_int extra) 51436af45170SJohn Baldwin { 51446af45170SJohn Baldwin u_int n; 51456af45170SJohn Baldwin 51466af45170SJohn Baldwin MPASS(nsegs > 0); 51476af45170SJohn Baldwin 51486af45170SJohn Baldwin nsegs--; /* first segment is part of ulptx_sgl */ 5149a4a4ad2dSNavdeep Parhar n = extra + sizeof(struct fw_eth_tx_pkt_vm_wr) + 51506af45170SJohn Baldwin sizeof(struct cpl_tx_pkt_core) + 51516af45170SJohn Baldwin sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 51526af45170SJohn Baldwin 51536af45170SJohn Baldwin return (howmany(n, 16)); 51546af45170SJohn Baldwin } 51556af45170SJohn Baldwin 5156a4a4ad2dSNavdeep Parhar static inline void 515730e3f2b4SNavdeep Parhar calculate_mbuf_len16(struct mbuf *m, bool vm_wr) 5158a4a4ad2dSNavdeep Parhar { 5159a4a4ad2dSNavdeep Parhar const int lso = sizeof(struct cpl_tx_pkt_lso_core); 5160a4a4ad2dSNavdeep Parhar const int tnl_lso = sizeof(struct cpl_tx_tnl_lso); 5161a4a4ad2dSNavdeep Parhar 516230e3f2b4SNavdeep Parhar if (vm_wr) { 5163a4a4ad2dSNavdeep Parhar if (needs_tso(m)) 5164a4a4ad2dSNavdeep Parhar set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), lso)); 5165a4a4ad2dSNavdeep Parhar else 5166a4a4ad2dSNavdeep Parhar set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), 0)); 5167a4a4ad2dSNavdeep Parhar return; 5168a4a4ad2dSNavdeep Parhar } 5169a4a4ad2dSNavdeep Parhar 5170a4a4ad2dSNavdeep Parhar if (needs_tso(m)) { 5171a4a4ad2dSNavdeep Parhar if (needs_vxlan_tso(m)) 5172a4a4ad2dSNavdeep Parhar set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), tnl_lso)); 5173a4a4ad2dSNavdeep Parhar else 5174a4a4ad2dSNavdeep Parhar set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), lso)); 5175a4a4ad2dSNavdeep Parhar } else 5176a4a4ad2dSNavdeep Parhar set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), 0)); 5177a4a4ad2dSNavdeep Parhar } 5178a4a4ad2dSNavdeep Parhar 51796af45170SJohn Baldwin /* 51807951040fSNavdeep Parhar * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work 51817951040fSNavdeep Parhar * request header. 51827951040fSNavdeep Parhar */ 51837951040fSNavdeep Parhar static inline u_int 51847951040fSNavdeep Parhar txpkts0_len16(u_int nsegs) 51857951040fSNavdeep Parhar { 51867951040fSNavdeep Parhar u_int n; 51877951040fSNavdeep Parhar 51887951040fSNavdeep Parhar MPASS(nsegs > 0); 51897951040fSNavdeep Parhar 51907951040fSNavdeep Parhar nsegs--; /* first segment is part of ulptx_sgl */ 51917951040fSNavdeep Parhar n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + 51927951040fSNavdeep Parhar sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 51937951040fSNavdeep Parhar 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 51947951040fSNavdeep Parhar 51957951040fSNavdeep Parhar return (howmany(n, 16)); 51967951040fSNavdeep Parhar } 51977951040fSNavdeep Parhar 51987951040fSNavdeep Parhar /* 51997951040fSNavdeep Parhar * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work 52007951040fSNavdeep Parhar * request header. 52017951040fSNavdeep Parhar */ 52027951040fSNavdeep Parhar static inline u_int 52037951040fSNavdeep Parhar txpkts1_len16(void) 52047951040fSNavdeep Parhar { 52057951040fSNavdeep Parhar u_int n; 52067951040fSNavdeep Parhar 52077951040fSNavdeep Parhar n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); 52087951040fSNavdeep Parhar 52097951040fSNavdeep Parhar return (howmany(n, 16)); 52107951040fSNavdeep Parhar } 52117951040fSNavdeep Parhar 52127951040fSNavdeep Parhar static inline u_int 52137951040fSNavdeep Parhar imm_payload(u_int ndesc) 52147951040fSNavdeep Parhar { 52157951040fSNavdeep Parhar u_int n; 52167951040fSNavdeep Parhar 52177951040fSNavdeep Parhar n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - 52187951040fSNavdeep Parhar sizeof(struct cpl_tx_pkt_core); 52197951040fSNavdeep Parhar 52207951040fSNavdeep Parhar return (n); 52217951040fSNavdeep Parhar } 52227951040fSNavdeep Parhar 5223c0236bd9SNavdeep Parhar static inline uint64_t 5224c0236bd9SNavdeep Parhar csum_to_ctrl(struct adapter *sc, struct mbuf *m) 5225c0236bd9SNavdeep Parhar { 5226c0236bd9SNavdeep Parhar uint64_t ctrl; 5227a4a4ad2dSNavdeep Parhar int csum_type, l2hlen, l3hlen; 5228a4a4ad2dSNavdeep Parhar int x, y; 5229a4a4ad2dSNavdeep Parhar static const int csum_types[3][2] = { 5230a4a4ad2dSNavdeep Parhar {TX_CSUM_TCPIP, TX_CSUM_TCPIP6}, 5231a4a4ad2dSNavdeep Parhar {TX_CSUM_UDPIP, TX_CSUM_UDPIP6}, 5232a4a4ad2dSNavdeep Parhar {TX_CSUM_IP, 0} 5233a4a4ad2dSNavdeep Parhar }; 5234c0236bd9SNavdeep Parhar 5235c0236bd9SNavdeep Parhar M_ASSERTPKTHDR(m); 5236c0236bd9SNavdeep Parhar 5237a4a4ad2dSNavdeep Parhar if (!needs_hwcsum(m)) 5238c0236bd9SNavdeep Parhar return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS); 5239c0236bd9SNavdeep Parhar 5240a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.l2hlen >= ETHER_HDR_LEN); 5241a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.l3hlen >= sizeof(struct ip)); 5242a4a4ad2dSNavdeep Parhar 5243a4a4ad2dSNavdeep Parhar if (needs_vxlan_csum(m)) { 5244a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.l4hlen > 0); 5245a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.l5hlen > 0); 5246a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.inner_l2hlen >= ETHER_HDR_LEN); 5247a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.inner_l3hlen >= sizeof(struct ip)); 5248a4a4ad2dSNavdeep Parhar 5249a4a4ad2dSNavdeep Parhar l2hlen = m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + 5250a4a4ad2dSNavdeep Parhar m->m_pkthdr.l4hlen + m->m_pkthdr.l5hlen + 5251a4a4ad2dSNavdeep Parhar m->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN; 5252a4a4ad2dSNavdeep Parhar l3hlen = m->m_pkthdr.inner_l3hlen; 5253a4a4ad2dSNavdeep Parhar } else { 5254a4a4ad2dSNavdeep Parhar l2hlen = m->m_pkthdr.l2hlen - ETHER_HDR_LEN; 5255a4a4ad2dSNavdeep Parhar l3hlen = m->m_pkthdr.l3hlen; 5256c0236bd9SNavdeep Parhar } 5257c0236bd9SNavdeep Parhar 5258a4a4ad2dSNavdeep Parhar ctrl = 0; 5259a4a4ad2dSNavdeep Parhar if (!needs_l3_csum(m)) 5260a4a4ad2dSNavdeep Parhar ctrl |= F_TXPKT_IPCSUM_DIS; 5261a4a4ad2dSNavdeep Parhar 5262a4a4ad2dSNavdeep Parhar if (m->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_INNER_IP_TCP | 5263a4a4ad2dSNavdeep Parhar CSUM_IP6_TCP | CSUM_INNER_IP6_TCP)) 5264a4a4ad2dSNavdeep Parhar x = 0; /* TCP */ 5265a4a4ad2dSNavdeep Parhar else if (m->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_INNER_IP_UDP | 5266a4a4ad2dSNavdeep Parhar CSUM_IP6_UDP | CSUM_INNER_IP6_UDP)) 5267a4a4ad2dSNavdeep Parhar x = 1; /* UDP */ 5268c0236bd9SNavdeep Parhar else 5269a4a4ad2dSNavdeep Parhar x = 2; 5270a4a4ad2dSNavdeep Parhar 5271a4a4ad2dSNavdeep Parhar if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP | 5272a4a4ad2dSNavdeep Parhar CSUM_INNER_IP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP)) 5273a4a4ad2dSNavdeep Parhar y = 0; /* IPv4 */ 5274a4a4ad2dSNavdeep Parhar else { 5275a4a4ad2dSNavdeep Parhar MPASS(m->m_pkthdr.csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | 5276a4a4ad2dSNavdeep Parhar CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP)); 5277a4a4ad2dSNavdeep Parhar y = 1; /* IPv6 */ 5278a4a4ad2dSNavdeep Parhar } 5279a4a4ad2dSNavdeep Parhar /* 5280a4a4ad2dSNavdeep Parhar * needs_hwcsum returned true earlier so there must be some kind of 5281a4a4ad2dSNavdeep Parhar * checksum to calculate. 5282a4a4ad2dSNavdeep Parhar */ 5283a4a4ad2dSNavdeep Parhar csum_type = csum_types[x][y]; 5284a4a4ad2dSNavdeep Parhar MPASS(csum_type != 0); 5285a4a4ad2dSNavdeep Parhar if (csum_type == TX_CSUM_IP) 5286a4a4ad2dSNavdeep Parhar ctrl |= F_TXPKT_L4CSUM_DIS; 5287a4a4ad2dSNavdeep Parhar ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | V_TXPKT_IPHDR_LEN(l3hlen); 5288a4a4ad2dSNavdeep Parhar if (chip_id(sc) <= CHELSIO_T5) 5289a4a4ad2dSNavdeep Parhar ctrl |= V_TXPKT_ETHHDR_LEN(l2hlen); 5290a4a4ad2dSNavdeep Parhar else 5291a4a4ad2dSNavdeep Parhar ctrl |= V_T6_TXPKT_ETHHDR_LEN(l2hlen); 5292c0236bd9SNavdeep Parhar 5293c0236bd9SNavdeep Parhar return (ctrl); 5294c0236bd9SNavdeep Parhar } 5295c0236bd9SNavdeep Parhar 5296a4a4ad2dSNavdeep Parhar static inline void * 5297a4a4ad2dSNavdeep Parhar write_lso_cpl(void *cpl, struct mbuf *m0) 5298a4a4ad2dSNavdeep Parhar { 5299a4a4ad2dSNavdeep Parhar struct cpl_tx_pkt_lso_core *lso; 5300a4a4ad2dSNavdeep Parhar uint32_t ctrl; 5301a4a4ad2dSNavdeep Parhar 5302a4a4ad2dSNavdeep Parhar KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 5303a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l4hlen > 0, 5304a4a4ad2dSNavdeep Parhar ("%s: mbuf %p needs TSO but missing header lengths", 5305a4a4ad2dSNavdeep Parhar __func__, m0)); 5306a4a4ad2dSNavdeep Parhar 5307a4a4ad2dSNavdeep Parhar ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | 5308a4a4ad2dSNavdeep Parhar F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | 5309a4a4ad2dSNavdeep Parhar V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) | 5310a4a4ad2dSNavdeep Parhar V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 5311a4a4ad2dSNavdeep Parhar V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 5312a4a4ad2dSNavdeep Parhar if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 5313a4a4ad2dSNavdeep Parhar ctrl |= F_LSO_IPV6; 5314a4a4ad2dSNavdeep Parhar 5315a4a4ad2dSNavdeep Parhar lso = cpl; 5316a4a4ad2dSNavdeep Parhar lso->lso_ctrl = htobe32(ctrl); 5317a4a4ad2dSNavdeep Parhar lso->ipid_ofst = htobe16(0); 5318a4a4ad2dSNavdeep Parhar lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 5319a4a4ad2dSNavdeep Parhar lso->seqno_offset = htobe32(0); 5320a4a4ad2dSNavdeep Parhar lso->len = htobe32(m0->m_pkthdr.len); 5321a4a4ad2dSNavdeep Parhar 5322a4a4ad2dSNavdeep Parhar return (lso + 1); 5323a4a4ad2dSNavdeep Parhar } 5324a4a4ad2dSNavdeep Parhar 5325a4a4ad2dSNavdeep Parhar static void * 5326a4a4ad2dSNavdeep Parhar write_tnl_lso_cpl(void *cpl, struct mbuf *m0) 5327a4a4ad2dSNavdeep Parhar { 5328a4a4ad2dSNavdeep Parhar struct cpl_tx_tnl_lso *tnl_lso = cpl; 5329a4a4ad2dSNavdeep Parhar uint32_t ctrl; 5330a4a4ad2dSNavdeep Parhar 5331a4a4ad2dSNavdeep Parhar KASSERT(m0->m_pkthdr.inner_l2hlen > 0 && 5332a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l3hlen > 0 && m0->m_pkthdr.inner_l4hlen > 0 && 5333a4a4ad2dSNavdeep Parhar m0->m_pkthdr.inner_l5hlen > 0, 5334a4a4ad2dSNavdeep Parhar ("%s: mbuf %p needs VXLAN_TSO but missing inner header lengths", 5335a4a4ad2dSNavdeep Parhar __func__, m0)); 5336a4a4ad2dSNavdeep Parhar KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 5337a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l4hlen > 0 && m0->m_pkthdr.l5hlen > 0, 5338a4a4ad2dSNavdeep Parhar ("%s: mbuf %p needs VXLAN_TSO but missing outer header lengths", 5339a4a4ad2dSNavdeep Parhar __func__, m0)); 5340a4a4ad2dSNavdeep Parhar 5341a4a4ad2dSNavdeep Parhar /* Outer headers. */ 5342a4a4ad2dSNavdeep Parhar ctrl = V_CPL_TX_TNL_LSO_OPCODE(CPL_TX_TNL_LSO) | 5343a4a4ad2dSNavdeep Parhar F_CPL_TX_TNL_LSO_FIRST | F_CPL_TX_TNL_LSO_LAST | 5344a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_ETHHDRLENOUT( 5345a4a4ad2dSNavdeep Parhar (m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) | 5346a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_IPHDRLENOUT(m0->m_pkthdr.l3hlen >> 2) | 5347a4a4ad2dSNavdeep Parhar F_CPL_TX_TNL_LSO_IPLENSETOUT; 5348a4a4ad2dSNavdeep Parhar if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 5349a4a4ad2dSNavdeep Parhar ctrl |= F_CPL_TX_TNL_LSO_IPV6OUT; 5350a4a4ad2dSNavdeep Parhar else { 5351a4a4ad2dSNavdeep Parhar ctrl |= F_CPL_TX_TNL_LSO_IPHDRCHKOUT | 5352a4a4ad2dSNavdeep Parhar F_CPL_TX_TNL_LSO_IPIDINCOUT; 5353a4a4ad2dSNavdeep Parhar } 5354a4a4ad2dSNavdeep Parhar tnl_lso->op_to_IpIdSplitOut = htobe32(ctrl); 5355a4a4ad2dSNavdeep Parhar tnl_lso->IpIdOffsetOut = 0; 5356a4a4ad2dSNavdeep Parhar tnl_lso->UdpLenSetOut_to_TnlHdrLen = 5357a4a4ad2dSNavdeep Parhar htobe16(F_CPL_TX_TNL_LSO_UDPCHKCLROUT | 5358a4a4ad2dSNavdeep Parhar F_CPL_TX_TNL_LSO_UDPLENSETOUT | 5359a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_TNLHDRLEN(m0->m_pkthdr.l2hlen + 5360a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen + 5361a4a4ad2dSNavdeep Parhar m0->m_pkthdr.l5hlen) | 5362a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_TNLTYPE(TX_TNL_TYPE_VXLAN)); 5363a4a4ad2dSNavdeep Parhar tnl_lso->r1 = 0; 5364a4a4ad2dSNavdeep Parhar 5365a4a4ad2dSNavdeep Parhar /* Inner headers. */ 5366a4a4ad2dSNavdeep Parhar ctrl = V_CPL_TX_TNL_LSO_ETHHDRLEN( 5367a4a4ad2dSNavdeep Parhar (m0->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN) >> 2) | 5368a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_IPHDRLEN(m0->m_pkthdr.inner_l3hlen >> 2) | 5369a4a4ad2dSNavdeep Parhar V_CPL_TX_TNL_LSO_TCPHDRLEN(m0->m_pkthdr.inner_l4hlen >> 2); 5370a4a4ad2dSNavdeep Parhar if (m0->m_pkthdr.inner_l3hlen == sizeof(struct ip6_hdr)) 5371a4a4ad2dSNavdeep Parhar ctrl |= F_CPL_TX_TNL_LSO_IPV6; 5372a4a4ad2dSNavdeep Parhar tnl_lso->Flow_to_TcpHdrLen = htobe32(ctrl); 5373a4a4ad2dSNavdeep Parhar tnl_lso->IpIdOffset = 0; 5374a4a4ad2dSNavdeep Parhar tnl_lso->IpIdSplit_to_Mss = 5375a4a4ad2dSNavdeep Parhar htobe16(V_CPL_TX_TNL_LSO_MSS(m0->m_pkthdr.tso_segsz)); 5376a4a4ad2dSNavdeep Parhar tnl_lso->TCPSeqOffset = 0; 5377a4a4ad2dSNavdeep Parhar tnl_lso->EthLenOffset_Size = 5378a4a4ad2dSNavdeep Parhar htobe32(V_CPL_TX_TNL_LSO_SIZE(m0->m_pkthdr.len)); 5379a4a4ad2dSNavdeep Parhar 5380a4a4ad2dSNavdeep Parhar return (tnl_lso + 1); 5381a4a4ad2dSNavdeep Parhar } 5382a4a4ad2dSNavdeep Parhar 5383800535c2SNavdeep Parhar #define VM_TX_L2HDR_LEN 16 /* ethmacdst to vlantci */ 5384800535c2SNavdeep Parhar 53857951040fSNavdeep Parhar /* 53866af45170SJohn Baldwin * Write a VM txpkt WR for this packet to the hardware descriptors, update the 53876af45170SJohn Baldwin * software descriptor, and advance the pidx. It is guaranteed that enough 53886af45170SJohn Baldwin * descriptors are available. 53896af45170SJohn Baldwin * 53906af45170SJohn Baldwin * The return value is the # of hardware descriptors used. 53916af45170SJohn Baldwin */ 53926af45170SJohn Baldwin static u_int 5393d735920dSNavdeep Parhar write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0) 53946af45170SJohn Baldwin { 5395d735920dSNavdeep Parhar struct sge_eq *eq; 5396d735920dSNavdeep Parhar struct fw_eth_tx_pkt_vm_wr *wr; 53976af45170SJohn Baldwin struct tx_sdesc *txsd; 53986af45170SJohn Baldwin struct cpl_tx_pkt_core *cpl; 53996af45170SJohn Baldwin uint32_t ctrl; /* used in many unrelated places */ 54006af45170SJohn Baldwin uint64_t ctrl1; 540139d5cbdcSNavdeep Parhar int len16, ndesc, pktlen; 54026af45170SJohn Baldwin caddr_t dst; 54036af45170SJohn Baldwin 54046af45170SJohn Baldwin TXQ_LOCK_ASSERT_OWNED(txq); 54056af45170SJohn Baldwin M_ASSERTPKTHDR(m0); 54066af45170SJohn Baldwin 54076af45170SJohn Baldwin len16 = mbuf_len16(m0); 54086af45170SJohn Baldwin pktlen = m0->m_pkthdr.len; 54096af45170SJohn Baldwin ctrl = sizeof(struct cpl_tx_pkt_core); 54106af45170SJohn Baldwin if (needs_tso(m0)) 54116af45170SJohn Baldwin ctrl += sizeof(struct cpl_tx_pkt_lso_core); 54120cadedfcSNavdeep Parhar ndesc = tx_len16_to_desc(len16); 54136af45170SJohn Baldwin 54146af45170SJohn Baldwin /* Firmware work request header */ 5415d735920dSNavdeep Parhar eq = &txq->eq; 5416d735920dSNavdeep Parhar wr = (void *)&eq->desc[eq->pidx]; 54176af45170SJohn Baldwin wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 54186af45170SJohn Baldwin V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 54196af45170SJohn Baldwin 54206af45170SJohn Baldwin ctrl = V_FW_WR_LEN16(len16); 54216af45170SJohn Baldwin wr->equiq_to_len16 = htobe32(ctrl); 54226af45170SJohn Baldwin wr->r3[0] = 0; 54236af45170SJohn Baldwin wr->r3[1] = 0; 54246af45170SJohn Baldwin 54256af45170SJohn Baldwin /* 54266af45170SJohn Baldwin * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. 54276af45170SJohn Baldwin * vlantci is ignored unless the ethtype is 0x8100, so it's 54286af45170SJohn Baldwin * simpler to always copy it rather than making it 54296af45170SJohn Baldwin * conditional. Also, it seems that we do not have to set 54306af45170SJohn Baldwin * vlantci or fake the ethtype when doing VLAN tag insertion. 54316af45170SJohn Baldwin */ 5432800535c2SNavdeep Parhar m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst); 54336af45170SJohn Baldwin 54346af45170SJohn Baldwin if (needs_tso(m0)) { 5435a4a4ad2dSNavdeep Parhar cpl = write_lso_cpl(wr + 1, m0); 54366af45170SJohn Baldwin txq->tso_wrs++; 5437c0236bd9SNavdeep Parhar } else 54386af45170SJohn Baldwin cpl = (void *)(wr + 1); 54396af45170SJohn Baldwin 54406af45170SJohn Baldwin /* Checksum offload */ 5441c0236bd9SNavdeep Parhar ctrl1 = csum_to_ctrl(sc, m0); 5442c0236bd9SNavdeep Parhar if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 54436af45170SJohn Baldwin txq->txcsum++; /* some hardware assistance provided */ 54446af45170SJohn Baldwin 54456af45170SJohn Baldwin /* VLAN tag insertion */ 54466af45170SJohn Baldwin if (needs_vlan_insertion(m0)) { 54476af45170SJohn Baldwin ctrl1 |= F_TXPKT_VLAN_VLD | 54486af45170SJohn Baldwin V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 54496af45170SJohn Baldwin txq->vlan_insertion++; 54506af45170SJohn Baldwin } 54516af45170SJohn Baldwin 54526af45170SJohn Baldwin /* CPL header */ 54536af45170SJohn Baldwin cpl->ctrl0 = txq->cpl_ctrl0; 54546af45170SJohn Baldwin cpl->pack = 0; 54556af45170SJohn Baldwin cpl->len = htobe16(pktlen); 54566af45170SJohn Baldwin cpl->ctrl1 = htobe64(ctrl1); 54576af45170SJohn Baldwin 54586af45170SJohn Baldwin /* SGL */ 54596af45170SJohn Baldwin dst = (void *)(cpl + 1); 54606af45170SJohn Baldwin 54616af45170SJohn Baldwin /* 54626af45170SJohn Baldwin * A packet using TSO will use up an entire descriptor for the 54636af45170SJohn Baldwin * firmware work request header, LSO CPL, and TX_PKT_XT CPL. 54646af45170SJohn Baldwin * If this descriptor is the last descriptor in the ring, wrap 54656af45170SJohn Baldwin * around to the front of the ring explicitly for the start of 54666af45170SJohn Baldwin * the sgl. 54676af45170SJohn Baldwin */ 54686af45170SJohn Baldwin if (dst == (void *)&eq->desc[eq->sidx]) { 54696af45170SJohn Baldwin dst = (void *)&eq->desc[0]; 54706af45170SJohn Baldwin write_gl_to_txd(txq, m0, &dst, 0); 54716af45170SJohn Baldwin } else 54726af45170SJohn Baldwin write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 54736af45170SJohn Baldwin txq->sgl_wrs++; 54746af45170SJohn Baldwin txq->txpkt_wrs++; 54756af45170SJohn Baldwin 54766af45170SJohn Baldwin txsd = &txq->sdesc[eq->pidx]; 54776af45170SJohn Baldwin txsd->m = m0; 54786af45170SJohn Baldwin txsd->desc_used = ndesc; 54796af45170SJohn Baldwin 54806af45170SJohn Baldwin return (ndesc); 54816af45170SJohn Baldwin } 54826af45170SJohn Baldwin 54836af45170SJohn Baldwin /* 54845cdaef71SJohn Baldwin * Write a raw WR to the hardware descriptors, update the software 54855cdaef71SJohn Baldwin * descriptor, and advance the pidx. It is guaranteed that enough 54865cdaef71SJohn Baldwin * descriptors are available. 54875cdaef71SJohn Baldwin * 54885cdaef71SJohn Baldwin * The return value is the # of hardware descriptors used. 54895cdaef71SJohn Baldwin */ 54905cdaef71SJohn Baldwin static u_int 54915cdaef71SJohn Baldwin write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available) 54925cdaef71SJohn Baldwin { 54935cdaef71SJohn Baldwin struct sge_eq *eq = &txq->eq; 54945cdaef71SJohn Baldwin struct tx_sdesc *txsd; 54955cdaef71SJohn Baldwin struct mbuf *m; 54965cdaef71SJohn Baldwin caddr_t dst; 54975cdaef71SJohn Baldwin int len16, ndesc; 54985cdaef71SJohn Baldwin 54995cdaef71SJohn Baldwin len16 = mbuf_len16(m0); 55000cadedfcSNavdeep Parhar ndesc = tx_len16_to_desc(len16); 55015cdaef71SJohn Baldwin MPASS(ndesc <= available); 55025cdaef71SJohn Baldwin 55035cdaef71SJohn Baldwin dst = wr; 55045cdaef71SJohn Baldwin for (m = m0; m != NULL; m = m->m_next) 55055cdaef71SJohn Baldwin copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 55065cdaef71SJohn Baldwin 55075cdaef71SJohn Baldwin txq->raw_wrs++; 55085cdaef71SJohn Baldwin 55095cdaef71SJohn Baldwin txsd = &txq->sdesc[eq->pidx]; 55105cdaef71SJohn Baldwin txsd->m = m0; 55115cdaef71SJohn Baldwin txsd->desc_used = ndesc; 55125cdaef71SJohn Baldwin 55135cdaef71SJohn Baldwin return (ndesc); 55145cdaef71SJohn Baldwin } 55155cdaef71SJohn Baldwin 55165cdaef71SJohn Baldwin /* 55177951040fSNavdeep Parhar * Write a txpkt WR for this packet to the hardware descriptors, update the 55187951040fSNavdeep Parhar * software descriptor, and advance the pidx. It is guaranteed that enough 55197951040fSNavdeep Parhar * descriptors are available. 552054e4ee71SNavdeep Parhar * 55217951040fSNavdeep Parhar * The return value is the # of hardware descriptors used. 552254e4ee71SNavdeep Parhar */ 55237951040fSNavdeep Parhar static u_int 5524d735920dSNavdeep Parhar write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0, 5525d735920dSNavdeep Parhar u_int available) 552654e4ee71SNavdeep Parhar { 5527d735920dSNavdeep Parhar struct sge_eq *eq; 5528d735920dSNavdeep Parhar struct fw_eth_tx_pkt_wr *wr; 55297951040fSNavdeep Parhar struct tx_sdesc *txsd; 553054e4ee71SNavdeep Parhar struct cpl_tx_pkt_core *cpl; 553154e4ee71SNavdeep Parhar uint32_t ctrl; /* used in many unrelated places */ 553254e4ee71SNavdeep Parhar uint64_t ctrl1; 55337951040fSNavdeep Parhar int len16, ndesc, pktlen, nsegs; 553454e4ee71SNavdeep Parhar caddr_t dst; 553554e4ee71SNavdeep Parhar 553654e4ee71SNavdeep Parhar TXQ_LOCK_ASSERT_OWNED(txq); 55377951040fSNavdeep Parhar M_ASSERTPKTHDR(m0); 553854e4ee71SNavdeep Parhar 55397951040fSNavdeep Parhar len16 = mbuf_len16(m0); 55407951040fSNavdeep Parhar nsegs = mbuf_nsegs(m0); 55417951040fSNavdeep Parhar pktlen = m0->m_pkthdr.len; 554254e4ee71SNavdeep Parhar ctrl = sizeof(struct cpl_tx_pkt_core); 5543a4a4ad2dSNavdeep Parhar if (needs_tso(m0)) { 5544a4a4ad2dSNavdeep Parhar if (needs_vxlan_tso(m0)) 5545a4a4ad2dSNavdeep Parhar ctrl += sizeof(struct cpl_tx_tnl_lso); 5546a4a4ad2dSNavdeep Parhar else 55472a5f6b0eSNavdeep Parhar ctrl += sizeof(struct cpl_tx_pkt_lso_core); 5548a4a4ad2dSNavdeep Parhar } else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) && 5549d76bbe17SJohn Baldwin available >= 2) { 55507951040fSNavdeep Parhar /* Immediate data. Recalculate len16 and set nsegs to 0. */ 5551ecb79ca4SNavdeep Parhar ctrl += pktlen; 55527951040fSNavdeep Parhar len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + 55537951040fSNavdeep Parhar sizeof(struct cpl_tx_pkt_core) + pktlen, 16); 55547951040fSNavdeep Parhar nsegs = 0; 555554e4ee71SNavdeep Parhar } 55560cadedfcSNavdeep Parhar ndesc = tx_len16_to_desc(len16); 55577951040fSNavdeep Parhar MPASS(ndesc <= available); 555854e4ee71SNavdeep Parhar 555954e4ee71SNavdeep Parhar /* Firmware work request header */ 5560d735920dSNavdeep Parhar eq = &txq->eq; 5561d735920dSNavdeep Parhar wr = (void *)&eq->desc[eq->pidx]; 556254e4ee71SNavdeep Parhar wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 5563733b9277SNavdeep Parhar V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 55646b49a4ecSNavdeep Parhar 55657951040fSNavdeep Parhar ctrl = V_FW_WR_LEN16(len16); 556654e4ee71SNavdeep Parhar wr->equiq_to_len16 = htobe32(ctrl); 556754e4ee71SNavdeep Parhar wr->r3 = 0; 556854e4ee71SNavdeep Parhar 55697951040fSNavdeep Parhar if (needs_tso(m0)) { 5570a4a4ad2dSNavdeep Parhar if (needs_vxlan_tso(m0)) { 5571a4a4ad2dSNavdeep Parhar cpl = write_tnl_lso_cpl(wr + 1, m0); 5572a4a4ad2dSNavdeep Parhar txq->vxlan_tso_wrs++; 5573a4a4ad2dSNavdeep Parhar } else { 5574a4a4ad2dSNavdeep Parhar cpl = write_lso_cpl(wr + 1, m0); 557554e4ee71SNavdeep Parhar txq->tso_wrs++; 5576a4a4ad2dSNavdeep Parhar } 557754e4ee71SNavdeep Parhar } else 557854e4ee71SNavdeep Parhar cpl = (void *)(wr + 1); 557954e4ee71SNavdeep Parhar 558054e4ee71SNavdeep Parhar /* Checksum offload */ 5581c0236bd9SNavdeep Parhar ctrl1 = csum_to_ctrl(sc, m0); 5582a4a4ad2dSNavdeep Parhar if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) { 5583a4a4ad2dSNavdeep Parhar /* some hardware assistance provided */ 5584a4a4ad2dSNavdeep Parhar if (needs_vxlan_csum(m0)) 5585a4a4ad2dSNavdeep Parhar txq->vxlan_txcsum++; 5586a4a4ad2dSNavdeep Parhar else 5587a4a4ad2dSNavdeep Parhar txq->txcsum++; 5588a4a4ad2dSNavdeep Parhar } 558954e4ee71SNavdeep Parhar 559054e4ee71SNavdeep Parhar /* VLAN tag insertion */ 55917951040fSNavdeep Parhar if (needs_vlan_insertion(m0)) { 5592a4a4ad2dSNavdeep Parhar ctrl1 |= F_TXPKT_VLAN_VLD | 5593a4a4ad2dSNavdeep Parhar V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 559454e4ee71SNavdeep Parhar txq->vlan_insertion++; 559554e4ee71SNavdeep Parhar } 559654e4ee71SNavdeep Parhar 559754e4ee71SNavdeep Parhar /* CPL header */ 55987951040fSNavdeep Parhar cpl->ctrl0 = txq->cpl_ctrl0; 559954e4ee71SNavdeep Parhar cpl->pack = 0; 5600ecb79ca4SNavdeep Parhar cpl->len = htobe16(pktlen); 560154e4ee71SNavdeep Parhar cpl->ctrl1 = htobe64(ctrl1); 560254e4ee71SNavdeep Parhar 560354e4ee71SNavdeep Parhar /* SGL */ 560454e4ee71SNavdeep Parhar dst = (void *)(cpl + 1); 5605a4a4ad2dSNavdeep Parhar if (__predict_false((uintptr_t)dst == (uintptr_t)&eq->desc[eq->sidx])) 5606a4a4ad2dSNavdeep Parhar dst = (caddr_t)&eq->desc[0]; 56077951040fSNavdeep Parhar if (nsegs > 0) { 56087951040fSNavdeep Parhar 56097951040fSNavdeep Parhar write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 561054e4ee71SNavdeep Parhar txq->sgl_wrs++; 561154e4ee71SNavdeep Parhar } else { 56127951040fSNavdeep Parhar struct mbuf *m; 56137951040fSNavdeep Parhar 56147951040fSNavdeep Parhar for (m = m0; m != NULL; m = m->m_next) { 561554e4ee71SNavdeep Parhar copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 5616ecb79ca4SNavdeep Parhar #ifdef INVARIANTS 5617ecb79ca4SNavdeep Parhar pktlen -= m->m_len; 5618ecb79ca4SNavdeep Parhar #endif 561954e4ee71SNavdeep Parhar } 5620ecb79ca4SNavdeep Parhar #ifdef INVARIANTS 5621ecb79ca4SNavdeep Parhar KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 5622ecb79ca4SNavdeep Parhar #endif 56237951040fSNavdeep Parhar txq->imm_wrs++; 562454e4ee71SNavdeep Parhar } 562554e4ee71SNavdeep Parhar 562654e4ee71SNavdeep Parhar txq->txpkt_wrs++; 562754e4ee71SNavdeep Parhar 5628f7dfe243SNavdeep Parhar txsd = &txq->sdesc[eq->pidx]; 56297951040fSNavdeep Parhar txsd->m = m0; 563054e4ee71SNavdeep Parhar txsd->desc_used = ndesc; 563154e4ee71SNavdeep Parhar 56327951040fSNavdeep Parhar return (ndesc); 563354e4ee71SNavdeep Parhar } 563454e4ee71SNavdeep Parhar 5635d735920dSNavdeep Parhar static inline bool 5636d735920dSNavdeep Parhar cmp_l2hdr(struct txpkts *txp, struct mbuf *m) 563754e4ee71SNavdeep Parhar { 5638d735920dSNavdeep Parhar int len; 56397951040fSNavdeep Parhar 5640d735920dSNavdeep Parhar MPASS(txp->npkt > 0); 5641800535c2SNavdeep Parhar MPASS(m->m_len >= VM_TX_L2HDR_LEN); 56427951040fSNavdeep Parhar 5643d735920dSNavdeep Parhar if (txp->ethtype == be16toh(ETHERTYPE_VLAN)) 5644800535c2SNavdeep Parhar len = VM_TX_L2HDR_LEN; 5645d735920dSNavdeep Parhar else 5646d735920dSNavdeep Parhar len = sizeof(struct ether_header); 5647d735920dSNavdeep Parhar 5648d735920dSNavdeep Parhar return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0); 56497951040fSNavdeep Parhar } 56507951040fSNavdeep Parhar 5651d735920dSNavdeep Parhar static inline void 5652d735920dSNavdeep Parhar save_l2hdr(struct txpkts *txp, struct mbuf *m) 5653d735920dSNavdeep Parhar { 5654800535c2SNavdeep Parhar MPASS(m->m_len >= VM_TX_L2HDR_LEN); 56557951040fSNavdeep Parhar 5656800535c2SNavdeep Parhar memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN); 5657d735920dSNavdeep Parhar } 56587951040fSNavdeep Parhar 5659d735920dSNavdeep Parhar static int 5660d735920dSNavdeep Parhar add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5661d735920dSNavdeep Parhar int avail, bool *send) 5662d735920dSNavdeep Parhar { 5663d735920dSNavdeep Parhar struct txpkts *txp = &txq->txp; 5664d735920dSNavdeep Parhar 5665d735920dSNavdeep Parhar /* Cannot have TSO and coalesce at the same time. */ 5666d735920dSNavdeep Parhar if (cannot_use_txpkts(m)) { 5667d735920dSNavdeep Parhar cannot_coalesce: 5668d735920dSNavdeep Parhar *send = txp->npkt > 0; 5669d735920dSNavdeep Parhar return (EINVAL); 5670d735920dSNavdeep Parhar } 5671d735920dSNavdeep Parhar 5672d735920dSNavdeep Parhar /* VF allows coalescing of type 1 (1 GL) only */ 5673d735920dSNavdeep Parhar if (mbuf_nsegs(m) > 1) 5674d735920dSNavdeep Parhar goto cannot_coalesce; 5675d735920dSNavdeep Parhar 5676d735920dSNavdeep Parhar *send = false; 5677d735920dSNavdeep Parhar if (txp->npkt > 0) { 5678d735920dSNavdeep Parhar MPASS(tx_len16_to_desc(txp->len16) <= avail); 5679d735920dSNavdeep Parhar MPASS(txp->npkt < txp->max_npkt); 5680d735920dSNavdeep Parhar MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5681d735920dSNavdeep Parhar 5682d735920dSNavdeep Parhar if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) { 5683d735920dSNavdeep Parhar retry_after_send: 5684d735920dSNavdeep Parhar *send = true; 5685d735920dSNavdeep Parhar return (EAGAIN); 5686d735920dSNavdeep Parhar } 5687d735920dSNavdeep Parhar if (m->m_pkthdr.len + txp->plen > 65535) 5688d735920dSNavdeep Parhar goto retry_after_send; 5689d735920dSNavdeep Parhar if (cmp_l2hdr(txp, m)) 5690d735920dSNavdeep Parhar goto retry_after_send; 5691d735920dSNavdeep Parhar 5692d735920dSNavdeep Parhar txp->len16 += txpkts1_len16(); 5693d735920dSNavdeep Parhar txp->plen += m->m_pkthdr.len; 5694d735920dSNavdeep Parhar txp->mb[txp->npkt++] = m; 5695d735920dSNavdeep Parhar if (txp->npkt == txp->max_npkt) 5696d735920dSNavdeep Parhar *send = true; 5697d735920dSNavdeep Parhar } else { 5698d735920dSNavdeep Parhar txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) + 5699d735920dSNavdeep Parhar txpkts1_len16(); 5700d735920dSNavdeep Parhar if (tx_len16_to_desc(txp->len16) > avail) 5701d735920dSNavdeep Parhar goto cannot_coalesce; 5702d735920dSNavdeep Parhar txp->npkt = 1; 5703d735920dSNavdeep Parhar txp->wr_type = 1; 5704d735920dSNavdeep Parhar txp->plen = m->m_pkthdr.len; 5705d735920dSNavdeep Parhar txp->mb[0] = m; 5706d735920dSNavdeep Parhar save_l2hdr(txp, m); 5707d735920dSNavdeep Parhar } 57087951040fSNavdeep Parhar return (0); 57097951040fSNavdeep Parhar } 57107951040fSNavdeep Parhar 57117951040fSNavdeep Parhar static int 5712d735920dSNavdeep Parhar add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5713d735920dSNavdeep Parhar int avail, bool *send) 57147951040fSNavdeep Parhar { 5715d735920dSNavdeep Parhar struct txpkts *txp = &txq->txp; 5716d735920dSNavdeep Parhar int nsegs; 5717d735920dSNavdeep Parhar 5718d735920dSNavdeep Parhar MPASS(!(sc->flags & IS_VF)); 5719d735920dSNavdeep Parhar 5720d735920dSNavdeep Parhar /* Cannot have TSO and coalesce at the same time. */ 5721d735920dSNavdeep Parhar if (cannot_use_txpkts(m)) { 5722d735920dSNavdeep Parhar cannot_coalesce: 5723d735920dSNavdeep Parhar *send = txp->npkt > 0; 5724d735920dSNavdeep Parhar return (EINVAL); 5725d735920dSNavdeep Parhar } 5726d735920dSNavdeep Parhar 5727d735920dSNavdeep Parhar *send = false; 5728d735920dSNavdeep Parhar nsegs = mbuf_nsegs(m); 5729d735920dSNavdeep Parhar if (txp->npkt == 0) { 5730d735920dSNavdeep Parhar if (m->m_pkthdr.len > 65535) 5731d735920dSNavdeep Parhar goto cannot_coalesce; 5732d735920dSNavdeep Parhar if (nsegs > 1) { 5733d735920dSNavdeep Parhar txp->wr_type = 0; 5734d735920dSNavdeep Parhar txp->len16 = 5735d735920dSNavdeep Parhar howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5736d735920dSNavdeep Parhar txpkts0_len16(nsegs); 5737d735920dSNavdeep Parhar } else { 5738d735920dSNavdeep Parhar txp->wr_type = 1; 5739d735920dSNavdeep Parhar txp->len16 = 5740d735920dSNavdeep Parhar howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5741d735920dSNavdeep Parhar txpkts1_len16(); 5742d735920dSNavdeep Parhar } 5743d735920dSNavdeep Parhar if (tx_len16_to_desc(txp->len16) > avail) 5744d735920dSNavdeep Parhar goto cannot_coalesce; 5745d735920dSNavdeep Parhar txp->npkt = 1; 5746d735920dSNavdeep Parhar txp->plen = m->m_pkthdr.len; 5747d735920dSNavdeep Parhar txp->mb[0] = m; 5748d735920dSNavdeep Parhar } else { 5749d735920dSNavdeep Parhar MPASS(tx_len16_to_desc(txp->len16) <= avail); 5750d735920dSNavdeep Parhar MPASS(txp->npkt < txp->max_npkt); 5751d735920dSNavdeep Parhar 5752d735920dSNavdeep Parhar if (m->m_pkthdr.len + txp->plen > 65535) { 5753d735920dSNavdeep Parhar retry_after_send: 5754d735920dSNavdeep Parhar *send = true; 5755d735920dSNavdeep Parhar return (EAGAIN); 5756d735920dSNavdeep Parhar } 57577951040fSNavdeep Parhar 57587951040fSNavdeep Parhar MPASS(txp->wr_type == 0 || txp->wr_type == 1); 5759d735920dSNavdeep Parhar if (txp->wr_type == 0) { 5760d735920dSNavdeep Parhar if (tx_len16_to_desc(txp->len16 + 5761d735920dSNavdeep Parhar txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC)) 5762d735920dSNavdeep Parhar goto retry_after_send; 5763d735920dSNavdeep Parhar txp->len16 += txpkts0_len16(nsegs); 5764d735920dSNavdeep Parhar } else { 5765d735920dSNavdeep Parhar if (nsegs != 1) 5766d735920dSNavdeep Parhar goto retry_after_send; 5767d735920dSNavdeep Parhar if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > 5768d735920dSNavdeep Parhar avail) 5769d735920dSNavdeep Parhar goto retry_after_send; 5770d735920dSNavdeep Parhar txp->len16 += txpkts1_len16(); 5771d735920dSNavdeep Parhar } 57727951040fSNavdeep Parhar 5773d735920dSNavdeep Parhar txp->plen += m->m_pkthdr.len; 5774d735920dSNavdeep Parhar txp->mb[txp->npkt++] = m; 5775d735920dSNavdeep Parhar if (txp->npkt == txp->max_npkt) 5776d735920dSNavdeep Parhar *send = true; 5777d735920dSNavdeep Parhar } 57787951040fSNavdeep Parhar return (0); 57797951040fSNavdeep Parhar } 57807951040fSNavdeep Parhar 57817951040fSNavdeep Parhar /* 57827951040fSNavdeep Parhar * Write a txpkts WR for the packets in txp to the hardware descriptors, update 57837951040fSNavdeep Parhar * the software descriptor, and advance the pidx. It is guaranteed that enough 57847951040fSNavdeep Parhar * descriptors are available. 57857951040fSNavdeep Parhar * 57867951040fSNavdeep Parhar * The return value is the # of hardware descriptors used. 57877951040fSNavdeep Parhar */ 57887951040fSNavdeep Parhar static u_int 5789d735920dSNavdeep Parhar write_txpkts_wr(struct adapter *sc, struct sge_txq *txq) 57907951040fSNavdeep Parhar { 5791d735920dSNavdeep Parhar const struct txpkts *txp = &txq->txp; 57927951040fSNavdeep Parhar struct sge_eq *eq = &txq->eq; 5793d735920dSNavdeep Parhar struct fw_eth_tx_pkts_wr *wr; 57947951040fSNavdeep Parhar struct tx_sdesc *txsd; 57957951040fSNavdeep Parhar struct cpl_tx_pkt_core *cpl; 57967951040fSNavdeep Parhar uint64_t ctrl1; 5797d735920dSNavdeep Parhar int ndesc, i, checkwrap; 5798d735920dSNavdeep Parhar struct mbuf *m, *last; 57997951040fSNavdeep Parhar void *flitp; 58007951040fSNavdeep Parhar 58017951040fSNavdeep Parhar TXQ_LOCK_ASSERT_OWNED(txq); 58027951040fSNavdeep Parhar MPASS(txp->npkt > 0); 58037951040fSNavdeep Parhar MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 58047951040fSNavdeep Parhar 5805d735920dSNavdeep Parhar wr = (void *)&eq->desc[eq->pidx]; 58067951040fSNavdeep Parhar wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 5807d735920dSNavdeep Parhar wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 58087951040fSNavdeep Parhar wr->plen = htobe16(txp->plen); 58097951040fSNavdeep Parhar wr->npkt = txp->npkt; 58107951040fSNavdeep Parhar wr->r3 = 0; 58117951040fSNavdeep Parhar wr->type = txp->wr_type; 58127951040fSNavdeep Parhar flitp = wr + 1; 58137951040fSNavdeep Parhar 58147951040fSNavdeep Parhar /* 58157951040fSNavdeep Parhar * At this point we are 16B into a hardware descriptor. If checkwrap is 58167951040fSNavdeep Parhar * set then we know the WR is going to wrap around somewhere. We'll 58177951040fSNavdeep Parhar * check for that at appropriate points. 58187951040fSNavdeep Parhar */ 5819d735920dSNavdeep Parhar ndesc = tx_len16_to_desc(txp->len16); 5820d735920dSNavdeep Parhar last = NULL; 58217951040fSNavdeep Parhar checkwrap = eq->sidx - ndesc < eq->pidx; 5822d735920dSNavdeep Parhar for (i = 0; i < txp->npkt; i++) { 5823d735920dSNavdeep Parhar m = txp->mb[i]; 58247951040fSNavdeep Parhar if (txp->wr_type == 0) { 582554e4ee71SNavdeep Parhar struct ulp_txpkt *ulpmc; 582654e4ee71SNavdeep Parhar struct ulptx_idata *ulpsc; 582754e4ee71SNavdeep Parhar 58287951040fSNavdeep Parhar /* ULP master command */ 58297951040fSNavdeep Parhar ulpmc = flitp; 58307951040fSNavdeep Parhar ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 58317951040fSNavdeep Parhar V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); 5832d735920dSNavdeep Parhar ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m))); 583354e4ee71SNavdeep Parhar 58347951040fSNavdeep Parhar /* ULP subcommand */ 58357951040fSNavdeep Parhar ulpsc = (void *)(ulpmc + 1); 58367951040fSNavdeep Parhar ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 58377951040fSNavdeep Parhar F_ULP_TX_SC_MORE); 58387951040fSNavdeep Parhar ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); 58397951040fSNavdeep Parhar 58407951040fSNavdeep Parhar cpl = (void *)(ulpsc + 1); 58417951040fSNavdeep Parhar if (checkwrap && 58427951040fSNavdeep Parhar (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) 58437951040fSNavdeep Parhar cpl = (void *)&eq->desc[0]; 58447951040fSNavdeep Parhar } else { 58457951040fSNavdeep Parhar cpl = flitp; 58467951040fSNavdeep Parhar } 584754e4ee71SNavdeep Parhar 584854e4ee71SNavdeep Parhar /* Checksum offload */ 5849c0236bd9SNavdeep Parhar ctrl1 = csum_to_ctrl(sc, m); 5850a4a4ad2dSNavdeep Parhar if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) { 5851a4a4ad2dSNavdeep Parhar /* some hardware assistance provided */ 5852a4a4ad2dSNavdeep Parhar if (needs_vxlan_csum(m)) 5853a4a4ad2dSNavdeep Parhar txq->vxlan_txcsum++; 5854a4a4ad2dSNavdeep Parhar else 5855a4a4ad2dSNavdeep Parhar txq->txcsum++; 5856a4a4ad2dSNavdeep Parhar } 585754e4ee71SNavdeep Parhar 585854e4ee71SNavdeep Parhar /* VLAN tag insertion */ 58597951040fSNavdeep Parhar if (needs_vlan_insertion(m)) { 58607951040fSNavdeep Parhar ctrl1 |= F_TXPKT_VLAN_VLD | 58617951040fSNavdeep Parhar V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 586254e4ee71SNavdeep Parhar txq->vlan_insertion++; 586354e4ee71SNavdeep Parhar } 586454e4ee71SNavdeep Parhar 58657951040fSNavdeep Parhar /* CPL header */ 58667951040fSNavdeep Parhar cpl->ctrl0 = txq->cpl_ctrl0; 586754e4ee71SNavdeep Parhar cpl->pack = 0; 586854e4ee71SNavdeep Parhar cpl->len = htobe16(m->m_pkthdr.len); 58697951040fSNavdeep Parhar cpl->ctrl1 = htobe64(ctrl1); 587054e4ee71SNavdeep Parhar 58717951040fSNavdeep Parhar flitp = cpl + 1; 58727951040fSNavdeep Parhar if (checkwrap && 58737951040fSNavdeep Parhar (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 58747951040fSNavdeep Parhar flitp = (void *)&eq->desc[0]; 587554e4ee71SNavdeep Parhar 58767951040fSNavdeep Parhar write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); 587754e4ee71SNavdeep Parhar 5878d735920dSNavdeep Parhar if (last != NULL) 5879d735920dSNavdeep Parhar last->m_nextpkt = m; 5880d735920dSNavdeep Parhar last = m; 58817951040fSNavdeep Parhar } 58827951040fSNavdeep Parhar 5883d735920dSNavdeep Parhar txq->sgl_wrs++; 5884a59a1477SNavdeep Parhar if (txp->wr_type == 0) { 5885a59a1477SNavdeep Parhar txq->txpkts0_pkts += txp->npkt; 5886a59a1477SNavdeep Parhar txq->txpkts0_wrs++; 5887a59a1477SNavdeep Parhar } else { 5888a59a1477SNavdeep Parhar txq->txpkts1_pkts += txp->npkt; 5889a59a1477SNavdeep Parhar txq->txpkts1_wrs++; 5890a59a1477SNavdeep Parhar } 5891a59a1477SNavdeep Parhar 58927951040fSNavdeep Parhar txsd = &txq->sdesc[eq->pidx]; 5893d735920dSNavdeep Parhar txsd->m = txp->mb[0]; 5894d735920dSNavdeep Parhar txsd->desc_used = ndesc; 5895d735920dSNavdeep Parhar 5896d735920dSNavdeep Parhar return (ndesc); 5897d735920dSNavdeep Parhar } 5898d735920dSNavdeep Parhar 5899d735920dSNavdeep Parhar static u_int 5900d735920dSNavdeep Parhar write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq) 5901d735920dSNavdeep Parhar { 5902d735920dSNavdeep Parhar const struct txpkts *txp = &txq->txp; 5903d735920dSNavdeep Parhar struct sge_eq *eq = &txq->eq; 5904d735920dSNavdeep Parhar struct fw_eth_tx_pkts_vm_wr *wr; 5905d735920dSNavdeep Parhar struct tx_sdesc *txsd; 5906d735920dSNavdeep Parhar struct cpl_tx_pkt_core *cpl; 5907d735920dSNavdeep Parhar uint64_t ctrl1; 5908d735920dSNavdeep Parhar int ndesc, i; 5909d735920dSNavdeep Parhar struct mbuf *m, *last; 5910d735920dSNavdeep Parhar void *flitp; 5911d735920dSNavdeep Parhar 5912d735920dSNavdeep Parhar TXQ_LOCK_ASSERT_OWNED(txq); 5913d735920dSNavdeep Parhar MPASS(txp->npkt > 0); 5914d735920dSNavdeep Parhar MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5915d735920dSNavdeep Parhar MPASS(txp->mb[0] != NULL); 5916d735920dSNavdeep Parhar MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 5917d735920dSNavdeep Parhar 5918d735920dSNavdeep Parhar wr = (void *)&eq->desc[eq->pidx]; 5919d735920dSNavdeep Parhar wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR)); 5920d735920dSNavdeep Parhar wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 5921d735920dSNavdeep Parhar wr->r3 = 0; 5922d735920dSNavdeep Parhar wr->plen = htobe16(txp->plen); 5923d735920dSNavdeep Parhar wr->npkt = txp->npkt; 5924d735920dSNavdeep Parhar wr->r4 = 0; 5925d735920dSNavdeep Parhar memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16); 5926d735920dSNavdeep Parhar flitp = wr + 1; 5927d735920dSNavdeep Parhar 5928d735920dSNavdeep Parhar /* 5929d735920dSNavdeep Parhar * At this point we are 32B into a hardware descriptor. Each mbuf in 5930d735920dSNavdeep Parhar * the WR will take 32B so we check for the end of the descriptor ring 5931d735920dSNavdeep Parhar * before writing odd mbufs (mb[1], 3, 5, ..) 5932d735920dSNavdeep Parhar */ 5933d735920dSNavdeep Parhar ndesc = tx_len16_to_desc(txp->len16); 5934d735920dSNavdeep Parhar last = NULL; 5935d735920dSNavdeep Parhar for (i = 0; i < txp->npkt; i++) { 5936d735920dSNavdeep Parhar m = txp->mb[i]; 5937d735920dSNavdeep Parhar if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 5938d735920dSNavdeep Parhar flitp = &eq->desc[0]; 5939d735920dSNavdeep Parhar cpl = flitp; 5940d735920dSNavdeep Parhar 5941d735920dSNavdeep Parhar /* Checksum offload */ 5942d735920dSNavdeep Parhar ctrl1 = csum_to_ctrl(sc, m); 5943d735920dSNavdeep Parhar if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 5944d735920dSNavdeep Parhar txq->txcsum++; /* some hardware assistance provided */ 5945d735920dSNavdeep Parhar 5946d735920dSNavdeep Parhar /* VLAN tag insertion */ 5947d735920dSNavdeep Parhar if (needs_vlan_insertion(m)) { 5948d735920dSNavdeep Parhar ctrl1 |= F_TXPKT_VLAN_VLD | 5949d735920dSNavdeep Parhar V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 5950d735920dSNavdeep Parhar txq->vlan_insertion++; 5951d735920dSNavdeep Parhar } 5952d735920dSNavdeep Parhar 5953d735920dSNavdeep Parhar /* CPL header */ 5954d735920dSNavdeep Parhar cpl->ctrl0 = txq->cpl_ctrl0; 5955d735920dSNavdeep Parhar cpl->pack = 0; 5956d735920dSNavdeep Parhar cpl->len = htobe16(m->m_pkthdr.len); 5957d735920dSNavdeep Parhar cpl->ctrl1 = htobe64(ctrl1); 5958d735920dSNavdeep Parhar 5959d735920dSNavdeep Parhar flitp = cpl + 1; 5960d735920dSNavdeep Parhar MPASS(mbuf_nsegs(m) == 1); 5961d735920dSNavdeep Parhar write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0); 5962d735920dSNavdeep Parhar 5963d735920dSNavdeep Parhar if (last != NULL) 5964d735920dSNavdeep Parhar last->m_nextpkt = m; 5965d735920dSNavdeep Parhar last = m; 5966d735920dSNavdeep Parhar } 5967d735920dSNavdeep Parhar 5968d735920dSNavdeep Parhar txq->sgl_wrs++; 5969d735920dSNavdeep Parhar txq->txpkts1_pkts += txp->npkt; 5970d735920dSNavdeep Parhar txq->txpkts1_wrs++; 5971d735920dSNavdeep Parhar 5972d735920dSNavdeep Parhar txsd = &txq->sdesc[eq->pidx]; 5973d735920dSNavdeep Parhar txsd->m = txp->mb[0]; 59747951040fSNavdeep Parhar txsd->desc_used = ndesc; 59757951040fSNavdeep Parhar 59767951040fSNavdeep Parhar return (ndesc); 597754e4ee71SNavdeep Parhar } 597854e4ee71SNavdeep Parhar 597954e4ee71SNavdeep Parhar /* 598054e4ee71SNavdeep Parhar * If the SGL ends on an address that is not 16 byte aligned, this function will 59817951040fSNavdeep Parhar * add a 0 filled flit at the end. 598254e4ee71SNavdeep Parhar */ 59837951040fSNavdeep Parhar static void 59847951040fSNavdeep Parhar write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) 598554e4ee71SNavdeep Parhar { 59867951040fSNavdeep Parhar struct sge_eq *eq = &txq->eq; 59877951040fSNavdeep Parhar struct sglist *gl = txq->gl; 59887951040fSNavdeep Parhar struct sglist_seg *seg; 59897951040fSNavdeep Parhar __be64 *flitp, *wrap; 599054e4ee71SNavdeep Parhar struct ulptx_sgl *usgl; 59917951040fSNavdeep Parhar int i, nflits, nsegs; 599254e4ee71SNavdeep Parhar 599354e4ee71SNavdeep Parhar KASSERT(((uintptr_t)(*to) & 0xf) == 0, 599454e4ee71SNavdeep Parhar ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); 59957951040fSNavdeep Parhar MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 59967951040fSNavdeep Parhar MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 599754e4ee71SNavdeep Parhar 59987951040fSNavdeep Parhar get_pkt_gl(m, gl); 59997951040fSNavdeep Parhar nsegs = gl->sg_nseg; 60007951040fSNavdeep Parhar MPASS(nsegs > 0); 60017951040fSNavdeep Parhar 60027951040fSNavdeep Parhar nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; 600354e4ee71SNavdeep Parhar flitp = (__be64 *)(*to); 60047951040fSNavdeep Parhar wrap = (__be64 *)(&eq->desc[eq->sidx]); 60057951040fSNavdeep Parhar seg = &gl->sg_segs[0]; 600654e4ee71SNavdeep Parhar usgl = (void *)flitp; 600754e4ee71SNavdeep Parhar 600854e4ee71SNavdeep Parhar /* 600954e4ee71SNavdeep Parhar * We start at a 16 byte boundary somewhere inside the tx descriptor 601054e4ee71SNavdeep Parhar * ring, so we're at least 16 bytes away from the status page. There is 601154e4ee71SNavdeep Parhar * no chance of a wrap around in the middle of usgl (which is 16 bytes). 601254e4ee71SNavdeep Parhar */ 601354e4ee71SNavdeep Parhar 601454e4ee71SNavdeep Parhar usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 60157951040fSNavdeep Parhar V_ULPTX_NSGE(nsegs)); 60167951040fSNavdeep Parhar usgl->len0 = htobe32(seg->ss_len); 60177951040fSNavdeep Parhar usgl->addr0 = htobe64(seg->ss_paddr); 601854e4ee71SNavdeep Parhar seg++; 601954e4ee71SNavdeep Parhar 60207951040fSNavdeep Parhar if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { 602154e4ee71SNavdeep Parhar 602254e4ee71SNavdeep Parhar /* Won't wrap around at all */ 602354e4ee71SNavdeep Parhar 60247951040fSNavdeep Parhar for (i = 0; i < nsegs - 1; i++, seg++) { 60257951040fSNavdeep Parhar usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 60267951040fSNavdeep Parhar usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); 602754e4ee71SNavdeep Parhar } 602854e4ee71SNavdeep Parhar if (i & 1) 602954e4ee71SNavdeep Parhar usgl->sge[i / 2].len[1] = htobe32(0); 60307951040fSNavdeep Parhar flitp += nflits; 603154e4ee71SNavdeep Parhar } else { 603254e4ee71SNavdeep Parhar 603354e4ee71SNavdeep Parhar /* Will wrap somewhere in the rest of the SGL */ 603454e4ee71SNavdeep Parhar 603554e4ee71SNavdeep Parhar /* 2 flits already written, write the rest flit by flit */ 603654e4ee71SNavdeep Parhar flitp = (void *)(usgl + 1); 60377951040fSNavdeep Parhar for (i = 0; i < nflits - 2; i++) { 60387951040fSNavdeep Parhar if (flitp == wrap) 603954e4ee71SNavdeep Parhar flitp = (void *)eq->desc; 60407951040fSNavdeep Parhar *flitp++ = get_flit(seg, nsegs - 1, i); 604154e4ee71SNavdeep Parhar } 604254e4ee71SNavdeep Parhar } 604354e4ee71SNavdeep Parhar 60447951040fSNavdeep Parhar if (nflits & 1) { 60457951040fSNavdeep Parhar MPASS(((uintptr_t)flitp) & 0xf); 60467951040fSNavdeep Parhar *flitp++ = 0; 60477951040fSNavdeep Parhar } 604854e4ee71SNavdeep Parhar 60497951040fSNavdeep Parhar MPASS((((uintptr_t)flitp) & 0xf) == 0); 60507951040fSNavdeep Parhar if (__predict_false(flitp == wrap)) 605154e4ee71SNavdeep Parhar *to = (void *)eq->desc; 605254e4ee71SNavdeep Parhar else 60537951040fSNavdeep Parhar *to = (void *)flitp; 605454e4ee71SNavdeep Parhar } 605554e4ee71SNavdeep Parhar 605654e4ee71SNavdeep Parhar static inline void 605754e4ee71SNavdeep Parhar copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 605854e4ee71SNavdeep Parhar { 60597951040fSNavdeep Parhar 60607951040fSNavdeep Parhar MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 60617951040fSNavdeep Parhar MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 60627951040fSNavdeep Parhar 60637951040fSNavdeep Parhar if (__predict_true((uintptr_t)(*to) + len <= 60647951040fSNavdeep Parhar (uintptr_t)&eq->desc[eq->sidx])) { 606554e4ee71SNavdeep Parhar bcopy(from, *to, len); 606654e4ee71SNavdeep Parhar (*to) += len; 606754e4ee71SNavdeep Parhar } else { 60687951040fSNavdeep Parhar int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); 606954e4ee71SNavdeep Parhar 607054e4ee71SNavdeep Parhar bcopy(from, *to, portion); 607154e4ee71SNavdeep Parhar from += portion; 607254e4ee71SNavdeep Parhar portion = len - portion; /* remaining */ 607354e4ee71SNavdeep Parhar bcopy(from, (void *)eq->desc, portion); 607454e4ee71SNavdeep Parhar (*to) = (caddr_t)eq->desc + portion; 607554e4ee71SNavdeep Parhar } 607654e4ee71SNavdeep Parhar } 607754e4ee71SNavdeep Parhar 607854e4ee71SNavdeep Parhar static inline void 60797951040fSNavdeep Parhar ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) 608054e4ee71SNavdeep Parhar { 60817951040fSNavdeep Parhar u_int db; 60827951040fSNavdeep Parhar 60837951040fSNavdeep Parhar MPASS(n > 0); 6084d14b0ac1SNavdeep Parhar 6085d14b0ac1SNavdeep Parhar db = eq->doorbells; 60867951040fSNavdeep Parhar if (n > 1) 608777ad3c41SNavdeep Parhar clrbit(&db, DOORBELL_WCWR); 6088d14b0ac1SNavdeep Parhar wmb(); 6089d14b0ac1SNavdeep Parhar 6090d14b0ac1SNavdeep Parhar switch (ffs(db) - 1) { 6091d14b0ac1SNavdeep Parhar case DOORBELL_UDB: 60927951040fSNavdeep Parhar *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 60937951040fSNavdeep Parhar break; 6094d14b0ac1SNavdeep Parhar 609577ad3c41SNavdeep Parhar case DOORBELL_WCWR: { 6096d14b0ac1SNavdeep Parhar volatile uint64_t *dst, *src; 6097d14b0ac1SNavdeep Parhar int i; 6098d14b0ac1SNavdeep Parhar 6099d14b0ac1SNavdeep Parhar /* 6100d14b0ac1SNavdeep Parhar * Queues whose 128B doorbell segment fits in the page do not 6101d14b0ac1SNavdeep Parhar * use relative qid (udb_qid is always 0). Only queues with 610277ad3c41SNavdeep Parhar * doorbell segments can do WCWR. 6103d14b0ac1SNavdeep Parhar */ 61047951040fSNavdeep Parhar KASSERT(eq->udb_qid == 0 && n == 1, 6105d14b0ac1SNavdeep Parhar ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", 61067951040fSNavdeep Parhar __func__, eq->doorbells, n, eq->dbidx, eq)); 6107d14b0ac1SNavdeep Parhar 6108d14b0ac1SNavdeep Parhar dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 6109d14b0ac1SNavdeep Parhar UDBS_DB_OFFSET); 61107951040fSNavdeep Parhar i = eq->dbidx; 6111d14b0ac1SNavdeep Parhar src = (void *)&eq->desc[i]; 6112d14b0ac1SNavdeep Parhar while (src != (void *)&eq->desc[i + 1]) 6113d14b0ac1SNavdeep Parhar *dst++ = *src++; 6114d14b0ac1SNavdeep Parhar wmb(); 61157951040fSNavdeep Parhar break; 6116d14b0ac1SNavdeep Parhar } 6117d14b0ac1SNavdeep Parhar 6118d14b0ac1SNavdeep Parhar case DOORBELL_UDBWC: 61197951040fSNavdeep Parhar *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 6120d14b0ac1SNavdeep Parhar wmb(); 61217951040fSNavdeep Parhar break; 6122d14b0ac1SNavdeep Parhar 6123d14b0ac1SNavdeep Parhar case DOORBELL_KDB: 6124315048f2SJohn Baldwin t4_write_reg(sc, sc->sge_kdoorbell_reg, 61257951040fSNavdeep Parhar V_QID(eq->cntxt_id) | V_PIDX(n)); 61267951040fSNavdeep Parhar break; 612754e4ee71SNavdeep Parhar } 612854e4ee71SNavdeep Parhar 61297951040fSNavdeep Parhar IDXINCR(eq->dbidx, n, eq->sidx); 61307951040fSNavdeep Parhar } 61317951040fSNavdeep Parhar 61327951040fSNavdeep Parhar static inline u_int 61337951040fSNavdeep Parhar reclaimable_tx_desc(struct sge_eq *eq) 613454e4ee71SNavdeep Parhar { 61357951040fSNavdeep Parhar uint16_t hw_cidx; 613654e4ee71SNavdeep Parhar 61377951040fSNavdeep Parhar hw_cidx = read_hw_cidx(eq); 61387951040fSNavdeep Parhar return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); 61397951040fSNavdeep Parhar } 614054e4ee71SNavdeep Parhar 61417951040fSNavdeep Parhar static inline u_int 61427951040fSNavdeep Parhar total_available_tx_desc(struct sge_eq *eq) 61437951040fSNavdeep Parhar { 61447951040fSNavdeep Parhar uint16_t hw_cidx, pidx; 61457951040fSNavdeep Parhar 61467951040fSNavdeep Parhar hw_cidx = read_hw_cidx(eq); 61477951040fSNavdeep Parhar pidx = eq->pidx; 61487951040fSNavdeep Parhar 61497951040fSNavdeep Parhar if (pidx == hw_cidx) 61507951040fSNavdeep Parhar return (eq->sidx - 1); 615154e4ee71SNavdeep Parhar else 61527951040fSNavdeep Parhar return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); 61537951040fSNavdeep Parhar } 61547951040fSNavdeep Parhar 61557951040fSNavdeep Parhar static inline uint16_t 61567951040fSNavdeep Parhar read_hw_cidx(struct sge_eq *eq) 61577951040fSNavdeep Parhar { 61587951040fSNavdeep Parhar struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 61597951040fSNavdeep Parhar uint16_t cidx = spg->cidx; /* stable snapshot */ 61607951040fSNavdeep Parhar 61617951040fSNavdeep Parhar return (be16toh(cidx)); 6162e874ff7aSNavdeep Parhar } 616354e4ee71SNavdeep Parhar 6164e874ff7aSNavdeep Parhar /* 61657951040fSNavdeep Parhar * Reclaim 'n' descriptors approximately. 6166e874ff7aSNavdeep Parhar */ 61677951040fSNavdeep Parhar static u_int 61687951040fSNavdeep Parhar reclaim_tx_descs(struct sge_txq *txq, u_int n) 6169e874ff7aSNavdeep Parhar { 6170e874ff7aSNavdeep Parhar struct tx_sdesc *txsd; 6171f7dfe243SNavdeep Parhar struct sge_eq *eq = &txq->eq; 61727951040fSNavdeep Parhar u_int can_reclaim, reclaimed; 617354e4ee71SNavdeep Parhar 6174733b9277SNavdeep Parhar TXQ_LOCK_ASSERT_OWNED(txq); 61757951040fSNavdeep Parhar MPASS(n > 0); 6176e874ff7aSNavdeep Parhar 61777951040fSNavdeep Parhar reclaimed = 0; 61787951040fSNavdeep Parhar can_reclaim = reclaimable_tx_desc(eq); 61797951040fSNavdeep Parhar while (can_reclaim && reclaimed < n) { 618054e4ee71SNavdeep Parhar int ndesc; 61817951040fSNavdeep Parhar struct mbuf *m, *nextpkt; 618254e4ee71SNavdeep Parhar 6183f7dfe243SNavdeep Parhar txsd = &txq->sdesc[eq->cidx]; 618454e4ee71SNavdeep Parhar ndesc = txsd->desc_used; 618554e4ee71SNavdeep Parhar 618654e4ee71SNavdeep Parhar /* Firmware doesn't return "partial" credits. */ 618754e4ee71SNavdeep Parhar KASSERT(can_reclaim >= ndesc, 618854e4ee71SNavdeep Parhar ("%s: unexpected number of credits: %d, %d", 618954e4ee71SNavdeep Parhar __func__, can_reclaim, ndesc)); 6190dcd50a20SJohn Baldwin KASSERT(ndesc != 0, 6191dcd50a20SJohn Baldwin ("%s: descriptor with no credits: cidx %d", 6192dcd50a20SJohn Baldwin __func__, eq->cidx)); 619354e4ee71SNavdeep Parhar 61947951040fSNavdeep Parhar for (m = txsd->m; m != NULL; m = nextpkt) { 61957951040fSNavdeep Parhar nextpkt = m->m_nextpkt; 61967951040fSNavdeep Parhar m->m_nextpkt = NULL; 61977951040fSNavdeep Parhar m_freem(m); 61987951040fSNavdeep Parhar } 619954e4ee71SNavdeep Parhar reclaimed += ndesc; 620054e4ee71SNavdeep Parhar can_reclaim -= ndesc; 62017951040fSNavdeep Parhar IDXINCR(eq->cidx, ndesc, eq->sidx); 620254e4ee71SNavdeep Parhar } 620354e4ee71SNavdeep Parhar 620454e4ee71SNavdeep Parhar return (reclaimed); 620554e4ee71SNavdeep Parhar } 620654e4ee71SNavdeep Parhar 620754e4ee71SNavdeep Parhar static void 62087951040fSNavdeep Parhar tx_reclaim(void *arg, int n) 620954e4ee71SNavdeep Parhar { 62107951040fSNavdeep Parhar struct sge_txq *txq = arg; 62117951040fSNavdeep Parhar struct sge_eq *eq = &txq->eq; 621254e4ee71SNavdeep Parhar 62137951040fSNavdeep Parhar do { 62147951040fSNavdeep Parhar if (TXQ_TRYLOCK(txq) == 0) 62157951040fSNavdeep Parhar break; 62167951040fSNavdeep Parhar n = reclaim_tx_descs(txq, 32); 62177951040fSNavdeep Parhar if (eq->cidx == eq->pidx) 62187951040fSNavdeep Parhar eq->equeqidx = eq->pidx; 62197951040fSNavdeep Parhar TXQ_UNLOCK(txq); 62207951040fSNavdeep Parhar } while (n > 0); 622154e4ee71SNavdeep Parhar } 622254e4ee71SNavdeep Parhar 622354e4ee71SNavdeep Parhar static __be64 62247951040fSNavdeep Parhar get_flit(struct sglist_seg *segs, int nsegs, int idx) 622554e4ee71SNavdeep Parhar { 622654e4ee71SNavdeep Parhar int i = (idx / 3) * 2; 622754e4ee71SNavdeep Parhar 622854e4ee71SNavdeep Parhar switch (idx % 3) { 622954e4ee71SNavdeep Parhar case 0: { 6230f078ecf6SWojciech Macek uint64_t rc; 623154e4ee71SNavdeep Parhar 6232f078ecf6SWojciech Macek rc = (uint64_t)segs[i].ss_len << 32; 623354e4ee71SNavdeep Parhar if (i + 1 < nsegs) 6234f078ecf6SWojciech Macek rc |= (uint64_t)(segs[i + 1].ss_len); 623554e4ee71SNavdeep Parhar 6236f078ecf6SWojciech Macek return (htobe64(rc)); 623754e4ee71SNavdeep Parhar } 623854e4ee71SNavdeep Parhar case 1: 62397951040fSNavdeep Parhar return (htobe64(segs[i].ss_paddr)); 624054e4ee71SNavdeep Parhar case 2: 62417951040fSNavdeep Parhar return (htobe64(segs[i + 1].ss_paddr)); 624254e4ee71SNavdeep Parhar } 624354e4ee71SNavdeep Parhar 624454e4ee71SNavdeep Parhar return (0); 624554e4ee71SNavdeep Parhar } 624654e4ee71SNavdeep Parhar 624746e1e307SNavdeep Parhar static int 624846e1e307SNavdeep Parhar find_refill_source(struct adapter *sc, int maxp, bool packing) 624954e4ee71SNavdeep Parhar { 625046e1e307SNavdeep Parhar int i, zidx = -1; 625146e1e307SNavdeep Parhar struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 625254e4ee71SNavdeep Parhar 625346e1e307SNavdeep Parhar if (packing) { 625446e1e307SNavdeep Parhar for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 625546e1e307SNavdeep Parhar if (rxb->hwidx2 == -1) 625646e1e307SNavdeep Parhar continue; 625746e1e307SNavdeep Parhar if (rxb->size1 < PAGE_SIZE && 625846e1e307SNavdeep Parhar rxb->size1 < largest_rx_cluster) 625946e1e307SNavdeep Parhar continue; 626046e1e307SNavdeep Parhar if (rxb->size1 > largest_rx_cluster) 626138035ed6SNavdeep Parhar break; 626246e1e307SNavdeep Parhar MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE); 626346e1e307SNavdeep Parhar if (rxb->size2 >= maxp) 626446e1e307SNavdeep Parhar return (i); 626546e1e307SNavdeep Parhar zidx = i; 626638035ed6SNavdeep Parhar } 626738035ed6SNavdeep Parhar } else { 626846e1e307SNavdeep Parhar for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 626946e1e307SNavdeep Parhar if (rxb->hwidx1 == -1) 627046e1e307SNavdeep Parhar continue; 627146e1e307SNavdeep Parhar if (rxb->size1 > largest_rx_cluster) 627238035ed6SNavdeep Parhar break; 627346e1e307SNavdeep Parhar if (rxb->size1 >= maxp) 627446e1e307SNavdeep Parhar return (i); 627546e1e307SNavdeep Parhar zidx = i; 627638035ed6SNavdeep Parhar } 627738035ed6SNavdeep Parhar } 627838035ed6SNavdeep Parhar 627946e1e307SNavdeep Parhar return (zidx); 628054e4ee71SNavdeep Parhar } 6281ecb79ca4SNavdeep Parhar 6282733b9277SNavdeep Parhar static void 6283733b9277SNavdeep Parhar add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 6284ecb79ca4SNavdeep Parhar { 6285733b9277SNavdeep Parhar mtx_lock(&sc->sfl_lock); 6286733b9277SNavdeep Parhar FL_LOCK(fl); 6287733b9277SNavdeep Parhar if ((fl->flags & FL_DOOMED) == 0) { 6288733b9277SNavdeep Parhar fl->flags |= FL_STARVING; 6289733b9277SNavdeep Parhar TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 6290733b9277SNavdeep Parhar callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 6291733b9277SNavdeep Parhar } 6292733b9277SNavdeep Parhar FL_UNLOCK(fl); 6293733b9277SNavdeep Parhar mtx_unlock(&sc->sfl_lock); 6294733b9277SNavdeep Parhar } 6295ecb79ca4SNavdeep Parhar 62967951040fSNavdeep Parhar static void 62977951040fSNavdeep Parhar handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) 62987951040fSNavdeep Parhar { 62997951040fSNavdeep Parhar struct sge_wrq *wrq = (void *)eq; 63007951040fSNavdeep Parhar 63017951040fSNavdeep Parhar atomic_readandclear_int(&eq->equiq); 63027951040fSNavdeep Parhar taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); 63037951040fSNavdeep Parhar } 63047951040fSNavdeep Parhar 63057951040fSNavdeep Parhar static void 63067951040fSNavdeep Parhar handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) 63077951040fSNavdeep Parhar { 63087951040fSNavdeep Parhar struct sge_txq *txq = (void *)eq; 63097951040fSNavdeep Parhar 631043bbae19SNavdeep Parhar MPASS(eq->type == EQ_ETH); 63117951040fSNavdeep Parhar 63127951040fSNavdeep Parhar atomic_readandclear_int(&eq->equiq); 6313d735920dSNavdeep Parhar if (mp_ring_is_idle(txq->r)) 63147951040fSNavdeep Parhar taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); 6315d735920dSNavdeep Parhar else 6316d735920dSNavdeep Parhar mp_ring_check_drainage(txq->r, 64); 63177951040fSNavdeep Parhar } 63187951040fSNavdeep Parhar 6319733b9277SNavdeep Parhar static int 6320733b9277SNavdeep Parhar handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 6321733b9277SNavdeep Parhar struct mbuf *m) 6322733b9277SNavdeep Parhar { 6323733b9277SNavdeep Parhar const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 6324733b9277SNavdeep Parhar unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 6325733b9277SNavdeep Parhar struct adapter *sc = iq->adapter; 6326733b9277SNavdeep Parhar struct sge *s = &sc->sge; 6327733b9277SNavdeep Parhar struct sge_eq *eq; 63287951040fSNavdeep Parhar static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, 63297951040fSNavdeep Parhar &handle_wrq_egr_update, &handle_eth_egr_update, 63307951040fSNavdeep Parhar &handle_wrq_egr_update}; 6331733b9277SNavdeep Parhar 6332733b9277SNavdeep Parhar KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6333733b9277SNavdeep Parhar rss->opcode)); 6334733b9277SNavdeep Parhar 6335ec55567cSJohn Baldwin eq = s->eqmap[qid - s->eq_start - s->eq_base]; 633643bbae19SNavdeep Parhar (*h[eq->type])(sc, eq); 6337ecb79ca4SNavdeep Parhar 6338ecb79ca4SNavdeep Parhar return (0); 6339ecb79ca4SNavdeep Parhar } 6340f7dfe243SNavdeep Parhar 63410abd31e2SNavdeep Parhar /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 63420abd31e2SNavdeep Parhar CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 63430abd31e2SNavdeep Parhar offsetof(struct cpl_fw6_msg, data)); 63440abd31e2SNavdeep Parhar 6345733b9277SNavdeep Parhar static int 63461b4cc91fSNavdeep Parhar handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 634756599263SNavdeep Parhar { 63481b4cc91fSNavdeep Parhar struct adapter *sc = iq->adapter; 634956599263SNavdeep Parhar const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 635056599263SNavdeep Parhar 6351733b9277SNavdeep Parhar KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6352733b9277SNavdeep Parhar rss->opcode)); 6353733b9277SNavdeep Parhar 63540abd31e2SNavdeep Parhar if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 63550abd31e2SNavdeep Parhar const struct rss_header *rss2; 63560abd31e2SNavdeep Parhar 63570abd31e2SNavdeep Parhar rss2 = (const struct rss_header *)&cpl->data[0]; 6358671bf2b8SNavdeep Parhar return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); 63590abd31e2SNavdeep Parhar } 63600abd31e2SNavdeep Parhar 6361671bf2b8SNavdeep Parhar return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); 6362f7dfe243SNavdeep Parhar } 6363af49c942SNavdeep Parhar 6364069af0ebSJohn Baldwin /** 6365069af0ebSJohn Baldwin * t4_handle_wrerr_rpl - process a FW work request error message 6366069af0ebSJohn Baldwin * @adap: the adapter 6367069af0ebSJohn Baldwin * @rpl: start of the FW message 6368069af0ebSJohn Baldwin */ 6369069af0ebSJohn Baldwin static int 6370069af0ebSJohn Baldwin t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) 6371069af0ebSJohn Baldwin { 6372069af0ebSJohn Baldwin u8 opcode = *(const u8 *)rpl; 6373069af0ebSJohn Baldwin const struct fw_error_cmd *e = (const void *)rpl; 6374069af0ebSJohn Baldwin unsigned int i; 6375069af0ebSJohn Baldwin 6376069af0ebSJohn Baldwin if (opcode != FW_ERROR_CMD) { 6377069af0ebSJohn Baldwin log(LOG_ERR, 6378069af0ebSJohn Baldwin "%s: Received WRERR_RPL message with opcode %#x\n", 6379069af0ebSJohn Baldwin device_get_nameunit(adap->dev), opcode); 6380069af0ebSJohn Baldwin return (EINVAL); 6381069af0ebSJohn Baldwin } 6382069af0ebSJohn Baldwin log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), 6383069af0ebSJohn Baldwin G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : 6384069af0ebSJohn Baldwin "non-fatal"); 6385069af0ebSJohn Baldwin switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { 6386069af0ebSJohn Baldwin case FW_ERROR_TYPE_EXCEPTION: 6387069af0ebSJohn Baldwin log(LOG_ERR, "exception info:\n"); 6388069af0ebSJohn Baldwin for (i = 0; i < nitems(e->u.exception.info); i++) 6389069af0ebSJohn Baldwin log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", 6390069af0ebSJohn Baldwin be32toh(e->u.exception.info[i])); 6391069af0ebSJohn Baldwin log(LOG_ERR, "\n"); 6392069af0ebSJohn Baldwin break; 6393069af0ebSJohn Baldwin case FW_ERROR_TYPE_HWMODULE: 6394069af0ebSJohn Baldwin log(LOG_ERR, "HW module regaddr %08x regval %08x\n", 6395069af0ebSJohn Baldwin be32toh(e->u.hwmodule.regaddr), 6396069af0ebSJohn Baldwin be32toh(e->u.hwmodule.regval)); 6397069af0ebSJohn Baldwin break; 6398069af0ebSJohn Baldwin case FW_ERROR_TYPE_WR: 6399069af0ebSJohn Baldwin log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", 6400069af0ebSJohn Baldwin be16toh(e->u.wr.cidx), 6401069af0ebSJohn Baldwin G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), 6402069af0ebSJohn Baldwin G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), 6403069af0ebSJohn Baldwin be32toh(e->u.wr.eqid)); 6404069af0ebSJohn Baldwin for (i = 0; i < nitems(e->u.wr.wrhdr); i++) 6405069af0ebSJohn Baldwin log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", 6406069af0ebSJohn Baldwin e->u.wr.wrhdr[i]); 6407069af0ebSJohn Baldwin log(LOG_ERR, "\n"); 6408069af0ebSJohn Baldwin break; 6409069af0ebSJohn Baldwin case FW_ERROR_TYPE_ACL: 6410069af0ebSJohn Baldwin log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", 6411069af0ebSJohn Baldwin be16toh(e->u.acl.cidx), 6412069af0ebSJohn Baldwin G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), 6413069af0ebSJohn Baldwin G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), 6414069af0ebSJohn Baldwin be32toh(e->u.acl.eqid), 6415069af0ebSJohn Baldwin G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : 6416069af0ebSJohn Baldwin "MAC"); 6417069af0ebSJohn Baldwin for (i = 0; i < nitems(e->u.acl.val); i++) 6418069af0ebSJohn Baldwin log(LOG_ERR, " %02x", e->u.acl.val[i]); 6419069af0ebSJohn Baldwin log(LOG_ERR, "\n"); 6420069af0ebSJohn Baldwin break; 6421069af0ebSJohn Baldwin default: 6422069af0ebSJohn Baldwin log(LOG_ERR, "type %#x\n", 6423069af0ebSJohn Baldwin G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); 6424069af0ebSJohn Baldwin return (EINVAL); 6425069af0ebSJohn Baldwin } 6426069af0ebSJohn Baldwin return (0); 6427069af0ebSJohn Baldwin } 6428069af0ebSJohn Baldwin 642946e1e307SNavdeep Parhar static inline bool 643046e1e307SNavdeep Parhar bufidx_used(struct adapter *sc, int idx) 643146e1e307SNavdeep Parhar { 643246e1e307SNavdeep Parhar struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 643346e1e307SNavdeep Parhar int i; 643446e1e307SNavdeep Parhar 643546e1e307SNavdeep Parhar for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 643646e1e307SNavdeep Parhar if (rxb->size1 > largest_rx_cluster) 643746e1e307SNavdeep Parhar continue; 643846e1e307SNavdeep Parhar if (rxb->hwidx1 == idx || rxb->hwidx2 == idx) 643946e1e307SNavdeep Parhar return (true); 644046e1e307SNavdeep Parhar } 644146e1e307SNavdeep Parhar 644246e1e307SNavdeep Parhar return (false); 644346e1e307SNavdeep Parhar } 644446e1e307SNavdeep Parhar 644538035ed6SNavdeep Parhar static int 644638035ed6SNavdeep Parhar sysctl_bufsizes(SYSCTL_HANDLER_ARGS) 644738035ed6SNavdeep Parhar { 644846e1e307SNavdeep Parhar struct adapter *sc = arg1; 644946e1e307SNavdeep Parhar struct sge_params *sp = &sc->params.sge; 645038035ed6SNavdeep Parhar int i, rc; 645138035ed6SNavdeep Parhar struct sbuf sb; 645238035ed6SNavdeep Parhar char c; 645338035ed6SNavdeep Parhar 645446e1e307SNavdeep Parhar sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND); 645546e1e307SNavdeep Parhar for (i = 0; i < SGE_FLBUF_SIZES; i++) { 645646e1e307SNavdeep Parhar if (bufidx_used(sc, i)) 645738035ed6SNavdeep Parhar c = '*'; 645838035ed6SNavdeep Parhar else 645938035ed6SNavdeep Parhar c = '\0'; 646038035ed6SNavdeep Parhar 646146e1e307SNavdeep Parhar sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c); 646238035ed6SNavdeep Parhar } 646338035ed6SNavdeep Parhar sbuf_trim(&sb); 646438035ed6SNavdeep Parhar sbuf_finish(&sb); 646538035ed6SNavdeep Parhar rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 646638035ed6SNavdeep Parhar sbuf_delete(&sb); 646738035ed6SNavdeep Parhar return (rc); 646838035ed6SNavdeep Parhar } 646902f972e8SNavdeep Parhar 6470786099deSNavdeep Parhar #ifdef RATELIMIT 6471ffbb373cSNavdeep Parhar #if defined(INET) || defined(INET6) 6472786099deSNavdeep Parhar /* 6473786099deSNavdeep Parhar * len16 for a txpkt WR with a GL. Includes the firmware work request header. 6474786099deSNavdeep Parhar */ 6475786099deSNavdeep Parhar static inline u_int 6476786099deSNavdeep Parhar txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso) 6477786099deSNavdeep Parhar { 6478786099deSNavdeep Parhar u_int n; 6479786099deSNavdeep Parhar 6480786099deSNavdeep Parhar MPASS(immhdrs > 0); 6481786099deSNavdeep Parhar 6482786099deSNavdeep Parhar n = roundup2(sizeof(struct fw_eth_tx_eo_wr) + 6483786099deSNavdeep Parhar sizeof(struct cpl_tx_pkt_core) + immhdrs, 16); 6484786099deSNavdeep Parhar if (__predict_false(nsegs == 0)) 6485786099deSNavdeep Parhar goto done; 6486786099deSNavdeep Parhar 6487786099deSNavdeep Parhar nsegs--; /* first segment is part of ulptx_sgl */ 6488786099deSNavdeep Parhar n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 6489786099deSNavdeep Parhar if (tso) 6490786099deSNavdeep Parhar n += sizeof(struct cpl_tx_pkt_lso_core); 6491786099deSNavdeep Parhar 6492786099deSNavdeep Parhar done: 6493786099deSNavdeep Parhar return (howmany(n, 16)); 6494786099deSNavdeep Parhar } 6495ffbb373cSNavdeep Parhar #endif 6496786099deSNavdeep Parhar 6497786099deSNavdeep Parhar #define ETID_FLOWC_NPARAMS 6 6498786099deSNavdeep Parhar #define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \ 6499786099deSNavdeep Parhar ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16)) 6500786099deSNavdeep Parhar #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16)) 6501786099deSNavdeep Parhar 6502786099deSNavdeep Parhar static int 6503e38a50e8SJohn Baldwin send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, 6504786099deSNavdeep Parhar struct vi_info *vi) 6505786099deSNavdeep Parhar { 6506786099deSNavdeep Parhar struct wrq_cookie cookie; 6507edb518f4SNavdeep Parhar u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN; 6508786099deSNavdeep Parhar struct fw_flowc_wr *flowc; 6509786099deSNavdeep Parhar 6510786099deSNavdeep Parhar mtx_assert(&cst->lock, MA_OWNED); 6511786099deSNavdeep Parhar MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) == 6512786099deSNavdeep Parhar EO_FLOWC_PENDING); 6513786099deSNavdeep Parhar 6514077ba6a8SJohn Baldwin flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie); 6515786099deSNavdeep Parhar if (__predict_false(flowc == NULL)) 6516786099deSNavdeep Parhar return (ENOMEM); 6517786099deSNavdeep Parhar 6518786099deSNavdeep Parhar bzero(flowc, ETID_FLOWC_LEN); 6519786099deSNavdeep Parhar flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 6520786099deSNavdeep Parhar V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0)); 6521786099deSNavdeep Parhar flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) | 6522786099deSNavdeep Parhar V_FW_WR_FLOWID(cst->etid)); 6523786099deSNavdeep Parhar flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 6524786099deSNavdeep Parhar flowc->mnemval[0].val = htobe32(pfvf); 6525786099deSNavdeep Parhar flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 6526786099deSNavdeep Parhar flowc->mnemval[1].val = htobe32(pi->tx_chan); 6527786099deSNavdeep Parhar flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 6528786099deSNavdeep Parhar flowc->mnemval[2].val = htobe32(pi->tx_chan); 6529786099deSNavdeep Parhar flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 6530786099deSNavdeep Parhar flowc->mnemval[3].val = htobe32(cst->iqid); 6531786099deSNavdeep Parhar flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE; 6532786099deSNavdeep Parhar flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED); 6533786099deSNavdeep Parhar flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; 6534786099deSNavdeep Parhar flowc->mnemval[5].val = htobe32(cst->schedcl); 6535786099deSNavdeep Parhar 6536077ba6a8SJohn Baldwin commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie); 6537786099deSNavdeep Parhar 6538786099deSNavdeep Parhar cst->flags &= ~EO_FLOWC_PENDING; 6539786099deSNavdeep Parhar cst->flags |= EO_FLOWC_RPL_PENDING; 6540786099deSNavdeep Parhar MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */ 6541786099deSNavdeep Parhar cst->tx_credits -= ETID_FLOWC_LEN16; 6542786099deSNavdeep Parhar 6543786099deSNavdeep Parhar return (0); 6544786099deSNavdeep Parhar } 6545786099deSNavdeep Parhar 6546786099deSNavdeep Parhar #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16)) 6547786099deSNavdeep Parhar 6548786099deSNavdeep Parhar void 6549e38a50e8SJohn Baldwin send_etid_flush_wr(struct cxgbe_rate_tag *cst) 6550786099deSNavdeep Parhar { 6551786099deSNavdeep Parhar struct fw_flowc_wr *flowc; 6552786099deSNavdeep Parhar struct wrq_cookie cookie; 6553786099deSNavdeep Parhar 6554786099deSNavdeep Parhar mtx_assert(&cst->lock, MA_OWNED); 6555786099deSNavdeep Parhar 6556077ba6a8SJohn Baldwin flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie); 6557786099deSNavdeep Parhar if (__predict_false(flowc == NULL)) 6558786099deSNavdeep Parhar CXGBE_UNIMPLEMENTED(__func__); 6559786099deSNavdeep Parhar 6560786099deSNavdeep Parhar bzero(flowc, ETID_FLUSH_LEN16 * 16); 6561786099deSNavdeep Parhar flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 6562786099deSNavdeep Parhar V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL); 6563786099deSNavdeep Parhar flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) | 6564786099deSNavdeep Parhar V_FW_WR_FLOWID(cst->etid)); 6565786099deSNavdeep Parhar 6566077ba6a8SJohn Baldwin commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie); 6567786099deSNavdeep Parhar 6568786099deSNavdeep Parhar cst->flags |= EO_FLUSH_RPL_PENDING; 6569786099deSNavdeep Parhar MPASS(cst->tx_credits >= ETID_FLUSH_LEN16); 6570786099deSNavdeep Parhar cst->tx_credits -= ETID_FLUSH_LEN16; 6571786099deSNavdeep Parhar cst->ncompl++; 6572786099deSNavdeep Parhar } 6573786099deSNavdeep Parhar 6574786099deSNavdeep Parhar static void 6575e38a50e8SJohn Baldwin write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, 6576786099deSNavdeep Parhar struct mbuf *m0, int compl) 6577786099deSNavdeep Parhar { 6578786099deSNavdeep Parhar struct cpl_tx_pkt_core *cpl; 6579786099deSNavdeep Parhar uint64_t ctrl1; 6580786099deSNavdeep Parhar uint32_t ctrl; /* used in many unrelated places */ 6581786099deSNavdeep Parhar int len16, pktlen, nsegs, immhdrs; 6582786099deSNavdeep Parhar uintptr_t p; 6583786099deSNavdeep Parhar struct ulptx_sgl *usgl; 6584786099deSNavdeep Parhar struct sglist sg; 6585786099deSNavdeep Parhar struct sglist_seg segs[38]; /* XXX: find real limit. XXX: get off the stack */ 6586786099deSNavdeep Parhar 6587786099deSNavdeep Parhar mtx_assert(&cst->lock, MA_OWNED); 6588786099deSNavdeep Parhar M_ASSERTPKTHDR(m0); 6589786099deSNavdeep Parhar KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 6590786099deSNavdeep Parhar m0->m_pkthdr.l4hlen > 0, 6591786099deSNavdeep Parhar ("%s: ethofld mbuf %p is missing header lengths", __func__, m0)); 6592786099deSNavdeep Parhar 6593786099deSNavdeep Parhar len16 = mbuf_eo_len16(m0); 6594786099deSNavdeep Parhar nsegs = mbuf_eo_nsegs(m0); 6595786099deSNavdeep Parhar pktlen = m0->m_pkthdr.len; 6596786099deSNavdeep Parhar ctrl = sizeof(struct cpl_tx_pkt_core); 6597786099deSNavdeep Parhar if (needs_tso(m0)) 6598786099deSNavdeep Parhar ctrl += sizeof(struct cpl_tx_pkt_lso_core); 6599786099deSNavdeep Parhar immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen; 6600786099deSNavdeep Parhar ctrl += immhdrs; 6601786099deSNavdeep Parhar 6602786099deSNavdeep Parhar wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) | 6603786099deSNavdeep Parhar V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl)); 6604786099deSNavdeep Parhar wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) | 6605786099deSNavdeep Parhar V_FW_WR_FLOWID(cst->etid)); 6606786099deSNavdeep Parhar wr->r3 = 0; 6607a4a4ad2dSNavdeep Parhar if (needs_outer_udp_csum(m0)) { 66086933902dSNavdeep Parhar wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG; 66096933902dSNavdeep Parhar wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen; 66106933902dSNavdeep Parhar wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 66116933902dSNavdeep Parhar wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen; 66126933902dSNavdeep Parhar wr->u.udpseg.rtplen = 0; 66136933902dSNavdeep Parhar wr->u.udpseg.r4 = 0; 66146933902dSNavdeep Parhar wr->u.udpseg.mss = htobe16(pktlen - immhdrs); 66156933902dSNavdeep Parhar wr->u.udpseg.schedpktsize = wr->u.udpseg.mss; 66166933902dSNavdeep Parhar wr->u.udpseg.plen = htobe32(pktlen - immhdrs); 66176933902dSNavdeep Parhar cpl = (void *)(wr + 1); 66186933902dSNavdeep Parhar } else { 6619a4a4ad2dSNavdeep Parhar MPASS(needs_outer_tcp_csum(m0)); 6620786099deSNavdeep Parhar wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG; 6621786099deSNavdeep Parhar wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen; 6622786099deSNavdeep Parhar wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 6623786099deSNavdeep Parhar wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen; 6624786099deSNavdeep Parhar wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0); 6625786099deSNavdeep Parhar wr->u.tcpseg.r4 = 0; 6626786099deSNavdeep Parhar wr->u.tcpseg.r5 = 0; 6627786099deSNavdeep Parhar wr->u.tcpseg.plen = htobe32(pktlen - immhdrs); 6628786099deSNavdeep Parhar 6629786099deSNavdeep Parhar if (needs_tso(m0)) { 6630786099deSNavdeep Parhar struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 6631786099deSNavdeep Parhar 6632786099deSNavdeep Parhar wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz); 6633786099deSNavdeep Parhar 66346933902dSNavdeep Parhar ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | 66356933902dSNavdeep Parhar F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | 6636c0236bd9SNavdeep Parhar V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - 6637c0236bd9SNavdeep Parhar ETHER_HDR_LEN) >> 2) | 66386933902dSNavdeep Parhar V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 66396933902dSNavdeep Parhar V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 6640786099deSNavdeep Parhar if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 6641786099deSNavdeep Parhar ctrl |= F_LSO_IPV6; 6642786099deSNavdeep Parhar lso->lso_ctrl = htobe32(ctrl); 6643786099deSNavdeep Parhar lso->ipid_ofst = htobe16(0); 6644786099deSNavdeep Parhar lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 6645786099deSNavdeep Parhar lso->seqno_offset = htobe32(0); 6646786099deSNavdeep Parhar lso->len = htobe32(pktlen); 6647786099deSNavdeep Parhar 6648786099deSNavdeep Parhar cpl = (void *)(lso + 1); 6649786099deSNavdeep Parhar } else { 6650786099deSNavdeep Parhar wr->u.tcpseg.mss = htobe16(0xffff); 6651786099deSNavdeep Parhar cpl = (void *)(wr + 1); 6652786099deSNavdeep Parhar } 66536933902dSNavdeep Parhar } 6654786099deSNavdeep Parhar 6655786099deSNavdeep Parhar /* Checksum offload must be requested for ethofld. */ 6656a4a4ad2dSNavdeep Parhar MPASS(needs_outer_l4_csum(m0)); 6657c0236bd9SNavdeep Parhar ctrl1 = csum_to_ctrl(cst->adapter, m0); 6658786099deSNavdeep Parhar 6659786099deSNavdeep Parhar /* VLAN tag insertion */ 6660786099deSNavdeep Parhar if (needs_vlan_insertion(m0)) { 6661786099deSNavdeep Parhar ctrl1 |= F_TXPKT_VLAN_VLD | 6662786099deSNavdeep Parhar V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 6663786099deSNavdeep Parhar } 6664786099deSNavdeep Parhar 6665786099deSNavdeep Parhar /* CPL header */ 6666786099deSNavdeep Parhar cpl->ctrl0 = cst->ctrl0; 6667786099deSNavdeep Parhar cpl->pack = 0; 6668786099deSNavdeep Parhar cpl->len = htobe16(pktlen); 6669786099deSNavdeep Parhar cpl->ctrl1 = htobe64(ctrl1); 6670786099deSNavdeep Parhar 66716933902dSNavdeep Parhar /* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */ 6672786099deSNavdeep Parhar p = (uintptr_t)(cpl + 1); 6673786099deSNavdeep Parhar m_copydata(m0, 0, immhdrs, (void *)p); 6674786099deSNavdeep Parhar 6675786099deSNavdeep Parhar /* SGL */ 6676786099deSNavdeep Parhar if (nsegs > 0) { 6677786099deSNavdeep Parhar int i, pad; 6678786099deSNavdeep Parhar 6679786099deSNavdeep Parhar /* zero-pad upto next 16Byte boundary, if not 16Byte aligned */ 6680786099deSNavdeep Parhar p += immhdrs; 6681786099deSNavdeep Parhar pad = 16 - (immhdrs & 0xf); 6682786099deSNavdeep Parhar bzero((void *)p, pad); 6683786099deSNavdeep Parhar 6684786099deSNavdeep Parhar usgl = (void *)(p + pad); 6685786099deSNavdeep Parhar usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 6686786099deSNavdeep Parhar V_ULPTX_NSGE(nsegs)); 6687786099deSNavdeep Parhar 6688786099deSNavdeep Parhar sglist_init(&sg, nitems(segs), segs); 6689786099deSNavdeep Parhar for (; m0 != NULL; m0 = m0->m_next) { 6690786099deSNavdeep Parhar if (__predict_false(m0->m_len == 0)) 6691786099deSNavdeep Parhar continue; 6692786099deSNavdeep Parhar if (immhdrs >= m0->m_len) { 6693786099deSNavdeep Parhar immhdrs -= m0->m_len; 6694786099deSNavdeep Parhar continue; 6695786099deSNavdeep Parhar } 66966edfd179SGleb Smirnoff if (m0->m_flags & M_EXTPG) 669749b6b60eSGleb Smirnoff sglist_append_mbuf_epg(&sg, m0, 669849b6b60eSGleb Smirnoff mtod(m0, vm_offset_t), m0->m_len); 669949b6b60eSGleb Smirnoff else 6700786099deSNavdeep Parhar sglist_append(&sg, mtod(m0, char *) + immhdrs, 6701786099deSNavdeep Parhar m0->m_len - immhdrs); 6702786099deSNavdeep Parhar immhdrs = 0; 6703786099deSNavdeep Parhar } 6704786099deSNavdeep Parhar MPASS(sg.sg_nseg == nsegs); 6705786099deSNavdeep Parhar 6706786099deSNavdeep Parhar /* 6707786099deSNavdeep Parhar * Zero pad last 8B in case the WR doesn't end on a 16B 6708786099deSNavdeep Parhar * boundary. 6709786099deSNavdeep Parhar */ 6710786099deSNavdeep Parhar *(uint64_t *)((char *)wr + len16 * 16 - 8) = 0; 6711786099deSNavdeep Parhar 6712786099deSNavdeep Parhar usgl->len0 = htobe32(segs[0].ss_len); 6713786099deSNavdeep Parhar usgl->addr0 = htobe64(segs[0].ss_paddr); 6714786099deSNavdeep Parhar for (i = 0; i < nsegs - 1; i++) { 6715786099deSNavdeep Parhar usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len); 6716786099deSNavdeep Parhar usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr); 6717786099deSNavdeep Parhar } 6718786099deSNavdeep Parhar if (i & 1) 6719786099deSNavdeep Parhar usgl->sge[i / 2].len[1] = htobe32(0); 6720786099deSNavdeep Parhar } 6721786099deSNavdeep Parhar 6722786099deSNavdeep Parhar } 6723786099deSNavdeep Parhar 6724786099deSNavdeep Parhar static void 6725e38a50e8SJohn Baldwin ethofld_tx(struct cxgbe_rate_tag *cst) 6726786099deSNavdeep Parhar { 6727786099deSNavdeep Parhar struct mbuf *m; 6728786099deSNavdeep Parhar struct wrq_cookie cookie; 6729786099deSNavdeep Parhar int next_credits, compl; 6730786099deSNavdeep Parhar struct fw_eth_tx_eo_wr *wr; 6731786099deSNavdeep Parhar 6732786099deSNavdeep Parhar mtx_assert(&cst->lock, MA_OWNED); 6733786099deSNavdeep Parhar 6734786099deSNavdeep Parhar while ((m = mbufq_first(&cst->pending_tx)) != NULL) { 6735786099deSNavdeep Parhar M_ASSERTPKTHDR(m); 6736786099deSNavdeep Parhar 6737786099deSNavdeep Parhar /* How many len16 credits do we need to send this mbuf. */ 6738786099deSNavdeep Parhar next_credits = mbuf_eo_len16(m); 6739786099deSNavdeep Parhar MPASS(next_credits > 0); 6740786099deSNavdeep Parhar if (next_credits > cst->tx_credits) { 6741786099deSNavdeep Parhar /* 6742786099deSNavdeep Parhar * Tx will make progress eventually because there is at 6743786099deSNavdeep Parhar * least one outstanding fw4_ack that will return 6744786099deSNavdeep Parhar * credits and kick the tx. 6745786099deSNavdeep Parhar */ 6746786099deSNavdeep Parhar MPASS(cst->ncompl > 0); 6747786099deSNavdeep Parhar return; 6748786099deSNavdeep Parhar } 6749077ba6a8SJohn Baldwin wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie); 6750786099deSNavdeep Parhar if (__predict_false(wr == NULL)) { 6751786099deSNavdeep Parhar /* XXX: wishful thinking, not a real assertion. */ 6752786099deSNavdeep Parhar MPASS(cst->ncompl > 0); 6753786099deSNavdeep Parhar return; 6754786099deSNavdeep Parhar } 6755786099deSNavdeep Parhar cst->tx_credits -= next_credits; 6756786099deSNavdeep Parhar cst->tx_nocompl += next_credits; 6757786099deSNavdeep Parhar compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2; 675856fb710fSJohn Baldwin ETHER_BPF_MTAP(cst->com.ifp, m); 6759786099deSNavdeep Parhar write_ethofld_wr(cst, wr, m, compl); 6760077ba6a8SJohn Baldwin commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie); 6761786099deSNavdeep Parhar if (compl) { 6762786099deSNavdeep Parhar cst->ncompl++; 6763786099deSNavdeep Parhar cst->tx_nocompl = 0; 6764786099deSNavdeep Parhar } 6765786099deSNavdeep Parhar (void) mbufq_dequeue(&cst->pending_tx); 6766fb3bc596SJohn Baldwin 6767fb3bc596SJohn Baldwin /* 6768fb3bc596SJohn Baldwin * Drop the mbuf's reference on the tag now rather 6769fb3bc596SJohn Baldwin * than waiting until m_freem(). This ensures that 6770e38a50e8SJohn Baldwin * cxgbe_rate_tag_free gets called when the inp drops 6771fb3bc596SJohn Baldwin * its reference on the tag and there are no more 6772fb3bc596SJohn Baldwin * mbufs in the pending_tx queue and can flush any 6773fb3bc596SJohn Baldwin * pending requests. Otherwise if the last mbuf 6774fb3bc596SJohn Baldwin * doesn't request a completion the etid will never be 6775fb3bc596SJohn Baldwin * released. 6776fb3bc596SJohn Baldwin */ 6777fb3bc596SJohn Baldwin m->m_pkthdr.snd_tag = NULL; 6778fb3bc596SJohn Baldwin m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 677956fb710fSJohn Baldwin m_snd_tag_rele(&cst->com); 6780fb3bc596SJohn Baldwin 6781786099deSNavdeep Parhar mbufq_enqueue(&cst->pending_fwack, m); 6782786099deSNavdeep Parhar } 6783786099deSNavdeep Parhar } 6784786099deSNavdeep Parhar 6785786099deSNavdeep Parhar int 6786786099deSNavdeep Parhar ethofld_transmit(struct ifnet *ifp, struct mbuf *m0) 6787786099deSNavdeep Parhar { 6788e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst; 6789786099deSNavdeep Parhar int rc; 6790786099deSNavdeep Parhar 6791786099deSNavdeep Parhar MPASS(m0->m_nextpkt == NULL); 6792fb3bc596SJohn Baldwin MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG); 6793786099deSNavdeep Parhar MPASS(m0->m_pkthdr.snd_tag != NULL); 6794e38a50e8SJohn Baldwin cst = mst_to_crt(m0->m_pkthdr.snd_tag); 6795786099deSNavdeep Parhar 6796786099deSNavdeep Parhar mtx_lock(&cst->lock); 6797786099deSNavdeep Parhar MPASS(cst->flags & EO_SND_TAG_REF); 6798786099deSNavdeep Parhar 6799786099deSNavdeep Parhar if (__predict_false(cst->flags & EO_FLOWC_PENDING)) { 6800786099deSNavdeep Parhar struct vi_info *vi = ifp->if_softc; 6801786099deSNavdeep Parhar struct port_info *pi = vi->pi; 6802786099deSNavdeep Parhar struct adapter *sc = pi->adapter; 6803786099deSNavdeep Parhar const uint32_t rss_mask = vi->rss_size - 1; 6804786099deSNavdeep Parhar uint32_t rss_hash; 6805786099deSNavdeep Parhar 6806786099deSNavdeep Parhar cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; 6807786099deSNavdeep Parhar if (M_HASHTYPE_ISHASH(m0)) 6808786099deSNavdeep Parhar rss_hash = m0->m_pkthdr.flowid; 6809786099deSNavdeep Parhar else 6810786099deSNavdeep Parhar rss_hash = arc4random(); 6811786099deSNavdeep Parhar /* We assume RSS hashing */ 6812786099deSNavdeep Parhar cst->iqid = vi->rss[rss_hash & rss_mask]; 6813786099deSNavdeep Parhar cst->eo_txq += rss_hash % vi->nofldtxq; 6814786099deSNavdeep Parhar rc = send_etid_flowc_wr(cst, pi, vi); 6815786099deSNavdeep Parhar if (rc != 0) 6816786099deSNavdeep Parhar goto done; 6817786099deSNavdeep Parhar } 6818786099deSNavdeep Parhar 6819786099deSNavdeep Parhar if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { 6820786099deSNavdeep Parhar rc = ENOBUFS; 6821786099deSNavdeep Parhar goto done; 6822786099deSNavdeep Parhar } 6823786099deSNavdeep Parhar 6824786099deSNavdeep Parhar mbufq_enqueue(&cst->pending_tx, m0); 6825786099deSNavdeep Parhar cst->plen += m0->m_pkthdr.len; 6826786099deSNavdeep Parhar 6827fb3bc596SJohn Baldwin /* 6828fb3bc596SJohn Baldwin * Hold an extra reference on the tag while generating work 6829fb3bc596SJohn Baldwin * requests to ensure that we don't try to free the tag during 6830fb3bc596SJohn Baldwin * ethofld_tx() in case we are sending the final mbuf after 6831fb3bc596SJohn Baldwin * the inp was freed. 6832fb3bc596SJohn Baldwin */ 683356fb710fSJohn Baldwin m_snd_tag_ref(&cst->com); 6834786099deSNavdeep Parhar ethofld_tx(cst); 6835fb3bc596SJohn Baldwin mtx_unlock(&cst->lock); 683656fb710fSJohn Baldwin m_snd_tag_rele(&cst->com); 6837fb3bc596SJohn Baldwin return (0); 6838fb3bc596SJohn Baldwin 6839786099deSNavdeep Parhar done: 6840786099deSNavdeep Parhar mtx_unlock(&cst->lock); 6841786099deSNavdeep Parhar if (__predict_false(rc != 0)) 6842786099deSNavdeep Parhar m_freem(m0); 6843786099deSNavdeep Parhar return (rc); 6844786099deSNavdeep Parhar } 6845786099deSNavdeep Parhar 6846786099deSNavdeep Parhar static int 6847786099deSNavdeep Parhar ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) 6848786099deSNavdeep Parhar { 6849786099deSNavdeep Parhar struct adapter *sc = iq->adapter; 6850786099deSNavdeep Parhar const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 6851786099deSNavdeep Parhar struct mbuf *m; 6852786099deSNavdeep Parhar u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 6853e38a50e8SJohn Baldwin struct cxgbe_rate_tag *cst; 6854786099deSNavdeep Parhar uint8_t credits = cpl->credits; 6855786099deSNavdeep Parhar 6856786099deSNavdeep Parhar cst = lookup_etid(sc, etid); 6857786099deSNavdeep Parhar mtx_lock(&cst->lock); 6858786099deSNavdeep Parhar if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) { 6859786099deSNavdeep Parhar MPASS(credits >= ETID_FLOWC_LEN16); 6860786099deSNavdeep Parhar credits -= ETID_FLOWC_LEN16; 6861786099deSNavdeep Parhar cst->flags &= ~EO_FLOWC_RPL_PENDING; 6862786099deSNavdeep Parhar } 6863786099deSNavdeep Parhar 6864786099deSNavdeep Parhar KASSERT(cst->ncompl > 0, 6865786099deSNavdeep Parhar ("%s: etid %u (%p) wasn't expecting completion.", 6866786099deSNavdeep Parhar __func__, etid, cst)); 6867786099deSNavdeep Parhar cst->ncompl--; 6868786099deSNavdeep Parhar 6869786099deSNavdeep Parhar while (credits > 0) { 6870786099deSNavdeep Parhar m = mbufq_dequeue(&cst->pending_fwack); 6871786099deSNavdeep Parhar if (__predict_false(m == NULL)) { 6872786099deSNavdeep Parhar /* 6873786099deSNavdeep Parhar * The remaining credits are for the final flush that 6874786099deSNavdeep Parhar * was issued when the tag was freed by the kernel. 6875786099deSNavdeep Parhar */ 6876786099deSNavdeep Parhar MPASS((cst->flags & 6877786099deSNavdeep Parhar (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) == 6878786099deSNavdeep Parhar EO_FLUSH_RPL_PENDING); 6879786099deSNavdeep Parhar MPASS(credits == ETID_FLUSH_LEN16); 6880786099deSNavdeep Parhar MPASS(cst->tx_credits + cpl->credits == cst->tx_total); 6881786099deSNavdeep Parhar MPASS(cst->ncompl == 0); 6882786099deSNavdeep Parhar 6883786099deSNavdeep Parhar cst->flags &= ~EO_FLUSH_RPL_PENDING; 6884786099deSNavdeep Parhar cst->tx_credits += cpl->credits; 6885e38a50e8SJohn Baldwin cxgbe_rate_tag_free_locked(cst); 6886786099deSNavdeep Parhar return (0); /* cst is gone. */ 6887786099deSNavdeep Parhar } 6888786099deSNavdeep Parhar KASSERT(m != NULL, 6889786099deSNavdeep Parhar ("%s: too many credits (%u, %u)", __func__, cpl->credits, 6890786099deSNavdeep Parhar credits)); 6891786099deSNavdeep Parhar KASSERT(credits >= mbuf_eo_len16(m), 6892786099deSNavdeep Parhar ("%s: too few credits (%u, %u, %u)", __func__, 6893786099deSNavdeep Parhar cpl->credits, credits, mbuf_eo_len16(m))); 6894786099deSNavdeep Parhar credits -= mbuf_eo_len16(m); 6895786099deSNavdeep Parhar cst->plen -= m->m_pkthdr.len; 6896786099deSNavdeep Parhar m_freem(m); 6897786099deSNavdeep Parhar } 6898786099deSNavdeep Parhar 6899786099deSNavdeep Parhar cst->tx_credits += cpl->credits; 6900786099deSNavdeep Parhar MPASS(cst->tx_credits <= cst->tx_total); 6901786099deSNavdeep Parhar 6902fb3bc596SJohn Baldwin if (cst->flags & EO_SND_TAG_REF) { 6903fb3bc596SJohn Baldwin /* 6904fb3bc596SJohn Baldwin * As with ethofld_transmit(), hold an extra reference 6905fb3bc596SJohn Baldwin * so that the tag is stable across ethold_tx(). 6906fb3bc596SJohn Baldwin */ 690756fb710fSJohn Baldwin m_snd_tag_ref(&cst->com); 6908786099deSNavdeep Parhar m = mbufq_first(&cst->pending_tx); 6909786099deSNavdeep Parhar if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m)) 6910786099deSNavdeep Parhar ethofld_tx(cst); 6911786099deSNavdeep Parhar mtx_unlock(&cst->lock); 691256fb710fSJohn Baldwin m_snd_tag_rele(&cst->com); 6913fb3bc596SJohn Baldwin } else { 6914fb3bc596SJohn Baldwin /* 6915fb3bc596SJohn Baldwin * There shouldn't be any pending packets if the tag 6916fb3bc596SJohn Baldwin * was freed by the kernel since any pending packet 6917fb3bc596SJohn Baldwin * should hold a reference to the tag. 6918fb3bc596SJohn Baldwin */ 6919fb3bc596SJohn Baldwin MPASS(mbufq_first(&cst->pending_tx) == NULL); 6920fb3bc596SJohn Baldwin mtx_unlock(&cst->lock); 6921fb3bc596SJohn Baldwin } 6922786099deSNavdeep Parhar 6923786099deSNavdeep Parhar return (0); 6924786099deSNavdeep Parhar } 6925786099deSNavdeep Parhar #endif 6926