xref: /freebsd/sys/dev/cxgbe/t4_sge.c (revision b406897911ea6c18c401907a076c4642dac46127)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 Chelsio Communications, Inc.
5  * All rights reserved.
6  * Written by: Navdeep Parhar <np@FreeBSD.org>
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 #include "opt_kern_tls.h"
36 #include "opt_ratelimit.h"
37 
38 #include <sys/types.h>
39 #include <sys/eventhandler.h>
40 #include <sys/mbuf.h>
41 #include <sys/socket.h>
42 #include <sys/kernel.h>
43 #include <sys/ktls.h>
44 #include <sys/malloc.h>
45 #include <sys/msan.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/taskqueue.h>
49 #include <sys/time.h>
50 #include <sys/sglist.h>
51 #include <sys/sysctl.h>
52 #include <sys/smp.h>
53 #include <sys/socketvar.h>
54 #include <sys/counter.h>
55 #include <net/bpf.h>
56 #include <net/ethernet.h>
57 #include <net/if.h>
58 #include <net/if_vlan_var.h>
59 #include <net/if_vxlan.h>
60 #include <netinet/in.h>
61 #include <netinet/ip.h>
62 #include <netinet/ip6.h>
63 #include <netinet/tcp.h>
64 #include <netinet/udp.h>
65 #include <machine/in_cksum.h>
66 #include <machine/md_var.h>
67 #include <vm/vm.h>
68 #include <vm/pmap.h>
69 #ifdef DEV_NETMAP
70 #include <machine/bus.h>
71 #include <sys/selinfo.h>
72 #include <net/if_var.h>
73 #include <net/netmap.h>
74 #include <dev/netmap/netmap_kern.h>
75 #endif
76 
77 #include "common/common.h"
78 #include "common/t4_regs.h"
79 #include "common/t4_regs_values.h"
80 #include "common/t4_msg.h"
81 #include "t4_l2t.h"
82 #include "t4_mp_ring.h"
83 
84 #ifdef T4_PKT_TIMESTAMP
85 #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
86 #else
87 #define RX_COPY_THRESHOLD MINCLSIZE
88 #endif
89 
90 /* Internal mbuf flags stored in PH_loc.eight[1]. */
91 #define	MC_NOMAP		0x01
92 #define	MC_RAW_WR		0x02
93 #define	MC_TLS			0x04
94 
95 /*
96  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
97  * 0-7 are valid values.
98  */
99 static int fl_pktshift = 0;
100 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0,
101     "payload DMA offset in rx buffer (bytes)");
102 
103 /*
104  * Pad ethernet payload up to this boundary.
105  * -1: driver should figure out a good value.
106  *  0: disable padding.
107  *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
108  */
109 int fl_pad = -1;
110 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0,
111     "payload pad boundary (bytes)");
112 
113 /*
114  * Status page length.
115  * -1: driver should figure out a good value.
116  *  64 or 128 are the only other valid values.
117  */
118 static int spg_len = -1;
119 SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0,
120     "status page size (bytes)");
121 
122 /*
123  * Congestion drops.
124  * -1: no congestion feedback (not recommended).
125  *  0: backpressure the channel instead of dropping packets right away.
126  *  1: no backpressure, drop packets for the congested queue immediately.
127  */
128 static int cong_drop = 0;
129 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0,
130     "Congestion control for RX queues (0 = backpressure, 1 = drop");
131 
132 /*
133  * Deliver multiple frames in the same free list buffer if they fit.
134  * -1: let the driver decide whether to enable buffer packing or not.
135  *  0: disable buffer packing.
136  *  1: enable buffer packing.
137  */
138 static int buffer_packing = -1;
139 SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing,
140     0, "Enable buffer packing");
141 
142 /*
143  * Start next frame in a packed buffer at this boundary.
144  * -1: driver should figure out a good value.
145  * T4: driver will ignore this and use the same value as fl_pad above.
146  * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
147  */
148 static int fl_pack = -1;
149 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0,
150     "payload pack boundary (bytes)");
151 
152 /*
153  * Largest rx cluster size that the driver is allowed to allocate.
154  */
155 static int largest_rx_cluster = MJUM16BYTES;
156 SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN,
157     &largest_rx_cluster, 0, "Largest rx cluster (bytes)");
158 
159 /*
160  * Size of cluster allocation that's most likely to succeed.  The driver will
161  * fall back to this size if it fails to allocate clusters larger than this.
162  */
163 static int safest_rx_cluster = PAGE_SIZE;
164 SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN,
165     &safest_rx_cluster, 0, "Safe rx cluster (bytes)");
166 
167 #ifdef RATELIMIT
168 /*
169  * Knob to control TCP timestamp rewriting, and the granularity of the tick used
170  * for rewriting.  -1 and 0-3 are all valid values.
171  * -1: hardware should leave the TCP timestamps alone.
172  * 0: 1ms
173  * 1: 100us
174  * 2: 10us
175  * 3: 1us
176  */
177 static int tsclk = -1;
178 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0,
179     "Control TCP timestamp rewriting when using pacing");
180 
181 static int eo_max_backlog = 1024 * 1024;
182 SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog,
183     0, "Maximum backlog of ratelimited data per flow");
184 #endif
185 
186 /*
187  * The interrupt holdoff timers are multiplied by this value on T6+.
188  * 1 and 3-17 (both inclusive) are legal values.
189  */
190 static int tscale = 1;
191 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0,
192     "Interrupt holdoff timer scale on T6+");
193 
194 /*
195  * Number of LRO entries in the lro_ctrl structure per rx queue.
196  */
197 static int lro_entries = TCP_LRO_ENTRIES;
198 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0,
199     "Number of LRO entries per RX queue");
200 
201 /*
202  * This enables presorting of frames before they're fed into tcp_lro_rx.
203  */
204 static int lro_mbufs = 0;
205 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0,
206     "Enable presorting of LRO frames");
207 
208 static counter_u64_t pullups;
209 SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups,
210     "Number of mbuf pullups performed");
211 
212 static counter_u64_t defrags;
213 SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, defrags, CTLFLAG_RD, &defrags,
214     "Number of mbuf defrags performed");
215 
216 static int t4_tx_coalesce = 1;
217 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce, CTLFLAG_RWTUN, &t4_tx_coalesce, 0,
218     "tx coalescing allowed");
219 
220 /*
221  * The driver will make aggressive attempts at tx coalescing if it sees these
222  * many packets eligible for coalescing in quick succession, with no more than
223  * the specified gap in between the eth_tx calls that delivered the packets.
224  */
225 static int t4_tx_coalesce_pkts = 32;
226 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_pkts, CTLFLAG_RWTUN,
227     &t4_tx_coalesce_pkts, 0,
228     "# of consecutive packets (1 - 255) that will trigger tx coalescing");
229 static int t4_tx_coalesce_gap = 5;
230 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_gap, CTLFLAG_RWTUN,
231     &t4_tx_coalesce_gap, 0, "tx gap (in microseconds)");
232 
233 static int service_iq(struct sge_iq *, int);
234 static int service_iq_fl(struct sge_iq *, int);
235 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
236 static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *,
237     u_int);
238 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
239     int, int);
240 static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
241 static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t,
242     struct sge_iq *, char *);
243 static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *,
244     struct sysctl_ctx_list *, struct sysctl_oid *);
245 static void free_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *);
246 static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
247     struct sge_iq *);
248 static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *,
249     struct sysctl_oid *, struct sge_fl *);
250 static int alloc_iq_fl_hwq(struct vi_info *, struct sge_iq *, struct sge_fl *);
251 static int free_iq_fl_hwq(struct adapter *, struct sge_iq *, struct sge_fl *);
252 static int alloc_fwq(struct adapter *);
253 static void free_fwq(struct adapter *);
254 static int alloc_ctrlq(struct adapter *, int);
255 static void free_ctrlq(struct adapter *, int);
256 static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, int);
257 static void free_rxq(struct vi_info *, struct sge_rxq *);
258 static void add_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
259     struct sge_rxq *);
260 #ifdef TCP_OFFLOAD
261 static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int,
262     int);
263 static void free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *);
264 static void add_ofld_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
265     struct sge_ofld_rxq *);
266 #endif
267 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
268 static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
269 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
270 static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
271 #endif
272 static int alloc_eq(struct adapter *, struct sge_eq *, struct sysctl_ctx_list *,
273     struct sysctl_oid *);
274 static void free_eq(struct adapter *, struct sge_eq *);
275 static void add_eq_sysctls(struct adapter *, struct sysctl_ctx_list *,
276     struct sysctl_oid *, struct sge_eq *);
277 static int alloc_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
278 static int free_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
279 static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *,
280     struct sysctl_ctx_list *, struct sysctl_oid *);
281 static void free_wrq(struct adapter *, struct sge_wrq *);
282 static void add_wrq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
283     struct sge_wrq *);
284 static int alloc_txq(struct vi_info *, struct sge_txq *, int);
285 static void free_txq(struct vi_info *, struct sge_txq *);
286 static void add_txq_sysctls(struct vi_info *, struct sysctl_ctx_list *,
287     struct sysctl_oid *, struct sge_txq *);
288 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
289 static int alloc_ofld_txq(struct vi_info *, struct sge_ofld_txq *, int);
290 static void free_ofld_txq(struct vi_info *, struct sge_ofld_txq *);
291 static void add_ofld_txq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
292     struct sge_ofld_txq *);
293 #endif
294 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
295 static inline void ring_fl_db(struct adapter *, struct sge_fl *);
296 static int refill_fl(struct adapter *, struct sge_fl *, int);
297 static void refill_sfl(void *);
298 static int find_refill_source(struct adapter *, int, bool);
299 static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
300 
301 static inline void get_pkt_gl(struct mbuf *, struct sglist *);
302 static inline u_int txpkt_len16(u_int, const u_int);
303 static inline u_int txpkt_vm_len16(u_int, const u_int);
304 static inline void calculate_mbuf_len16(struct mbuf *, bool);
305 static inline u_int txpkts0_len16(u_int);
306 static inline u_int txpkts1_len16(void);
307 static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int);
308 static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *,
309     u_int);
310 static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *,
311     struct mbuf *);
312 static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *,
313     int, bool *);
314 static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *,
315     int, bool *);
316 static u_int write_txpkts_wr(struct adapter *, struct sge_txq *);
317 static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *);
318 static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
319 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
320 static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
321 static inline uint16_t read_hw_cidx(struct sge_eq *);
322 static inline u_int reclaimable_tx_desc(struct sge_eq *);
323 static inline u_int total_available_tx_desc(struct sge_eq *);
324 static u_int reclaim_tx_descs(struct sge_txq *, u_int);
325 static void tx_reclaim(void *, int);
326 static __be64 get_flit(struct sglist_seg *, int, int);
327 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
328     struct mbuf *);
329 static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
330     struct mbuf *);
331 static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *);
332 static void wrq_tx_drain(void *, int);
333 static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *);
334 
335 static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
336 #ifdef RATELIMIT
337 #if defined(INET) || defined(INET6)
338 static inline u_int txpkt_eo_len16(u_int, u_int, u_int);
339 #endif
340 static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *,
341     struct mbuf *);
342 #endif
343 
344 static counter_u64_t extfree_refs;
345 static counter_u64_t extfree_rels;
346 
347 an_handler_t t4_an_handler;
348 fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
349 cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
350 cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES];
351 cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES];
352 cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES];
353 cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES];
354 cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES];
355 
356 void
357 t4_register_an_handler(an_handler_t h)
358 {
359 	uintptr_t *loc;
360 
361 	MPASS(h == NULL || t4_an_handler == NULL);
362 
363 	loc = (uintptr_t *)&t4_an_handler;
364 	atomic_store_rel_ptr(loc, (uintptr_t)h);
365 }
366 
367 void
368 t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
369 {
370 	uintptr_t *loc;
371 
372 	MPASS(type < nitems(t4_fw_msg_handler));
373 	MPASS(h == NULL || t4_fw_msg_handler[type] == NULL);
374 	/*
375 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
376 	 * handler dispatch table.  Reject any attempt to install a handler for
377 	 * this subtype.
378 	 */
379 	MPASS(type != FW_TYPE_RSSCPL);
380 	MPASS(type != FW6_TYPE_RSSCPL);
381 
382 	loc = (uintptr_t *)&t4_fw_msg_handler[type];
383 	atomic_store_rel_ptr(loc, (uintptr_t)h);
384 }
385 
386 void
387 t4_register_cpl_handler(int opcode, cpl_handler_t h)
388 {
389 	uintptr_t *loc;
390 
391 	MPASS(opcode < nitems(t4_cpl_handler));
392 	MPASS(h == NULL || t4_cpl_handler[opcode] == NULL);
393 
394 	loc = (uintptr_t *)&t4_cpl_handler[opcode];
395 	atomic_store_rel_ptr(loc, (uintptr_t)h);
396 }
397 
398 static int
399 set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
400     struct mbuf *m)
401 {
402 	const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
403 	u_int tid;
404 	int cookie;
405 
406 	MPASS(m == NULL);
407 
408 	tid = GET_TID(cpl);
409 	if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) {
410 		/*
411 		 * The return code for filter-write is put in the CPL cookie so
412 		 * we have to rely on the hardware tid (is_ftid) to determine
413 		 * that this is a response to a filter.
414 		 */
415 		cookie = CPL_COOKIE_FILTER;
416 	} else {
417 		cookie = G_COOKIE(cpl->cookie);
418 	}
419 	MPASS(cookie > CPL_COOKIE_RESERVED);
420 	MPASS(cookie < nitems(set_tcb_rpl_handlers));
421 
422 	return (set_tcb_rpl_handlers[cookie](iq, rss, m));
423 }
424 
425 static int
426 l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
427     struct mbuf *m)
428 {
429 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
430 	unsigned int cookie;
431 
432 	MPASS(m == NULL);
433 
434 	cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER;
435 	return (l2t_write_rpl_handlers[cookie](iq, rss, m));
436 }
437 
438 static int
439 act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
440     struct mbuf *m)
441 {
442 	const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
443 	u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status)));
444 
445 	MPASS(m == NULL);
446 	MPASS(cookie != CPL_COOKIE_RESERVED);
447 
448 	return (act_open_rpl_handlers[cookie](iq, rss, m));
449 }
450 
451 static int
452 abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss,
453     struct mbuf *m)
454 {
455 	struct adapter *sc = iq->adapter;
456 	u_int cookie;
457 
458 	MPASS(m == NULL);
459 	if (is_hashfilter(sc))
460 		cookie = CPL_COOKIE_HASHFILTER;
461 	else
462 		cookie = CPL_COOKIE_TOM;
463 
464 	return (abort_rpl_rss_handlers[cookie](iq, rss, m));
465 }
466 
467 static int
468 fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
469 {
470 	struct adapter *sc = iq->adapter;
471 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
472 	unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
473 	u_int cookie;
474 
475 	MPASS(m == NULL);
476 	if (is_etid(sc, tid))
477 		cookie = CPL_COOKIE_ETHOFLD;
478 	else
479 		cookie = CPL_COOKIE_TOM;
480 
481 	return (fw4_ack_handlers[cookie](iq, rss, m));
482 }
483 
484 static void
485 t4_init_shared_cpl_handlers(void)
486 {
487 
488 	t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler);
489 	t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler);
490 	t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler);
491 	t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler);
492 	t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler);
493 }
494 
495 void
496 t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
497 {
498 	uintptr_t *loc;
499 
500 	MPASS(opcode < nitems(t4_cpl_handler));
501 	MPASS(cookie > CPL_COOKIE_RESERVED);
502 	MPASS(cookie < NUM_CPL_COOKIES);
503 	MPASS(t4_cpl_handler[opcode] != NULL);
504 
505 	switch (opcode) {
506 	case CPL_SET_TCB_RPL:
507 		loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie];
508 		break;
509 	case CPL_L2T_WRITE_RPL:
510 		loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie];
511 		break;
512 	case CPL_ACT_OPEN_RPL:
513 		loc = (uintptr_t *)&act_open_rpl_handlers[cookie];
514 		break;
515 	case CPL_ABORT_RPL_RSS:
516 		loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie];
517 		break;
518 	case CPL_FW4_ACK:
519 		loc = (uintptr_t *)&fw4_ack_handlers[cookie];
520 		break;
521 	default:
522 		MPASS(0);
523 		return;
524 	}
525 	MPASS(h == NULL || *loc == (uintptr_t)NULL);
526 	atomic_store_rel_ptr(loc, (uintptr_t)h);
527 }
528 
529 /*
530  * Called on MOD_LOAD.  Validates and calculates the SGE tunables.
531  */
532 void
533 t4_sge_modload(void)
534 {
535 
536 	if (fl_pktshift < 0 || fl_pktshift > 7) {
537 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
538 		    " using 0 instead.\n", fl_pktshift);
539 		fl_pktshift = 0;
540 	}
541 
542 	if (spg_len != 64 && spg_len != 128) {
543 		int len;
544 
545 #if defined(__i386__) || defined(__amd64__)
546 		len = cpu_clflush_line_size > 64 ? 128 : 64;
547 #else
548 		len = 64;
549 #endif
550 		if (spg_len != -1) {
551 			printf("Invalid hw.cxgbe.spg_len value (%d),"
552 			    " using %d instead.\n", spg_len, len);
553 		}
554 		spg_len = len;
555 	}
556 
557 	if (cong_drop < -1 || cong_drop > 1) {
558 		printf("Invalid hw.cxgbe.cong_drop value (%d),"
559 		    " using 0 instead.\n", cong_drop);
560 		cong_drop = 0;
561 	}
562 
563 	if (tscale != 1 && (tscale < 3 || tscale > 17)) {
564 		printf("Invalid hw.cxgbe.tscale value (%d),"
565 		    " using 1 instead.\n", tscale);
566 		tscale = 1;
567 	}
568 
569 	if (largest_rx_cluster != MCLBYTES &&
570 #if MJUMPAGESIZE != MCLBYTES
571 	    largest_rx_cluster != MJUMPAGESIZE &&
572 #endif
573 	    largest_rx_cluster != MJUM9BYTES &&
574 	    largest_rx_cluster != MJUM16BYTES) {
575 		printf("Invalid hw.cxgbe.largest_rx_cluster value (%d),"
576 		    " using %d instead.\n", largest_rx_cluster, MJUM16BYTES);
577 		largest_rx_cluster = MJUM16BYTES;
578 	}
579 
580 	if (safest_rx_cluster != MCLBYTES &&
581 #if MJUMPAGESIZE != MCLBYTES
582 	    safest_rx_cluster != MJUMPAGESIZE &&
583 #endif
584 	    safest_rx_cluster != MJUM9BYTES &&
585 	    safest_rx_cluster != MJUM16BYTES) {
586 		printf("Invalid hw.cxgbe.safest_rx_cluster value (%d),"
587 		    " using %d instead.\n", safest_rx_cluster, MJUMPAGESIZE);
588 		safest_rx_cluster = MJUMPAGESIZE;
589 	}
590 
591 	extfree_refs = counter_u64_alloc(M_WAITOK);
592 	extfree_rels = counter_u64_alloc(M_WAITOK);
593 	pullups = counter_u64_alloc(M_WAITOK);
594 	defrags = counter_u64_alloc(M_WAITOK);
595 	counter_u64_zero(extfree_refs);
596 	counter_u64_zero(extfree_rels);
597 	counter_u64_zero(pullups);
598 	counter_u64_zero(defrags);
599 
600 	t4_init_shared_cpl_handlers();
601 	t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
602 	t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
603 	t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
604 #ifdef RATELIMIT
605 	t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack,
606 	    CPL_COOKIE_ETHOFLD);
607 #endif
608 	t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
609 	t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl);
610 }
611 
612 void
613 t4_sge_modunload(void)
614 {
615 
616 	counter_u64_free(extfree_refs);
617 	counter_u64_free(extfree_rels);
618 	counter_u64_free(pullups);
619 	counter_u64_free(defrags);
620 }
621 
622 uint64_t
623 t4_sge_extfree_refs(void)
624 {
625 	uint64_t refs, rels;
626 
627 	rels = counter_u64_fetch(extfree_rels);
628 	refs = counter_u64_fetch(extfree_refs);
629 
630 	return (refs - rels);
631 }
632 
633 /* max 4096 */
634 #define MAX_PACK_BOUNDARY 512
635 
636 static inline void
637 setup_pad_and_pack_boundaries(struct adapter *sc)
638 {
639 	uint32_t v, m;
640 	int pad, pack, pad_shift;
641 
642 	pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT :
643 	    X_INGPADBOUNDARY_SHIFT;
644 	pad = fl_pad;
645 	if (fl_pad < (1 << pad_shift) ||
646 	    fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) ||
647 	    !powerof2(fl_pad)) {
648 		/*
649 		 * If there is any chance that we might use buffer packing and
650 		 * the chip is a T4, then pick 64 as the pad/pack boundary.  Set
651 		 * it to the minimum allowed in all other cases.
652 		 */
653 		pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift;
654 
655 		/*
656 		 * For fl_pad = 0 we'll still write a reasonable value to the
657 		 * register but all the freelists will opt out of padding.
658 		 * We'll complain here only if the user tried to set it to a
659 		 * value greater than 0 that was invalid.
660 		 */
661 		if (fl_pad > 0) {
662 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
663 			    " (%d), using %d instead.\n", fl_pad, pad);
664 		}
665 	}
666 	m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
667 	v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift);
668 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
669 
670 	if (is_t4(sc)) {
671 		if (fl_pack != -1 && fl_pack != pad) {
672 			/* Complain but carry on. */
673 			device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
674 			    " using %d instead.\n", fl_pack, pad);
675 		}
676 		return;
677 	}
678 
679 	pack = fl_pack;
680 	if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
681 	    !powerof2(fl_pack)) {
682 		if (sc->params.pci.mps > MAX_PACK_BOUNDARY)
683 			pack = MAX_PACK_BOUNDARY;
684 		else
685 			pack = max(sc->params.pci.mps, CACHE_LINE_SIZE);
686 		MPASS(powerof2(pack));
687 		if (pack < 16)
688 			pack = 16;
689 		if (pack == 32)
690 			pack = 64;
691 		if (pack > 4096)
692 			pack = 4096;
693 		if (fl_pack != -1) {
694 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value"
695 			    " (%d), using %d instead.\n", fl_pack, pack);
696 		}
697 	}
698 	m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
699 	if (pack == 16)
700 		v = V_INGPACKBOUNDARY(0);
701 	else
702 		v = V_INGPACKBOUNDARY(ilog2(pack) - 5);
703 
704 	MPASS(!is_t4(sc));	/* T4 doesn't have SGE_CONTROL2 */
705 	t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
706 }
707 
708 /*
709  * adap->params.vpd.cclk must be set up before this is called.
710  */
711 void
712 t4_tweak_chip_settings(struct adapter *sc)
713 {
714 	int i, reg;
715 	uint32_t v, m;
716 	int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
717 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
718 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
719 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
720 	static int sw_buf_sizes[] = {
721 		MCLBYTES,
722 #if MJUMPAGESIZE != MCLBYTES
723 		MJUMPAGESIZE,
724 #endif
725 		MJUM9BYTES,
726 		MJUM16BYTES
727 	};
728 
729 	KASSERT(sc->flags & MASTER_PF,
730 	    ("%s: trying to change chip settings when not master.", __func__));
731 
732 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
733 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
734 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
735 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
736 
737 	setup_pad_and_pack_boundaries(sc);
738 
739 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
740 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
741 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
742 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
743 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
744 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
745 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
746 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
747 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
748 
749 	t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096);
750 	t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536);
751 	reg = A_SGE_FL_BUFFER_SIZE2;
752 	for (i = 0; i < nitems(sw_buf_sizes); i++) {
753 		MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
754 		t4_write_reg(sc, reg, sw_buf_sizes[i]);
755 		reg += 4;
756 		MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
757 		t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE);
758 		reg += 4;
759 	}
760 
761 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
762 	    V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
763 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
764 
765 	KASSERT(intr_timer[0] <= timer_max,
766 	    ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
767 	    timer_max));
768 	for (i = 1; i < nitems(intr_timer); i++) {
769 		KASSERT(intr_timer[i] >= intr_timer[i - 1],
770 		    ("%s: timers not listed in increasing order (%d)",
771 		    __func__, i));
772 
773 		while (intr_timer[i] > timer_max) {
774 			if (i == nitems(intr_timer) - 1) {
775 				intr_timer[i] = timer_max;
776 				break;
777 			}
778 			intr_timer[i] += intr_timer[i - 1];
779 			intr_timer[i] /= 2;
780 		}
781 	}
782 
783 	v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
784 	    V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
785 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
786 	v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
787 	    V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
788 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
789 	v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
790 	    V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
791 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
792 
793 	if (chip_id(sc) >= CHELSIO_T6) {
794 		m = V_TSCALE(M_TSCALE);
795 		if (tscale == 1)
796 			v = 0;
797 		else
798 			v = V_TSCALE(tscale - 2);
799 		t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v);
800 
801 		if (sc->debug_flags & DF_DISABLE_TCB_CACHE) {
802 			m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN |
803 			    V_WRTHRTHRESH(M_WRTHRTHRESH);
804 			t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1);
805 			v &= ~m;
806 			v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN |
807 			    V_WRTHRTHRESH(16);
808 			t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1);
809 		}
810 	}
811 
812 	/* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */
813 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
814 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
815 
816 	/*
817 	 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP.  These have been
818 	 * chosen with MAXPHYS = 128K in mind.  The largest DDP buffer that we
819 	 * may have to deal with is MAXPHYS + 1 page.
820 	 */
821 	v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4);
822 	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v);
823 
824 	/* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
825 	m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
826 	t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
827 
828 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
829 	    F_RESETDDPOFFSET;
830 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
831 	t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
832 }
833 
834 /*
835  * SGE wants the buffer to be at least 64B and then a multiple of 16.  Its
836  * address mut be 16B aligned.  If padding is in use the buffer's start and end
837  * need to be aligned to the pad boundary as well.  We'll just make sure that
838  * the size is a multiple of the pad boundary here, it is up to the buffer
839  * allocation code to make sure the start of the buffer is aligned.
840  */
841 static inline int
842 hwsz_ok(struct adapter *sc, int hwsz)
843 {
844 	int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1;
845 
846 	return (hwsz >= 64 && (hwsz & mask) == 0);
847 }
848 
849 /*
850  * Initialize the rx buffer sizes and figure out which zones the buffers will
851  * be allocated from.
852  */
853 void
854 t4_init_rx_buf_info(struct adapter *sc)
855 {
856 	struct sge *s = &sc->sge;
857 	struct sge_params *sp = &sc->params.sge;
858 	int i, j, n;
859 	static int sw_buf_sizes[] = {	/* Sorted by size */
860 		MCLBYTES,
861 #if MJUMPAGESIZE != MCLBYTES
862 		MJUMPAGESIZE,
863 #endif
864 		MJUM9BYTES,
865 		MJUM16BYTES
866 	};
867 	struct rx_buf_info *rxb;
868 
869 	s->safe_zidx = -1;
870 	rxb = &s->rx_buf_info[0];
871 	for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
872 		rxb->size1 = sw_buf_sizes[i];
873 		rxb->zone = m_getzone(rxb->size1);
874 		rxb->type = m_gettype(rxb->size1);
875 		rxb->size2 = 0;
876 		rxb->hwidx1 = -1;
877 		rxb->hwidx2 = -1;
878 		for (j = 0; j < SGE_FLBUF_SIZES; j++) {
879 			int hwsize = sp->sge_fl_buffer_size[j];
880 
881 			if (!hwsz_ok(sc, hwsize))
882 				continue;
883 
884 			/* hwidx for size1 */
885 			if (rxb->hwidx1 == -1 && rxb->size1 == hwsize)
886 				rxb->hwidx1 = j;
887 
888 			/* hwidx for size2 (buffer packing) */
889 			if (rxb->size1 - CL_METADATA_SIZE < hwsize)
890 				continue;
891 			n = rxb->size1 - hwsize - CL_METADATA_SIZE;
892 			if (n == 0) {
893 				rxb->hwidx2 = j;
894 				rxb->size2 = hwsize;
895 				break;	/* stop looking */
896 			}
897 			if (rxb->hwidx2 != -1) {
898 				if (n < sp->sge_fl_buffer_size[rxb->hwidx2] -
899 				    hwsize - CL_METADATA_SIZE) {
900 					rxb->hwidx2 = j;
901 					rxb->size2 = hwsize;
902 				}
903 			} else if (n <= 2 * CL_METADATA_SIZE) {
904 				rxb->hwidx2 = j;
905 				rxb->size2 = hwsize;
906 			}
907 		}
908 		if (rxb->hwidx2 != -1)
909 			sc->flags |= BUF_PACKING_OK;
910 		if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster)
911 			s->safe_zidx = i;
912 	}
913 }
914 
915 /*
916  * Verify some basic SGE settings for the PF and VF driver, and other
917  * miscellaneous settings for the PF driver.
918  */
919 int
920 t4_verify_chip_settings(struct adapter *sc)
921 {
922 	struct sge_params *sp = &sc->params.sge;
923 	uint32_t m, v, r;
924 	int rc = 0;
925 	const uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
926 
927 	m = F_RXPKTCPLMODE;
928 	v = F_RXPKTCPLMODE;
929 	r = sp->sge_control;
930 	if ((r & m) != v) {
931 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
932 		rc = EINVAL;
933 	}
934 
935 	/*
936 	 * If this changes then every single use of PAGE_SHIFT in the driver
937 	 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift.
938 	 */
939 	if (sp->page_shift != PAGE_SHIFT) {
940 		device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
941 		rc = EINVAL;
942 	}
943 
944 	if (sc->flags & IS_VF)
945 		return (0);
946 
947 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
948 	r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
949 	if (r != v) {
950 		device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
951 		if (sc->vres.ddp.size != 0)
952 			rc = EINVAL;
953 	}
954 
955 	m = v = F_TDDPTAGTCB;
956 	r = t4_read_reg(sc, A_ULP_RX_CTL);
957 	if ((r & m) != v) {
958 		device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
959 		if (sc->vres.ddp.size != 0)
960 			rc = EINVAL;
961 	}
962 
963 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
964 	    F_RESETDDPOFFSET;
965 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
966 	r = t4_read_reg(sc, A_TP_PARA_REG5);
967 	if ((r & m) != v) {
968 		device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
969 		if (sc->vres.ddp.size != 0)
970 			rc = EINVAL;
971 	}
972 
973 	return (rc);
974 }
975 
976 int
977 t4_create_dma_tag(struct adapter *sc)
978 {
979 	int rc;
980 
981 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
982 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
983 	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
984 	    NULL, &sc->dmat);
985 	if (rc != 0) {
986 		device_printf(sc->dev,
987 		    "failed to create main DMA tag: %d\n", rc);
988 	}
989 
990 	return (rc);
991 }
992 
993 void
994 t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
995     struct sysctl_oid_list *children)
996 {
997 	struct sge_params *sp = &sc->params.sge;
998 
999 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes",
1000 	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
1001 	    sysctl_bufsizes, "A", "freelist buffer sizes");
1002 
1003 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD,
1004 	    NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)");
1005 
1006 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD,
1007 	    NULL, sp->pad_boundary, "payload pad boundary (bytes)");
1008 
1009 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD,
1010 	    NULL, sp->spg_len, "status page size (bytes)");
1011 
1012 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
1013 	    NULL, cong_drop, "congestion drop setting");
1014 
1015 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
1016 	    NULL, sp->pack_boundary, "payload pack boundary (bytes)");
1017 }
1018 
1019 int
1020 t4_destroy_dma_tag(struct adapter *sc)
1021 {
1022 	if (sc->dmat)
1023 		bus_dma_tag_destroy(sc->dmat);
1024 
1025 	return (0);
1026 }
1027 
1028 /*
1029  * Allocate and initialize the firmware event queue, control queues, and special
1030  * purpose rx queues owned by the adapter.
1031  *
1032  * Returns errno on failure.  Resources allocated up to that point may still be
1033  * allocated.  Caller is responsible for cleanup in case this function fails.
1034  */
1035 int
1036 t4_setup_adapter_queues(struct adapter *sc)
1037 {
1038 	int rc, i;
1039 
1040 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1041 
1042 	/*
1043 	 * Firmware event queue
1044 	 */
1045 	rc = alloc_fwq(sc);
1046 	if (rc != 0)
1047 		return (rc);
1048 
1049 	/*
1050 	 * That's all for the VF driver.
1051 	 */
1052 	if (sc->flags & IS_VF)
1053 		return (rc);
1054 
1055 	/*
1056 	 * XXX: General purpose rx queues, one per port.
1057 	 */
1058 
1059 	/*
1060 	 * Control queues, one per port.
1061 	 */
1062 	for_each_port(sc, i) {
1063 		rc = alloc_ctrlq(sc, i);
1064 		if (rc != 0)
1065 			return (rc);
1066 	}
1067 
1068 	return (rc);
1069 }
1070 
1071 /*
1072  * Idempotent
1073  */
1074 int
1075 t4_teardown_adapter_queues(struct adapter *sc)
1076 {
1077 	int i;
1078 
1079 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1080 
1081 	if (sc->sge.ctrlq != NULL) {
1082 		MPASS(!(sc->flags & IS_VF));	/* VFs don't allocate ctrlq. */
1083 		for_each_port(sc, i)
1084 			free_ctrlq(sc, i);
1085 	}
1086 	free_fwq(sc);
1087 
1088 	return (0);
1089 }
1090 
1091 /* Maximum payload that could arrive with a single iq descriptor. */
1092 static inline int
1093 max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld)
1094 {
1095 	int maxp;
1096 
1097 	/* large enough even when hw VLAN extraction is disabled */
1098 	maxp = sc->params.sge.fl_pktshift + ETHER_HDR_LEN +
1099 	    ETHER_VLAN_ENCAP_LEN + ifp->if_mtu;
1100 	if (ofld && sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS &&
1101 	    maxp < sc->params.tp.max_rx_pdu)
1102 		maxp = sc->params.tp.max_rx_pdu;
1103 	return (maxp);
1104 }
1105 
1106 int
1107 t4_setup_vi_queues(struct vi_info *vi)
1108 {
1109 	int rc = 0, i, intr_idx;
1110 	struct sge_rxq *rxq;
1111 	struct sge_txq *txq;
1112 #ifdef TCP_OFFLOAD
1113 	struct sge_ofld_rxq *ofld_rxq;
1114 #endif
1115 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1116 	struct sge_ofld_txq *ofld_txq;
1117 #endif
1118 #ifdef DEV_NETMAP
1119 	int saved_idx, iqidx;
1120 	struct sge_nm_rxq *nm_rxq;
1121 	struct sge_nm_txq *nm_txq;
1122 #endif
1123 	struct adapter *sc = vi->adapter;
1124 	struct ifnet *ifp = vi->ifp;
1125 	int maxp;
1126 
1127 	/* Interrupt vector to start from (when using multiple vectors) */
1128 	intr_idx = vi->first_intr;
1129 
1130 #ifdef DEV_NETMAP
1131 	saved_idx = intr_idx;
1132 	if (ifp->if_capabilities & IFCAP_NETMAP) {
1133 
1134 		/* netmap is supported with direct interrupts only. */
1135 		MPASS(!forwarding_intr_to_fwq(sc));
1136 		MPASS(vi->first_intr >= 0);
1137 
1138 		/*
1139 		 * We don't have buffers to back the netmap rx queues
1140 		 * right now so we create the queues in a way that
1141 		 * doesn't set off any congestion signal in the chip.
1142 		 */
1143 		for_each_nm_rxq(vi, i, nm_rxq) {
1144 			rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i);
1145 			if (rc != 0)
1146 				goto done;
1147 			intr_idx++;
1148 		}
1149 
1150 		for_each_nm_txq(vi, i, nm_txq) {
1151 			iqidx = vi->first_nm_rxq + (i % vi->nnmrxq);
1152 			rc = alloc_nm_txq(vi, nm_txq, iqidx, i);
1153 			if (rc != 0)
1154 				goto done;
1155 		}
1156 	}
1157 
1158 	/* Normal rx queues and netmap rx queues share the same interrupts. */
1159 	intr_idx = saved_idx;
1160 #endif
1161 
1162 	/*
1163 	 * Allocate rx queues first because a default iqid is required when
1164 	 * creating a tx queue.
1165 	 */
1166 	maxp = max_rx_payload(sc, ifp, false);
1167 	for_each_rxq(vi, i, rxq) {
1168 		rc = alloc_rxq(vi, rxq, i, intr_idx, maxp);
1169 		if (rc != 0)
1170 			goto done;
1171 		if (!forwarding_intr_to_fwq(sc))
1172 			intr_idx++;
1173 	}
1174 #ifdef DEV_NETMAP
1175 	if (ifp->if_capabilities & IFCAP_NETMAP)
1176 		intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq);
1177 #endif
1178 #ifdef TCP_OFFLOAD
1179 	maxp = max_rx_payload(sc, ifp, true);
1180 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1181 		rc = alloc_ofld_rxq(vi, ofld_rxq, i, intr_idx, maxp);
1182 		if (rc != 0)
1183 			goto done;
1184 		if (!forwarding_intr_to_fwq(sc))
1185 			intr_idx++;
1186 	}
1187 #endif
1188 
1189 	/*
1190 	 * Now the tx queues.
1191 	 */
1192 	for_each_txq(vi, i, txq) {
1193 		rc = alloc_txq(vi, txq, i);
1194 		if (rc != 0)
1195 			goto done;
1196 	}
1197 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1198 	for_each_ofld_txq(vi, i, ofld_txq) {
1199 		rc = alloc_ofld_txq(vi, ofld_txq, i);
1200 		if (rc != 0)
1201 			goto done;
1202 	}
1203 #endif
1204 done:
1205 	if (rc)
1206 		t4_teardown_vi_queues(vi);
1207 
1208 	return (rc);
1209 }
1210 
1211 /*
1212  * Idempotent
1213  */
1214 int
1215 t4_teardown_vi_queues(struct vi_info *vi)
1216 {
1217 	int i;
1218 	struct sge_rxq *rxq;
1219 	struct sge_txq *txq;
1220 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1221 	struct sge_ofld_txq *ofld_txq;
1222 #endif
1223 #ifdef TCP_OFFLOAD
1224 	struct sge_ofld_rxq *ofld_rxq;
1225 #endif
1226 #ifdef DEV_NETMAP
1227 	struct sge_nm_rxq *nm_rxq;
1228 	struct sge_nm_txq *nm_txq;
1229 #endif
1230 
1231 #ifdef DEV_NETMAP
1232 	if (vi->ifp->if_capabilities & IFCAP_NETMAP) {
1233 		for_each_nm_txq(vi, i, nm_txq) {
1234 			free_nm_txq(vi, nm_txq);
1235 		}
1236 
1237 		for_each_nm_rxq(vi, i, nm_rxq) {
1238 			free_nm_rxq(vi, nm_rxq);
1239 		}
1240 	}
1241 #endif
1242 
1243 	/*
1244 	 * Take down all the tx queues first, as they reference the rx queues
1245 	 * (for egress updates, etc.).
1246 	 */
1247 
1248 	for_each_txq(vi, i, txq) {
1249 		free_txq(vi, txq);
1250 	}
1251 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1252 	for_each_ofld_txq(vi, i, ofld_txq) {
1253 		free_ofld_txq(vi, ofld_txq);
1254 	}
1255 #endif
1256 
1257 	/*
1258 	 * Then take down the rx queues.
1259 	 */
1260 
1261 	for_each_rxq(vi, i, rxq) {
1262 		free_rxq(vi, rxq);
1263 	}
1264 #ifdef TCP_OFFLOAD
1265 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1266 		free_ofld_rxq(vi, ofld_rxq);
1267 	}
1268 #endif
1269 
1270 	return (0);
1271 }
1272 
1273 /*
1274  * Interrupt handler when the driver is using only 1 interrupt.  This is a very
1275  * unusual scenario.
1276  *
1277  * a) Deals with errors, if any.
1278  * b) Services firmware event queue, which is taking interrupts for all other
1279  *    queues.
1280  */
1281 void
1282 t4_intr_all(void *arg)
1283 {
1284 	struct adapter *sc = arg;
1285 	struct sge_iq *fwq = &sc->sge.fwq;
1286 
1287 	MPASS(sc->intr_count == 1);
1288 
1289 	if (sc->intr_type == INTR_INTX)
1290 		t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
1291 
1292 	t4_intr_err(arg);
1293 	t4_intr_evt(fwq);
1294 }
1295 
1296 /*
1297  * Interrupt handler for errors (installed directly when multiple interrupts are
1298  * being used, or called by t4_intr_all).
1299  */
1300 void
1301 t4_intr_err(void *arg)
1302 {
1303 	struct adapter *sc = arg;
1304 	uint32_t v;
1305 	const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
1306 
1307 	if (sc->flags & ADAP_ERR)
1308 		return;
1309 
1310 	v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE));
1311 	if (v & F_PFSW) {
1312 		sc->swintr++;
1313 		t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v);
1314 	}
1315 
1316 	t4_slow_intr_handler(sc, verbose);
1317 }
1318 
1319 /*
1320  * Interrupt handler for iq-only queues.  The firmware event queue is the only
1321  * such queue right now.
1322  */
1323 void
1324 t4_intr_evt(void *arg)
1325 {
1326 	struct sge_iq *iq = arg;
1327 
1328 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1329 		service_iq(iq, 0);
1330 		(void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1331 	}
1332 }
1333 
1334 /*
1335  * Interrupt handler for iq+fl queues.
1336  */
1337 void
1338 t4_intr(void *arg)
1339 {
1340 	struct sge_iq *iq = arg;
1341 
1342 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1343 		service_iq_fl(iq, 0);
1344 		(void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1345 	}
1346 }
1347 
1348 #ifdef DEV_NETMAP
1349 /*
1350  * Interrupt handler for netmap rx queues.
1351  */
1352 void
1353 t4_nm_intr(void *arg)
1354 {
1355 	struct sge_nm_rxq *nm_rxq = arg;
1356 
1357 	if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
1358 		service_nm_rxq(nm_rxq);
1359 		(void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
1360 	}
1361 }
1362 
1363 /*
1364  * Interrupt handler for vectors shared between NIC and netmap rx queues.
1365  */
1366 void
1367 t4_vi_intr(void *arg)
1368 {
1369 	struct irq *irq = arg;
1370 
1371 	MPASS(irq->nm_rxq != NULL);
1372 	t4_nm_intr(irq->nm_rxq);
1373 
1374 	MPASS(irq->rxq != NULL);
1375 	t4_intr(irq->rxq);
1376 }
1377 #endif
1378 
1379 /*
1380  * Deals with interrupts on an iq-only (no freelist) queue.
1381  */
1382 static int
1383 service_iq(struct sge_iq *iq, int budget)
1384 {
1385 	struct sge_iq *q;
1386 	struct adapter *sc = iq->adapter;
1387 	struct iq_desc *d = &iq->desc[iq->cidx];
1388 	int ndescs = 0, limit;
1389 	int rsp_type;
1390 	uint32_t lq;
1391 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1392 
1393 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1394 	KASSERT((iq->flags & IQ_HAS_FL) == 0,
1395 	    ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
1396 	    iq->flags));
1397 	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
1398 	MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
1399 
1400 	limit = budget ? budget : iq->qsize / 16;
1401 
1402 	/*
1403 	 * We always come back and check the descriptor ring for new indirect
1404 	 * interrupts and other responses after running a single handler.
1405 	 */
1406 	for (;;) {
1407 		while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1408 
1409 			rmb();
1410 
1411 			rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1412 			lq = be32toh(d->rsp.pldbuflen_qid);
1413 
1414 			switch (rsp_type) {
1415 			case X_RSPD_TYPE_FLBUF:
1416 				panic("%s: data for an iq (%p) with no freelist",
1417 				    __func__, iq);
1418 
1419 				/* NOTREACHED */
1420 
1421 			case X_RSPD_TYPE_CPL:
1422 				KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1423 				    ("%s: bad opcode %02x.", __func__,
1424 				    d->rss.opcode));
1425 				t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL);
1426 				break;
1427 
1428 			case X_RSPD_TYPE_INTR:
1429 				/*
1430 				 * There are 1K interrupt-capable queues (qids 0
1431 				 * through 1023).  A response type indicating a
1432 				 * forwarded interrupt with a qid >= 1K is an
1433 				 * iWARP async notification.
1434 				 */
1435 				if (__predict_true(lq >= 1024)) {
1436 					t4_an_handler(iq, &d->rsp);
1437 					break;
1438 				}
1439 
1440 				q = sc->sge.iqmap[lq - sc->sge.iq_start -
1441 				    sc->sge.iq_base];
1442 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
1443 				    IQS_BUSY)) {
1444 					if (service_iq_fl(q, q->qsize / 16) == 0) {
1445 						(void) atomic_cmpset_int(&q->state,
1446 						    IQS_BUSY, IQS_IDLE);
1447 					} else {
1448 						STAILQ_INSERT_TAIL(&iql, q,
1449 						    link);
1450 					}
1451 				}
1452 				break;
1453 
1454 			default:
1455 				KASSERT(0,
1456 				    ("%s: illegal response type %d on iq %p",
1457 				    __func__, rsp_type, iq));
1458 				log(LOG_ERR,
1459 				    "%s: illegal response type %d on iq %p",
1460 				    device_get_nameunit(sc->dev), rsp_type, iq);
1461 				break;
1462 			}
1463 
1464 			d++;
1465 			if (__predict_false(++iq->cidx == iq->sidx)) {
1466 				iq->cidx = 0;
1467 				iq->gen ^= F_RSPD_GEN;
1468 				d = &iq->desc[0];
1469 			}
1470 			if (__predict_false(++ndescs == limit)) {
1471 				t4_write_reg(sc, sc->sge_gts_reg,
1472 				    V_CIDXINC(ndescs) |
1473 				    V_INGRESSQID(iq->cntxt_id) |
1474 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1475 				ndescs = 0;
1476 
1477 				if (budget) {
1478 					return (EINPROGRESS);
1479 				}
1480 			}
1481 		}
1482 
1483 		if (STAILQ_EMPTY(&iql))
1484 			break;
1485 
1486 		/*
1487 		 * Process the head only, and send it to the back of the list if
1488 		 * it's still not done.
1489 		 */
1490 		q = STAILQ_FIRST(&iql);
1491 		STAILQ_REMOVE_HEAD(&iql, link);
1492 		if (service_iq_fl(q, q->qsize / 8) == 0)
1493 			(void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
1494 		else
1495 			STAILQ_INSERT_TAIL(&iql, q, link);
1496 	}
1497 
1498 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1499 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1500 
1501 	return (0);
1502 }
1503 
1504 #if defined(INET) || defined(INET6)
1505 static inline int
1506 sort_before_lro(struct lro_ctrl *lro)
1507 {
1508 
1509 	return (lro->lro_mbuf_max != 0);
1510 }
1511 #endif
1512 
1513 static inline uint64_t
1514 last_flit_to_ns(struct adapter *sc, uint64_t lf)
1515 {
1516 	uint64_t n = be64toh(lf) & 0xfffffffffffffff;	/* 60b, not 64b. */
1517 
1518 	if (n > UINT64_MAX / 1000000)
1519 		return (n / sc->params.vpd.cclk * 1000000);
1520 	else
1521 		return (n * 1000000 / sc->params.vpd.cclk);
1522 }
1523 
1524 static inline void
1525 move_to_next_rxbuf(struct sge_fl *fl)
1526 {
1527 
1528 	fl->rx_offset = 0;
1529 	if (__predict_false((++fl->cidx & 7) == 0)) {
1530 		uint16_t cidx = fl->cidx >> 3;
1531 
1532 		if (__predict_false(cidx == fl->sidx))
1533 			fl->cidx = cidx = 0;
1534 		fl->hw_cidx = cidx;
1535 	}
1536 }
1537 
1538 /*
1539  * Deals with interrupts on an iq+fl queue.
1540  */
1541 static int
1542 service_iq_fl(struct sge_iq *iq, int budget)
1543 {
1544 	struct sge_rxq *rxq = iq_to_rxq(iq);
1545 	struct sge_fl *fl;
1546 	struct adapter *sc = iq->adapter;
1547 	struct iq_desc *d = &iq->desc[iq->cidx];
1548 	int ndescs, limit;
1549 	int rsp_type, starved;
1550 	uint32_t lq;
1551 	uint16_t fl_hw_cidx;
1552 	struct mbuf *m0;
1553 #if defined(INET) || defined(INET6)
1554 	const struct timeval lro_timeout = {0, sc->lro_timeout};
1555 	struct lro_ctrl *lro = &rxq->lro;
1556 #endif
1557 
1558 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1559 	MPASS(iq->flags & IQ_HAS_FL);
1560 
1561 	ndescs = 0;
1562 #if defined(INET) || defined(INET6)
1563 	if (iq->flags & IQ_ADJ_CREDIT) {
1564 		MPASS(sort_before_lro(lro));
1565 		iq->flags &= ~IQ_ADJ_CREDIT;
1566 		if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
1567 			tcp_lro_flush_all(lro);
1568 			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
1569 			    V_INGRESSQID((u32)iq->cntxt_id) |
1570 			    V_SEINTARM(iq->intr_params));
1571 			return (0);
1572 		}
1573 		ndescs = 1;
1574 	}
1575 #else
1576 	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
1577 #endif
1578 
1579 	limit = budget ? budget : iq->qsize / 16;
1580 	fl = &rxq->fl;
1581 	fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
1582 	while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1583 
1584 		rmb();
1585 
1586 		m0 = NULL;
1587 		rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1588 		lq = be32toh(d->rsp.pldbuflen_qid);
1589 
1590 		switch (rsp_type) {
1591 		case X_RSPD_TYPE_FLBUF:
1592 			if (lq & F_RSPD_NEWBUF) {
1593 				if (fl->rx_offset > 0)
1594 					move_to_next_rxbuf(fl);
1595 				lq = G_RSPD_LEN(lq);
1596 			}
1597 			if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) {
1598 				FL_LOCK(fl);
1599 				refill_fl(sc, fl, 64);
1600 				FL_UNLOCK(fl);
1601 				fl_hw_cidx = fl->hw_cidx;
1602 			}
1603 
1604 			if (d->rss.opcode == CPL_RX_PKT) {
1605 				if (__predict_true(eth_rx(sc, rxq, d, lq) == 0))
1606 					break;
1607 				goto out;
1608 			}
1609 			m0 = get_fl_payload(sc, fl, lq);
1610 			if (__predict_false(m0 == NULL))
1611 				goto out;
1612 
1613 			/* fall through */
1614 
1615 		case X_RSPD_TYPE_CPL:
1616 			KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1617 			    ("%s: bad opcode %02x.", __func__, d->rss.opcode));
1618 			t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
1619 			break;
1620 
1621 		case X_RSPD_TYPE_INTR:
1622 
1623 			/*
1624 			 * There are 1K interrupt-capable queues (qids 0
1625 			 * through 1023).  A response type indicating a
1626 			 * forwarded interrupt with a qid >= 1K is an
1627 			 * iWARP async notification.  That is the only
1628 			 * acceptable indirect interrupt on this queue.
1629 			 */
1630 			if (__predict_false(lq < 1024)) {
1631 				panic("%s: indirect interrupt on iq_fl %p "
1632 				    "with qid %u", __func__, iq, lq);
1633 			}
1634 
1635 			t4_an_handler(iq, &d->rsp);
1636 			break;
1637 
1638 		default:
1639 			KASSERT(0, ("%s: illegal response type %d on iq %p",
1640 			    __func__, rsp_type, iq));
1641 			log(LOG_ERR, "%s: illegal response type %d on iq %p",
1642 			    device_get_nameunit(sc->dev), rsp_type, iq);
1643 			break;
1644 		}
1645 
1646 		d++;
1647 		if (__predict_false(++iq->cidx == iq->sidx)) {
1648 			iq->cidx = 0;
1649 			iq->gen ^= F_RSPD_GEN;
1650 			d = &iq->desc[0];
1651 		}
1652 		if (__predict_false(++ndescs == limit)) {
1653 			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1654 			    V_INGRESSQID(iq->cntxt_id) |
1655 			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1656 
1657 #if defined(INET) || defined(INET6)
1658 			if (iq->flags & IQ_LRO_ENABLED &&
1659 			    !sort_before_lro(lro) &&
1660 			    sc->lro_timeout != 0) {
1661 				tcp_lro_flush_inactive(lro, &lro_timeout);
1662 			}
1663 #endif
1664 			if (budget)
1665 				return (EINPROGRESS);
1666 			ndescs = 0;
1667 		}
1668 	}
1669 out:
1670 #if defined(INET) || defined(INET6)
1671 	if (iq->flags & IQ_LRO_ENABLED) {
1672 		if (ndescs > 0 && lro->lro_mbuf_count > 8) {
1673 			MPASS(sort_before_lro(lro));
1674 			/* hold back one credit and don't flush LRO state */
1675 			iq->flags |= IQ_ADJ_CREDIT;
1676 			ndescs--;
1677 		} else {
1678 			tcp_lro_flush_all(lro);
1679 		}
1680 	}
1681 #endif
1682 
1683 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1684 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1685 
1686 	FL_LOCK(fl);
1687 	starved = refill_fl(sc, fl, 64);
1688 	FL_UNLOCK(fl);
1689 	if (__predict_false(starved != 0))
1690 		add_fl_to_sfl(sc, fl);
1691 
1692 	return (0);
1693 }
1694 
1695 static inline struct cluster_metadata *
1696 cl_metadata(struct fl_sdesc *sd)
1697 {
1698 
1699 	return ((void *)(sd->cl + sd->moff));
1700 }
1701 
1702 static void
1703 rxb_free(struct mbuf *m)
1704 {
1705 	struct cluster_metadata *clm = m->m_ext.ext_arg1;
1706 
1707 	uma_zfree(clm->zone, clm->cl);
1708 	counter_u64_add(extfree_rels, 1);
1709 }
1710 
1711 /*
1712  * The mbuf returned comes from zone_muf and carries the payload in one of these
1713  * ways
1714  * a) complete frame inside the mbuf
1715  * b) m_cljset (for clusters without metadata)
1716  * d) m_extaddref (cluster with metadata)
1717  */
1718 static struct mbuf *
1719 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1720     int remaining)
1721 {
1722 	struct mbuf *m;
1723 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1724 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1725 	struct cluster_metadata *clm;
1726 	int len, blen;
1727 	caddr_t payload;
1728 
1729 	if (fl->flags & FL_BUF_PACKING) {
1730 		u_int l, pad;
1731 
1732 		blen = rxb->size2 - fl->rx_offset;	/* max possible in this buf */
1733 		len = min(remaining, blen);
1734 		payload = sd->cl + fl->rx_offset;
1735 
1736 		l = fr_offset + len;
1737 		pad = roundup2(l, fl->buf_boundary) - l;
1738 		if (fl->rx_offset + len + pad < rxb->size2)
1739 			blen = len + pad;
1740 		MPASS(fl->rx_offset + blen <= rxb->size2);
1741 	} else {
1742 		MPASS(fl->rx_offset == 0);	/* not packing */
1743 		blen = rxb->size1;
1744 		len = min(remaining, blen);
1745 		payload = sd->cl;
1746 	}
1747 
1748 	if (fr_offset == 0) {
1749 		m = m_gethdr(M_NOWAIT, MT_DATA);
1750 		if (__predict_false(m == NULL))
1751 			return (NULL);
1752 		m->m_pkthdr.len = remaining;
1753 	} else {
1754 		m = m_get(M_NOWAIT, MT_DATA);
1755 		if (__predict_false(m == NULL))
1756 			return (NULL);
1757 	}
1758 	m->m_len = len;
1759 	kmsan_mark(payload, len, KMSAN_STATE_INITED);
1760 
1761 	if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
1762 		/* copy data to mbuf */
1763 		bcopy(payload, mtod(m, caddr_t), len);
1764 		if (fl->flags & FL_BUF_PACKING) {
1765 			fl->rx_offset += blen;
1766 			MPASS(fl->rx_offset <= rxb->size2);
1767 			if (fl->rx_offset < rxb->size2)
1768 				return (m);	/* without advancing the cidx */
1769 		}
1770 	} else if (fl->flags & FL_BUF_PACKING) {
1771 		clm = cl_metadata(sd);
1772 		if (sd->nmbuf++ == 0) {
1773 			clm->refcount = 1;
1774 			clm->zone = rxb->zone;
1775 			clm->cl = sd->cl;
1776 			counter_u64_add(extfree_refs, 1);
1777 		}
1778 		m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm,
1779 		    NULL);
1780 
1781 		fl->rx_offset += blen;
1782 		MPASS(fl->rx_offset <= rxb->size2);
1783 		if (fl->rx_offset < rxb->size2)
1784 			return (m);	/* without advancing the cidx */
1785 	} else {
1786 		m_cljset(m, sd->cl, rxb->type);
1787 		sd->cl = NULL;	/* consumed, not a recycle candidate */
1788 	}
1789 
1790 	move_to_next_rxbuf(fl);
1791 
1792 	return (m);
1793 }
1794 
1795 static struct mbuf *
1796 get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen)
1797 {
1798 	struct mbuf *m0, *m, **pnext;
1799 	u_int remaining;
1800 
1801 	if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1802 		M_ASSERTPKTHDR(fl->m0);
1803 		MPASS(fl->m0->m_pkthdr.len == plen);
1804 		MPASS(fl->remaining < plen);
1805 
1806 		m0 = fl->m0;
1807 		pnext = fl->pnext;
1808 		remaining = fl->remaining;
1809 		fl->flags &= ~FL_BUF_RESUME;
1810 		goto get_segment;
1811 	}
1812 
1813 	/*
1814 	 * Payload starts at rx_offset in the current hw buffer.  Its length is
1815 	 * 'len' and it may span multiple hw buffers.
1816 	 */
1817 
1818 	m0 = get_scatter_segment(sc, fl, 0, plen);
1819 	if (m0 == NULL)
1820 		return (NULL);
1821 	remaining = plen - m0->m_len;
1822 	pnext = &m0->m_next;
1823 	while (remaining > 0) {
1824 get_segment:
1825 		MPASS(fl->rx_offset == 0);
1826 		m = get_scatter_segment(sc, fl, plen - remaining, remaining);
1827 		if (__predict_false(m == NULL)) {
1828 			fl->m0 = m0;
1829 			fl->pnext = pnext;
1830 			fl->remaining = remaining;
1831 			fl->flags |= FL_BUF_RESUME;
1832 			return (NULL);
1833 		}
1834 		*pnext = m;
1835 		pnext = &m->m_next;
1836 		remaining -= m->m_len;
1837 	}
1838 	*pnext = NULL;
1839 
1840 	M_ASSERTPKTHDR(m0);
1841 	return (m0);
1842 }
1843 
1844 static int
1845 skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1846     int remaining)
1847 {
1848 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1849 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1850 	int len, blen;
1851 
1852 	if (fl->flags & FL_BUF_PACKING) {
1853 		u_int l, pad;
1854 
1855 		blen = rxb->size2 - fl->rx_offset;	/* max possible in this buf */
1856 		len = min(remaining, blen);
1857 
1858 		l = fr_offset + len;
1859 		pad = roundup2(l, fl->buf_boundary) - l;
1860 		if (fl->rx_offset + len + pad < rxb->size2)
1861 			blen = len + pad;
1862 		fl->rx_offset += blen;
1863 		MPASS(fl->rx_offset <= rxb->size2);
1864 		if (fl->rx_offset < rxb->size2)
1865 			return (len);	/* without advancing the cidx */
1866 	} else {
1867 		MPASS(fl->rx_offset == 0);	/* not packing */
1868 		blen = rxb->size1;
1869 		len = min(remaining, blen);
1870 	}
1871 	move_to_next_rxbuf(fl);
1872 	return (len);
1873 }
1874 
1875 static inline void
1876 skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen)
1877 {
1878 	int remaining, fr_offset, len;
1879 
1880 	fr_offset = 0;
1881 	remaining = plen;
1882 	while (remaining > 0) {
1883 		len = skip_scatter_segment(sc, fl, fr_offset, remaining);
1884 		fr_offset += len;
1885 		remaining -= len;
1886 	}
1887 }
1888 
1889 static inline int
1890 get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen)
1891 {
1892 	int len;
1893 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1894 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1895 
1896 	if (fl->flags & FL_BUF_PACKING)
1897 		len = rxb->size2 - fl->rx_offset;
1898 	else
1899 		len = rxb->size1;
1900 
1901 	return (min(plen, len));
1902 }
1903 
1904 static int
1905 eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d,
1906     u_int plen)
1907 {
1908 	struct mbuf *m0;
1909 	struct ifnet *ifp = rxq->ifp;
1910 	struct sge_fl *fl = &rxq->fl;
1911 	struct vi_info *vi = ifp->if_softc;
1912 	const struct cpl_rx_pkt *cpl;
1913 #if defined(INET) || defined(INET6)
1914 	struct lro_ctrl *lro = &rxq->lro;
1915 #endif
1916 	uint16_t err_vec, tnl_type, tnlhdr_len;
1917 	static const int sw_hashtype[4][2] = {
1918 		{M_HASHTYPE_NONE, M_HASHTYPE_NONE},
1919 		{M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6},
1920 		{M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6},
1921 		{M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6},
1922 	};
1923 	static const int sw_csum_flags[2][2] = {
1924 		{
1925 			/* IP, inner IP */
1926 			CSUM_ENCAP_VXLAN |
1927 			    CSUM_L3_CALC | CSUM_L3_VALID |
1928 			    CSUM_L4_CALC | CSUM_L4_VALID |
1929 			    CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1930 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1931 
1932 			/* IP, inner IP6 */
1933 			CSUM_ENCAP_VXLAN |
1934 			    CSUM_L3_CALC | CSUM_L3_VALID |
1935 			    CSUM_L4_CALC | CSUM_L4_VALID |
1936 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1937 		},
1938 		{
1939 			/* IP6, inner IP */
1940 			CSUM_ENCAP_VXLAN |
1941 			    CSUM_L4_CALC | CSUM_L4_VALID |
1942 			    CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1943 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1944 
1945 			/* IP6, inner IP6 */
1946 			CSUM_ENCAP_VXLAN |
1947 			    CSUM_L4_CALC | CSUM_L4_VALID |
1948 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1949 		},
1950 	};
1951 
1952 	MPASS(plen > sc->params.sge.fl_pktshift);
1953 	if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) &&
1954 	    __predict_true((fl->flags & FL_BUF_RESUME) == 0)) {
1955 		struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1956 		caddr_t frame;
1957 		int rc, slen;
1958 
1959 		slen = get_segment_len(sc, fl, plen) -
1960 		    sc->params.sge.fl_pktshift;
1961 		frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift;
1962 		CURVNET_SET_QUIET(ifp->if_vnet);
1963 		rc = pfil_run_hooks(vi->pfil, frame, ifp,
1964 		    slen | PFIL_MEMPTR | PFIL_IN, NULL);
1965 		CURVNET_RESTORE();
1966 		if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) {
1967 			skip_fl_payload(sc, fl, plen);
1968 			return (0);
1969 		}
1970 		if (rc == PFIL_REALLOCED) {
1971 			skip_fl_payload(sc, fl, plen);
1972 			m0 = pfil_mem2mbuf(frame);
1973 			goto have_mbuf;
1974 		}
1975 	}
1976 
1977 	m0 = get_fl_payload(sc, fl, plen);
1978 	if (__predict_false(m0 == NULL))
1979 		return (ENOMEM);
1980 
1981 	m0->m_pkthdr.len -= sc->params.sge.fl_pktshift;
1982 	m0->m_len -= sc->params.sge.fl_pktshift;
1983 	m0->m_data += sc->params.sge.fl_pktshift;
1984 
1985 have_mbuf:
1986 	m0->m_pkthdr.rcvif = ifp;
1987 	M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]);
1988 	m0->m_pkthdr.flowid = be32toh(d->rss.hash_val);
1989 
1990 	cpl = (const void *)(&d->rss + 1);
1991 	if (sc->params.tp.rx_pkt_encap) {
1992 		const uint16_t ev = be16toh(cpl->err_vec);
1993 
1994 		err_vec = G_T6_COMPR_RXERR_VEC(ev);
1995 		tnl_type = G_T6_RX_TNL_TYPE(ev);
1996 		tnlhdr_len = G_T6_RX_TNLHDR_LEN(ev);
1997 	} else {
1998 		err_vec = be16toh(cpl->err_vec);
1999 		tnl_type = 0;
2000 		tnlhdr_len = 0;
2001 	}
2002 	if (cpl->csum_calc && err_vec == 0) {
2003 		int ipv6 = !!(cpl->l2info & htobe32(F_RXF_IP6));
2004 
2005 		/* checksum(s) calculated and found to be correct. */
2006 
2007 		MPASS((cpl->l2info & htobe32(F_RXF_IP)) ^
2008 		    (cpl->l2info & htobe32(F_RXF_IP6)));
2009 		m0->m_pkthdr.csum_data = be16toh(cpl->csum);
2010 		if (tnl_type == 0) {
2011 	    		if (!ipv6 && ifp->if_capenable & IFCAP_RXCSUM) {
2012 				m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2013 				    CSUM_L3_VALID | CSUM_L4_CALC |
2014 				    CSUM_L4_VALID;
2015 			} else if (ipv6 && ifp->if_capenable & IFCAP_RXCSUM_IPV6) {
2016 				m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2017 				    CSUM_L4_VALID;
2018 			}
2019 			rxq->rxcsum++;
2020 		} else {
2021 			MPASS(tnl_type == RX_PKT_TNL_TYPE_VXLAN);
2022 
2023 			M_HASHTYPE_SETINNER(m0);
2024 			if (__predict_false(cpl->ip_frag)) {
2025 				/*
2026 				 * csum_data is for the inner frame (which is an
2027 				 * IP fragment) and is not 0xffff.  There is no
2028 				 * way to pass the inner csum_data to the stack.
2029 				 * We don't want the stack to use the inner
2030 				 * csum_data to validate the outer frame or it
2031 				 * will get rejected.  So we fix csum_data here
2032 				 * and let sw do the checksum of inner IP
2033 				 * fragments.
2034 				 *
2035 				 * XXX: Need 32b for csum_data2 in an rx mbuf.
2036 				 * Maybe stuff it into rcv_tstmp?
2037 				 */
2038 				m0->m_pkthdr.csum_data = 0xffff;
2039 				if (ipv6) {
2040 					m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2041 					    CSUM_L4_VALID;
2042 				} else {
2043 					m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2044 					    CSUM_L3_VALID | CSUM_L4_CALC |
2045 					    CSUM_L4_VALID;
2046 				}
2047 			} else {
2048 				int outer_ipv6;
2049 
2050 				MPASS(m0->m_pkthdr.csum_data == 0xffff);
2051 
2052 				outer_ipv6 = tnlhdr_len >=
2053 				    sizeof(struct ether_header) +
2054 				    sizeof(struct ip6_hdr);
2055 				m0->m_pkthdr.csum_flags =
2056 				    sw_csum_flags[outer_ipv6][ipv6];
2057 			}
2058 			rxq->vxlan_rxcsum++;
2059 		}
2060 	}
2061 
2062 	if (cpl->vlan_ex) {
2063 		m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
2064 		m0->m_flags |= M_VLANTAG;
2065 		rxq->vlan_extraction++;
2066 	}
2067 
2068 	if (rxq->iq.flags & IQ_RX_TIMESTAMP) {
2069 		/*
2070 		 * Fill up rcv_tstmp but do not set M_TSTMP.
2071 		 * rcv_tstmp is not in the format that the
2072 		 * kernel expects and we don't want to mislead
2073 		 * it.  For now this is only for custom code
2074 		 * that knows how to interpret cxgbe's stamp.
2075 		 */
2076 		m0->m_pkthdr.rcv_tstmp =
2077 		    last_flit_to_ns(sc, d->rsp.u.last_flit);
2078 #ifdef notyet
2079 		m0->m_flags |= M_TSTMP;
2080 #endif
2081 	}
2082 
2083 #ifdef NUMA
2084 	m0->m_pkthdr.numa_domain = ifp->if_numa_domain;
2085 #endif
2086 #if defined(INET) || defined(INET6)
2087 	if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 &&
2088 	    (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 ||
2089 	    M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) {
2090 		if (sort_before_lro(lro)) {
2091 			tcp_lro_queue_mbuf(lro, m0);
2092 			return (0); /* queued for sort, then LRO */
2093 		}
2094 		if (tcp_lro_rx(lro, m0, 0) == 0)
2095 			return (0); /* queued for LRO */
2096 	}
2097 #endif
2098 	ifp->if_input(ifp, m0);
2099 
2100 	return (0);
2101 }
2102 
2103 /*
2104  * Must drain the wrq or make sure that someone else will.
2105  */
2106 static void
2107 wrq_tx_drain(void *arg, int n)
2108 {
2109 	struct sge_wrq *wrq = arg;
2110 	struct sge_eq *eq = &wrq->eq;
2111 
2112 	EQ_LOCK(eq);
2113 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2114 		drain_wrq_wr_list(wrq->adapter, wrq);
2115 	EQ_UNLOCK(eq);
2116 }
2117 
2118 static void
2119 drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq)
2120 {
2121 	struct sge_eq *eq = &wrq->eq;
2122 	u_int available, dbdiff;	/* # of hardware descriptors */
2123 	u_int n;
2124 	struct wrqe *wr;
2125 	struct fw_eth_tx_pkt_wr *dst;	/* any fw WR struct will do */
2126 
2127 	EQ_LOCK_ASSERT_OWNED(eq);
2128 	MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
2129 	wr = STAILQ_FIRST(&wrq->wr_list);
2130 	MPASS(wr != NULL);	/* Must be called with something useful to do */
2131 	MPASS(eq->pidx == eq->dbidx);
2132 	dbdiff = 0;
2133 
2134 	do {
2135 		eq->cidx = read_hw_cidx(eq);
2136 		if (eq->pidx == eq->cidx)
2137 			available = eq->sidx - 1;
2138 		else
2139 			available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2140 
2141 		MPASS(wr->wrq == wrq);
2142 		n = howmany(wr->wr_len, EQ_ESIZE);
2143 		if (available < n)
2144 			break;
2145 
2146 		dst = (void *)&eq->desc[eq->pidx];
2147 		if (__predict_true(eq->sidx - eq->pidx > n)) {
2148 			/* Won't wrap, won't end exactly at the status page. */
2149 			bcopy(&wr->wr[0], dst, wr->wr_len);
2150 			eq->pidx += n;
2151 		} else {
2152 			int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE;
2153 
2154 			bcopy(&wr->wr[0], dst, first_portion);
2155 			if (wr->wr_len > first_portion) {
2156 				bcopy(&wr->wr[first_portion], &eq->desc[0],
2157 				    wr->wr_len - first_portion);
2158 			}
2159 			eq->pidx = n - (eq->sidx - eq->pidx);
2160 		}
2161 		wrq->tx_wrs_copied++;
2162 
2163 		if (available < eq->sidx / 4 &&
2164 		    atomic_cmpset_int(&eq->equiq, 0, 1)) {
2165 				/*
2166 				 * XXX: This is not 100% reliable with some
2167 				 * types of WRs.  But this is a very unusual
2168 				 * situation for an ofld/ctrl queue anyway.
2169 				 */
2170 			dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
2171 			    F_FW_WR_EQUEQ);
2172 		}
2173 
2174 		dbdiff += n;
2175 		if (dbdiff >= 16) {
2176 			ring_eq_db(sc, eq, dbdiff);
2177 			dbdiff = 0;
2178 		}
2179 
2180 		STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
2181 		free_wrqe(wr);
2182 		MPASS(wrq->nwr_pending > 0);
2183 		wrq->nwr_pending--;
2184 		MPASS(wrq->ndesc_needed >= n);
2185 		wrq->ndesc_needed -= n;
2186 	} while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL);
2187 
2188 	if (dbdiff)
2189 		ring_eq_db(sc, eq, dbdiff);
2190 }
2191 
2192 /*
2193  * Doesn't fail.  Holds on to work requests it can't send right away.
2194  */
2195 void
2196 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
2197 {
2198 #ifdef INVARIANTS
2199 	struct sge_eq *eq = &wrq->eq;
2200 #endif
2201 
2202 	EQ_LOCK_ASSERT_OWNED(eq);
2203 	MPASS(wr != NULL);
2204 	MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN);
2205 	MPASS((wr->wr_len & 0x7) == 0);
2206 
2207 	STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
2208 	wrq->nwr_pending++;
2209 	wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE);
2210 
2211 	if (!TAILQ_EMPTY(&wrq->incomplete_wrs))
2212 		return;	/* commit_wrq_wr will drain wr_list as well. */
2213 
2214 	drain_wrq_wr_list(sc, wrq);
2215 
2216 	/* Doorbell must have caught up to the pidx. */
2217 	MPASS(eq->pidx == eq->dbidx);
2218 }
2219 
2220 void
2221 t4_update_fl_bufsize(struct ifnet *ifp)
2222 {
2223 	struct vi_info *vi = ifp->if_softc;
2224 	struct adapter *sc = vi->adapter;
2225 	struct sge_rxq *rxq;
2226 #ifdef TCP_OFFLOAD
2227 	struct sge_ofld_rxq *ofld_rxq;
2228 #endif
2229 	struct sge_fl *fl;
2230 	int i, maxp;
2231 
2232 	maxp = max_rx_payload(sc, ifp, false);
2233 	for_each_rxq(vi, i, rxq) {
2234 		fl = &rxq->fl;
2235 
2236 		FL_LOCK(fl);
2237 		fl->zidx = find_refill_source(sc, maxp,
2238 		    fl->flags & FL_BUF_PACKING);
2239 		FL_UNLOCK(fl);
2240 	}
2241 #ifdef TCP_OFFLOAD
2242 	maxp = max_rx_payload(sc, ifp, true);
2243 	for_each_ofld_rxq(vi, i, ofld_rxq) {
2244 		fl = &ofld_rxq->fl;
2245 
2246 		FL_LOCK(fl);
2247 		fl->zidx = find_refill_source(sc, maxp,
2248 		    fl->flags & FL_BUF_PACKING);
2249 		FL_UNLOCK(fl);
2250 	}
2251 #endif
2252 }
2253 
2254 static inline int
2255 mbuf_nsegs(struct mbuf *m)
2256 {
2257 
2258 	M_ASSERTPKTHDR(m);
2259 	KASSERT(m->m_pkthdr.inner_l5hlen > 0,
2260 	    ("%s: mbuf %p missing information on # of segments.", __func__, m));
2261 
2262 	return (m->m_pkthdr.inner_l5hlen);
2263 }
2264 
2265 static inline void
2266 set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
2267 {
2268 
2269 	M_ASSERTPKTHDR(m);
2270 	m->m_pkthdr.inner_l5hlen = nsegs;
2271 }
2272 
2273 static inline int
2274 mbuf_cflags(struct mbuf *m)
2275 {
2276 
2277 	M_ASSERTPKTHDR(m);
2278 	return (m->m_pkthdr.PH_loc.eight[4]);
2279 }
2280 
2281 static inline void
2282 set_mbuf_cflags(struct mbuf *m, uint8_t flags)
2283 {
2284 
2285 	M_ASSERTPKTHDR(m);
2286 	m->m_pkthdr.PH_loc.eight[4] = flags;
2287 }
2288 
2289 static inline int
2290 mbuf_len16(struct mbuf *m)
2291 {
2292 	int n;
2293 
2294 	M_ASSERTPKTHDR(m);
2295 	n = m->m_pkthdr.PH_loc.eight[0];
2296 	if (!(mbuf_cflags(m) & MC_TLS))
2297 		MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2298 
2299 	return (n);
2300 }
2301 
2302 static inline void
2303 set_mbuf_len16(struct mbuf *m, uint8_t len16)
2304 {
2305 
2306 	M_ASSERTPKTHDR(m);
2307 	if (!(mbuf_cflags(m) & MC_TLS))
2308 		MPASS(len16 > 0 && len16 <= SGE_MAX_WR_LEN / 16);
2309 	m->m_pkthdr.PH_loc.eight[0] = len16;
2310 }
2311 
2312 #ifdef RATELIMIT
2313 static inline int
2314 mbuf_eo_nsegs(struct mbuf *m)
2315 {
2316 
2317 	M_ASSERTPKTHDR(m);
2318 	return (m->m_pkthdr.PH_loc.eight[1]);
2319 }
2320 
2321 #if defined(INET) || defined(INET6)
2322 static inline void
2323 set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs)
2324 {
2325 
2326 	M_ASSERTPKTHDR(m);
2327 	m->m_pkthdr.PH_loc.eight[1] = nsegs;
2328 }
2329 #endif
2330 
2331 static inline int
2332 mbuf_eo_len16(struct mbuf *m)
2333 {
2334 	int n;
2335 
2336 	M_ASSERTPKTHDR(m);
2337 	n = m->m_pkthdr.PH_loc.eight[2];
2338 	MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2339 
2340 	return (n);
2341 }
2342 
2343 #if defined(INET) || defined(INET6)
2344 static inline void
2345 set_mbuf_eo_len16(struct mbuf *m, uint8_t len16)
2346 {
2347 
2348 	M_ASSERTPKTHDR(m);
2349 	m->m_pkthdr.PH_loc.eight[2] = len16;
2350 }
2351 #endif
2352 
2353 static inline int
2354 mbuf_eo_tsclk_tsoff(struct mbuf *m)
2355 {
2356 
2357 	M_ASSERTPKTHDR(m);
2358 	return (m->m_pkthdr.PH_loc.eight[3]);
2359 }
2360 
2361 #if defined(INET) || defined(INET6)
2362 static inline void
2363 set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff)
2364 {
2365 
2366 	M_ASSERTPKTHDR(m);
2367 	m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff;
2368 }
2369 #endif
2370 
2371 static inline int
2372 needs_eo(struct m_snd_tag *mst)
2373 {
2374 
2375 	return (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT);
2376 }
2377 #endif
2378 
2379 /*
2380  * Try to allocate an mbuf to contain a raw work request.  To make it
2381  * easy to construct the work request, don't allocate a chain but a
2382  * single mbuf.
2383  */
2384 struct mbuf *
2385 alloc_wr_mbuf(int len, int how)
2386 {
2387 	struct mbuf *m;
2388 
2389 	if (len <= MHLEN)
2390 		m = m_gethdr(how, MT_DATA);
2391 	else if (len <= MCLBYTES)
2392 		m = m_getcl(how, MT_DATA, M_PKTHDR);
2393 	else
2394 		m = NULL;
2395 	if (m == NULL)
2396 		return (NULL);
2397 	m->m_pkthdr.len = len;
2398 	m->m_len = len;
2399 	set_mbuf_cflags(m, MC_RAW_WR);
2400 	set_mbuf_len16(m, howmany(len, 16));
2401 	return (m);
2402 }
2403 
2404 static inline bool
2405 needs_hwcsum(struct mbuf *m)
2406 {
2407 	const uint32_t csum_flags = CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP |
2408 	    CSUM_IP_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP |
2409 	    CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_IP6_UDP |
2410 	    CSUM_IP6_TCP | CSUM_IP6_TSO | CSUM_INNER_IP6_UDP |
2411 	    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO;
2412 
2413 	M_ASSERTPKTHDR(m);
2414 
2415 	return (m->m_pkthdr.csum_flags & csum_flags);
2416 }
2417 
2418 static inline bool
2419 needs_tso(struct mbuf *m)
2420 {
2421 	const uint32_t csum_flags = CSUM_IP_TSO | CSUM_IP6_TSO |
2422 	    CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
2423 
2424 	M_ASSERTPKTHDR(m);
2425 
2426 	return (m->m_pkthdr.csum_flags & csum_flags);
2427 }
2428 
2429 static inline bool
2430 needs_vxlan_csum(struct mbuf *m)
2431 {
2432 
2433 	M_ASSERTPKTHDR(m);
2434 
2435 	return (m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN);
2436 }
2437 
2438 static inline bool
2439 needs_vxlan_tso(struct mbuf *m)
2440 {
2441 	const uint32_t csum_flags = CSUM_ENCAP_VXLAN | CSUM_INNER_IP_TSO |
2442 	    CSUM_INNER_IP6_TSO;
2443 
2444 	M_ASSERTPKTHDR(m);
2445 
2446 	return ((m->m_pkthdr.csum_flags & csum_flags) != 0 &&
2447 	    (m->m_pkthdr.csum_flags & csum_flags) != CSUM_ENCAP_VXLAN);
2448 }
2449 
2450 #if defined(INET) || defined(INET6)
2451 static inline bool
2452 needs_inner_tcp_csum(struct mbuf *m)
2453 {
2454 	const uint32_t csum_flags = CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
2455 
2456 	M_ASSERTPKTHDR(m);
2457 
2458 	return (m->m_pkthdr.csum_flags & csum_flags);
2459 }
2460 #endif
2461 
2462 static inline bool
2463 needs_l3_csum(struct mbuf *m)
2464 {
2465 	const uint32_t csum_flags = CSUM_IP | CSUM_IP_TSO | CSUM_INNER_IP |
2466 	    CSUM_INNER_IP_TSO;
2467 
2468 	M_ASSERTPKTHDR(m);
2469 
2470 	return (m->m_pkthdr.csum_flags & csum_flags);
2471 }
2472 
2473 static inline bool
2474 needs_outer_tcp_csum(struct mbuf *m)
2475 {
2476 	const uint32_t csum_flags = CSUM_IP_TCP | CSUM_IP_TSO | CSUM_IP6_TCP |
2477 	    CSUM_IP6_TSO;
2478 
2479 	M_ASSERTPKTHDR(m);
2480 
2481 	return (m->m_pkthdr.csum_flags & csum_flags);
2482 }
2483 
2484 #ifdef RATELIMIT
2485 static inline bool
2486 needs_outer_l4_csum(struct mbuf *m)
2487 {
2488 	const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_TSO |
2489 	    CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_TSO;
2490 
2491 	M_ASSERTPKTHDR(m);
2492 
2493 	return (m->m_pkthdr.csum_flags & csum_flags);
2494 }
2495 
2496 static inline bool
2497 needs_outer_udp_csum(struct mbuf *m)
2498 {
2499 	const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP6_UDP;
2500 
2501 	M_ASSERTPKTHDR(m);
2502 
2503 	return (m->m_pkthdr.csum_flags & csum_flags);
2504 }
2505 #endif
2506 
2507 static inline bool
2508 needs_vlan_insertion(struct mbuf *m)
2509 {
2510 
2511 	M_ASSERTPKTHDR(m);
2512 
2513 	return (m->m_flags & M_VLANTAG);
2514 }
2515 
2516 static void *
2517 m_advance(struct mbuf **pm, int *poffset, int len)
2518 {
2519 	struct mbuf *m = *pm;
2520 	int offset = *poffset;
2521 	uintptr_t p = 0;
2522 
2523 	MPASS(len > 0);
2524 
2525 	for (;;) {
2526 		if (offset + len < m->m_len) {
2527 			offset += len;
2528 			p = mtod(m, uintptr_t) + offset;
2529 			break;
2530 		}
2531 		len -= m->m_len - offset;
2532 		m = m->m_next;
2533 		offset = 0;
2534 		MPASS(m != NULL);
2535 	}
2536 	*poffset = offset;
2537 	*pm = m;
2538 	return ((void *)p);
2539 }
2540 
2541 static inline int
2542 count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
2543 {
2544 	vm_paddr_t paddr;
2545 	int i, len, off, pglen, pgoff, seglen, segoff;
2546 	int nsegs = 0;
2547 
2548 	M_ASSERTEXTPG(m);
2549 	off = mtod(m, vm_offset_t);
2550 	len = m->m_len;
2551 	off += skip;
2552 	len -= skip;
2553 
2554 	if (m->m_epg_hdrlen != 0) {
2555 		if (off >= m->m_epg_hdrlen) {
2556 			off -= m->m_epg_hdrlen;
2557 		} else {
2558 			seglen = m->m_epg_hdrlen - off;
2559 			segoff = off;
2560 			seglen = min(seglen, len);
2561 			off = 0;
2562 			len -= seglen;
2563 			paddr = pmap_kextract(
2564 			    (vm_offset_t)&m->m_epg_hdr[segoff]);
2565 			if (*nextaddr != paddr)
2566 				nsegs++;
2567 			*nextaddr = paddr + seglen;
2568 		}
2569 	}
2570 	pgoff = m->m_epg_1st_off;
2571 	for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
2572 		pglen = m_epg_pagelen(m, i, pgoff);
2573 		if (off >= pglen) {
2574 			off -= pglen;
2575 			pgoff = 0;
2576 			continue;
2577 		}
2578 		seglen = pglen - off;
2579 		segoff = pgoff + off;
2580 		off = 0;
2581 		seglen = min(seglen, len);
2582 		len -= seglen;
2583 		paddr = m->m_epg_pa[i] + segoff;
2584 		if (*nextaddr != paddr)
2585 			nsegs++;
2586 		*nextaddr = paddr + seglen;
2587 		pgoff = 0;
2588 	};
2589 	if (len != 0) {
2590 		seglen = min(len, m->m_epg_trllen - off);
2591 		len -= seglen;
2592 		paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
2593 		if (*nextaddr != paddr)
2594 			nsegs++;
2595 		*nextaddr = paddr + seglen;
2596 	}
2597 
2598 	return (nsegs);
2599 }
2600 
2601 
2602 /*
2603  * Can deal with empty mbufs in the chain that have m_len = 0, but the chain
2604  * must have at least one mbuf that's not empty.  It is possible for this
2605  * routine to return 0 if skip accounts for all the contents of the mbuf chain.
2606  */
2607 static inline int
2608 count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags)
2609 {
2610 	vm_paddr_t nextaddr, paddr;
2611 	vm_offset_t va;
2612 	int len, nsegs;
2613 
2614 	M_ASSERTPKTHDR(m);
2615 	MPASS(m->m_pkthdr.len > 0);
2616 	MPASS(m->m_pkthdr.len >= skip);
2617 
2618 	nsegs = 0;
2619 	nextaddr = 0;
2620 	for (; m; m = m->m_next) {
2621 		len = m->m_len;
2622 		if (__predict_false(len == 0))
2623 			continue;
2624 		if (skip >= len) {
2625 			skip -= len;
2626 			continue;
2627 		}
2628 		if ((m->m_flags & M_EXTPG) != 0) {
2629 			*cflags |= MC_NOMAP;
2630 			nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr);
2631 			skip = 0;
2632 			continue;
2633 		}
2634 		va = mtod(m, vm_offset_t) + skip;
2635 		len -= skip;
2636 		skip = 0;
2637 		paddr = pmap_kextract(va);
2638 		nsegs += sglist_count((void *)(uintptr_t)va, len);
2639 		if (paddr == nextaddr)
2640 			nsegs--;
2641 		nextaddr = pmap_kextract(va + len - 1) + 1;
2642 	}
2643 
2644 	return (nsegs);
2645 }
2646 
2647 /*
2648  * The maximum number of segments that can fit in a WR.
2649  */
2650 static int
2651 max_nsegs_allowed(struct mbuf *m, bool vm_wr)
2652 {
2653 
2654 	if (vm_wr) {
2655 		if (needs_tso(m))
2656 			return (TX_SGL_SEGS_VM_TSO);
2657 		return (TX_SGL_SEGS_VM);
2658 	}
2659 
2660 	if (needs_tso(m)) {
2661 		if (needs_vxlan_tso(m))
2662 			return (TX_SGL_SEGS_VXLAN_TSO);
2663 		else
2664 			return (TX_SGL_SEGS_TSO);
2665 	}
2666 
2667 	return (TX_SGL_SEGS);
2668 }
2669 
2670 static struct timeval txerr_ratecheck = {0};
2671 static const struct timeval txerr_interval = {3, 0};
2672 
2673 /*
2674  * Analyze the mbuf to determine its tx needs.  The mbuf passed in may change:
2675  * a) caller can assume it's been freed if this function returns with an error.
2676  * b) it may get defragged up if the gather list is too long for the hardware.
2677  */
2678 int
2679 parse_pkt(struct mbuf **mp, bool vm_wr)
2680 {
2681 	struct mbuf *m0 = *mp, *m;
2682 	int rc, nsegs, defragged = 0, offset;
2683 	struct ether_header *eh;
2684 	void *l3hdr;
2685 #if defined(INET) || defined(INET6)
2686 	struct tcphdr *tcp;
2687 #endif
2688 #if defined(KERN_TLS) || defined(RATELIMIT)
2689 	struct m_snd_tag *mst;
2690 #endif
2691 	uint16_t eh_type;
2692 	uint8_t cflags;
2693 
2694 	cflags = 0;
2695 	M_ASSERTPKTHDR(m0);
2696 	if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) {
2697 		rc = EINVAL;
2698 fail:
2699 		m_freem(m0);
2700 		*mp = NULL;
2701 		return (rc);
2702 	}
2703 restart:
2704 	/*
2705 	 * First count the number of gather list segments in the payload.
2706 	 * Defrag the mbuf if nsegs exceeds the hardware limit.
2707 	 */
2708 	M_ASSERTPKTHDR(m0);
2709 	MPASS(m0->m_pkthdr.len > 0);
2710 	nsegs = count_mbuf_nsegs(m0, 0, &cflags);
2711 #if defined(KERN_TLS) || defined(RATELIMIT)
2712 	if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG)
2713 		mst = m0->m_pkthdr.snd_tag;
2714 	else
2715 		mst = NULL;
2716 #endif
2717 #ifdef KERN_TLS
2718 	if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) {
2719 		int len16;
2720 
2721 		cflags |= MC_TLS;
2722 		set_mbuf_cflags(m0, cflags);
2723 		rc = t6_ktls_parse_pkt(m0, &nsegs, &len16);
2724 		if (rc != 0)
2725 			goto fail;
2726 		set_mbuf_nsegs(m0, nsegs);
2727 		set_mbuf_len16(m0, len16);
2728 		return (0);
2729 	}
2730 #endif
2731 	if (nsegs > max_nsegs_allowed(m0, vm_wr)) {
2732 		if (defragged++ > 0) {
2733 			rc = EFBIG;
2734 			goto fail;
2735 		}
2736 		counter_u64_add(defrags, 1);
2737 		if ((m = m_defrag(m0, M_NOWAIT)) == NULL) {
2738 			rc = ENOMEM;
2739 			goto fail;
2740 		}
2741 		*mp = m0 = m;	/* update caller's copy after defrag */
2742 		goto restart;
2743 	}
2744 
2745 	if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN &&
2746 	    !(cflags & MC_NOMAP))) {
2747 		counter_u64_add(pullups, 1);
2748 		m0 = m_pullup(m0, m0->m_pkthdr.len);
2749 		if (m0 == NULL) {
2750 			/* Should have left well enough alone. */
2751 			rc = EFBIG;
2752 			goto fail;
2753 		}
2754 		*mp = m0;	/* update caller's copy after pullup */
2755 		goto restart;
2756 	}
2757 	set_mbuf_nsegs(m0, nsegs);
2758 	set_mbuf_cflags(m0, cflags);
2759 	calculate_mbuf_len16(m0, vm_wr);
2760 
2761 #ifdef RATELIMIT
2762 	/*
2763 	 * Ethofld is limited to TCP and UDP for now, and only when L4 hw
2764 	 * checksumming is enabled.  needs_outer_l4_csum happens to check for
2765 	 * all the right things.
2766 	 */
2767 	if (__predict_false(needs_eo(mst) && !needs_outer_l4_csum(m0))) {
2768 		m_snd_tag_rele(m0->m_pkthdr.snd_tag);
2769 		m0->m_pkthdr.snd_tag = NULL;
2770 		m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
2771 		mst = NULL;
2772 	}
2773 #endif
2774 
2775 	if (!needs_hwcsum(m0)
2776 #ifdef RATELIMIT
2777    		 && !needs_eo(mst)
2778 #endif
2779 	)
2780 		return (0);
2781 
2782 	m = m0;
2783 	eh = mtod(m, struct ether_header *);
2784 	eh_type = ntohs(eh->ether_type);
2785 	if (eh_type == ETHERTYPE_VLAN) {
2786 		struct ether_vlan_header *evh = (void *)eh;
2787 
2788 		eh_type = ntohs(evh->evl_proto);
2789 		m0->m_pkthdr.l2hlen = sizeof(*evh);
2790 	} else
2791 		m0->m_pkthdr.l2hlen = sizeof(*eh);
2792 
2793 	offset = 0;
2794 	l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
2795 
2796 	switch (eh_type) {
2797 #ifdef INET6
2798 	case ETHERTYPE_IPV6:
2799 		m0->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
2800 		break;
2801 #endif
2802 #ifdef INET
2803 	case ETHERTYPE_IP:
2804 	{
2805 		struct ip *ip = l3hdr;
2806 
2807 		if (needs_vxlan_csum(m0)) {
2808 			/* Driver will do the outer IP hdr checksum. */
2809 			ip->ip_sum = 0;
2810 			if (needs_vxlan_tso(m0)) {
2811 				const uint16_t ipl = ip->ip_len;
2812 
2813 				ip->ip_len = 0;
2814 				ip->ip_sum = ~in_cksum_hdr(ip);
2815 				ip->ip_len = ipl;
2816 			} else
2817 				ip->ip_sum = in_cksum_hdr(ip);
2818 		}
2819 		m0->m_pkthdr.l3hlen = ip->ip_hl << 2;
2820 		break;
2821 	}
2822 #endif
2823 	default:
2824 		if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2825 			log(LOG_ERR, "%s: ethertype 0x%04x unknown.  "
2826 			    "if_cxgbe must be compiled with the same "
2827 			    "INET/INET6 options as the kernel.\n", __func__,
2828 			    eh_type);
2829 		}
2830 		rc = EINVAL;
2831 		goto fail;
2832 	}
2833 
2834 	if (needs_vxlan_csum(m0)) {
2835 		m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2836 		m0->m_pkthdr.l5hlen = sizeof(struct vxlan_header);
2837 
2838 		/* Inner headers. */
2839 		eh = m_advance(&m, &offset, m0->m_pkthdr.l3hlen +
2840 		    sizeof(struct udphdr) + sizeof(struct vxlan_header));
2841 		eh_type = ntohs(eh->ether_type);
2842 		if (eh_type == ETHERTYPE_VLAN) {
2843 			struct ether_vlan_header *evh = (void *)eh;
2844 
2845 			eh_type = ntohs(evh->evl_proto);
2846 			m0->m_pkthdr.inner_l2hlen = sizeof(*evh);
2847 		} else
2848 			m0->m_pkthdr.inner_l2hlen = sizeof(*eh);
2849 		l3hdr = m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen);
2850 
2851 		switch (eh_type) {
2852 #ifdef INET6
2853 		case ETHERTYPE_IPV6:
2854 			m0->m_pkthdr.inner_l3hlen = sizeof(struct ip6_hdr);
2855 			break;
2856 #endif
2857 #ifdef INET
2858 		case ETHERTYPE_IP:
2859 		{
2860 			struct ip *ip = l3hdr;
2861 
2862 			m0->m_pkthdr.inner_l3hlen = ip->ip_hl << 2;
2863 			break;
2864 		}
2865 #endif
2866 		default:
2867 			if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2868 				log(LOG_ERR, "%s: VXLAN hw offload requested"
2869 				    "with unknown ethertype 0x%04x.  if_cxgbe "
2870 				    "must be compiled with the same INET/INET6 "
2871 				    "options as the kernel.\n", __func__,
2872 				    eh_type);
2873 			}
2874 			rc = EINVAL;
2875 			goto fail;
2876 		}
2877 #if defined(INET) || defined(INET6)
2878 		if (needs_inner_tcp_csum(m0)) {
2879 			tcp = m_advance(&m, &offset, m0->m_pkthdr.inner_l3hlen);
2880 			m0->m_pkthdr.inner_l4hlen = tcp->th_off * 4;
2881 		}
2882 #endif
2883 		MPASS((m0->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
2884 		m0->m_pkthdr.csum_flags &= CSUM_INNER_IP6_UDP |
2885 		    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP |
2886 		    CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO |
2887 		    CSUM_ENCAP_VXLAN;
2888 	}
2889 
2890 #if defined(INET) || defined(INET6)
2891 	if (needs_outer_tcp_csum(m0)) {
2892 		tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen);
2893 		m0->m_pkthdr.l4hlen = tcp->th_off * 4;
2894 #ifdef RATELIMIT
2895 		if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) {
2896 			set_mbuf_eo_tsclk_tsoff(m0,
2897 			    V_FW_ETH_TX_EO_WR_TSCLK(tsclk) |
2898 			    V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1));
2899 		} else
2900 			set_mbuf_eo_tsclk_tsoff(m0, 0);
2901 	} else if (needs_outer_udp_csum(m0)) {
2902 		m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2903 #endif
2904 	}
2905 #ifdef RATELIMIT
2906 	if (needs_eo(mst)) {
2907 		u_int immhdrs;
2908 
2909 		/* EO WRs have the headers in the WR and not the GL. */
2910 		immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen +
2911 		    m0->m_pkthdr.l4hlen;
2912 		cflags = 0;
2913 		nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags);
2914 		MPASS(cflags == mbuf_cflags(m0));
2915 		set_mbuf_eo_nsegs(m0, nsegs);
2916 		set_mbuf_eo_len16(m0,
2917 		    txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0)));
2918 	}
2919 #endif
2920 #endif
2921 	MPASS(m0 == *mp);
2922 	return (0);
2923 }
2924 
2925 void *
2926 start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
2927 {
2928 	struct sge_eq *eq = &wrq->eq;
2929 	struct adapter *sc = wrq->adapter;
2930 	int ndesc, available;
2931 	struct wrqe *wr;
2932 	void *w;
2933 
2934 	MPASS(len16 > 0);
2935 	ndesc = tx_len16_to_desc(len16);
2936 	MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
2937 
2938 	EQ_LOCK(eq);
2939 
2940 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2941 		drain_wrq_wr_list(sc, wrq);
2942 
2943 	if (!STAILQ_EMPTY(&wrq->wr_list)) {
2944 slowpath:
2945 		EQ_UNLOCK(eq);
2946 		wr = alloc_wrqe(len16 * 16, wrq);
2947 		if (__predict_false(wr == NULL))
2948 			return (NULL);
2949 		cookie->pidx = -1;
2950 		cookie->ndesc = ndesc;
2951 		return (&wr->wr);
2952 	}
2953 
2954 	eq->cidx = read_hw_cidx(eq);
2955 	if (eq->pidx == eq->cidx)
2956 		available = eq->sidx - 1;
2957 	else
2958 		available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2959 	if (available < ndesc)
2960 		goto slowpath;
2961 
2962 	cookie->pidx = eq->pidx;
2963 	cookie->ndesc = ndesc;
2964 	TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link);
2965 
2966 	w = &eq->desc[eq->pidx];
2967 	IDXINCR(eq->pidx, ndesc, eq->sidx);
2968 	if (__predict_false(cookie->pidx + ndesc > eq->sidx)) {
2969 		w = &wrq->ss[0];
2970 		wrq->ss_pidx = cookie->pidx;
2971 		wrq->ss_len = len16 * 16;
2972 	}
2973 
2974 	EQ_UNLOCK(eq);
2975 
2976 	return (w);
2977 }
2978 
2979 void
2980 commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
2981 {
2982 	struct sge_eq *eq = &wrq->eq;
2983 	struct adapter *sc = wrq->adapter;
2984 	int ndesc, pidx;
2985 	struct wrq_cookie *prev, *next;
2986 
2987 	if (cookie->pidx == -1) {
2988 		struct wrqe *wr = __containerof(w, struct wrqe, wr);
2989 
2990 		t4_wrq_tx(sc, wr);
2991 		return;
2992 	}
2993 
2994 	if (__predict_false(w == &wrq->ss[0])) {
2995 		int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE;
2996 
2997 		MPASS(wrq->ss_len > n);	/* WR had better wrap around. */
2998 		bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n);
2999 		bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n);
3000 		wrq->tx_wrs_ss++;
3001 	} else
3002 		wrq->tx_wrs_direct++;
3003 
3004 	EQ_LOCK(eq);
3005 	ndesc = cookie->ndesc;	/* Can be more than SGE_MAX_WR_NDESC here. */
3006 	pidx = cookie->pidx;
3007 	MPASS(pidx >= 0 && pidx < eq->sidx);
3008 	prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link);
3009 	next = TAILQ_NEXT(cookie, link);
3010 	if (prev == NULL) {
3011 		MPASS(pidx == eq->dbidx);
3012 		if (next == NULL || ndesc >= 16) {
3013 			int available;
3014 			struct fw_eth_tx_pkt_wr *dst;	/* any fw WR struct will do */
3015 
3016 			/*
3017 			 * Note that the WR via which we'll request tx updates
3018 			 * is at pidx and not eq->pidx, which has moved on
3019 			 * already.
3020 			 */
3021 			dst = (void *)&eq->desc[pidx];
3022 			available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
3023 			if (available < eq->sidx / 4 &&
3024 			    atomic_cmpset_int(&eq->equiq, 0, 1)) {
3025 				/*
3026 				 * XXX: This is not 100% reliable with some
3027 				 * types of WRs.  But this is a very unusual
3028 				 * situation for an ofld/ctrl queue anyway.
3029 				 */
3030 				dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
3031 				    F_FW_WR_EQUEQ);
3032 			}
3033 
3034 			ring_eq_db(wrq->adapter, eq, ndesc);
3035 		} else {
3036 			MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
3037 			next->pidx = pidx;
3038 			next->ndesc += ndesc;
3039 		}
3040 	} else {
3041 		MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc);
3042 		prev->ndesc += ndesc;
3043 	}
3044 	TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link);
3045 
3046 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
3047 		drain_wrq_wr_list(sc, wrq);
3048 
3049 #ifdef INVARIANTS
3050 	if (TAILQ_EMPTY(&wrq->incomplete_wrs)) {
3051 		/* Doorbell must have caught up to the pidx. */
3052 		MPASS(wrq->eq.pidx == wrq->eq.dbidx);
3053 	}
3054 #endif
3055 	EQ_UNLOCK(eq);
3056 }
3057 
3058 static u_int
3059 can_resume_eth_tx(struct mp_ring *r)
3060 {
3061 	struct sge_eq *eq = r->cookie;
3062 
3063 	return (total_available_tx_desc(eq) > eq->sidx / 8);
3064 }
3065 
3066 static inline bool
3067 cannot_use_txpkts(struct mbuf *m)
3068 {
3069 	/* maybe put a GL limit too, to avoid silliness? */
3070 
3071 	return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0);
3072 }
3073 
3074 static inline int
3075 discard_tx(struct sge_eq *eq)
3076 {
3077 
3078 	return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED);
3079 }
3080 
3081 static inline int
3082 wr_can_update_eq(void *p)
3083 {
3084 	struct fw_eth_tx_pkts_wr *wr = p;
3085 
3086 	switch (G_FW_WR_OP(be32toh(wr->op_pkd))) {
3087 	case FW_ULPTX_WR:
3088 	case FW_ETH_TX_PKT_WR:
3089 	case FW_ETH_TX_PKTS_WR:
3090 	case FW_ETH_TX_PKTS2_WR:
3091 	case FW_ETH_TX_PKT_VM_WR:
3092 	case FW_ETH_TX_PKTS_VM_WR:
3093 		return (1);
3094 	default:
3095 		return (0);
3096 	}
3097 }
3098 
3099 static inline void
3100 set_txupdate_flags(struct sge_txq *txq, u_int avail,
3101     struct fw_eth_tx_pkt_wr *wr)
3102 {
3103 	struct sge_eq *eq = &txq->eq;
3104 	struct txpkts *txp = &txq->txp;
3105 
3106 	if ((txp->npkt > 0 || avail < eq->sidx / 2) &&
3107 	    atomic_cmpset_int(&eq->equiq, 0, 1)) {
3108 		wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3109 		eq->equeqidx = eq->pidx;
3110 	} else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
3111 		wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
3112 		eq->equeqidx = eq->pidx;
3113 	}
3114 }
3115 
3116 #if defined(__i386__) || defined(__amd64__)
3117 extern uint64_t tsc_freq;
3118 #endif
3119 
3120 static inline bool
3121 record_eth_tx_time(struct sge_txq *txq)
3122 {
3123 	const uint64_t cycles = get_cyclecount();
3124 	const uint64_t last_tx = txq->last_tx;
3125 #if defined(__i386__) || defined(__amd64__)
3126 	const uint64_t itg = tsc_freq * t4_tx_coalesce_gap / 1000000;
3127 #else
3128 	const uint64_t itg = 0;
3129 #endif
3130 
3131 	MPASS(cycles >= last_tx);
3132 	txq->last_tx = cycles;
3133 	return (cycles - last_tx < itg);
3134 }
3135 
3136 /*
3137  * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to
3138  * be consumed.  Return the actual number consumed.  0 indicates a stall.
3139  */
3140 static u_int
3141 eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing)
3142 {
3143 	struct sge_txq *txq = r->cookie;
3144 	struct ifnet *ifp = txq->ifp;
3145 	struct sge_eq *eq = &txq->eq;
3146 	struct txpkts *txp = &txq->txp;
3147 	struct vi_info *vi = ifp->if_softc;
3148 	struct adapter *sc = vi->adapter;
3149 	u_int total, remaining;		/* # of packets */
3150 	u_int n, avail, dbdiff;		/* # of hardware descriptors */
3151 	int i, rc;
3152 	struct mbuf *m0;
3153 	bool snd, recent_tx;
3154 	void *wr;	/* start of the last WR written to the ring */
3155 
3156 	TXQ_LOCK_ASSERT_OWNED(txq);
3157 	recent_tx = record_eth_tx_time(txq);
3158 
3159 	remaining = IDXDIFF(pidx, cidx, r->size);
3160 	if (__predict_false(discard_tx(eq))) {
3161 		for (i = 0; i < txp->npkt; i++)
3162 			m_freem(txp->mb[i]);
3163 		txp->npkt = 0;
3164 		while (cidx != pidx) {
3165 			m0 = r->items[cidx];
3166 			m_freem(m0);
3167 			if (++cidx == r->size)
3168 				cidx = 0;
3169 		}
3170 		reclaim_tx_descs(txq, eq->sidx);
3171 		*coalescing = false;
3172 		return (remaining);	/* emptied */
3173 	}
3174 
3175 	/* How many hardware descriptors do we have readily available. */
3176 	if (eq->pidx == eq->cidx)
3177 		avail = eq->sidx - 1;
3178 	else
3179 		avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
3180 
3181 	total = 0;
3182 	if (remaining == 0) {
3183 		txp->score = 0;
3184 		txq->txpkts_flush++;
3185 		goto send_txpkts;
3186 	}
3187 
3188 	dbdiff = 0;
3189 	MPASS(remaining > 0);
3190 	while (remaining > 0) {
3191 		m0 = r->items[cidx];
3192 		M_ASSERTPKTHDR(m0);
3193 		MPASS(m0->m_nextpkt == NULL);
3194 
3195 		if (avail < 2 * SGE_MAX_WR_NDESC)
3196 			avail += reclaim_tx_descs(txq, 64);
3197 
3198 		if (t4_tx_coalesce == 0 && txp->npkt == 0)
3199 			goto skip_coalescing;
3200 		if (cannot_use_txpkts(m0))
3201 			txp->score = 0;
3202 		else if (recent_tx) {
3203 			if (++txp->score == 0)
3204 				txp->score = UINT8_MAX;
3205 		} else
3206 			txp->score = 1;
3207 		if (txp->npkt > 0 || remaining > 1 ||
3208 		    txp->score >= t4_tx_coalesce_pkts ||
3209 		    atomic_load_int(&txq->eq.equiq) != 0) {
3210 			if (vi->flags & TX_USES_VM_WR)
3211 				rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd);
3212 			else
3213 				rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd);
3214 		} else {
3215 			snd = false;
3216 			rc = EINVAL;
3217 		}
3218 		if (snd) {
3219 			MPASS(txp->npkt > 0);
3220 			for (i = 0; i < txp->npkt; i++)
3221 				ETHER_BPF_MTAP(ifp, txp->mb[i]);
3222 			if (txp->npkt > 1) {
3223 				MPASS(avail >= tx_len16_to_desc(txp->len16));
3224 				if (vi->flags & TX_USES_VM_WR)
3225 					n = write_txpkts_vm_wr(sc, txq);
3226 				else
3227 					n = write_txpkts_wr(sc, txq);
3228 			} else {
3229 				MPASS(avail >=
3230 				    tx_len16_to_desc(mbuf_len16(txp->mb[0])));
3231 				if (vi->flags & TX_USES_VM_WR)
3232 					n = write_txpkt_vm_wr(sc, txq,
3233 					    txp->mb[0]);
3234 				else
3235 					n = write_txpkt_wr(sc, txq, txp->mb[0],
3236 					    avail);
3237 			}
3238 			MPASS(n <= SGE_MAX_WR_NDESC);
3239 			avail -= n;
3240 			dbdiff += n;
3241 			wr = &eq->desc[eq->pidx];
3242 			IDXINCR(eq->pidx, n, eq->sidx);
3243 			txp->npkt = 0;	/* emptied */
3244 		}
3245 		if (rc == 0) {
3246 			/* m0 was coalesced into txq->txpkts. */
3247 			goto next_mbuf;
3248 		}
3249 		if (rc == EAGAIN) {
3250 			/*
3251 			 * m0 is suitable for tx coalescing but could not be
3252 			 * combined with the existing txq->txpkts, which has now
3253 			 * been transmitted.  Start a new txpkts with m0.
3254 			 */
3255 			MPASS(snd);
3256 			MPASS(txp->npkt == 0);
3257 			continue;
3258 		}
3259 
3260 		MPASS(rc != 0 && rc != EAGAIN);
3261 		MPASS(txp->npkt == 0);
3262 skip_coalescing:
3263 		n = tx_len16_to_desc(mbuf_len16(m0));
3264 		if (__predict_false(avail < n)) {
3265 			avail += reclaim_tx_descs(txq, min(n, 32));
3266 			if (avail < n)
3267 				break;	/* out of descriptors */
3268 		}
3269 
3270 		wr = &eq->desc[eq->pidx];
3271 		if (mbuf_cflags(m0) & MC_RAW_WR) {
3272 			n = write_raw_wr(txq, wr, m0, avail);
3273 #ifdef KERN_TLS
3274 		} else if (mbuf_cflags(m0) & MC_TLS) {
3275 			ETHER_BPF_MTAP(ifp, m0);
3276 			n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0),
3277 			    avail);
3278 #endif
3279 		} else {
3280 			ETHER_BPF_MTAP(ifp, m0);
3281 			if (vi->flags & TX_USES_VM_WR)
3282 				n = write_txpkt_vm_wr(sc, txq, m0);
3283 			else
3284 				n = write_txpkt_wr(sc, txq, m0, avail);
3285 		}
3286 		MPASS(n >= 1 && n <= avail);
3287 		if (!(mbuf_cflags(m0) & MC_TLS))
3288 			MPASS(n <= SGE_MAX_WR_NDESC);
3289 
3290 		avail -= n;
3291 		dbdiff += n;
3292 		IDXINCR(eq->pidx, n, eq->sidx);
3293 
3294 		if (dbdiff >= 512 / EQ_ESIZE) {	/* X_FETCHBURSTMAX_512B */
3295 			if (wr_can_update_eq(wr))
3296 				set_txupdate_flags(txq, avail, wr);
3297 			ring_eq_db(sc, eq, dbdiff);
3298 			avail += reclaim_tx_descs(txq, 32);
3299 			dbdiff = 0;
3300 		}
3301 next_mbuf:
3302 		total++;
3303 		remaining--;
3304 		if (__predict_false(++cidx == r->size))
3305 			cidx = 0;
3306 	}
3307 	if (dbdiff != 0) {
3308 		if (wr_can_update_eq(wr))
3309 			set_txupdate_flags(txq, avail, wr);
3310 		ring_eq_db(sc, eq, dbdiff);
3311 		reclaim_tx_descs(txq, 32);
3312 	} else if (eq->pidx == eq->cidx && txp->npkt > 0 &&
3313 	    atomic_load_int(&txq->eq.equiq) == 0) {
3314 		/*
3315 		 * If nothing was submitted to the chip for tx (it was coalesced
3316 		 * into txpkts instead) and there is no tx update outstanding
3317 		 * then we need to send txpkts now.
3318 		 */
3319 send_txpkts:
3320 		MPASS(txp->npkt > 0);
3321 		for (i = 0; i < txp->npkt; i++)
3322 			ETHER_BPF_MTAP(ifp, txp->mb[i]);
3323 		if (txp->npkt > 1) {
3324 			MPASS(avail >= tx_len16_to_desc(txp->len16));
3325 			if (vi->flags & TX_USES_VM_WR)
3326 				n = write_txpkts_vm_wr(sc, txq);
3327 			else
3328 				n = write_txpkts_wr(sc, txq);
3329 		} else {
3330 			MPASS(avail >=
3331 			    tx_len16_to_desc(mbuf_len16(txp->mb[0])));
3332 			if (vi->flags & TX_USES_VM_WR)
3333 				n = write_txpkt_vm_wr(sc, txq, txp->mb[0]);
3334 			else
3335 				n = write_txpkt_wr(sc, txq, txp->mb[0], avail);
3336 		}
3337 		MPASS(n <= SGE_MAX_WR_NDESC);
3338 		wr = &eq->desc[eq->pidx];
3339 		IDXINCR(eq->pidx, n, eq->sidx);
3340 		txp->npkt = 0;	/* emptied */
3341 
3342 		MPASS(wr_can_update_eq(wr));
3343 		set_txupdate_flags(txq, avail - n, wr);
3344 		ring_eq_db(sc, eq, n);
3345 		reclaim_tx_descs(txq, 32);
3346 	}
3347 	*coalescing = txp->npkt > 0;
3348 
3349 	return (total);
3350 }
3351 
3352 static inline void
3353 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
3354     int qsize, int intr_idx, int cong)
3355 {
3356 
3357 	KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
3358 	    ("%s: bad tmr_idx %d", __func__, tmr_idx));
3359 	KASSERT(pktc_idx < SGE_NCOUNTERS,	/* -ve is ok, means don't use */
3360 	    ("%s: bad pktc_idx %d", __func__, pktc_idx));
3361 	KASSERT(intr_idx >= -1 && intr_idx < sc->intr_count,
3362 	    ("%s: bad intr_idx %d", __func__, intr_idx));
3363 
3364 	iq->flags = 0;
3365 	iq->state = IQS_DISABLED;
3366 	iq->adapter = sc;
3367 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
3368 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
3369 	if (pktc_idx >= 0) {
3370 		iq->intr_params |= F_QINTR_CNT_EN;
3371 		iq->intr_pktc_idx = pktc_idx;
3372 	}
3373 	iq->qsize = roundup2(qsize, 16);	/* See FW_IQ_CMD/iqsize */
3374 	iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE;
3375 	iq->intr_idx = intr_idx;
3376 	iq->cong = cong;
3377 }
3378 
3379 static inline void
3380 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
3381 {
3382 	struct sge_params *sp = &sc->params.sge;
3383 
3384 	fl->qsize = qsize;
3385 	fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
3386 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
3387 	mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
3388 	if (sc->flags & BUF_PACKING_OK &&
3389 	    ((!is_t4(sc) && buffer_packing) ||	/* T5+: enabled unless 0 */
3390 	    (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */
3391 		fl->flags |= FL_BUF_PACKING;
3392 	fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING);
3393 	fl->safe_zidx = sc->sge.safe_zidx;
3394 	if (fl->flags & FL_BUF_PACKING) {
3395 		fl->lowat = roundup2(sp->fl_starve_threshold2, 8);
3396 		fl->buf_boundary = sp->pack_boundary;
3397 	} else {
3398 		fl->lowat = roundup2(sp->fl_starve_threshold, 8);
3399 		fl->buf_boundary = 16;
3400 	}
3401 	if (fl_pad && fl->buf_boundary < sp->pad_boundary)
3402 		fl->buf_boundary = sp->pad_boundary;
3403 }
3404 
3405 static inline void
3406 init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize,
3407     uint8_t tx_chan, struct sge_iq *iq, char *name)
3408 {
3409 	KASSERT(eqtype >= EQ_CTRL && eqtype <= EQ_OFLD,
3410 	    ("%s: bad qtype %d", __func__, eqtype));
3411 
3412 	eq->type = eqtype;
3413 	eq->tx_chan = tx_chan;
3414 	eq->iq = iq;
3415 	eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
3416 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
3417 	mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
3418 }
3419 
3420 int
3421 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
3422     bus_dmamap_t *map, bus_addr_t *pa, void **va)
3423 {
3424 	int rc;
3425 
3426 	rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
3427 	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
3428 	if (rc != 0) {
3429 		CH_ERR(sc, "cannot allocate DMA tag: %d\n", rc);
3430 		goto done;
3431 	}
3432 
3433 	rc = bus_dmamem_alloc(*tag, va,
3434 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
3435 	if (rc != 0) {
3436 		CH_ERR(sc, "cannot allocate DMA memory: %d\n", rc);
3437 		goto done;
3438 	}
3439 
3440 	rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
3441 	if (rc != 0) {
3442 		CH_ERR(sc, "cannot load DMA map: %d\n", rc);
3443 		goto done;
3444 	}
3445 done:
3446 	if (rc)
3447 		free_ring(sc, *tag, *map, *pa, *va);
3448 
3449 	return (rc);
3450 }
3451 
3452 int
3453 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
3454     bus_addr_t pa, void *va)
3455 {
3456 	if (pa)
3457 		bus_dmamap_unload(tag, map);
3458 	if (va)
3459 		bus_dmamem_free(tag, va, map);
3460 	if (tag)
3461 		bus_dma_tag_destroy(tag);
3462 
3463 	return (0);
3464 }
3465 
3466 /*
3467  * Allocates the software resources (mainly memory and sysctl nodes) for an
3468  * ingress queue and an optional freelist.
3469  *
3470  * Sets IQ_SW_ALLOCATED and returns 0 on success.
3471  */
3472 static int
3473 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,
3474     struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
3475 {
3476 	int rc;
3477 	size_t len;
3478 	struct adapter *sc = vi->adapter;
3479 
3480 	MPASS(!(iq->flags & IQ_SW_ALLOCATED));
3481 
3482 	len = iq->qsize * IQ_ESIZE;
3483 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
3484 	    (void **)&iq->desc);
3485 	if (rc != 0)
3486 		return (rc);
3487 
3488 	if (fl) {
3489 		len = fl->qsize * EQ_ESIZE;
3490 		rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
3491 		    &fl->ba, (void **)&fl->desc);
3492 		if (rc) {
3493 			free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba,
3494 			    iq->desc);
3495 			return (rc);
3496 		}
3497 
3498 		/* Allocate space for one software descriptor per buffer. */
3499 		fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc),
3500 		    M_CXGBE, M_ZERO | M_WAITOK);
3501 
3502 		add_fl_sysctls(sc, ctx, oid, fl);
3503 		iq->flags |= IQ_HAS_FL;
3504 	}
3505 	add_iq_sysctls(ctx, oid, iq);
3506 	iq->flags |= IQ_SW_ALLOCATED;
3507 
3508 	return (0);
3509 }
3510 
3511 /*
3512  * Frees all software resources (memory and locks) associated with an ingress
3513  * queue and an optional freelist.
3514  */
3515 static void
3516 free_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
3517 {
3518 	MPASS(iq->flags & IQ_SW_ALLOCATED);
3519 
3520 	if (fl) {
3521 		MPASS(iq->flags & IQ_HAS_FL);
3522 		free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc);
3523 		free_fl_buffers(sc, fl);
3524 		free(fl->sdesc, M_CXGBE);
3525 		mtx_destroy(&fl->fl_lock);
3526 		bzero(fl, sizeof(*fl));
3527 	}
3528 	free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
3529 	bzero(iq, sizeof(*iq));
3530 }
3531 
3532 /*
3533  * Allocates a hardware ingress queue and an optional freelist that will be
3534  * associated with it.
3535  *
3536  * Returns errno on failure.  Resources allocated up to that point may still be
3537  * allocated.  Caller is responsible for cleanup in case this function fails.
3538  */
3539 static int
3540 alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)
3541 {
3542 	int rc, i, cntxt_id;
3543 	struct fw_iq_cmd c;
3544 	struct adapter *sc = vi->adapter;
3545 	__be32 v = 0;
3546 
3547 	MPASS (!(iq->flags & IQ_HW_ALLOCATED));
3548 
3549 	bzero(&c, sizeof(c));
3550 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
3551 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
3552 	    V_FW_IQ_CMD_VFN(0));
3553 
3554 	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
3555 	    FW_LEN16(c));
3556 
3557 	/* Special handling for firmware event queue */
3558 	if (iq == &sc->sge.fwq)
3559 		v |= F_FW_IQ_CMD_IQASYNCH;
3560 
3561 	if (iq->intr_idx < 0) {
3562 		/* Forwarded interrupts, all headed to fwq */
3563 		v |= F_FW_IQ_CMD_IQANDST;
3564 		v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id);
3565 	} else {
3566 		KASSERT(iq->intr_idx < sc->intr_count,
3567 		    ("%s: invalid direct intr_idx %d", __func__, iq->intr_idx));
3568 		v |= V_FW_IQ_CMD_IQANDSTINDEX(iq->intr_idx);
3569 	}
3570 
3571 	bzero(iq->desc, iq->qsize * IQ_ESIZE);
3572 	c.type_to_iqandstindex = htobe32(v |
3573 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
3574 	    V_FW_IQ_CMD_VIID(vi->viid) |
3575 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
3576 	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(vi->pi->tx_chan) |
3577 	    F_FW_IQ_CMD_IQGTSMODE |
3578 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
3579 	    V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
3580 	c.iqsize = htobe16(iq->qsize);
3581 	c.iqaddr = htobe64(iq->ba);
3582 	if (iq->cong >= 0)
3583 		c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
3584 
3585 	if (fl) {
3586 		bzero(fl->desc, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len);
3587 		c.iqns_to_fl0congen |=
3588 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
3589 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
3590 			(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
3591 			(fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
3592 			    0));
3593 		if (iq->cong >= 0) {
3594 			c.iqns_to_fl0congen |=
3595 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(iq->cong) |
3596 				    F_FW_IQ_CMD_FL0CONGCIF |
3597 				    F_FW_IQ_CMD_FL0CONGEN);
3598 		}
3599 		c.fl0dcaen_to_fl0cidxfthresh =
3600 		    htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ?
3601 			X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) |
3602 			V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ?
3603 			X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
3604 		c.fl0size = htobe16(fl->qsize);
3605 		c.fl0addr = htobe64(fl->ba);
3606 	}
3607 
3608 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3609 	if (rc != 0) {
3610 		CH_ERR(sc, "failed to create hw ingress queue: %d\n", rc);
3611 		return (rc);
3612 	}
3613 
3614 	iq->cidx = 0;
3615 	iq->gen = F_RSPD_GEN;
3616 	iq->cntxt_id = be16toh(c.iqid);
3617 	iq->abs_id = be16toh(c.physiqid);
3618 
3619 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
3620 	if (cntxt_id >= sc->sge.iqmap_sz) {
3621 		panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
3622 		    cntxt_id, sc->sge.iqmap_sz - 1);
3623 	}
3624 	sc->sge.iqmap[cntxt_id] = iq;
3625 
3626 	if (fl) {
3627 		u_int qid;
3628 #ifdef INVARIANTS
3629 		MPASS(!(fl->flags & FL_BUF_RESUME));
3630 		for (i = 0; i < fl->sidx * 8; i++)
3631 			MPASS(fl->sdesc[i].cl == NULL);
3632 #endif
3633 		fl->cntxt_id = be16toh(c.fl0id);
3634 		fl->pidx = fl->cidx = fl->hw_cidx = fl->dbidx = 0;
3635 		fl->rx_offset = 0;
3636 		fl->flags &= ~(FL_STARVING | FL_DOOMED);
3637 
3638 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
3639 		if (cntxt_id >= sc->sge.eqmap_sz) {
3640 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
3641 			    __func__, cntxt_id, sc->sge.eqmap_sz - 1);
3642 		}
3643 		sc->sge.eqmap[cntxt_id] = (void *)fl;
3644 
3645 		qid = fl->cntxt_id;
3646 		if (isset(&sc->doorbells, DOORBELL_UDB)) {
3647 			uint32_t s_qpp = sc->params.sge.eq_s_qpp;
3648 			uint32_t mask = (1 << s_qpp) - 1;
3649 			volatile uint8_t *udb;
3650 
3651 			udb = sc->udbs_base + UDBS_DB_OFFSET;
3652 			udb += (qid >> s_qpp) << PAGE_SHIFT;
3653 			qid &= mask;
3654 			if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
3655 				udb += qid << UDBS_SEG_SHIFT;
3656 				qid = 0;
3657 			}
3658 			fl->udb = (volatile void *)udb;
3659 		}
3660 		fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
3661 
3662 		FL_LOCK(fl);
3663 		/* Enough to make sure the SGE doesn't think it's starved */
3664 		refill_fl(sc, fl, fl->lowat);
3665 		FL_UNLOCK(fl);
3666 	}
3667 
3668 	if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && iq->cong >= 0) {
3669 		uint32_t param, val;
3670 
3671 		param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
3672 		    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3673 		    V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
3674 		if (iq->cong == 0)
3675 			val = 1 << 19;
3676 		else {
3677 			val = 2 << 19;
3678 			for (i = 0; i < 4; i++) {
3679 				if (iq->cong & (1 << i))
3680 					val |= 1 << (i << 2);
3681 			}
3682 		}
3683 
3684 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3685 		if (rc != 0) {
3686 			/* report error but carry on */
3687 			CH_ERR(sc, "failed to set congestion manager context "
3688 			    "for ingress queue %d: %d\n", iq->cntxt_id, rc);
3689 		}
3690 	}
3691 
3692 	/* Enable IQ interrupts */
3693 	atomic_store_rel_int(&iq->state, IQS_IDLE);
3694 	t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) |
3695 	    V_INGRESSQID(iq->cntxt_id));
3696 
3697 	iq->flags |= IQ_HW_ALLOCATED;
3698 
3699 	return (0);
3700 }
3701 
3702 static int
3703 free_iq_fl_hwq(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
3704 {
3705 	int rc;
3706 
3707 	MPASS(iq->flags & IQ_HW_ALLOCATED);
3708 	rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
3709 	    iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff);
3710 	if (rc != 0) {
3711 		CH_ERR(sc, "failed to free iq %p: %d\n", iq, rc);
3712 		return (rc);
3713 	}
3714 	iq->flags &= ~IQ_HW_ALLOCATED;
3715 
3716 	return (0);
3717 }
3718 
3719 static void
3720 add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
3721     struct sge_iq *iq)
3722 {
3723 	struct sysctl_oid_list *children;
3724 
3725 	if (ctx == NULL || oid == NULL)
3726 		return;
3727 
3728 	children = SYSCTL_CHILDREN(oid);
3729 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba,
3730 	    "bus address of descriptor ring");
3731 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3732 	    iq->qsize * IQ_ESIZE, "descriptor ring size in bytes");
3733 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
3734 	    &iq->abs_id, 0, "absolute id of the queue");
3735 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3736 	    &iq->cntxt_id, 0, "SGE context id of the queue");
3737 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &iq->cidx,
3738 	    0, "consumer index");
3739 }
3740 
3741 static void
3742 add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
3743     struct sysctl_oid *oid, struct sge_fl *fl)
3744 {
3745 	struct sysctl_oid_list *children;
3746 
3747 	if (ctx == NULL || oid == NULL)
3748 		return;
3749 
3750 	children = SYSCTL_CHILDREN(oid);
3751 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl",
3752 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist");
3753 	children = SYSCTL_CHILDREN(oid);
3754 
3755 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD,
3756 	    &fl->ba, "bus address of descriptor ring");
3757 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3758 	    fl->sidx * EQ_ESIZE + sc->params.sge.spg_len,
3759 	    "desc ring size in bytes");
3760 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3761 	    &fl->cntxt_id, 0, "SGE context id of the freelist");
3762 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL,
3763 	    fl_pad ? 1 : 0, "padding enabled");
3764 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL,
3765 	    fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled");
3766 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
3767 	    0, "consumer index");
3768 	if (fl->flags & FL_BUF_PACKING) {
3769 		SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset",
3770 		    CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
3771 	}
3772 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
3773 	    0, "producer index");
3774 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated",
3775 	    CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
3776 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled",
3777 	    CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
3778 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled",
3779 	    CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
3780 }
3781 
3782 /*
3783  * Idempotent.
3784  */
3785 static int
3786 alloc_fwq(struct adapter *sc)
3787 {
3788 	int rc, intr_idx;
3789 	struct sge_iq *fwq = &sc->sge.fwq;
3790 	struct vi_info *vi = &sc->port[0]->vi[0];
3791 
3792 	if (!(fwq->flags & IQ_SW_ALLOCATED)) {
3793 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3794 
3795 		if (sc->flags & IS_VF)
3796 			intr_idx = 0;
3797 		else
3798 			intr_idx = sc->intr_count > 1 ? 1 : 0;
3799 		init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, intr_idx, -1);
3800 		rc = alloc_iq_fl(vi, fwq, NULL, &sc->ctx, sc->fwq_oid);
3801 		if (rc != 0) {
3802 			CH_ERR(sc, "failed to allocate fwq: %d\n", rc);
3803 			return (rc);
3804 		}
3805 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
3806 	}
3807 
3808 	if (!(fwq->flags & IQ_HW_ALLOCATED)) {
3809 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
3810 
3811 		rc = alloc_iq_fl_hwq(vi, fwq, NULL);
3812 		if (rc != 0) {
3813 			CH_ERR(sc, "failed to create hw fwq: %d\n", rc);
3814 			return (rc);
3815 		}
3816 		MPASS(fwq->flags & IQ_HW_ALLOCATED);
3817 	}
3818 
3819 	return (0);
3820 }
3821 
3822 /*
3823  * Idempotent.
3824  */
3825 static void
3826 free_fwq(struct adapter *sc)
3827 {
3828 	struct sge_iq *fwq = &sc->sge.fwq;
3829 
3830 	if (fwq->flags & IQ_HW_ALLOCATED) {
3831 		MPASS(fwq->flags & IQ_SW_ALLOCATED);
3832 		free_iq_fl_hwq(sc, fwq, NULL);
3833 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3834 	}
3835 
3836 	if (fwq->flags & IQ_SW_ALLOCATED) {
3837 		MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3838 		free_iq_fl(sc, fwq, NULL);
3839 		MPASS(!(fwq->flags & IQ_SW_ALLOCATED));
3840 	}
3841 }
3842 
3843 /*
3844  * Idempotent.
3845  */
3846 static int
3847 alloc_ctrlq(struct adapter *sc, int idx)
3848 {
3849 	int rc;
3850 	char name[16];
3851 	struct sysctl_oid *oid;
3852 	struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
3853 
3854 	MPASS(idx < sc->params.nports);
3855 
3856 	if (!(ctrlq->eq.flags & EQ_SW_ALLOCATED)) {
3857 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3858 
3859 		snprintf(name, sizeof(name), "%d", idx);
3860 		oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(sc->ctrlq_oid),
3861 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3862 		    "ctrl queue");
3863 
3864 		snprintf(name, sizeof(name), "%s ctrlq%d",
3865 		    device_get_nameunit(sc->dev), idx);
3866 		init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE,
3867 		    sc->port[idx]->tx_chan, &sc->sge.fwq, name);
3868 		rc = alloc_wrq(sc, NULL, ctrlq, &sc->ctx, oid);
3869 		if (rc != 0) {
3870 			CH_ERR(sc, "failed to allocate ctrlq%d: %d\n", idx, rc);
3871 			sysctl_remove_oid(oid, 1, 1);
3872 			return (rc);
3873 		}
3874 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3875 	}
3876 
3877 	if (!(ctrlq->eq.flags & EQ_HW_ALLOCATED)) {
3878 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3879 
3880 		rc = alloc_eq_hwq(sc, NULL, &ctrlq->eq);
3881 		if (rc != 0) {
3882 			CH_ERR(sc, "failed to create hw ctrlq%d: %d\n", idx, rc);
3883 			return (rc);
3884 		}
3885 		MPASS(ctrlq->eq.flags & EQ_HW_ALLOCATED);
3886 	}
3887 
3888 	return (0);
3889 }
3890 
3891 /*
3892  * Idempotent.
3893  */
3894 static void
3895 free_ctrlq(struct adapter *sc, int idx)
3896 {
3897 	struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
3898 
3899 	if (ctrlq->eq.flags & EQ_HW_ALLOCATED) {
3900 		MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3901 		free_eq_hwq(sc, NULL, &ctrlq->eq);
3902 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3903 	}
3904 
3905 	if (ctrlq->eq.flags & EQ_SW_ALLOCATED) {
3906 		MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3907 		free_wrq(sc, ctrlq);
3908 		MPASS(!(ctrlq->eq.flags & EQ_SW_ALLOCATED));
3909 	}
3910 }
3911 
3912 int
3913 tnl_cong(struct port_info *pi, int drop)
3914 {
3915 
3916 	if (drop == -1)
3917 		return (-1);
3918 	else if (drop == 1)
3919 		return (0);
3920 	else
3921 		return (pi->rx_e_chan_map);
3922 }
3923 
3924 /*
3925  * Idempotent.
3926  */
3927 static int
3928 alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int idx, int intr_idx,
3929     int maxp)
3930 {
3931 	int rc;
3932 	struct adapter *sc = vi->adapter;
3933 	struct ifnet *ifp = vi->ifp;
3934 	struct sysctl_oid *oid;
3935 	char name[16];
3936 
3937 	if (!(rxq->iq.flags & IQ_SW_ALLOCATED)) {
3938 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
3939 #if defined(INET) || defined(INET6)
3940 		rc = tcp_lro_init_args(&rxq->lro, ifp, lro_entries, lro_mbufs);
3941 		if (rc != 0)
3942 			return (rc);
3943 		MPASS(rxq->lro.ifp == ifp);	/* also indicates LRO init'ed */
3944 #endif
3945 		rxq->ifp = ifp;
3946 
3947 		snprintf(name, sizeof(name), "%d", idx);
3948 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->rxq_oid),
3949 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3950 		    "rx queue");
3951 
3952 		init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq,
3953 		    intr_idx, tnl_cong(vi->pi, cong_drop));
3954 #if defined(INET) || defined(INET6)
3955 		if (ifp->if_capenable & IFCAP_LRO)
3956 			rxq->iq.flags |= IQ_LRO_ENABLED;
3957 #endif
3958 		if (ifp->if_capenable & IFCAP_HWRXTSTMP)
3959 			rxq->iq.flags |= IQ_RX_TIMESTAMP;
3960 		snprintf(name, sizeof(name), "%s rxq%d-fl",
3961 		    device_get_nameunit(vi->dev), idx);
3962 		init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);
3963 		rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid);
3964 		if (rc != 0) {
3965 			CH_ERR(vi, "failed to allocate rxq%d: %d\n", idx, rc);
3966 			sysctl_remove_oid(oid, 1, 1);
3967 #if defined(INET) || defined(INET6)
3968 			tcp_lro_free(&rxq->lro);
3969 			rxq->lro.ifp = NULL;
3970 #endif
3971 			return (rc);
3972 		}
3973 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
3974 		add_rxq_sysctls(&vi->ctx, oid, rxq);
3975 	}
3976 
3977 	if (!(rxq->iq.flags & IQ_HW_ALLOCATED)) {
3978 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
3979 		rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl);
3980 		if (rc != 0) {
3981 			CH_ERR(vi, "failed to create hw rxq%d: %d\n", idx, rc);
3982 			return (rc);
3983 		}
3984 		MPASS(rxq->iq.flags & IQ_HW_ALLOCATED);
3985 
3986 		if (idx == 0)
3987 			sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id;
3988 		else
3989 			KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id,
3990 			    ("iq_base mismatch"));
3991 		KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF,
3992 		    ("PF with non-zero iq_base"));
3993 
3994 		/*
3995 		 * The freelist is just barely above the starvation threshold
3996 		 * right now, fill it up a bit more.
3997 		 */
3998 		FL_LOCK(&rxq->fl);
3999 		refill_fl(sc, &rxq->fl, 128);
4000 		FL_UNLOCK(&rxq->fl);
4001 	}
4002 
4003 	return (0);
4004 }
4005 
4006 /*
4007  * Idempotent.
4008  */
4009 static void
4010 free_rxq(struct vi_info *vi, struct sge_rxq *rxq)
4011 {
4012 	if (rxq->iq.flags & IQ_HW_ALLOCATED) {
4013 		MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
4014 		free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl);
4015 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
4016 	}
4017 
4018 	if (rxq->iq.flags & IQ_SW_ALLOCATED) {
4019 		MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
4020 #if defined(INET) || defined(INET6)
4021 		tcp_lro_free(&rxq->lro);
4022 #endif
4023 		free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl);
4024 		MPASS(!(rxq->iq.flags & IQ_SW_ALLOCATED));
4025 		bzero(rxq, sizeof(*rxq));
4026 	}
4027 }
4028 
4029 static void
4030 add_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4031     struct sge_rxq *rxq)
4032 {
4033 	struct sysctl_oid_list *children;
4034 
4035 	if (ctx == NULL || oid == NULL)
4036 		return;
4037 
4038 	children = SYSCTL_CHILDREN(oid);
4039 #if defined(INET) || defined(INET6)
4040 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
4041 	    &rxq->lro.lro_queued, 0, NULL);
4042 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
4043 	    &rxq->lro.lro_flushed, 0, NULL);
4044 #endif
4045 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
4046 	    &rxq->rxcsum, "# of times hardware assisted with checksum");
4047 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD,
4048 	    &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag");
4049 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_rxcsum", CTLFLAG_RD,
4050 	    &rxq->vxlan_rxcsum,
4051 	    "# of times hardware assisted with inner checksum (VXLAN)");
4052 }
4053 
4054 #ifdef TCP_OFFLOAD
4055 /*
4056  * Idempotent.
4057  */
4058 static int
4059 alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,
4060     int intr_idx, int maxp)
4061 {
4062 	int rc;
4063 	struct adapter *sc = vi->adapter;
4064 	struct sysctl_oid *oid;
4065 	char name[16];
4066 
4067 	if (!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)) {
4068 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4069 
4070 		snprintf(name, sizeof(name), "%d", idx);
4071 		oid = SYSCTL_ADD_NODE(&vi->ctx,
4072 		    SYSCTL_CHILDREN(vi->ofld_rxq_oid), OID_AUTO, name,
4073 		    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload rx queue");
4074 
4075 		init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx,
4076 		    vi->qsize_rxq, intr_idx, 0);
4077 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
4078 		    device_get_nameunit(vi->dev), idx);
4079 		init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);
4080 		rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx,
4081 		    oid);
4082 		if (rc != 0) {
4083 			CH_ERR(vi, "failed to allocate ofld_rxq%d: %d\n", idx,
4084 			    rc);
4085 			sysctl_remove_oid(oid, 1, 1);
4086 			return (rc);
4087 		}
4088 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4089 		ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
4090 		ofld_rxq->rx_iscsi_ddp_setup_error =
4091 		    counter_u64_alloc(M_WAITOK);
4092 		add_ofld_rxq_sysctls(&vi->ctx, oid, ofld_rxq);
4093 	}
4094 
4095 	if (!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)) {
4096 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4097 		rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl);
4098 		if (rc != 0) {
4099 			CH_ERR(vi, "failed to create hw ofld_rxq%d: %d\n", idx,
4100 			    rc);
4101 			return (rc);
4102 		}
4103 		MPASS(ofld_rxq->iq.flags & IQ_HW_ALLOCATED);
4104 	}
4105 	return (rc);
4106 }
4107 
4108 /*
4109  * Idempotent.
4110  */
4111 static void
4112 free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
4113 {
4114 	if (ofld_rxq->iq.flags & IQ_HW_ALLOCATED) {
4115 		MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4116 		free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
4117 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4118 	}
4119 
4120 	if (ofld_rxq->iq.flags & IQ_SW_ALLOCATED) {
4121 		MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4122 		free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
4123 		MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED));
4124 		counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok);
4125 		counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error);
4126 		bzero(ofld_rxq, sizeof(*ofld_rxq));
4127 	}
4128 }
4129 
4130 static void
4131 add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4132     struct sge_ofld_rxq *ofld_rxq)
4133 {
4134 	struct sysctl_oid_list *children;
4135 
4136 	if (ctx == NULL || oid == NULL)
4137 		return;
4138 
4139 	children = SYSCTL_CHILDREN(oid);
4140 	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
4141 	    "rx_toe_tls_records", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_records,
4142 	    "# of TOE TLS records received");
4143 	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
4144 	    "rx_toe_tls_octets", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_octets,
4145 	    "# of payload octets in received TOE TLS records");
4146 
4147 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "iscsi",
4148 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE iSCSI statistics");
4149 	children = SYSCTL_CHILDREN(oid);
4150 
4151 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok",
4152 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_ok,
4153 	    "# of times DDP buffer was setup successfully.");
4154 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error",
4155 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_error,
4156 	    "# of times DDP buffer setup failed.");
4157 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_octets",
4158 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_octets, 0,
4159 	    "# of octets placed directly");
4160 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_pdus",
4161 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_pdus, 0,
4162 	    "# of PDUs with data placed directly.");
4163 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_octets",
4164 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_octets, 0,
4165 	    "# of data octets delivered in freelist");
4166 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_pdus",
4167 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_pdus, 0,
4168 	    "# of PDUs with data delivered in freelist");
4169 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "padding_errors",
4170 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_padding_errors, 0,
4171 	    "# of PDUs with invalid padding");
4172 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "header_digest_errors",
4173 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_header_digest_errors, 0,
4174 	    "# of PDUs with invalid header digests");
4175 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors",
4176 	    CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0,
4177 	    "# of PDUs with invalid data digests");
4178 }
4179 #endif
4180 
4181 /*
4182  * Returns a reasonable automatic cidx flush threshold for a given queue size.
4183  */
4184 static u_int
4185 qsize_to_fthresh(int qsize)
4186 {
4187 	u_int fthresh;
4188 
4189 	while (!powerof2(qsize))
4190 		qsize++;
4191 	fthresh = ilog2(qsize);
4192 	if (fthresh > X_CIDXFLUSHTHRESH_128)
4193 		fthresh = X_CIDXFLUSHTHRESH_128;
4194 
4195 	return (fthresh);
4196 }
4197 
4198 static int
4199 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
4200 {
4201 	int rc, cntxt_id;
4202 	struct fw_eq_ctrl_cmd c;
4203 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4204 
4205 	bzero(&c, sizeof(c));
4206 
4207 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
4208 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
4209 	    V_FW_EQ_CTRL_CMD_VFN(0));
4210 	c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
4211 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
4212 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
4213 	c.physeqid_pkd = htobe32(0);
4214 	c.fetchszm_to_iqid =
4215 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4216 		V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
4217 		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
4218 	c.dcaen_to_eqsize =
4219 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4220 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4221 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4222 		V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
4223 		V_FW_EQ_CTRL_CMD_EQSIZE(qsize));
4224 	c.eqaddr = htobe64(eq->ba);
4225 
4226 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4227 	if (rc != 0) {
4228 		CH_ERR(sc, "failed to create hw ctrlq for tx_chan %d: %d\n",
4229 		    eq->tx_chan, rc);
4230 		return (rc);
4231 	}
4232 
4233 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
4234 	eq->abs_id = G_FW_EQ_CTRL_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4235 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4236 	if (cntxt_id >= sc->sge.eqmap_sz)
4237 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4238 		cntxt_id, sc->sge.eqmap_sz - 1);
4239 	sc->sge.eqmap[cntxt_id] = eq;
4240 
4241 	return (rc);
4242 }
4243 
4244 static int
4245 eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4246 {
4247 	int rc, cntxt_id;
4248 	struct fw_eq_eth_cmd c;
4249 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4250 
4251 	bzero(&c, sizeof(c));
4252 
4253 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
4254 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
4255 	    V_FW_EQ_ETH_CMD_VFN(0));
4256 	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
4257 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
4258 	c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
4259 	    F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
4260 	c.fetchszm_to_iqid =
4261 	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
4262 		V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
4263 		V_FW_EQ_ETH_CMD_IQID(eq->iqid));
4264 	c.dcaen_to_eqsize =
4265 	    htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4266 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4267 		V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4268 		V_FW_EQ_ETH_CMD_EQSIZE(qsize));
4269 	c.eqaddr = htobe64(eq->ba);
4270 
4271 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4272 	if (rc != 0) {
4273 		device_printf(vi->dev,
4274 		    "failed to create Ethernet egress queue: %d\n", rc);
4275 		return (rc);
4276 	}
4277 
4278 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
4279 	eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4280 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4281 	if (cntxt_id >= sc->sge.eqmap_sz)
4282 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4283 		cntxt_id, sc->sge.eqmap_sz - 1);
4284 	sc->sge.eqmap[cntxt_id] = eq;
4285 
4286 	return (rc);
4287 }
4288 
4289 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4290 static int
4291 ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4292 {
4293 	int rc, cntxt_id;
4294 	struct fw_eq_ofld_cmd c;
4295 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4296 
4297 	bzero(&c, sizeof(c));
4298 
4299 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
4300 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
4301 	    V_FW_EQ_OFLD_CMD_VFN(0));
4302 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
4303 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
4304 	c.fetchszm_to_iqid =
4305 		htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4306 		    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
4307 		    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
4308 	c.dcaen_to_eqsize =
4309 	    htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4310 		X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4311 		V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4312 		V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
4313 		V_FW_EQ_OFLD_CMD_EQSIZE(qsize));
4314 	c.eqaddr = htobe64(eq->ba);
4315 
4316 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4317 	if (rc != 0) {
4318 		device_printf(vi->dev,
4319 		    "failed to create egress queue for TCP offload: %d\n", rc);
4320 		return (rc);
4321 	}
4322 
4323 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
4324 	eq->abs_id = G_FW_EQ_OFLD_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4325 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4326 	if (cntxt_id >= sc->sge.eqmap_sz)
4327 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4328 		cntxt_id, sc->sge.eqmap_sz - 1);
4329 	sc->sge.eqmap[cntxt_id] = eq;
4330 
4331 	return (rc);
4332 }
4333 #endif
4334 
4335 /* SW only */
4336 static int
4337 alloc_eq(struct adapter *sc, struct sge_eq *eq, struct sysctl_ctx_list *ctx,
4338     struct sysctl_oid *oid)
4339 {
4340 	int rc, qsize;
4341 	size_t len;
4342 
4343 	MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4344 
4345 	qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4346 	len = qsize * EQ_ESIZE;
4347 	rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba,
4348 	    (void **)&eq->desc);
4349 	if (rc)
4350 		return (rc);
4351 	if (ctx != NULL && oid != NULL)
4352 		add_eq_sysctls(sc, ctx, oid, eq);
4353 	eq->flags |= EQ_SW_ALLOCATED;
4354 
4355 	return (0);
4356 }
4357 
4358 /* SW only */
4359 static void
4360 free_eq(struct adapter *sc, struct sge_eq *eq)
4361 {
4362 	MPASS(eq->flags & EQ_SW_ALLOCATED);
4363 	if (eq->type == EQ_ETH)
4364 		MPASS(eq->pidx == eq->cidx);
4365 
4366 	free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
4367 	mtx_destroy(&eq->eq_lock);
4368 	bzero(eq, sizeof(*eq));
4369 }
4370 
4371 static void
4372 add_eq_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
4373     struct sysctl_oid *oid, struct sge_eq *eq)
4374 {
4375 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
4376 
4377 	SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba,
4378 	    "bus address of descriptor ring");
4379 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
4380 	    eq->sidx * EQ_ESIZE + sc->params.sge.spg_len,
4381 	    "desc ring size in bytes");
4382 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
4383 	    &eq->abs_id, 0, "absolute id of the queue");
4384 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
4385 	    &eq->cntxt_id, 0, "SGE context id of the queue");
4386 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &eq->cidx,
4387 	    0, "consumer index");
4388 	SYSCTL_ADD_U16(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &eq->pidx,
4389 	    0, "producer index");
4390 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL,
4391 	    eq->sidx, "status page index");
4392 }
4393 
4394 static int
4395 alloc_eq_hwq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4396 {
4397 	int rc;
4398 
4399 	MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4400 
4401 	eq->iqid = eq->iq->cntxt_id;
4402 	eq->pidx = eq->cidx = eq->dbidx = 0;
4403 	/* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */
4404 	eq->equeqidx = 0;
4405 	eq->doorbells = sc->doorbells;
4406 	bzero(eq->desc, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len);
4407 
4408 	switch (eq->type) {
4409 	case EQ_CTRL:
4410 		rc = ctrl_eq_alloc(sc, eq);
4411 		break;
4412 
4413 	case EQ_ETH:
4414 		rc = eth_eq_alloc(sc, vi, eq);
4415 		break;
4416 
4417 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4418 	case EQ_OFLD:
4419 		rc = ofld_eq_alloc(sc, vi, eq);
4420 		break;
4421 #endif
4422 
4423 	default:
4424 		panic("%s: invalid eq type %d.", __func__, eq->type);
4425 	}
4426 	if (rc != 0) {
4427 		CH_ERR(sc, "failed to allocate egress queue(%d): %d\n",
4428 		    eq->type, rc);
4429 		return (rc);
4430 	}
4431 
4432 	if (isset(&eq->doorbells, DOORBELL_UDB) ||
4433 	    isset(&eq->doorbells, DOORBELL_UDBWC) ||
4434 	    isset(&eq->doorbells, DOORBELL_WCWR)) {
4435 		uint32_t s_qpp = sc->params.sge.eq_s_qpp;
4436 		uint32_t mask = (1 << s_qpp) - 1;
4437 		volatile uint8_t *udb;
4438 
4439 		udb = sc->udbs_base + UDBS_DB_OFFSET;
4440 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;	/* pg offset */
4441 		eq->udb_qid = eq->cntxt_id & mask;		/* id in page */
4442 		if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
4443 	    		clrbit(&eq->doorbells, DOORBELL_WCWR);
4444 		else {
4445 			udb += eq->udb_qid << UDBS_SEG_SHIFT;	/* seg offset */
4446 			eq->udb_qid = 0;
4447 		}
4448 		eq->udb = (volatile void *)udb;
4449 	}
4450 
4451 	eq->flags |= EQ_HW_ALLOCATED;
4452 	return (0);
4453 }
4454 
4455 static int
4456 free_eq_hwq(struct adapter *sc, struct vi_info *vi __unused, struct sge_eq *eq)
4457 {
4458 	int rc;
4459 
4460 	MPASS(eq->flags & EQ_HW_ALLOCATED);
4461 
4462 	switch (eq->type) {
4463 	case EQ_CTRL:
4464 		rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4465 		break;
4466 	case EQ_ETH:
4467 		rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4468 		break;
4469 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4470 	case EQ_OFLD:
4471 		rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4472 		break;
4473 #endif
4474 	default:
4475 		panic("%s: invalid eq type %d.", __func__, eq->type);
4476 	}
4477 	if (rc != 0) {
4478 		CH_ERR(sc, "failed to free eq (type %d): %d\n", eq->type, rc);
4479 		return (rc);
4480 	}
4481 	eq->flags &= ~EQ_HW_ALLOCATED;
4482 
4483 	return (0);
4484 }
4485 
4486 static int
4487 alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq,
4488     struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
4489 {
4490 	struct sge_eq *eq = &wrq->eq;
4491 	int rc;
4492 
4493 	MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4494 
4495 	rc = alloc_eq(sc, eq, ctx, oid);
4496 	if (rc)
4497 		return (rc);
4498 	MPASS(eq->flags & EQ_SW_ALLOCATED);
4499 	/* Can't fail after this. */
4500 
4501 	wrq->adapter = sc;
4502 	TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq);
4503 	TAILQ_INIT(&wrq->incomplete_wrs);
4504 	STAILQ_INIT(&wrq->wr_list);
4505 	wrq->nwr_pending = 0;
4506 	wrq->ndesc_needed = 0;
4507 	add_wrq_sysctls(ctx, oid, wrq);
4508 
4509 	return (0);
4510 }
4511 
4512 static void
4513 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
4514 {
4515 	free_eq(sc, &wrq->eq);
4516 	MPASS(wrq->nwr_pending == 0);
4517 	MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
4518 	MPASS(STAILQ_EMPTY(&wrq->wr_list));
4519 	bzero(wrq, sizeof(*wrq));
4520 }
4521 
4522 static void
4523 add_wrq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4524     struct sge_wrq *wrq)
4525 {
4526 	struct sysctl_oid_list *children;
4527 
4528 	if (ctx == NULL || oid == NULL)
4529 		return;
4530 
4531 	children = SYSCTL_CHILDREN(oid);
4532 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD,
4533 	    &wrq->tx_wrs_direct, "# of work requests (direct)");
4534 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD,
4535 	    &wrq->tx_wrs_copied, "# of work requests (copied)");
4536 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD,
4537 	    &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)");
4538 }
4539 
4540 /*
4541  * Idempotent.
4542  */
4543 static int
4544 alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx)
4545 {
4546 	int rc, iqidx;
4547 	struct port_info *pi = vi->pi;
4548 	struct adapter *sc = vi->adapter;
4549 	struct sge_eq *eq = &txq->eq;
4550 	struct txpkts *txp;
4551 	char name[16];
4552 	struct sysctl_oid *oid;
4553 
4554 	if (!(eq->flags & EQ_SW_ALLOCATED)) {
4555 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4556 
4557 		snprintf(name, sizeof(name), "%d", idx);
4558 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->txq_oid),
4559 		    OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
4560 		    "tx queue");
4561 
4562 		iqidx = vi->first_rxq + (idx % vi->nrxq);
4563 		snprintf(name, sizeof(name), "%s txq%d",
4564 		    device_get_nameunit(vi->dev), idx);
4565 		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan,
4566 		    &sc->sge.rxq[iqidx].iq, name);
4567 
4568 		rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx,
4569 		    can_resume_eth_tx, M_CXGBE, &eq->eq_lock, M_WAITOK);
4570 		if (rc != 0) {
4571 			CH_ERR(vi, "failed to allocate mp_ring for txq%d: %d\n",
4572 			    idx, rc);
4573 failed:
4574 			sysctl_remove_oid(oid, 1, 1);
4575 			return (rc);
4576 		}
4577 
4578 		rc = alloc_eq(sc, eq, &vi->ctx, oid);
4579 		if (rc) {
4580 			CH_ERR(vi, "failed to allocate txq%d: %d\n", idx, rc);
4581 			mp_ring_free(txq->r);
4582 			goto failed;
4583 		}
4584 		MPASS(eq->flags & EQ_SW_ALLOCATED);
4585 		/* Can't fail after this point. */
4586 
4587 		TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
4588 		txq->ifp = vi->ifp;
4589 		txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
4590 		txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
4591 		    M_ZERO | M_WAITOK);
4592 
4593 		add_txq_sysctls(vi, &vi->ctx, oid, txq);
4594 	}
4595 
4596 	if (!(eq->flags & EQ_HW_ALLOCATED)) {
4597 		MPASS(eq->flags & EQ_SW_ALLOCATED);
4598 		rc = alloc_eq_hwq(sc, vi, eq);
4599 		if (rc != 0) {
4600 			CH_ERR(vi, "failed to create hw txq%d: %d\n", idx, rc);
4601 			return (rc);
4602 		}
4603 		MPASS(eq->flags & EQ_HW_ALLOCATED);
4604 		/* Can't fail after this point. */
4605 
4606 		if (idx == 0)
4607 			sc->sge.eq_base = eq->abs_id - eq->cntxt_id;
4608 		else
4609 			KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id,
4610 			    ("eq_base mismatch"));
4611 		KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF,
4612 		    ("PF with non-zero eq_base"));
4613 
4614 		txp = &txq->txp;
4615 		MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr);
4616 		txq->txp.max_npkt = min(nitems(txp->mb),
4617 		    sc->params.max_pkts_per_eth_tx_pkts_wr);
4618 		if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF))
4619 			txq->txp.max_npkt--;
4620 
4621 		if (vi->flags & TX_USES_VM_WR)
4622 			txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
4623 			    V_TXPKT_INTF(pi->tx_chan));
4624 		else
4625 			txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
4626 			    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
4627 			    V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
4628 
4629 		txq->tc_idx = -1;
4630 	}
4631 
4632 	return (0);
4633 }
4634 
4635 /*
4636  * Idempotent.
4637  */
4638 static void
4639 free_txq(struct vi_info *vi, struct sge_txq *txq)
4640 {
4641 	struct adapter *sc = vi->adapter;
4642 	struct sge_eq *eq = &txq->eq;
4643 
4644 	if (eq->flags & EQ_HW_ALLOCATED) {
4645 		MPASS(eq->flags & EQ_SW_ALLOCATED);
4646 		free_eq_hwq(sc, NULL, eq);
4647 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4648 	}
4649 
4650 	if (eq->flags & EQ_SW_ALLOCATED) {
4651 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4652 		sglist_free(txq->gl);
4653 		free(txq->sdesc, M_CXGBE);
4654 		mp_ring_free(txq->r);
4655 		free_eq(sc, eq);
4656 		MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4657 		bzero(txq, sizeof(*txq));
4658 	}
4659 }
4660 
4661 static void
4662 add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx,
4663     struct sysctl_oid *oid, struct sge_txq *txq)
4664 {
4665 	struct adapter *sc;
4666 	struct sysctl_oid_list *children;
4667 
4668 	if (ctx == NULL || oid == NULL)
4669 		return;
4670 
4671 	sc = vi->adapter;
4672 	children = SYSCTL_CHILDREN(oid);
4673 
4674 	mp_ring_sysctls(txq->r, ctx, children);
4675 
4676 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tc",
4677 	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, txq - sc->sge.txq,
4678 	    sysctl_tc, "I", "traffic class (-1 means none)");
4679 
4680 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
4681 	    &txq->txcsum, "# of times hardware assisted with checksum");
4682 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD,
4683 	    &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag");
4684 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
4685 	    &txq->tso_wrs, "# of TSO work requests");
4686 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
4687 	    &txq->imm_wrs, "# of work requests with immediate data");
4688 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
4689 	    &txq->sgl_wrs, "# of work requests with direct SGL");
4690 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
4691 	    &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
4692 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD,
4693 	    &txq->txpkts0_wrs, "# of txpkts (type 0) work requests");
4694 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD,
4695 	    &txq->txpkts1_wrs, "# of txpkts (type 1) work requests");
4696 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD,
4697 	    &txq->txpkts0_pkts,
4698 	    "# of frames tx'd using type0 txpkts work requests");
4699 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD,
4700 	    &txq->txpkts1_pkts,
4701 	    "# of frames tx'd using type1 txpkts work requests");
4702 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts_flush", CTLFLAG_RD,
4703 	    &txq->txpkts_flush,
4704 	    "# of times txpkts had to be flushed out by an egress-update");
4705 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD,
4706 	    &txq->raw_wrs, "# of raw work requests (non-packets)");
4707 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_tso_wrs", CTLFLAG_RD,
4708 	    &txq->vxlan_tso_wrs, "# of VXLAN TSO work requests");
4709 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_txcsum", CTLFLAG_RD,
4710 	    &txq->vxlan_txcsum,
4711 	    "# of times hardware assisted with inner checksums (VXLAN)");
4712 
4713 #ifdef KERN_TLS
4714 	if (is_ktls(sc)) {
4715 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_records",
4716 		    CTLFLAG_RD, &txq->kern_tls_records,
4717 		    "# of NIC TLS records transmitted");
4718 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_short",
4719 		    CTLFLAG_RD, &txq->kern_tls_short,
4720 		    "# of short NIC TLS records transmitted");
4721 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_partial",
4722 		    CTLFLAG_RD, &txq->kern_tls_partial,
4723 		    "# of partial NIC TLS records transmitted");
4724 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_full",
4725 		    CTLFLAG_RD, &txq->kern_tls_full,
4726 		    "# of full NIC TLS records transmitted");
4727 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_octets",
4728 		    CTLFLAG_RD, &txq->kern_tls_octets,
4729 		    "# of payload octets in transmitted NIC TLS records");
4730 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_waste",
4731 		    CTLFLAG_RD, &txq->kern_tls_waste,
4732 		    "# of octets DMAd but not transmitted in NIC TLS records");
4733 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_options",
4734 		    CTLFLAG_RD, &txq->kern_tls_options,
4735 		    "# of NIC TLS options-only packets transmitted");
4736 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_header",
4737 		    CTLFLAG_RD, &txq->kern_tls_header,
4738 		    "# of NIC TLS header-only packets transmitted");
4739 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin",
4740 		    CTLFLAG_RD, &txq->kern_tls_fin,
4741 		    "# of NIC TLS FIN-only packets transmitted");
4742 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin_short",
4743 		    CTLFLAG_RD, &txq->kern_tls_fin_short,
4744 		    "# of NIC TLS padded FIN packets on short TLS records");
4745 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_cbc",
4746 		    CTLFLAG_RD, &txq->kern_tls_cbc,
4747 		    "# of NIC TLS sessions using AES-CBC");
4748 		SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_gcm",
4749 		    CTLFLAG_RD, &txq->kern_tls_gcm,
4750 		    "# of NIC TLS sessions using AES-GCM");
4751 	}
4752 #endif
4753 }
4754 
4755 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4756 /*
4757  * Idempotent.
4758  */
4759 static int
4760 alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)
4761 {
4762 	struct sysctl_oid *oid;
4763 	struct port_info *pi = vi->pi;
4764 	struct adapter *sc = vi->adapter;
4765 	struct sge_eq *eq = &ofld_txq->wrq.eq;
4766 	int rc, iqidx;
4767 	char name[16];
4768 
4769 	MPASS(idx >= 0);
4770 	MPASS(idx < vi->nofldtxq);
4771 
4772 	if (!(eq->flags & EQ_SW_ALLOCATED)) {
4773 		snprintf(name, sizeof(name), "%d", idx);
4774 		oid = SYSCTL_ADD_NODE(&vi->ctx,
4775 		    SYSCTL_CHILDREN(vi->ofld_txq_oid), OID_AUTO, name,
4776 		    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue");
4777 
4778 		snprintf(name, sizeof(name), "%s ofld_txq%d",
4779 		    device_get_nameunit(vi->dev), idx);
4780 		if (vi->nofldrxq > 0) {
4781 			iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq);
4782 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
4783 			    &sc->sge.ofld_rxq[iqidx].iq, name);
4784 		} else {
4785 			iqidx = vi->first_rxq + (idx % vi->nrxq);
4786 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
4787 			    &sc->sge.rxq[iqidx].iq, name);
4788 		}
4789 
4790 		rc = alloc_wrq(sc, vi, &ofld_txq->wrq, &vi->ctx, oid);
4791 		if (rc != 0) {
4792 			CH_ERR(vi, "failed to allocate ofld_txq%d: %d\n", idx,
4793 			    rc);
4794 			sysctl_remove_oid(oid, 1, 1);
4795 			return (rc);
4796 		}
4797 		MPASS(eq->flags & EQ_SW_ALLOCATED);
4798 		/* Can't fail after this point. */
4799 
4800 		ofld_txq->tx_iscsi_pdus = counter_u64_alloc(M_WAITOK);
4801 		ofld_txq->tx_iscsi_octets = counter_u64_alloc(M_WAITOK);
4802 		ofld_txq->tx_iscsi_iso_wrs = counter_u64_alloc(M_WAITOK);
4803 		ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK);
4804 		ofld_txq->tx_toe_tls_octets = counter_u64_alloc(M_WAITOK);
4805 		add_ofld_txq_sysctls(&vi->ctx, oid, ofld_txq);
4806 	}
4807 
4808 	if (!(eq->flags & EQ_HW_ALLOCATED)) {
4809 		rc = alloc_eq_hwq(sc, vi, eq);
4810 		if (rc != 0) {
4811 			CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx,
4812 			    rc);
4813 			return (rc);
4814 		}
4815 		MPASS(eq->flags & EQ_HW_ALLOCATED);
4816 	}
4817 
4818 	return (0);
4819 }
4820 
4821 /*
4822  * Idempotent.
4823  */
4824 static void
4825 free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)
4826 {
4827 	struct adapter *sc = vi->adapter;
4828 	struct sge_eq *eq = &ofld_txq->wrq.eq;
4829 
4830 	if (eq->flags & EQ_HW_ALLOCATED) {
4831 		MPASS(eq->flags & EQ_SW_ALLOCATED);
4832 		free_eq_hwq(sc, NULL, eq);
4833 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4834 	}
4835 
4836 	if (eq->flags & EQ_SW_ALLOCATED) {
4837 		MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4838 		counter_u64_free(ofld_txq->tx_iscsi_pdus);
4839 		counter_u64_free(ofld_txq->tx_iscsi_octets);
4840 		counter_u64_free(ofld_txq->tx_iscsi_iso_wrs);
4841 		counter_u64_free(ofld_txq->tx_toe_tls_records);
4842 		counter_u64_free(ofld_txq->tx_toe_tls_octets);
4843 		free_wrq(sc, &ofld_txq->wrq);
4844 		MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4845 		bzero(ofld_txq, sizeof(*ofld_txq));
4846 	}
4847 }
4848 
4849 static void
4850 add_ofld_txq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4851     struct sge_ofld_txq *ofld_txq)
4852 {
4853 	struct sysctl_oid_list *children;
4854 
4855 	if (ctx == NULL || oid == NULL)
4856 		return;
4857 
4858 	children = SYSCTL_CHILDREN(oid);
4859 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_pdus",
4860 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_pdus,
4861 	    "# of iSCSI PDUs transmitted");
4862 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_octets",
4863 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_octets,
4864 	    "# of payload octets in transmitted iSCSI PDUs");
4865 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_iso_wrs",
4866 	    CTLFLAG_RD, &ofld_txq->tx_iscsi_iso_wrs,
4867 	    "# of iSCSI segmentation offload work requests");
4868 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_records",
4869 	    CTLFLAG_RD, &ofld_txq->tx_toe_tls_records,
4870 	    "# of TOE TLS records transmitted");
4871 	SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_octets",
4872 	    CTLFLAG_RD, &ofld_txq->tx_toe_tls_octets,
4873 	    "# of payload octets in transmitted TOE TLS records");
4874 }
4875 #endif
4876 
4877 static void
4878 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4879 {
4880 	bus_addr_t *ba = arg;
4881 
4882 	KASSERT(nseg == 1,
4883 	    ("%s meant for single segment mappings only.", __func__));
4884 
4885 	*ba = error ? 0 : segs->ds_addr;
4886 }
4887 
4888 static inline void
4889 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
4890 {
4891 	uint32_t n, v;
4892 
4893 	n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx);
4894 	MPASS(n > 0);
4895 
4896 	wmb();
4897 	v = fl->dbval | V_PIDX(n);
4898 	if (fl->udb)
4899 		*fl->udb = htole32(v);
4900 	else
4901 		t4_write_reg(sc, sc->sge_kdoorbell_reg, v);
4902 	IDXINCR(fl->dbidx, n, fl->sidx);
4903 }
4904 
4905 /*
4906  * Fills up the freelist by allocating up to 'n' buffers.  Buffers that are
4907  * recycled do not count towards this allocation budget.
4908  *
4909  * Returns non-zero to indicate that this freelist should be added to the list
4910  * of starving freelists.
4911  */
4912 static int
4913 refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
4914 {
4915 	__be64 *d;
4916 	struct fl_sdesc *sd;
4917 	uintptr_t pa;
4918 	caddr_t cl;
4919 	struct rx_buf_info *rxb;
4920 	struct cluster_metadata *clm;
4921 	uint16_t max_pidx, zidx = fl->zidx;
4922 	uint16_t hw_cidx = fl->hw_cidx;		/* stable snapshot */
4923 
4924 	FL_LOCK_ASSERT_OWNED(fl);
4925 
4926 	/*
4927 	 * We always stop at the beginning of the hardware descriptor that's just
4928 	 * before the one with the hw cidx.  This is to avoid hw pidx = hw cidx,
4929 	 * which would mean an empty freelist to the chip.
4930 	 */
4931 	max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
4932 	if (fl->pidx == max_pidx * 8)
4933 		return (0);
4934 
4935 	d = &fl->desc[fl->pidx];
4936 	sd = &fl->sdesc[fl->pidx];
4937 	rxb = &sc->sge.rx_buf_info[zidx];
4938 
4939 	while (n > 0) {
4940 
4941 		if (sd->cl != NULL) {
4942 
4943 			if (sd->nmbuf == 0) {
4944 				/*
4945 				 * Fast recycle without involving any atomics on
4946 				 * the cluster's metadata (if the cluster has
4947 				 * metadata).  This happens when all frames
4948 				 * received in the cluster were small enough to
4949 				 * fit within a single mbuf each.
4950 				 */
4951 				fl->cl_fast_recycled++;
4952 				goto recycled;
4953 			}
4954 
4955 			/*
4956 			 * Cluster is guaranteed to have metadata.  Clusters
4957 			 * without metadata always take the fast recycle path
4958 			 * when they're recycled.
4959 			 */
4960 			clm = cl_metadata(sd);
4961 			MPASS(clm != NULL);
4962 
4963 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
4964 				fl->cl_recycled++;
4965 				counter_u64_add(extfree_rels, 1);
4966 				goto recycled;
4967 			}
4968 			sd->cl = NULL;	/* gave up my reference */
4969 		}
4970 		MPASS(sd->cl == NULL);
4971 		cl = uma_zalloc(rxb->zone, M_NOWAIT);
4972 		if (__predict_false(cl == NULL)) {
4973 			if (zidx != fl->safe_zidx) {
4974 				zidx = fl->safe_zidx;
4975 				rxb = &sc->sge.rx_buf_info[zidx];
4976 				cl = uma_zalloc(rxb->zone, M_NOWAIT);
4977 			}
4978 			if (cl == NULL)
4979 				break;
4980 		}
4981 		fl->cl_allocated++;
4982 		n--;
4983 
4984 		pa = pmap_kextract((vm_offset_t)cl);
4985 		sd->cl = cl;
4986 		sd->zidx = zidx;
4987 
4988 		if (fl->flags & FL_BUF_PACKING) {
4989 			*d = htobe64(pa | rxb->hwidx2);
4990 			sd->moff = rxb->size2;
4991 		} else {
4992 			*d = htobe64(pa | rxb->hwidx1);
4993 			sd->moff = 0;
4994 		}
4995 recycled:
4996 		sd->nmbuf = 0;
4997 		d++;
4998 		sd++;
4999 		if (__predict_false((++fl->pidx & 7) == 0)) {
5000 			uint16_t pidx = fl->pidx >> 3;
5001 
5002 			if (__predict_false(pidx == fl->sidx)) {
5003 				fl->pidx = 0;
5004 				pidx = 0;
5005 				sd = fl->sdesc;
5006 				d = fl->desc;
5007 			}
5008 			if (n < 8 || pidx == max_pidx)
5009 				break;
5010 
5011 			if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
5012 				ring_fl_db(sc, fl);
5013 		}
5014 	}
5015 
5016 	if ((fl->pidx >> 3) != fl->dbidx)
5017 		ring_fl_db(sc, fl);
5018 
5019 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
5020 }
5021 
5022 /*
5023  * Attempt to refill all starving freelists.
5024  */
5025 static void
5026 refill_sfl(void *arg)
5027 {
5028 	struct adapter *sc = arg;
5029 	struct sge_fl *fl, *fl_temp;
5030 
5031 	mtx_assert(&sc->sfl_lock, MA_OWNED);
5032 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
5033 		FL_LOCK(fl);
5034 		refill_fl(sc, fl, 64);
5035 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
5036 			TAILQ_REMOVE(&sc->sfl, fl, link);
5037 			fl->flags &= ~FL_STARVING;
5038 		}
5039 		FL_UNLOCK(fl);
5040 	}
5041 
5042 	if (!TAILQ_EMPTY(&sc->sfl))
5043 		callout_schedule(&sc->sfl_callout, hz / 5);
5044 }
5045 
5046 /*
5047  * Release the driver's reference on all buffers in the given freelist.  Buffers
5048  * with kernel references cannot be freed and will prevent the driver from being
5049  * unloaded safely.
5050  */
5051 void
5052 free_fl_buffers(struct adapter *sc, struct sge_fl *fl)
5053 {
5054 	struct fl_sdesc *sd;
5055 	struct cluster_metadata *clm;
5056 	int i;
5057 
5058 	sd = fl->sdesc;
5059 	for (i = 0; i < fl->sidx * 8; i++, sd++) {
5060 		if (sd->cl == NULL)
5061 			continue;
5062 
5063 		if (sd->nmbuf == 0)
5064 			uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl);
5065 		else if (fl->flags & FL_BUF_PACKING) {
5066 			clm = cl_metadata(sd);
5067 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
5068 				uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone,
5069 				    sd->cl);
5070 				counter_u64_add(extfree_rels, 1);
5071 			}
5072 		}
5073 		sd->cl = NULL;
5074 	}
5075 
5076 	if (fl->flags & FL_BUF_RESUME) {
5077 		m_freem(fl->m0);
5078 		fl->flags &= ~FL_BUF_RESUME;
5079 	}
5080 }
5081 
5082 static inline void
5083 get_pkt_gl(struct mbuf *m, struct sglist *gl)
5084 {
5085 	int rc;
5086 
5087 	M_ASSERTPKTHDR(m);
5088 
5089 	sglist_reset(gl);
5090 	rc = sglist_append_mbuf(gl, m);
5091 	if (__predict_false(rc != 0)) {
5092 		panic("%s: mbuf %p (%d segs) was vetted earlier but now fails "
5093 		    "with %d.", __func__, m, mbuf_nsegs(m), rc);
5094 	}
5095 
5096 	KASSERT(gl->sg_nseg == mbuf_nsegs(m),
5097 	    ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m,
5098 	    mbuf_nsegs(m), gl->sg_nseg));
5099 #if 0	/* vm_wr not readily available here. */
5100 	KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= max_nsegs_allowed(m, vm_wr),
5101 	    ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__,
5102 		gl->sg_nseg, max_nsegs_allowed(m, vm_wr)));
5103 #endif
5104 }
5105 
5106 /*
5107  * len16 for a txpkt WR with a GL.  Includes the firmware work request header.
5108  */
5109 static inline u_int
5110 txpkt_len16(u_int nsegs, const u_int extra)
5111 {
5112 	u_int n;
5113 
5114 	MPASS(nsegs > 0);
5115 
5116 	nsegs--; /* first segment is part of ulptx_sgl */
5117 	n = extra + sizeof(struct fw_eth_tx_pkt_wr) +
5118 	    sizeof(struct cpl_tx_pkt_core) +
5119 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
5120 
5121 	return (howmany(n, 16));
5122 }
5123 
5124 /*
5125  * len16 for a txpkt_vm WR with a GL.  Includes the firmware work
5126  * request header.
5127  */
5128 static inline u_int
5129 txpkt_vm_len16(u_int nsegs, const u_int extra)
5130 {
5131 	u_int n;
5132 
5133 	MPASS(nsegs > 0);
5134 
5135 	nsegs--; /* first segment is part of ulptx_sgl */
5136 	n = extra + sizeof(struct fw_eth_tx_pkt_vm_wr) +
5137 	    sizeof(struct cpl_tx_pkt_core) +
5138 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
5139 
5140 	return (howmany(n, 16));
5141 }
5142 
5143 static inline void
5144 calculate_mbuf_len16(struct mbuf *m, bool vm_wr)
5145 {
5146 	const int lso = sizeof(struct cpl_tx_pkt_lso_core);
5147 	const int tnl_lso = sizeof(struct cpl_tx_tnl_lso);
5148 
5149 	if (vm_wr) {
5150 		if (needs_tso(m))
5151 			set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), lso));
5152 		else
5153 			set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), 0));
5154 		return;
5155 	}
5156 
5157 	if (needs_tso(m)) {
5158 		if (needs_vxlan_tso(m))
5159 			set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), tnl_lso));
5160 		else
5161 			set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), lso));
5162 	} else
5163 		set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), 0));
5164 }
5165 
5166 /*
5167  * len16 for a txpkts type 0 WR with a GL.  Does not include the firmware work
5168  * request header.
5169  */
5170 static inline u_int
5171 txpkts0_len16(u_int nsegs)
5172 {
5173 	u_int n;
5174 
5175 	MPASS(nsegs > 0);
5176 
5177 	nsegs--; /* first segment is part of ulptx_sgl */
5178 	n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) +
5179 	    sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) +
5180 	    8 * ((3 * nsegs) / 2 + (nsegs & 1));
5181 
5182 	return (howmany(n, 16));
5183 }
5184 
5185 /*
5186  * len16 for a txpkts type 1 WR with a GL.  Does not include the firmware work
5187  * request header.
5188  */
5189 static inline u_int
5190 txpkts1_len16(void)
5191 {
5192 	u_int n;
5193 
5194 	n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl);
5195 
5196 	return (howmany(n, 16));
5197 }
5198 
5199 static inline u_int
5200 imm_payload(u_int ndesc)
5201 {
5202 	u_int n;
5203 
5204 	n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) -
5205 	    sizeof(struct cpl_tx_pkt_core);
5206 
5207 	return (n);
5208 }
5209 
5210 static inline uint64_t
5211 csum_to_ctrl(struct adapter *sc, struct mbuf *m)
5212 {
5213 	uint64_t ctrl;
5214 	int csum_type, l2hlen, l3hlen;
5215 	int x, y;
5216 	static const int csum_types[3][2] = {
5217 		{TX_CSUM_TCPIP, TX_CSUM_TCPIP6},
5218 		{TX_CSUM_UDPIP, TX_CSUM_UDPIP6},
5219 		{TX_CSUM_IP, 0}
5220 	};
5221 
5222 	M_ASSERTPKTHDR(m);
5223 
5224 	if (!needs_hwcsum(m))
5225 		return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS);
5226 
5227 	MPASS(m->m_pkthdr.l2hlen >= ETHER_HDR_LEN);
5228 	MPASS(m->m_pkthdr.l3hlen >= sizeof(struct ip));
5229 
5230 	if (needs_vxlan_csum(m)) {
5231 		MPASS(m->m_pkthdr.l4hlen > 0);
5232 		MPASS(m->m_pkthdr.l5hlen > 0);
5233 		MPASS(m->m_pkthdr.inner_l2hlen >= ETHER_HDR_LEN);
5234 		MPASS(m->m_pkthdr.inner_l3hlen >= sizeof(struct ip));
5235 
5236 		l2hlen = m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
5237 		    m->m_pkthdr.l4hlen + m->m_pkthdr.l5hlen +
5238 		    m->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN;
5239 		l3hlen = m->m_pkthdr.inner_l3hlen;
5240 	} else {
5241 		l2hlen = m->m_pkthdr.l2hlen - ETHER_HDR_LEN;
5242 		l3hlen = m->m_pkthdr.l3hlen;
5243 	}
5244 
5245 	ctrl = 0;
5246 	if (!needs_l3_csum(m))
5247 		ctrl |= F_TXPKT_IPCSUM_DIS;
5248 
5249 	if (m->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_INNER_IP_TCP |
5250 	    CSUM_IP6_TCP | CSUM_INNER_IP6_TCP))
5251 		x = 0;	/* TCP */
5252 	else if (m->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_INNER_IP_UDP |
5253 	    CSUM_IP6_UDP | CSUM_INNER_IP6_UDP))
5254 		x = 1;	/* UDP */
5255 	else
5256 		x = 2;
5257 
5258 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP |
5259 	    CSUM_INNER_IP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP))
5260 		y = 0;	/* IPv4 */
5261 	else {
5262 		MPASS(m->m_pkthdr.csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP |
5263 		    CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP));
5264 		y = 1;	/* IPv6 */
5265 	}
5266 	/*
5267 	 * needs_hwcsum returned true earlier so there must be some kind of
5268 	 * checksum to calculate.
5269 	 */
5270 	csum_type = csum_types[x][y];
5271 	MPASS(csum_type != 0);
5272 	if (csum_type == TX_CSUM_IP)
5273 		ctrl |= F_TXPKT_L4CSUM_DIS;
5274 	ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | V_TXPKT_IPHDR_LEN(l3hlen);
5275 	if (chip_id(sc) <= CHELSIO_T5)
5276 		ctrl |= V_TXPKT_ETHHDR_LEN(l2hlen);
5277 	else
5278 		ctrl |= V_T6_TXPKT_ETHHDR_LEN(l2hlen);
5279 
5280 	return (ctrl);
5281 }
5282 
5283 static inline void *
5284 write_lso_cpl(void *cpl, struct mbuf *m0)
5285 {
5286 	struct cpl_tx_pkt_lso_core *lso;
5287 	uint32_t ctrl;
5288 
5289 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5290 	    m0->m_pkthdr.l4hlen > 0,
5291 	    ("%s: mbuf %p needs TSO but missing header lengths",
5292 		__func__, m0));
5293 
5294 	ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
5295 	    F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
5296 	    V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5297 	    V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
5298 	    V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
5299 	if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5300 		ctrl |= F_LSO_IPV6;
5301 
5302 	lso = cpl;
5303 	lso->lso_ctrl = htobe32(ctrl);
5304 	lso->ipid_ofst = htobe16(0);
5305 	lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
5306 	lso->seqno_offset = htobe32(0);
5307 	lso->len = htobe32(m0->m_pkthdr.len);
5308 
5309 	return (lso + 1);
5310 }
5311 
5312 static void *
5313 write_tnl_lso_cpl(void *cpl, struct mbuf *m0)
5314 {
5315 	struct cpl_tx_tnl_lso *tnl_lso = cpl;
5316 	uint32_t ctrl;
5317 
5318 	KASSERT(m0->m_pkthdr.inner_l2hlen > 0 &&
5319 	    m0->m_pkthdr.inner_l3hlen > 0 && m0->m_pkthdr.inner_l4hlen > 0 &&
5320 	    m0->m_pkthdr.inner_l5hlen > 0,
5321 	    ("%s: mbuf %p needs VXLAN_TSO but missing inner header lengths",
5322 		__func__, m0));
5323 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5324 	    m0->m_pkthdr.l4hlen > 0 && m0->m_pkthdr.l5hlen > 0,
5325 	    ("%s: mbuf %p needs VXLAN_TSO but missing outer header lengths",
5326 		__func__, m0));
5327 
5328 	/* Outer headers. */
5329 	ctrl = V_CPL_TX_TNL_LSO_OPCODE(CPL_TX_TNL_LSO) |
5330 	    F_CPL_TX_TNL_LSO_FIRST | F_CPL_TX_TNL_LSO_LAST |
5331 	    V_CPL_TX_TNL_LSO_ETHHDRLENOUT(
5332 		(m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5333 	    V_CPL_TX_TNL_LSO_IPHDRLENOUT(m0->m_pkthdr.l3hlen >> 2) |
5334 	    F_CPL_TX_TNL_LSO_IPLENSETOUT;
5335 	if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5336 		ctrl |= F_CPL_TX_TNL_LSO_IPV6OUT;
5337 	else {
5338 		ctrl |= F_CPL_TX_TNL_LSO_IPHDRCHKOUT |
5339 		    F_CPL_TX_TNL_LSO_IPIDINCOUT;
5340 	}
5341 	tnl_lso->op_to_IpIdSplitOut = htobe32(ctrl);
5342 	tnl_lso->IpIdOffsetOut = 0;
5343 	tnl_lso->UdpLenSetOut_to_TnlHdrLen =
5344 		htobe16(F_CPL_TX_TNL_LSO_UDPCHKCLROUT |
5345 		    F_CPL_TX_TNL_LSO_UDPLENSETOUT |
5346 		    V_CPL_TX_TNL_LSO_TNLHDRLEN(m0->m_pkthdr.l2hlen +
5347 			m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen +
5348 			m0->m_pkthdr.l5hlen) |
5349 		    V_CPL_TX_TNL_LSO_TNLTYPE(TX_TNL_TYPE_VXLAN));
5350 	tnl_lso->r1 = 0;
5351 
5352 	/* Inner headers. */
5353 	ctrl = V_CPL_TX_TNL_LSO_ETHHDRLEN(
5354 	    (m0->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN) >> 2) |
5355 	    V_CPL_TX_TNL_LSO_IPHDRLEN(m0->m_pkthdr.inner_l3hlen >> 2) |
5356 	    V_CPL_TX_TNL_LSO_TCPHDRLEN(m0->m_pkthdr.inner_l4hlen >> 2);
5357 	if (m0->m_pkthdr.inner_l3hlen == sizeof(struct ip6_hdr))
5358 		ctrl |= F_CPL_TX_TNL_LSO_IPV6;
5359 	tnl_lso->Flow_to_TcpHdrLen = htobe32(ctrl);
5360 	tnl_lso->IpIdOffset = 0;
5361 	tnl_lso->IpIdSplit_to_Mss =
5362 	    htobe16(V_CPL_TX_TNL_LSO_MSS(m0->m_pkthdr.tso_segsz));
5363 	tnl_lso->TCPSeqOffset = 0;
5364 	tnl_lso->EthLenOffset_Size =
5365 	    htobe32(V_CPL_TX_TNL_LSO_SIZE(m0->m_pkthdr.len));
5366 
5367 	return (tnl_lso + 1);
5368 }
5369 
5370 #define VM_TX_L2HDR_LEN	16	/* ethmacdst to vlantci */
5371 
5372 /*
5373  * Write a VM txpkt WR for this packet to the hardware descriptors, update the
5374  * software descriptor, and advance the pidx.  It is guaranteed that enough
5375  * descriptors are available.
5376  *
5377  * The return value is the # of hardware descriptors used.
5378  */
5379 static u_int
5380 write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0)
5381 {
5382 	struct sge_eq *eq;
5383 	struct fw_eth_tx_pkt_vm_wr *wr;
5384 	struct tx_sdesc *txsd;
5385 	struct cpl_tx_pkt_core *cpl;
5386 	uint32_t ctrl;	/* used in many unrelated places */
5387 	uint64_t ctrl1;
5388 	int len16, ndesc, pktlen, nsegs;
5389 	caddr_t dst;
5390 
5391 	TXQ_LOCK_ASSERT_OWNED(txq);
5392 	M_ASSERTPKTHDR(m0);
5393 
5394 	len16 = mbuf_len16(m0);
5395 	nsegs = mbuf_nsegs(m0);
5396 	pktlen = m0->m_pkthdr.len;
5397 	ctrl = sizeof(struct cpl_tx_pkt_core);
5398 	if (needs_tso(m0))
5399 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
5400 	ndesc = tx_len16_to_desc(len16);
5401 
5402 	/* Firmware work request header */
5403 	eq = &txq->eq;
5404 	wr = (void *)&eq->desc[eq->pidx];
5405 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
5406 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
5407 
5408 	ctrl = V_FW_WR_LEN16(len16);
5409 	wr->equiq_to_len16 = htobe32(ctrl);
5410 	wr->r3[0] = 0;
5411 	wr->r3[1] = 0;
5412 
5413 	/*
5414 	 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci.
5415 	 * vlantci is ignored unless the ethtype is 0x8100, so it's
5416 	 * simpler to always copy it rather than making it
5417 	 * conditional.  Also, it seems that we do not have to set
5418 	 * vlantci or fake the ethtype when doing VLAN tag insertion.
5419 	 */
5420 	m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst);
5421 
5422 	if (needs_tso(m0)) {
5423 		cpl = write_lso_cpl(wr + 1, m0);
5424 		txq->tso_wrs++;
5425 	} else
5426 		cpl = (void *)(wr + 1);
5427 
5428 	/* Checksum offload */
5429 	ctrl1 = csum_to_ctrl(sc, m0);
5430 	if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
5431 		txq->txcsum++;	/* some hardware assistance provided */
5432 
5433 	/* VLAN tag insertion */
5434 	if (needs_vlan_insertion(m0)) {
5435 		ctrl1 |= F_TXPKT_VLAN_VLD |
5436 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
5437 		txq->vlan_insertion++;
5438 	}
5439 
5440 	/* CPL header */
5441 	cpl->ctrl0 = txq->cpl_ctrl0;
5442 	cpl->pack = 0;
5443 	cpl->len = htobe16(pktlen);
5444 	cpl->ctrl1 = htobe64(ctrl1);
5445 
5446 	/* SGL */
5447 	dst = (void *)(cpl + 1);
5448 
5449 	/*
5450 	 * A packet using TSO will use up an entire descriptor for the
5451 	 * firmware work request header, LSO CPL, and TX_PKT_XT CPL.
5452 	 * If this descriptor is the last descriptor in the ring, wrap
5453 	 * around to the front of the ring explicitly for the start of
5454 	 * the sgl.
5455 	 */
5456 	if (dst == (void *)&eq->desc[eq->sidx]) {
5457 		dst = (void *)&eq->desc[0];
5458 		write_gl_to_txd(txq, m0, &dst, 0);
5459 	} else
5460 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
5461 	txq->sgl_wrs++;
5462 	txq->txpkt_wrs++;
5463 
5464 	txsd = &txq->sdesc[eq->pidx];
5465 	txsd->m = m0;
5466 	txsd->desc_used = ndesc;
5467 
5468 	return (ndesc);
5469 }
5470 
5471 /*
5472  * Write a raw WR to the hardware descriptors, update the software
5473  * descriptor, and advance the pidx.  It is guaranteed that enough
5474  * descriptors are available.
5475  *
5476  * The return value is the # of hardware descriptors used.
5477  */
5478 static u_int
5479 write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available)
5480 {
5481 	struct sge_eq *eq = &txq->eq;
5482 	struct tx_sdesc *txsd;
5483 	struct mbuf *m;
5484 	caddr_t dst;
5485 	int len16, ndesc;
5486 
5487 	len16 = mbuf_len16(m0);
5488 	ndesc = tx_len16_to_desc(len16);
5489 	MPASS(ndesc <= available);
5490 
5491 	dst = wr;
5492 	for (m = m0; m != NULL; m = m->m_next)
5493 		copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
5494 
5495 	txq->raw_wrs++;
5496 
5497 	txsd = &txq->sdesc[eq->pidx];
5498 	txsd->m = m0;
5499 	txsd->desc_used = ndesc;
5500 
5501 	return (ndesc);
5502 }
5503 
5504 /*
5505  * Write a txpkt WR for this packet to the hardware descriptors, update the
5506  * software descriptor, and advance the pidx.  It is guaranteed that enough
5507  * descriptors are available.
5508  *
5509  * The return value is the # of hardware descriptors used.
5510  */
5511 static u_int
5512 write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0,
5513     u_int available)
5514 {
5515 	struct sge_eq *eq;
5516 	struct fw_eth_tx_pkt_wr *wr;
5517 	struct tx_sdesc *txsd;
5518 	struct cpl_tx_pkt_core *cpl;
5519 	uint32_t ctrl;	/* used in many unrelated places */
5520 	uint64_t ctrl1;
5521 	int len16, ndesc, pktlen, nsegs;
5522 	caddr_t dst;
5523 
5524 	TXQ_LOCK_ASSERT_OWNED(txq);
5525 	M_ASSERTPKTHDR(m0);
5526 
5527 	len16 = mbuf_len16(m0);
5528 	nsegs = mbuf_nsegs(m0);
5529 	pktlen = m0->m_pkthdr.len;
5530 	ctrl = sizeof(struct cpl_tx_pkt_core);
5531 	if (needs_tso(m0)) {
5532 		if (needs_vxlan_tso(m0))
5533 			ctrl += sizeof(struct cpl_tx_tnl_lso);
5534 		else
5535 			ctrl += sizeof(struct cpl_tx_pkt_lso_core);
5536 	} else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) &&
5537 	    available >= 2) {
5538 		/* Immediate data.  Recalculate len16 and set nsegs to 0. */
5539 		ctrl += pktlen;
5540 		len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) +
5541 		    sizeof(struct cpl_tx_pkt_core) + pktlen, 16);
5542 		nsegs = 0;
5543 	}
5544 	ndesc = tx_len16_to_desc(len16);
5545 	MPASS(ndesc <= available);
5546 
5547 	/* Firmware work request header */
5548 	eq = &txq->eq;
5549 	wr = (void *)&eq->desc[eq->pidx];
5550 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
5551 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
5552 
5553 	ctrl = V_FW_WR_LEN16(len16);
5554 	wr->equiq_to_len16 = htobe32(ctrl);
5555 	wr->r3 = 0;
5556 
5557 	if (needs_tso(m0)) {
5558 		if (needs_vxlan_tso(m0)) {
5559 			cpl = write_tnl_lso_cpl(wr + 1, m0);
5560 			txq->vxlan_tso_wrs++;
5561 		} else {
5562 			cpl = write_lso_cpl(wr + 1, m0);
5563 			txq->tso_wrs++;
5564 		}
5565 	} else
5566 		cpl = (void *)(wr + 1);
5567 
5568 	/* Checksum offload */
5569 	ctrl1 = csum_to_ctrl(sc, m0);
5570 	if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5571 		/* some hardware assistance provided */
5572 		if (needs_vxlan_csum(m0))
5573 			txq->vxlan_txcsum++;
5574 		else
5575 			txq->txcsum++;
5576 	}
5577 
5578 	/* VLAN tag insertion */
5579 	if (needs_vlan_insertion(m0)) {
5580 		ctrl1 |= F_TXPKT_VLAN_VLD |
5581 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
5582 		txq->vlan_insertion++;
5583 	}
5584 
5585 	/* CPL header */
5586 	cpl->ctrl0 = txq->cpl_ctrl0;
5587 	cpl->pack = 0;
5588 	cpl->len = htobe16(pktlen);
5589 	cpl->ctrl1 = htobe64(ctrl1);
5590 
5591 	/* SGL */
5592 	dst = (void *)(cpl + 1);
5593 	if (__predict_false((uintptr_t)dst == (uintptr_t)&eq->desc[eq->sidx]))
5594 		dst = (caddr_t)&eq->desc[0];
5595 	if (nsegs > 0) {
5596 
5597 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
5598 		txq->sgl_wrs++;
5599 	} else {
5600 		struct mbuf *m;
5601 
5602 		for (m = m0; m != NULL; m = m->m_next) {
5603 			copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
5604 #ifdef INVARIANTS
5605 			pktlen -= m->m_len;
5606 #endif
5607 		}
5608 #ifdef INVARIANTS
5609 		KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
5610 #endif
5611 		txq->imm_wrs++;
5612 	}
5613 
5614 	txq->txpkt_wrs++;
5615 
5616 	txsd = &txq->sdesc[eq->pidx];
5617 	txsd->m = m0;
5618 	txsd->desc_used = ndesc;
5619 
5620 	return (ndesc);
5621 }
5622 
5623 static inline bool
5624 cmp_l2hdr(struct txpkts *txp, struct mbuf *m)
5625 {
5626 	int len;
5627 
5628 	MPASS(txp->npkt > 0);
5629 	MPASS(m->m_len >= VM_TX_L2HDR_LEN);
5630 
5631 	if (txp->ethtype == be16toh(ETHERTYPE_VLAN))
5632 		len = VM_TX_L2HDR_LEN;
5633 	else
5634 		len = sizeof(struct ether_header);
5635 
5636 	return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0);
5637 }
5638 
5639 static inline void
5640 save_l2hdr(struct txpkts *txp, struct mbuf *m)
5641 {
5642 	MPASS(m->m_len >= VM_TX_L2HDR_LEN);
5643 
5644 	memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN);
5645 }
5646 
5647 static int
5648 add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5649     int avail, bool *send)
5650 {
5651 	struct txpkts *txp = &txq->txp;
5652 
5653 	/* Cannot have TSO and coalesce at the same time. */
5654 	if (cannot_use_txpkts(m)) {
5655 cannot_coalesce:
5656 		*send = txp->npkt > 0;
5657 		return (EINVAL);
5658 	}
5659 
5660 	/* VF allows coalescing of type 1 (1 GL) only */
5661 	if (mbuf_nsegs(m) > 1)
5662 		goto cannot_coalesce;
5663 
5664 	*send = false;
5665 	if (txp->npkt > 0) {
5666 		MPASS(tx_len16_to_desc(txp->len16) <= avail);
5667 		MPASS(txp->npkt < txp->max_npkt);
5668 		MPASS(txp->wr_type == 1);	/* VF supports type 1 only */
5669 
5670 		if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) {
5671 retry_after_send:
5672 			*send = true;
5673 			return (EAGAIN);
5674 		}
5675 		if (m->m_pkthdr.len + txp->plen > 65535)
5676 			goto retry_after_send;
5677 		if (cmp_l2hdr(txp, m))
5678 			goto retry_after_send;
5679 
5680 		txp->len16 += txpkts1_len16();
5681 		txp->plen += m->m_pkthdr.len;
5682 		txp->mb[txp->npkt++] = m;
5683 		if (txp->npkt == txp->max_npkt)
5684 			*send = true;
5685 	} else {
5686 		txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) +
5687 		    txpkts1_len16();
5688 		if (tx_len16_to_desc(txp->len16) > avail)
5689 			goto cannot_coalesce;
5690 		txp->npkt = 1;
5691 		txp->wr_type = 1;
5692 		txp->plen = m->m_pkthdr.len;
5693 		txp->mb[0] = m;
5694 		save_l2hdr(txp, m);
5695 	}
5696 	return (0);
5697 }
5698 
5699 static int
5700 add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5701     int avail, bool *send)
5702 {
5703 	struct txpkts *txp = &txq->txp;
5704 	int nsegs;
5705 
5706 	MPASS(!(sc->flags & IS_VF));
5707 
5708 	/* Cannot have TSO and coalesce at the same time. */
5709 	if (cannot_use_txpkts(m)) {
5710 cannot_coalesce:
5711 		*send = txp->npkt > 0;
5712 		return (EINVAL);
5713 	}
5714 
5715 	*send = false;
5716 	nsegs = mbuf_nsegs(m);
5717 	if (txp->npkt == 0) {
5718 		if (m->m_pkthdr.len > 65535)
5719 			goto cannot_coalesce;
5720 		if (nsegs > 1) {
5721 			txp->wr_type = 0;
5722 			txp->len16 =
5723 			    howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5724 			    txpkts0_len16(nsegs);
5725 		} else {
5726 			txp->wr_type = 1;
5727 			txp->len16 =
5728 			    howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5729 			    txpkts1_len16();
5730 		}
5731 		if (tx_len16_to_desc(txp->len16) > avail)
5732 			goto cannot_coalesce;
5733 		txp->npkt = 1;
5734 		txp->plen = m->m_pkthdr.len;
5735 		txp->mb[0] = m;
5736 	} else {
5737 		MPASS(tx_len16_to_desc(txp->len16) <= avail);
5738 		MPASS(txp->npkt < txp->max_npkt);
5739 
5740 		if (m->m_pkthdr.len + txp->plen > 65535) {
5741 retry_after_send:
5742 			*send = true;
5743 			return (EAGAIN);
5744 		}
5745 
5746 		MPASS(txp->wr_type == 0 || txp->wr_type == 1);
5747 		if (txp->wr_type == 0) {
5748 			if (tx_len16_to_desc(txp->len16 +
5749 			    txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC))
5750 				goto retry_after_send;
5751 			txp->len16 += txpkts0_len16(nsegs);
5752 		} else {
5753 			if (nsegs != 1)
5754 				goto retry_after_send;
5755 			if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) >
5756 			    avail)
5757 				goto retry_after_send;
5758 			txp->len16 += txpkts1_len16();
5759 		}
5760 
5761 		txp->plen += m->m_pkthdr.len;
5762 		txp->mb[txp->npkt++] = m;
5763 		if (txp->npkt == txp->max_npkt)
5764 			*send = true;
5765 	}
5766 	return (0);
5767 }
5768 
5769 /*
5770  * Write a txpkts WR for the packets in txp to the hardware descriptors, update
5771  * the software descriptor, and advance the pidx.  It is guaranteed that enough
5772  * descriptors are available.
5773  *
5774  * The return value is the # of hardware descriptors used.
5775  */
5776 static u_int
5777 write_txpkts_wr(struct adapter *sc, struct sge_txq *txq)
5778 {
5779 	const struct txpkts *txp = &txq->txp;
5780 	struct sge_eq *eq = &txq->eq;
5781 	struct fw_eth_tx_pkts_wr *wr;
5782 	struct tx_sdesc *txsd;
5783 	struct cpl_tx_pkt_core *cpl;
5784 	uint64_t ctrl1;
5785 	int ndesc, i, checkwrap;
5786 	struct mbuf *m, *last;
5787 	void *flitp;
5788 
5789 	TXQ_LOCK_ASSERT_OWNED(txq);
5790 	MPASS(txp->npkt > 0);
5791 	MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
5792 
5793 	wr = (void *)&eq->desc[eq->pidx];
5794 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
5795 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
5796 	wr->plen = htobe16(txp->plen);
5797 	wr->npkt = txp->npkt;
5798 	wr->r3 = 0;
5799 	wr->type = txp->wr_type;
5800 	flitp = wr + 1;
5801 
5802 	/*
5803 	 * At this point we are 16B into a hardware descriptor.  If checkwrap is
5804 	 * set then we know the WR is going to wrap around somewhere.  We'll
5805 	 * check for that at appropriate points.
5806 	 */
5807 	ndesc = tx_len16_to_desc(txp->len16);
5808 	last = NULL;
5809 	checkwrap = eq->sidx - ndesc < eq->pidx;
5810 	for (i = 0; i < txp->npkt; i++) {
5811 		m = txp->mb[i];
5812 		if (txp->wr_type == 0) {
5813 			struct ulp_txpkt *ulpmc;
5814 			struct ulptx_idata *ulpsc;
5815 
5816 			/* ULP master command */
5817 			ulpmc = flitp;
5818 			ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
5819 			    V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid));
5820 			ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m)));
5821 
5822 			/* ULP subcommand */
5823 			ulpsc = (void *)(ulpmc + 1);
5824 			ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
5825 			    F_ULP_TX_SC_MORE);
5826 			ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
5827 
5828 			cpl = (void *)(ulpsc + 1);
5829 			if (checkwrap &&
5830 			    (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx])
5831 				cpl = (void *)&eq->desc[0];
5832 		} else {
5833 			cpl = flitp;
5834 		}
5835 
5836 		/* Checksum offload */
5837 		ctrl1 = csum_to_ctrl(sc, m);
5838 		if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5839 			/* some hardware assistance provided */
5840 			if (needs_vxlan_csum(m))
5841 				txq->vxlan_txcsum++;
5842 			else
5843 				txq->txcsum++;
5844 		}
5845 
5846 		/* VLAN tag insertion */
5847 		if (needs_vlan_insertion(m)) {
5848 			ctrl1 |= F_TXPKT_VLAN_VLD |
5849 			    V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
5850 			txq->vlan_insertion++;
5851 		}
5852 
5853 		/* CPL header */
5854 		cpl->ctrl0 = txq->cpl_ctrl0;
5855 		cpl->pack = 0;
5856 		cpl->len = htobe16(m->m_pkthdr.len);
5857 		cpl->ctrl1 = htobe64(ctrl1);
5858 
5859 		flitp = cpl + 1;
5860 		if (checkwrap &&
5861 		    (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
5862 			flitp = (void *)&eq->desc[0];
5863 
5864 		write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap);
5865 
5866 		if (last != NULL)
5867 			last->m_nextpkt = m;
5868 		last = m;
5869 	}
5870 
5871 	txq->sgl_wrs++;
5872 	if (txp->wr_type == 0) {
5873 		txq->txpkts0_pkts += txp->npkt;
5874 		txq->txpkts0_wrs++;
5875 	} else {
5876 		txq->txpkts1_pkts += txp->npkt;
5877 		txq->txpkts1_wrs++;
5878 	}
5879 
5880 	txsd = &txq->sdesc[eq->pidx];
5881 	txsd->m = txp->mb[0];
5882 	txsd->desc_used = ndesc;
5883 
5884 	return (ndesc);
5885 }
5886 
5887 static u_int
5888 write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq)
5889 {
5890 	const struct txpkts *txp = &txq->txp;
5891 	struct sge_eq *eq = &txq->eq;
5892 	struct fw_eth_tx_pkts_vm_wr *wr;
5893 	struct tx_sdesc *txsd;
5894 	struct cpl_tx_pkt_core *cpl;
5895 	uint64_t ctrl1;
5896 	int ndesc, i;
5897 	struct mbuf *m, *last;
5898 	void *flitp;
5899 
5900 	TXQ_LOCK_ASSERT_OWNED(txq);
5901 	MPASS(txp->npkt > 0);
5902 	MPASS(txp->wr_type == 1);	/* VF supports type 1 only */
5903 	MPASS(txp->mb[0] != NULL);
5904 	MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
5905 
5906 	wr = (void *)&eq->desc[eq->pidx];
5907 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR));
5908 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
5909 	wr->r3 = 0;
5910 	wr->plen = htobe16(txp->plen);
5911 	wr->npkt = txp->npkt;
5912 	wr->r4 = 0;
5913 	memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16);
5914 	flitp = wr + 1;
5915 
5916 	/*
5917 	 * At this point we are 32B into a hardware descriptor.  Each mbuf in
5918 	 * the WR will take 32B so we check for the end of the descriptor ring
5919 	 * before writing odd mbufs (mb[1], 3, 5, ..)
5920 	 */
5921 	ndesc = tx_len16_to_desc(txp->len16);
5922 	last = NULL;
5923 	for (i = 0; i < txp->npkt; i++) {
5924 		m = txp->mb[i];
5925 		if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
5926 			flitp = &eq->desc[0];
5927 		cpl = flitp;
5928 
5929 		/* Checksum offload */
5930 		ctrl1 = csum_to_ctrl(sc, m);
5931 		if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
5932 			txq->txcsum++;	/* some hardware assistance provided */
5933 
5934 		/* VLAN tag insertion */
5935 		if (needs_vlan_insertion(m)) {
5936 			ctrl1 |= F_TXPKT_VLAN_VLD |
5937 			    V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
5938 			txq->vlan_insertion++;
5939 		}
5940 
5941 		/* CPL header */
5942 		cpl->ctrl0 = txq->cpl_ctrl0;
5943 		cpl->pack = 0;
5944 		cpl->len = htobe16(m->m_pkthdr.len);
5945 		cpl->ctrl1 = htobe64(ctrl1);
5946 
5947 		flitp = cpl + 1;
5948 		MPASS(mbuf_nsegs(m) == 1);
5949 		write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0);
5950 
5951 		if (last != NULL)
5952 			last->m_nextpkt = m;
5953 		last = m;
5954 	}
5955 
5956 	txq->sgl_wrs++;
5957 	txq->txpkts1_pkts += txp->npkt;
5958 	txq->txpkts1_wrs++;
5959 
5960 	txsd = &txq->sdesc[eq->pidx];
5961 	txsd->m = txp->mb[0];
5962 	txsd->desc_used = ndesc;
5963 
5964 	return (ndesc);
5965 }
5966 
5967 /*
5968  * If the SGL ends on an address that is not 16 byte aligned, this function will
5969  * add a 0 filled flit at the end.
5970  */
5971 static void
5972 write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap)
5973 {
5974 	struct sge_eq *eq = &txq->eq;
5975 	struct sglist *gl = txq->gl;
5976 	struct sglist_seg *seg;
5977 	__be64 *flitp, *wrap;
5978 	struct ulptx_sgl *usgl;
5979 	int i, nflits, nsegs;
5980 
5981 	KASSERT(((uintptr_t)(*to) & 0xf) == 0,
5982 	    ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
5983 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
5984 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
5985 
5986 	get_pkt_gl(m, gl);
5987 	nsegs = gl->sg_nseg;
5988 	MPASS(nsegs > 0);
5989 
5990 	nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
5991 	flitp = (__be64 *)(*to);
5992 	wrap = (__be64 *)(&eq->desc[eq->sidx]);
5993 	seg = &gl->sg_segs[0];
5994 	usgl = (void *)flitp;
5995 
5996 	/*
5997 	 * We start at a 16 byte boundary somewhere inside the tx descriptor
5998 	 * ring, so we're at least 16 bytes away from the status page.  There is
5999 	 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
6000 	 */
6001 
6002 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
6003 	    V_ULPTX_NSGE(nsegs));
6004 	usgl->len0 = htobe32(seg->ss_len);
6005 	usgl->addr0 = htobe64(seg->ss_paddr);
6006 	seg++;
6007 
6008 	if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) {
6009 
6010 		/* Won't wrap around at all */
6011 
6012 		for (i = 0; i < nsegs - 1; i++, seg++) {
6013 			usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
6014 			usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
6015 		}
6016 		if (i & 1)
6017 			usgl->sge[i / 2].len[1] = htobe32(0);
6018 		flitp += nflits;
6019 	} else {
6020 
6021 		/* Will wrap somewhere in the rest of the SGL */
6022 
6023 		/* 2 flits already written, write the rest flit by flit */
6024 		flitp = (void *)(usgl + 1);
6025 		for (i = 0; i < nflits - 2; i++) {
6026 			if (flitp == wrap)
6027 				flitp = (void *)eq->desc;
6028 			*flitp++ = get_flit(seg, nsegs - 1, i);
6029 		}
6030 	}
6031 
6032 	if (nflits & 1) {
6033 		MPASS(((uintptr_t)flitp) & 0xf);
6034 		*flitp++ = 0;
6035 	}
6036 
6037 	MPASS((((uintptr_t)flitp) & 0xf) == 0);
6038 	if (__predict_false(flitp == wrap))
6039 		*to = (void *)eq->desc;
6040 	else
6041 		*to = (void *)flitp;
6042 }
6043 
6044 static inline void
6045 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
6046 {
6047 
6048 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
6049 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
6050 
6051 	if (__predict_true((uintptr_t)(*to) + len <=
6052 	    (uintptr_t)&eq->desc[eq->sidx])) {
6053 		bcopy(from, *to, len);
6054 		(*to) += len;
6055 	} else {
6056 		int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
6057 
6058 		bcopy(from, *to, portion);
6059 		from += portion;
6060 		portion = len - portion;	/* remaining */
6061 		bcopy(from, (void *)eq->desc, portion);
6062 		(*to) = (caddr_t)eq->desc + portion;
6063 	}
6064 }
6065 
6066 static inline void
6067 ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n)
6068 {
6069 	u_int db;
6070 
6071 	MPASS(n > 0);
6072 
6073 	db = eq->doorbells;
6074 	if (n > 1)
6075 		clrbit(&db, DOORBELL_WCWR);
6076 	wmb();
6077 
6078 	switch (ffs(db) - 1) {
6079 	case DOORBELL_UDB:
6080 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
6081 		break;
6082 
6083 	case DOORBELL_WCWR: {
6084 		volatile uint64_t *dst, *src;
6085 		int i;
6086 
6087 		/*
6088 		 * Queues whose 128B doorbell segment fits in the page do not
6089 		 * use relative qid (udb_qid is always 0).  Only queues with
6090 		 * doorbell segments can do WCWR.
6091 		 */
6092 		KASSERT(eq->udb_qid == 0 && n == 1,
6093 		    ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
6094 		    __func__, eq->doorbells, n, eq->dbidx, eq));
6095 
6096 		dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
6097 		    UDBS_DB_OFFSET);
6098 		i = eq->dbidx;
6099 		src = (void *)&eq->desc[i];
6100 		while (src != (void *)&eq->desc[i + 1])
6101 			*dst++ = *src++;
6102 		wmb();
6103 		break;
6104 	}
6105 
6106 	case DOORBELL_UDBWC:
6107 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
6108 		wmb();
6109 		break;
6110 
6111 	case DOORBELL_KDB:
6112 		t4_write_reg(sc, sc->sge_kdoorbell_reg,
6113 		    V_QID(eq->cntxt_id) | V_PIDX(n));
6114 		break;
6115 	}
6116 
6117 	IDXINCR(eq->dbidx, n, eq->sidx);
6118 }
6119 
6120 static inline u_int
6121 reclaimable_tx_desc(struct sge_eq *eq)
6122 {
6123 	uint16_t hw_cidx;
6124 
6125 	hw_cidx = read_hw_cidx(eq);
6126 	return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx));
6127 }
6128 
6129 static inline u_int
6130 total_available_tx_desc(struct sge_eq *eq)
6131 {
6132 	uint16_t hw_cidx, pidx;
6133 
6134 	hw_cidx = read_hw_cidx(eq);
6135 	pidx = eq->pidx;
6136 
6137 	if (pidx == hw_cidx)
6138 		return (eq->sidx - 1);
6139 	else
6140 		return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1);
6141 }
6142 
6143 static inline uint16_t
6144 read_hw_cidx(struct sge_eq *eq)
6145 {
6146 	struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
6147 	uint16_t cidx = spg->cidx;	/* stable snapshot */
6148 
6149 	return (be16toh(cidx));
6150 }
6151 
6152 /*
6153  * Reclaim 'n' descriptors approximately.
6154  */
6155 static u_int
6156 reclaim_tx_descs(struct sge_txq *txq, u_int n)
6157 {
6158 	struct tx_sdesc *txsd;
6159 	struct sge_eq *eq = &txq->eq;
6160 	u_int can_reclaim, reclaimed;
6161 
6162 	TXQ_LOCK_ASSERT_OWNED(txq);
6163 	MPASS(n > 0);
6164 
6165 	reclaimed = 0;
6166 	can_reclaim = reclaimable_tx_desc(eq);
6167 	while (can_reclaim && reclaimed < n) {
6168 		int ndesc;
6169 		struct mbuf *m, *nextpkt;
6170 
6171 		txsd = &txq->sdesc[eq->cidx];
6172 		ndesc = txsd->desc_used;
6173 
6174 		/* Firmware doesn't return "partial" credits. */
6175 		KASSERT(can_reclaim >= ndesc,
6176 		    ("%s: unexpected number of credits: %d, %d",
6177 		    __func__, can_reclaim, ndesc));
6178 		KASSERT(ndesc != 0,
6179 		    ("%s: descriptor with no credits: cidx %d",
6180 		    __func__, eq->cidx));
6181 
6182 		for (m = txsd->m; m != NULL; m = nextpkt) {
6183 			nextpkt = m->m_nextpkt;
6184 			m->m_nextpkt = NULL;
6185 			m_freem(m);
6186 		}
6187 		reclaimed += ndesc;
6188 		can_reclaim -= ndesc;
6189 		IDXINCR(eq->cidx, ndesc, eq->sidx);
6190 	}
6191 
6192 	return (reclaimed);
6193 }
6194 
6195 static void
6196 tx_reclaim(void *arg, int n)
6197 {
6198 	struct sge_txq *txq = arg;
6199 	struct sge_eq *eq = &txq->eq;
6200 
6201 	do {
6202 		if (TXQ_TRYLOCK(txq) == 0)
6203 			break;
6204 		n = reclaim_tx_descs(txq, 32);
6205 		if (eq->cidx == eq->pidx)
6206 			eq->equeqidx = eq->pidx;
6207 		TXQ_UNLOCK(txq);
6208 	} while (n > 0);
6209 }
6210 
6211 static __be64
6212 get_flit(struct sglist_seg *segs, int nsegs, int idx)
6213 {
6214 	int i = (idx / 3) * 2;
6215 
6216 	switch (idx % 3) {
6217 	case 0: {
6218 		uint64_t rc;
6219 
6220 		rc = (uint64_t)segs[i].ss_len << 32;
6221 		if (i + 1 < nsegs)
6222 			rc |= (uint64_t)(segs[i + 1].ss_len);
6223 
6224 		return (htobe64(rc));
6225 	}
6226 	case 1:
6227 		return (htobe64(segs[i].ss_paddr));
6228 	case 2:
6229 		return (htobe64(segs[i + 1].ss_paddr));
6230 	}
6231 
6232 	return (0);
6233 }
6234 
6235 static int
6236 find_refill_source(struct adapter *sc, int maxp, bool packing)
6237 {
6238 	int i, zidx = -1;
6239 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
6240 
6241 	if (packing) {
6242 		for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6243 			if (rxb->hwidx2 == -1)
6244 				continue;
6245 			if (rxb->size1 < PAGE_SIZE &&
6246 			    rxb->size1 < largest_rx_cluster)
6247 				continue;
6248 			if (rxb->size1 > largest_rx_cluster)
6249 				break;
6250 			MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE);
6251 			if (rxb->size2 >= maxp)
6252 				return (i);
6253 			zidx = i;
6254 		}
6255 	} else {
6256 		for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6257 			if (rxb->hwidx1 == -1)
6258 				continue;
6259 			if (rxb->size1 > largest_rx_cluster)
6260 				break;
6261 			if (rxb->size1 >= maxp)
6262 				return (i);
6263 			zidx = i;
6264 		}
6265 	}
6266 
6267 	return (zidx);
6268 }
6269 
6270 static void
6271 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
6272 {
6273 	mtx_lock(&sc->sfl_lock);
6274 	FL_LOCK(fl);
6275 	if ((fl->flags & FL_DOOMED) == 0) {
6276 		fl->flags |= FL_STARVING;
6277 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
6278 		callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
6279 	}
6280 	FL_UNLOCK(fl);
6281 	mtx_unlock(&sc->sfl_lock);
6282 }
6283 
6284 static void
6285 handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
6286 {
6287 	struct sge_wrq *wrq = (void *)eq;
6288 
6289 	atomic_readandclear_int(&eq->equiq);
6290 	taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
6291 }
6292 
6293 static void
6294 handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
6295 {
6296 	struct sge_txq *txq = (void *)eq;
6297 
6298 	MPASS(eq->type == EQ_ETH);
6299 
6300 	atomic_readandclear_int(&eq->equiq);
6301 	if (mp_ring_is_idle(txq->r))
6302 		taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
6303 	else
6304 		mp_ring_check_drainage(txq->r, 64);
6305 }
6306 
6307 static int
6308 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
6309     struct mbuf *m)
6310 {
6311 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
6312 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
6313 	struct adapter *sc = iq->adapter;
6314 	struct sge *s = &sc->sge;
6315 	struct sge_eq *eq;
6316 	static void (*h[])(struct adapter *, struct sge_eq *) = {NULL,
6317 		&handle_wrq_egr_update, &handle_eth_egr_update,
6318 		&handle_wrq_egr_update};
6319 
6320 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6321 	    rss->opcode));
6322 
6323 	eq = s->eqmap[qid - s->eq_start - s->eq_base];
6324 	(*h[eq->type])(sc, eq);
6325 
6326 	return (0);
6327 }
6328 
6329 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
6330 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
6331     offsetof(struct cpl_fw6_msg, data));
6332 
6333 static int
6334 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
6335 {
6336 	struct adapter *sc = iq->adapter;
6337 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
6338 
6339 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6340 	    rss->opcode));
6341 
6342 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
6343 		const struct rss_header *rss2;
6344 
6345 		rss2 = (const struct rss_header *)&cpl->data[0];
6346 		return (t4_cpl_handler[rss2->opcode](iq, rss2, m));
6347 	}
6348 
6349 	return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0]));
6350 }
6351 
6352 /**
6353  *	t4_handle_wrerr_rpl - process a FW work request error message
6354  *	@adap: the adapter
6355  *	@rpl: start of the FW message
6356  */
6357 static int
6358 t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl)
6359 {
6360 	u8 opcode = *(const u8 *)rpl;
6361 	const struct fw_error_cmd *e = (const void *)rpl;
6362 	unsigned int i;
6363 
6364 	if (opcode != FW_ERROR_CMD) {
6365 		log(LOG_ERR,
6366 		    "%s: Received WRERR_RPL message with opcode %#x\n",
6367 		    device_get_nameunit(adap->dev), opcode);
6368 		return (EINVAL);
6369 	}
6370 	log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev),
6371 	    G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" :
6372 	    "non-fatal");
6373 	switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) {
6374 	case FW_ERROR_TYPE_EXCEPTION:
6375 		log(LOG_ERR, "exception info:\n");
6376 		for (i = 0; i < nitems(e->u.exception.info); i++)
6377 			log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ",
6378 			    be32toh(e->u.exception.info[i]));
6379 		log(LOG_ERR, "\n");
6380 		break;
6381 	case FW_ERROR_TYPE_HWMODULE:
6382 		log(LOG_ERR, "HW module regaddr %08x regval %08x\n",
6383 		    be32toh(e->u.hwmodule.regaddr),
6384 		    be32toh(e->u.hwmodule.regval));
6385 		break;
6386 	case FW_ERROR_TYPE_WR:
6387 		log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n",
6388 		    be16toh(e->u.wr.cidx),
6389 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)),
6390 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)),
6391 		    be32toh(e->u.wr.eqid));
6392 		for (i = 0; i < nitems(e->u.wr.wrhdr); i++)
6393 			log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ",
6394 			    e->u.wr.wrhdr[i]);
6395 		log(LOG_ERR, "\n");
6396 		break;
6397 	case FW_ERROR_TYPE_ACL:
6398 		log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s",
6399 		    be16toh(e->u.acl.cidx),
6400 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)),
6401 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)),
6402 		    be32toh(e->u.acl.eqid),
6403 		    G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" :
6404 		    "MAC");
6405 		for (i = 0; i < nitems(e->u.acl.val); i++)
6406 			log(LOG_ERR, " %02x", e->u.acl.val[i]);
6407 		log(LOG_ERR, "\n");
6408 		break;
6409 	default:
6410 		log(LOG_ERR, "type %#x\n",
6411 		    G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type)));
6412 		return (EINVAL);
6413 	}
6414 	return (0);
6415 }
6416 
6417 static inline bool
6418 bufidx_used(struct adapter *sc, int idx)
6419 {
6420 	struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
6421 	int i;
6422 
6423 	for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6424 		if (rxb->size1 > largest_rx_cluster)
6425 			continue;
6426 		if (rxb->hwidx1 == idx || rxb->hwidx2 == idx)
6427 			return (true);
6428 	}
6429 
6430 	return (false);
6431 }
6432 
6433 static int
6434 sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
6435 {
6436 	struct adapter *sc = arg1;
6437 	struct sge_params *sp = &sc->params.sge;
6438 	int i, rc;
6439 	struct sbuf sb;
6440 	char c;
6441 
6442 	sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND);
6443 	for (i = 0; i < SGE_FLBUF_SIZES; i++) {
6444 		if (bufidx_used(sc, i))
6445 			c = '*';
6446 		else
6447 			c = '\0';
6448 
6449 		sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c);
6450 	}
6451 	sbuf_trim(&sb);
6452 	sbuf_finish(&sb);
6453 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
6454 	sbuf_delete(&sb);
6455 	return (rc);
6456 }
6457 
6458 #ifdef RATELIMIT
6459 #if defined(INET) || defined(INET6)
6460 /*
6461  * len16 for a txpkt WR with a GL.  Includes the firmware work request header.
6462  */
6463 static inline u_int
6464 txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso)
6465 {
6466 	u_int n;
6467 
6468 	MPASS(immhdrs > 0);
6469 
6470 	n = roundup2(sizeof(struct fw_eth_tx_eo_wr) +
6471 	    sizeof(struct cpl_tx_pkt_core) + immhdrs, 16);
6472 	if (__predict_false(nsegs == 0))
6473 		goto done;
6474 
6475 	nsegs--; /* first segment is part of ulptx_sgl */
6476 	n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
6477 	if (tso)
6478 		n += sizeof(struct cpl_tx_pkt_lso_core);
6479 
6480 done:
6481 	return (howmany(n, 16));
6482 }
6483 #endif
6484 
6485 #define ETID_FLOWC_NPARAMS 6
6486 #define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \
6487     ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16))
6488 #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16))
6489 
6490 static int
6491 send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi,
6492     struct vi_info *vi)
6493 {
6494 	struct wrq_cookie cookie;
6495 	u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN;
6496 	struct fw_flowc_wr *flowc;
6497 
6498 	mtx_assert(&cst->lock, MA_OWNED);
6499 	MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) ==
6500 	    EO_FLOWC_PENDING);
6501 
6502 	flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie);
6503 	if (__predict_false(flowc == NULL))
6504 		return (ENOMEM);
6505 
6506 	bzero(flowc, ETID_FLOWC_LEN);
6507 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6508 	    V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0));
6509 	flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) |
6510 	    V_FW_WR_FLOWID(cst->etid));
6511 	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
6512 	flowc->mnemval[0].val = htobe32(pfvf);
6513 	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
6514 	flowc->mnemval[1].val = htobe32(pi->tx_chan);
6515 	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
6516 	flowc->mnemval[2].val = htobe32(pi->tx_chan);
6517 	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
6518 	flowc->mnemval[3].val = htobe32(cst->iqid);
6519 	flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE;
6520 	flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);
6521 	flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
6522 	flowc->mnemval[5].val = htobe32(cst->schedcl);
6523 
6524 	commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6525 
6526 	cst->flags &= ~EO_FLOWC_PENDING;
6527 	cst->flags |= EO_FLOWC_RPL_PENDING;
6528 	MPASS(cst->tx_credits >= ETID_FLOWC_LEN16);	/* flowc is first WR. */
6529 	cst->tx_credits -= ETID_FLOWC_LEN16;
6530 
6531 	return (0);
6532 }
6533 
6534 #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))
6535 
6536 void
6537 send_etid_flush_wr(struct cxgbe_rate_tag *cst)
6538 {
6539 	struct fw_flowc_wr *flowc;
6540 	struct wrq_cookie cookie;
6541 
6542 	mtx_assert(&cst->lock, MA_OWNED);
6543 
6544 	flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie);
6545 	if (__predict_false(flowc == NULL))
6546 		CXGBE_UNIMPLEMENTED(__func__);
6547 
6548 	bzero(flowc, ETID_FLUSH_LEN16 * 16);
6549 	flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6550 	    V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL);
6551 	flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) |
6552 	    V_FW_WR_FLOWID(cst->etid));
6553 
6554 	commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6555 
6556 	cst->flags |= EO_FLUSH_RPL_PENDING;
6557 	MPASS(cst->tx_credits >= ETID_FLUSH_LEN16);
6558 	cst->tx_credits -= ETID_FLUSH_LEN16;
6559 	cst->ncompl++;
6560 }
6561 
6562 static void
6563 write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr,
6564     struct mbuf *m0, int compl)
6565 {
6566 	struct cpl_tx_pkt_core *cpl;
6567 	uint64_t ctrl1;
6568 	uint32_t ctrl;	/* used in many unrelated places */
6569 	int len16, pktlen, nsegs, immhdrs;
6570 	caddr_t dst;
6571 	uintptr_t p;
6572 	struct ulptx_sgl *usgl;
6573 	struct sglist sg;
6574 	struct sglist_seg segs[38];	/* XXX: find real limit.  XXX: get off the stack */
6575 
6576 	mtx_assert(&cst->lock, MA_OWNED);
6577 	M_ASSERTPKTHDR(m0);
6578 	KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
6579 	    m0->m_pkthdr.l4hlen > 0,
6580 	    ("%s: ethofld mbuf %p is missing header lengths", __func__, m0));
6581 
6582 	len16 = mbuf_eo_len16(m0);
6583 	nsegs = mbuf_eo_nsegs(m0);
6584 	pktlen = m0->m_pkthdr.len;
6585 	ctrl = sizeof(struct cpl_tx_pkt_core);
6586 	if (needs_tso(m0))
6587 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
6588 	immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen;
6589 	ctrl += immhdrs;
6590 
6591 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) |
6592 	    V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl));
6593 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) |
6594 	    V_FW_WR_FLOWID(cst->etid));
6595 	wr->r3 = 0;
6596 	if (needs_outer_udp_csum(m0)) {
6597 		wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG;
6598 		wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen;
6599 		wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
6600 		wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen;
6601 		wr->u.udpseg.rtplen = 0;
6602 		wr->u.udpseg.r4 = 0;
6603 		wr->u.udpseg.mss = htobe16(pktlen - immhdrs);
6604 		wr->u.udpseg.schedpktsize = wr->u.udpseg.mss;
6605 		wr->u.udpseg.plen = htobe32(pktlen - immhdrs);
6606 		cpl = (void *)(wr + 1);
6607 	} else {
6608 		MPASS(needs_outer_tcp_csum(m0));
6609 		wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG;
6610 		wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen;
6611 		wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
6612 		wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen;
6613 		wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0);
6614 		wr->u.tcpseg.r4 = 0;
6615 		wr->u.tcpseg.r5 = 0;
6616 		wr->u.tcpseg.plen = htobe32(pktlen - immhdrs);
6617 
6618 		if (needs_tso(m0)) {
6619 			struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
6620 
6621 			wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz);
6622 
6623 			ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
6624 			    F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
6625 			    V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen -
6626 				ETHER_HDR_LEN) >> 2) |
6627 			    V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
6628 			    V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
6629 			if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
6630 				ctrl |= F_LSO_IPV6;
6631 			lso->lso_ctrl = htobe32(ctrl);
6632 			lso->ipid_ofst = htobe16(0);
6633 			lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
6634 			lso->seqno_offset = htobe32(0);
6635 			lso->len = htobe32(pktlen);
6636 
6637 			cpl = (void *)(lso + 1);
6638 		} else {
6639 			wr->u.tcpseg.mss = htobe16(0xffff);
6640 			cpl = (void *)(wr + 1);
6641 		}
6642 	}
6643 
6644 	/* Checksum offload must be requested for ethofld. */
6645 	MPASS(needs_outer_l4_csum(m0));
6646 	ctrl1 = csum_to_ctrl(cst->adapter, m0);
6647 
6648 	/* VLAN tag insertion */
6649 	if (needs_vlan_insertion(m0)) {
6650 		ctrl1 |= F_TXPKT_VLAN_VLD |
6651 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
6652 	}
6653 
6654 	/* CPL header */
6655 	cpl->ctrl0 = cst->ctrl0;
6656 	cpl->pack = 0;
6657 	cpl->len = htobe16(pktlen);
6658 	cpl->ctrl1 = htobe64(ctrl1);
6659 
6660 	/* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */
6661 	p = (uintptr_t)(cpl + 1);
6662 	m_copydata(m0, 0, immhdrs, (void *)p);
6663 
6664 	/* SGL */
6665 	dst = (void *)(cpl + 1);
6666 	if (nsegs > 0) {
6667 		int i, pad;
6668 
6669 		/* zero-pad upto next 16Byte boundary, if not 16Byte aligned */
6670 		p += immhdrs;
6671 		pad = 16 - (immhdrs & 0xf);
6672 		bzero((void *)p, pad);
6673 
6674 		usgl = (void *)(p + pad);
6675 		usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
6676 		    V_ULPTX_NSGE(nsegs));
6677 
6678 		sglist_init(&sg, nitems(segs), segs);
6679 		for (; m0 != NULL; m0 = m0->m_next) {
6680 			if (__predict_false(m0->m_len == 0))
6681 				continue;
6682 			if (immhdrs >= m0->m_len) {
6683 				immhdrs -= m0->m_len;
6684 				continue;
6685 			}
6686 			if (m0->m_flags & M_EXTPG)
6687 				sglist_append_mbuf_epg(&sg, m0,
6688 				    mtod(m0, vm_offset_t), m0->m_len);
6689                         else
6690 				sglist_append(&sg, mtod(m0, char *) + immhdrs,
6691 				    m0->m_len - immhdrs);
6692 			immhdrs = 0;
6693 		}
6694 		MPASS(sg.sg_nseg == nsegs);
6695 
6696 		/*
6697 		 * Zero pad last 8B in case the WR doesn't end on a 16B
6698 		 * boundary.
6699 		 */
6700 		*(uint64_t *)((char *)wr + len16 * 16 - 8) = 0;
6701 
6702 		usgl->len0 = htobe32(segs[0].ss_len);
6703 		usgl->addr0 = htobe64(segs[0].ss_paddr);
6704 		for (i = 0; i < nsegs - 1; i++) {
6705 			usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len);
6706 			usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr);
6707 		}
6708 		if (i & 1)
6709 			usgl->sge[i / 2].len[1] = htobe32(0);
6710 	}
6711 
6712 }
6713 
6714 static void
6715 ethofld_tx(struct cxgbe_rate_tag *cst)
6716 {
6717 	struct mbuf *m;
6718 	struct wrq_cookie cookie;
6719 	int next_credits, compl;
6720 	struct fw_eth_tx_eo_wr *wr;
6721 
6722 	mtx_assert(&cst->lock, MA_OWNED);
6723 
6724 	while ((m = mbufq_first(&cst->pending_tx)) != NULL) {
6725 		M_ASSERTPKTHDR(m);
6726 
6727 		/* How many len16 credits do we need to send this mbuf. */
6728 		next_credits = mbuf_eo_len16(m);
6729 		MPASS(next_credits > 0);
6730 		if (next_credits > cst->tx_credits) {
6731 			/*
6732 			 * Tx will make progress eventually because there is at
6733 			 * least one outstanding fw4_ack that will return
6734 			 * credits and kick the tx.
6735 			 */
6736 			MPASS(cst->ncompl > 0);
6737 			return;
6738 		}
6739 		wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie);
6740 		if (__predict_false(wr == NULL)) {
6741 			/* XXX: wishful thinking, not a real assertion. */
6742 			MPASS(cst->ncompl > 0);
6743 			return;
6744 		}
6745 		cst->tx_credits -= next_credits;
6746 		cst->tx_nocompl += next_credits;
6747 		compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2;
6748 		ETHER_BPF_MTAP(cst->com.ifp, m);
6749 		write_ethofld_wr(cst, wr, m, compl);
6750 		commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie);
6751 		if (compl) {
6752 			cst->ncompl++;
6753 			cst->tx_nocompl	= 0;
6754 		}
6755 		(void) mbufq_dequeue(&cst->pending_tx);
6756 
6757 		/*
6758 		 * Drop the mbuf's reference on the tag now rather
6759 		 * than waiting until m_freem().  This ensures that
6760 		 * cxgbe_rate_tag_free gets called when the inp drops
6761 		 * its reference on the tag and there are no more
6762 		 * mbufs in the pending_tx queue and can flush any
6763 		 * pending requests.  Otherwise if the last mbuf
6764 		 * doesn't request a completion the etid will never be
6765 		 * released.
6766 		 */
6767 		m->m_pkthdr.snd_tag = NULL;
6768 		m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
6769 		m_snd_tag_rele(&cst->com);
6770 
6771 		mbufq_enqueue(&cst->pending_fwack, m);
6772 	}
6773 }
6774 
6775 int
6776 ethofld_transmit(struct ifnet *ifp, struct mbuf *m0)
6777 {
6778 	struct cxgbe_rate_tag *cst;
6779 	int rc;
6780 
6781 	MPASS(m0->m_nextpkt == NULL);
6782 	MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG);
6783 	MPASS(m0->m_pkthdr.snd_tag != NULL);
6784 	cst = mst_to_crt(m0->m_pkthdr.snd_tag);
6785 
6786 	mtx_lock(&cst->lock);
6787 	MPASS(cst->flags & EO_SND_TAG_REF);
6788 
6789 	if (__predict_false(cst->flags & EO_FLOWC_PENDING)) {
6790 		struct vi_info *vi = ifp->if_softc;
6791 		struct port_info *pi = vi->pi;
6792 		struct adapter *sc = pi->adapter;
6793 		const uint32_t rss_mask = vi->rss_size - 1;
6794 		uint32_t rss_hash;
6795 
6796 		cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];
6797 		if (M_HASHTYPE_ISHASH(m0))
6798 			rss_hash = m0->m_pkthdr.flowid;
6799 		else
6800 			rss_hash = arc4random();
6801 		/* We assume RSS hashing */
6802 		cst->iqid = vi->rss[rss_hash & rss_mask];
6803 		cst->eo_txq += rss_hash % vi->nofldtxq;
6804 		rc = send_etid_flowc_wr(cst, pi, vi);
6805 		if (rc != 0)
6806 			goto done;
6807 	}
6808 
6809 	if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) {
6810 		rc = ENOBUFS;
6811 		goto done;
6812 	}
6813 
6814 	mbufq_enqueue(&cst->pending_tx, m0);
6815 	cst->plen += m0->m_pkthdr.len;
6816 
6817 	/*
6818 	 * Hold an extra reference on the tag while generating work
6819 	 * requests to ensure that we don't try to free the tag during
6820 	 * ethofld_tx() in case we are sending the final mbuf after
6821 	 * the inp was freed.
6822 	 */
6823 	m_snd_tag_ref(&cst->com);
6824 	ethofld_tx(cst);
6825 	mtx_unlock(&cst->lock);
6826 	m_snd_tag_rele(&cst->com);
6827 	return (0);
6828 
6829 done:
6830 	mtx_unlock(&cst->lock);
6831 	if (__predict_false(rc != 0))
6832 		m_freem(m0);
6833 	return (rc);
6834 }
6835 
6836 static int
6837 ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
6838 {
6839 	struct adapter *sc = iq->adapter;
6840 	const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
6841 	struct mbuf *m;
6842 	u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
6843 	struct cxgbe_rate_tag *cst;
6844 	uint8_t credits = cpl->credits;
6845 
6846 	cst = lookup_etid(sc, etid);
6847 	mtx_lock(&cst->lock);
6848 	if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) {
6849 		MPASS(credits >= ETID_FLOWC_LEN16);
6850 		credits -= ETID_FLOWC_LEN16;
6851 		cst->flags &= ~EO_FLOWC_RPL_PENDING;
6852 	}
6853 
6854 	KASSERT(cst->ncompl > 0,
6855 	    ("%s: etid %u (%p) wasn't expecting completion.",
6856 	    __func__, etid, cst));
6857 	cst->ncompl--;
6858 
6859 	while (credits > 0) {
6860 		m = mbufq_dequeue(&cst->pending_fwack);
6861 		if (__predict_false(m == NULL)) {
6862 			/*
6863 			 * The remaining credits are for the final flush that
6864 			 * was issued when the tag was freed by the kernel.
6865 			 */
6866 			MPASS((cst->flags &
6867 			    (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) ==
6868 			    EO_FLUSH_RPL_PENDING);
6869 			MPASS(credits == ETID_FLUSH_LEN16);
6870 			MPASS(cst->tx_credits + cpl->credits == cst->tx_total);
6871 			MPASS(cst->ncompl == 0);
6872 
6873 			cst->flags &= ~EO_FLUSH_RPL_PENDING;
6874 			cst->tx_credits += cpl->credits;
6875 			cxgbe_rate_tag_free_locked(cst);
6876 			return (0);	/* cst is gone. */
6877 		}
6878 		KASSERT(m != NULL,
6879 		    ("%s: too many credits (%u, %u)", __func__, cpl->credits,
6880 		    credits));
6881 		KASSERT(credits >= mbuf_eo_len16(m),
6882 		    ("%s: too few credits (%u, %u, %u)", __func__,
6883 		    cpl->credits, credits, mbuf_eo_len16(m)));
6884 		credits -= mbuf_eo_len16(m);
6885 		cst->plen -= m->m_pkthdr.len;
6886 		m_freem(m);
6887 	}
6888 
6889 	cst->tx_credits += cpl->credits;
6890 	MPASS(cst->tx_credits <= cst->tx_total);
6891 
6892 	if (cst->flags & EO_SND_TAG_REF) {
6893 		/*
6894 		 * As with ethofld_transmit(), hold an extra reference
6895 		 * so that the tag is stable across ethold_tx().
6896 		 */
6897 		m_snd_tag_ref(&cst->com);
6898 		m = mbufq_first(&cst->pending_tx);
6899 		if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m))
6900 			ethofld_tx(cst);
6901 		mtx_unlock(&cst->lock);
6902 		m_snd_tag_rele(&cst->com);
6903 	} else {
6904 		/*
6905 		 * There shouldn't be any pending packets if the tag
6906 		 * was freed by the kernel since any pending packet
6907 		 * should hold a reference to the tag.
6908 		 */
6909 		MPASS(mbufq_first(&cst->pending_tx) == NULL);
6910 		mtx_unlock(&cst->lock);
6911 	}
6912 
6913 	return (0);
6914 }
6915 #endif
6916