xref: /freebsd/sys/dev/cxgbe/t4_sge.c (revision 04708d25e0f89d975f40757248bade347d5dc994)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/types.h>
35 #include <sys/eventhandler.h>
36 #include <sys/mbuf.h>
37 #include <sys/socket.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/queue.h>
41 #include <sys/sbuf.h>
42 #include <sys/taskqueue.h>
43 #include <sys/time.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46 #include <sys/smp.h>
47 #include <sys/counter.h>
48 #include <net/bpf.h>
49 #include <net/ethernet.h>
50 #include <net/if.h>
51 #include <net/if_vlan_var.h>
52 #include <netinet/in.h>
53 #include <netinet/ip.h>
54 #include <netinet/ip6.h>
55 #include <netinet/tcp.h>
56 #include <machine/in_cksum.h>
57 #include <machine/md_var.h>
58 #include <vm/vm.h>
59 #include <vm/pmap.h>
60 #ifdef DEV_NETMAP
61 #include <machine/bus.h>
62 #include <sys/selinfo.h>
63 #include <net/if_var.h>
64 #include <net/netmap.h>
65 #include <dev/netmap/netmap_kern.h>
66 #endif
67 
68 #include "common/common.h"
69 #include "common/t4_regs.h"
70 #include "common/t4_regs_values.h"
71 #include "common/t4_msg.h"
72 #include "t4_l2t.h"
73 #include "t4_mp_ring.h"
74 
75 #ifdef T4_PKT_TIMESTAMP
76 #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
77 #else
78 #define RX_COPY_THRESHOLD MINCLSIZE
79 #endif
80 
81 /*
82  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
83  * 0-7 are valid values.
84  */
85 static int fl_pktshift = 2;
86 TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
87 
88 /*
89  * Pad ethernet payload up to this boundary.
90  * -1: driver should figure out a good value.
91  *  0: disable padding.
92  *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
93  */
94 int fl_pad = -1;
95 TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
96 
97 /*
98  * Status page length.
99  * -1: driver should figure out a good value.
100  *  64 or 128 are the only other valid values.
101  */
102 static int spg_len = -1;
103 TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
104 
105 /*
106  * Congestion drops.
107  * -1: no congestion feedback (not recommended).
108  *  0: backpressure the channel instead of dropping packets right away.
109  *  1: no backpressure, drop packets for the congested queue immediately.
110  */
111 static int cong_drop = 0;
112 TUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop);
113 
114 /*
115  * Deliver multiple frames in the same free list buffer if they fit.
116  * -1: let the driver decide whether to enable buffer packing or not.
117  *  0: disable buffer packing.
118  *  1: enable buffer packing.
119  */
120 static int buffer_packing = -1;
121 TUNABLE_INT("hw.cxgbe.buffer_packing", &buffer_packing);
122 
123 /*
124  * Start next frame in a packed buffer at this boundary.
125  * -1: driver should figure out a good value.
126  * T4: driver will ignore this and use the same value as fl_pad above.
127  * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
128  */
129 static int fl_pack = -1;
130 TUNABLE_INT("hw.cxgbe.fl_pack", &fl_pack);
131 
132 /*
133  * Allow the driver to create mbuf(s) in a cluster allocated for rx.
134  * 0: never; always allocate mbufs from the zone_mbuf UMA zone.
135  * 1: ok to create mbuf(s) within a cluster if there is room.
136  */
137 static int allow_mbufs_in_cluster = 1;
138 TUNABLE_INT("hw.cxgbe.allow_mbufs_in_cluster", &allow_mbufs_in_cluster);
139 
140 /*
141  * Largest rx cluster size that the driver is allowed to allocate.
142  */
143 static int largest_rx_cluster = MJUM16BYTES;
144 TUNABLE_INT("hw.cxgbe.largest_rx_cluster", &largest_rx_cluster);
145 
146 /*
147  * Size of cluster allocation that's most likely to succeed.  The driver will
148  * fall back to this size if it fails to allocate clusters larger than this.
149  */
150 static int safest_rx_cluster = PAGE_SIZE;
151 TUNABLE_INT("hw.cxgbe.safest_rx_cluster", &safest_rx_cluster);
152 
153 struct txpkts {
154 	u_int wr_type;		/* type 0 or type 1 */
155 	u_int npkt;		/* # of packets in this work request */
156 	u_int plen;		/* total payload (sum of all packets) */
157 	u_int len16;		/* # of 16B pieces used by this work request */
158 };
159 
160 /* A packet's SGL.  This + m_pkthdr has all info needed for tx */
161 struct sgl {
162 	struct sglist sg;
163 	struct sglist_seg seg[TX_SGL_SEGS];
164 };
165 
166 static int service_iq(struct sge_iq *, int);
167 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
168 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
169 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
170 static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
171 static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t,
172     uint16_t, char *);
173 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
174     bus_addr_t *, void **);
175 static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
176     void *);
177 static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *,
178     int, int);
179 static int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *);
180 static void add_fl_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
181     struct sge_fl *);
182 static int alloc_fwq(struct adapter *);
183 static int free_fwq(struct adapter *);
184 static int alloc_mgmtq(struct adapter *);
185 static int free_mgmtq(struct adapter *);
186 static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int,
187     struct sysctl_oid *);
188 static int free_rxq(struct vi_info *, struct sge_rxq *);
189 #ifdef TCP_OFFLOAD
190 static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int,
191     struct sysctl_oid *);
192 static int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *);
193 #endif
194 #ifdef DEV_NETMAP
195 static int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int,
196     struct sysctl_oid *);
197 static int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *);
198 static int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int,
199     struct sysctl_oid *);
200 static int free_nm_txq(struct vi_info *, struct sge_nm_txq *);
201 #endif
202 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
203 static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
204 #ifdef TCP_OFFLOAD
205 static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
206 #endif
207 static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *);
208 static int free_eq(struct adapter *, struct sge_eq *);
209 static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *,
210     struct sysctl_oid *);
211 static int free_wrq(struct adapter *, struct sge_wrq *);
212 static int alloc_txq(struct vi_info *, struct sge_txq *, int,
213     struct sysctl_oid *);
214 static int free_txq(struct vi_info *, struct sge_txq *);
215 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
216 static inline void ring_fl_db(struct adapter *, struct sge_fl *);
217 static int refill_fl(struct adapter *, struct sge_fl *, int);
218 static void refill_sfl(void *);
219 static int alloc_fl_sdesc(struct sge_fl *);
220 static void free_fl_sdesc(struct adapter *, struct sge_fl *);
221 static void find_best_refill_source(struct adapter *, struct sge_fl *, int);
222 static void find_safe_refill_source(struct adapter *, struct sge_fl *);
223 static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
224 
225 static inline void get_pkt_gl(struct mbuf *, struct sglist *);
226 static inline u_int txpkt_len16(u_int, u_int);
227 static inline u_int txpkt_vm_len16(u_int, u_int);
228 static inline u_int txpkts0_len16(u_int);
229 static inline u_int txpkts1_len16(void);
230 static u_int write_txpkt_wr(struct sge_txq *, struct fw_eth_tx_pkt_wr *,
231     struct mbuf *, u_int);
232 static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *,
233     struct fw_eth_tx_pkt_vm_wr *, struct mbuf *, u_int);
234 static int try_txpkts(struct mbuf *, struct mbuf *, struct txpkts *, u_int);
235 static int add_to_txpkts(struct mbuf *, struct txpkts *, u_int);
236 static u_int write_txpkts_wr(struct sge_txq *, struct fw_eth_tx_pkts_wr *,
237     struct mbuf *, const struct txpkts *, u_int);
238 static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
239 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
240 static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
241 static inline uint16_t read_hw_cidx(struct sge_eq *);
242 static inline u_int reclaimable_tx_desc(struct sge_eq *);
243 static inline u_int total_available_tx_desc(struct sge_eq *);
244 static u_int reclaim_tx_descs(struct sge_txq *, u_int);
245 static void tx_reclaim(void *, int);
246 static __be64 get_flit(struct sglist_seg *, int, int);
247 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
248     struct mbuf *);
249 static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
250     struct mbuf *);
251 static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *);
252 static void wrq_tx_drain(void *, int);
253 static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *);
254 
255 static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
256 static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
257 static int sysctl_tc(SYSCTL_HANDLER_ARGS);
258 
259 static counter_u64_t extfree_refs;
260 static counter_u64_t extfree_rels;
261 
262 an_handler_t t4_an_handler;
263 fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
264 cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
265 
266 
267 static int
268 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
269 {
270 
271 #ifdef INVARIANTS
272 	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
273 #else
274 	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
275 	    __func__, iq, ctrl);
276 #endif
277 	return (EDOOFUS);
278 }
279 
280 int
281 t4_register_an_handler(an_handler_t h)
282 {
283 	uintptr_t *loc, new;
284 
285 	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
286 	loc = (uintptr_t *) &t4_an_handler;
287 	atomic_store_rel_ptr(loc, new);
288 
289 	return (0);
290 }
291 
292 static int
293 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
294 {
295 	const struct cpl_fw6_msg *cpl =
296 	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
297 
298 #ifdef INVARIANTS
299 	panic("%s: fw_msg type %d", __func__, cpl->type);
300 #else
301 	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
302 #endif
303 	return (EDOOFUS);
304 }
305 
306 int
307 t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
308 {
309 	uintptr_t *loc, new;
310 
311 	if (type >= nitems(t4_fw_msg_handler))
312 		return (EINVAL);
313 
314 	/*
315 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
316 	 * handler dispatch table.  Reject any attempt to install a handler for
317 	 * this subtype.
318 	 */
319 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
320 		return (EINVAL);
321 
322 	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
323 	loc = (uintptr_t *) &t4_fw_msg_handler[type];
324 	atomic_store_rel_ptr(loc, new);
325 
326 	return (0);
327 }
328 
329 static int
330 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
331 {
332 
333 #ifdef INVARIANTS
334 	panic("%s: opcode 0x%02x on iq %p with payload %p",
335 	    __func__, rss->opcode, iq, m);
336 #else
337 	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
338 	    __func__, rss->opcode, iq, m);
339 	m_freem(m);
340 #endif
341 	return (EDOOFUS);
342 }
343 
344 int
345 t4_register_cpl_handler(int opcode, cpl_handler_t h)
346 {
347 	uintptr_t *loc, new;
348 
349 	if (opcode >= nitems(t4_cpl_handler))
350 		return (EINVAL);
351 
352 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
353 	loc = (uintptr_t *) &t4_cpl_handler[opcode];
354 	atomic_store_rel_ptr(loc, new);
355 
356 	return (0);
357 }
358 
359 /*
360  * Called on MOD_LOAD.  Validates and calculates the SGE tunables.
361  */
362 void
363 t4_sge_modload(void)
364 {
365 	int i;
366 
367 	if (fl_pktshift < 0 || fl_pktshift > 7) {
368 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
369 		    " using 2 instead.\n", fl_pktshift);
370 		fl_pktshift = 2;
371 	}
372 
373 	if (spg_len != 64 && spg_len != 128) {
374 		int len;
375 
376 #if defined(__i386__) || defined(__amd64__)
377 		len = cpu_clflush_line_size > 64 ? 128 : 64;
378 #else
379 		len = 64;
380 #endif
381 		if (spg_len != -1) {
382 			printf("Invalid hw.cxgbe.spg_len value (%d),"
383 			    " using %d instead.\n", spg_len, len);
384 		}
385 		spg_len = len;
386 	}
387 
388 	if (cong_drop < -1 || cong_drop > 1) {
389 		printf("Invalid hw.cxgbe.cong_drop value (%d),"
390 		    " using 0 instead.\n", cong_drop);
391 		cong_drop = 0;
392 	}
393 
394 	extfree_refs = counter_u64_alloc(M_WAITOK);
395 	extfree_rels = counter_u64_alloc(M_WAITOK);
396 	counter_u64_zero(extfree_refs);
397 	counter_u64_zero(extfree_rels);
398 
399 	t4_an_handler = an_not_handled;
400 	for (i = 0; i < nitems(t4_fw_msg_handler); i++)
401 		t4_fw_msg_handler[i] = fw_msg_not_handled;
402 	for (i = 0; i < nitems(t4_cpl_handler); i++)
403 		t4_cpl_handler[i] = cpl_not_handled;
404 
405 	t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
406 	t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
407 	t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
408 	t4_register_cpl_handler(CPL_RX_PKT, t4_eth_rx);
409 	t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
410 	t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl);
411 }
412 
413 void
414 t4_sge_modunload(void)
415 {
416 
417 	counter_u64_free(extfree_refs);
418 	counter_u64_free(extfree_rels);
419 }
420 
421 uint64_t
422 t4_sge_extfree_refs(void)
423 {
424 	uint64_t refs, rels;
425 
426 	rels = counter_u64_fetch(extfree_rels);
427 	refs = counter_u64_fetch(extfree_refs);
428 
429 	return (refs - rels);
430 }
431 
432 static inline void
433 setup_pad_and_pack_boundaries(struct adapter *sc)
434 {
435 	uint32_t v, m;
436 	int pad, pack, pad_shift;
437 
438 	pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT :
439 	    X_INGPADBOUNDARY_SHIFT;
440 	pad = fl_pad;
441 	if (fl_pad < (1 << pad_shift) ||
442 	    fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) ||
443 	    !powerof2(fl_pad)) {
444 		/*
445 		 * If there is any chance that we might use buffer packing and
446 		 * the chip is a T4, then pick 64 as the pad/pack boundary.  Set
447 		 * it to the minimum allowed in all other cases.
448 		 */
449 		pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift;
450 
451 		/*
452 		 * For fl_pad = 0 we'll still write a reasonable value to the
453 		 * register but all the freelists will opt out of padding.
454 		 * We'll complain here only if the user tried to set it to a
455 		 * value greater than 0 that was invalid.
456 		 */
457 		if (fl_pad > 0) {
458 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
459 			    " (%d), using %d instead.\n", fl_pad, pad);
460 		}
461 	}
462 	m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
463 	v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift);
464 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
465 
466 	if (is_t4(sc)) {
467 		if (fl_pack != -1 && fl_pack != pad) {
468 			/* Complain but carry on. */
469 			device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
470 			    " using %d instead.\n", fl_pack, pad);
471 		}
472 		return;
473 	}
474 
475 	pack = fl_pack;
476 	if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
477 	    !powerof2(fl_pack)) {
478 		pack = max(sc->params.pci.mps, CACHE_LINE_SIZE);
479 		MPASS(powerof2(pack));
480 		if (pack < 16)
481 			pack = 16;
482 		if (pack == 32)
483 			pack = 64;
484 		if (pack > 4096)
485 			pack = 4096;
486 		if (fl_pack != -1) {
487 			device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value"
488 			    " (%d), using %d instead.\n", fl_pack, pack);
489 		}
490 	}
491 	m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
492 	if (pack == 16)
493 		v = V_INGPACKBOUNDARY(0);
494 	else
495 		v = V_INGPACKBOUNDARY(ilog2(pack) - 5);
496 
497 	MPASS(!is_t4(sc));	/* T4 doesn't have SGE_CONTROL2 */
498 	t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
499 }
500 
501 /*
502  * adap->params.vpd.cclk must be set up before this is called.
503  */
504 void
505 t4_tweak_chip_settings(struct adapter *sc)
506 {
507 	int i;
508 	uint32_t v, m;
509 	int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
510 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
511 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
512 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
513 	static int sge_flbuf_sizes[] = {
514 		MCLBYTES,
515 #if MJUMPAGESIZE != MCLBYTES
516 		MJUMPAGESIZE,
517 		MJUMPAGESIZE - CL_METADATA_SIZE,
518 		MJUMPAGESIZE - 2 * MSIZE - CL_METADATA_SIZE,
519 #endif
520 		MJUM9BYTES,
521 		MJUM16BYTES,
522 		MCLBYTES - MSIZE - CL_METADATA_SIZE,
523 		MJUM9BYTES - CL_METADATA_SIZE,
524 		MJUM16BYTES - CL_METADATA_SIZE,
525 	};
526 
527 	KASSERT(sc->flags & MASTER_PF,
528 	    ("%s: trying to change chip settings when not master.", __func__));
529 
530 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
531 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
532 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
533 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
534 
535 	setup_pad_and_pack_boundaries(sc);
536 
537 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
538 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
539 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
540 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
541 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
542 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
543 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
544 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
545 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
546 
547 	KASSERT(nitems(sge_flbuf_sizes) <= SGE_FLBUF_SIZES,
548 	    ("%s: hw buffer size table too big", __func__));
549 	for (i = 0; i < min(nitems(sge_flbuf_sizes), SGE_FLBUF_SIZES); i++) {
550 		t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
551 		    sge_flbuf_sizes[i]);
552 	}
553 
554 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
555 	    V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
556 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
557 
558 	KASSERT(intr_timer[0] <= timer_max,
559 	    ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
560 	    timer_max));
561 	for (i = 1; i < nitems(intr_timer); i++) {
562 		KASSERT(intr_timer[i] >= intr_timer[i - 1],
563 		    ("%s: timers not listed in increasing order (%d)",
564 		    __func__, i));
565 
566 		while (intr_timer[i] > timer_max) {
567 			if (i == nitems(intr_timer) - 1) {
568 				intr_timer[i] = timer_max;
569 				break;
570 			}
571 			intr_timer[i] += intr_timer[i - 1];
572 			intr_timer[i] /= 2;
573 		}
574 	}
575 
576 	v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
577 	    V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
578 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
579 	v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
580 	    V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
581 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
582 	v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
583 	    V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
584 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
585 
586 	/* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */
587 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
588 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
589 
590 	/*
591 	 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP.  These have been
592 	 * chosen with MAXPHYS = 128K in mind.  The largest DDP buffer that we
593 	 * may have to deal with is MAXPHYS + 1 page.
594 	 */
595 	v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4);
596 	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v);
597 
598 	/* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
599 	m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
600 	t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
601 
602 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
603 	    F_RESETDDPOFFSET;
604 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
605 	t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
606 }
607 
608 /*
609  * SGE wants the buffer to be at least 64B and then a multiple of 16.  If
610  * padding is in use, the buffer's start and end need to be aligned to the pad
611  * boundary as well.  We'll just make sure that the size is a multiple of the
612  * boundary here, it is up to the buffer allocation code to make sure the start
613  * of the buffer is aligned as well.
614  */
615 static inline int
616 hwsz_ok(struct adapter *sc, int hwsz)
617 {
618 	int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1;
619 
620 	return (hwsz >= 64 && (hwsz & mask) == 0);
621 }
622 
623 /*
624  * XXX: driver really should be able to deal with unexpected settings.
625  */
626 int
627 t4_read_chip_settings(struct adapter *sc)
628 {
629 	struct sge *s = &sc->sge;
630 	struct sge_params *sp = &sc->params.sge;
631 	int i, j, n, rc = 0;
632 	uint32_t m, v, r;
633 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
634 	static int sw_buf_sizes[] = {	/* Sorted by size */
635 		MCLBYTES,
636 #if MJUMPAGESIZE != MCLBYTES
637 		MJUMPAGESIZE,
638 #endif
639 		MJUM9BYTES,
640 		MJUM16BYTES
641 	};
642 	struct sw_zone_info *swz, *safe_swz;
643 	struct hw_buf_info *hwb;
644 
645 	m = F_RXPKTCPLMODE;
646 	v = F_RXPKTCPLMODE;
647 	r = sc->params.sge.sge_control;
648 	if ((r & m) != v) {
649 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
650 		rc = EINVAL;
651 	}
652 
653 	/*
654 	 * If this changes then every single use of PAGE_SHIFT in the driver
655 	 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift.
656 	 */
657 	if (sp->page_shift != PAGE_SHIFT) {
658 		device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
659 		rc = EINVAL;
660 	}
661 
662 	/* Filter out unusable hw buffer sizes entirely (mark with -2). */
663 	hwb = &s->hw_buf_info[0];
664 	for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) {
665 		r = sc->params.sge.sge_fl_buffer_size[i];
666 		hwb->size = r;
667 		hwb->zidx = hwsz_ok(sc, r) ? -1 : -2;
668 		hwb->next = -1;
669 	}
670 
671 	/*
672 	 * Create a sorted list in decreasing order of hw buffer sizes (and so
673 	 * increasing order of spare area) for each software zone.
674 	 *
675 	 * If padding is enabled then the start and end of the buffer must align
676 	 * to the pad boundary; if packing is enabled then they must align with
677 	 * the pack boundary as well.  Allocations from the cluster zones are
678 	 * aligned to min(size, 4K), so the buffer starts at that alignment and
679 	 * ends at hwb->size alignment.  If mbuf inlining is allowed the
680 	 * starting alignment will be reduced to MSIZE and the driver will
681 	 * exercise appropriate caution when deciding on the best buffer layout
682 	 * to use.
683 	 */
684 	n = 0;	/* no usable buffer size to begin with */
685 	swz = &s->sw_zone_info[0];
686 	safe_swz = NULL;
687 	for (i = 0; i < SW_ZONE_SIZES; i++, swz++) {
688 		int8_t head = -1, tail = -1;
689 
690 		swz->size = sw_buf_sizes[i];
691 		swz->zone = m_getzone(swz->size);
692 		swz->type = m_gettype(swz->size);
693 
694 		if (swz->size < PAGE_SIZE) {
695 			MPASS(powerof2(swz->size));
696 			if (fl_pad && (swz->size % sp->pad_boundary != 0))
697 				continue;
698 		}
699 
700 		if (swz->size == safest_rx_cluster)
701 			safe_swz = swz;
702 
703 		hwb = &s->hw_buf_info[0];
704 		for (j = 0; j < SGE_FLBUF_SIZES; j++, hwb++) {
705 			if (hwb->zidx != -1 || hwb->size > swz->size)
706 				continue;
707 #ifdef INVARIANTS
708 			if (fl_pad)
709 				MPASS(hwb->size % sp->pad_boundary == 0);
710 #endif
711 			hwb->zidx = i;
712 			if (head == -1)
713 				head = tail = j;
714 			else if (hwb->size < s->hw_buf_info[tail].size) {
715 				s->hw_buf_info[tail].next = j;
716 				tail = j;
717 			} else {
718 				int8_t *cur;
719 				struct hw_buf_info *t;
720 
721 				for (cur = &head; *cur != -1; cur = &t->next) {
722 					t = &s->hw_buf_info[*cur];
723 					if (hwb->size == t->size) {
724 						hwb->zidx = -2;
725 						break;
726 					}
727 					if (hwb->size > t->size) {
728 						hwb->next = *cur;
729 						*cur = j;
730 						break;
731 					}
732 				}
733 			}
734 		}
735 		swz->head_hwidx = head;
736 		swz->tail_hwidx = tail;
737 
738 		if (tail != -1) {
739 			n++;
740 			if (swz->size - s->hw_buf_info[tail].size >=
741 			    CL_METADATA_SIZE)
742 				sc->flags |= BUF_PACKING_OK;
743 		}
744 	}
745 	if (n == 0) {
746 		device_printf(sc->dev, "no usable SGE FL buffer size.\n");
747 		rc = EINVAL;
748 	}
749 
750 	s->safe_hwidx1 = -1;
751 	s->safe_hwidx2 = -1;
752 	if (safe_swz != NULL) {
753 		s->safe_hwidx1 = safe_swz->head_hwidx;
754 		for (i = safe_swz->head_hwidx; i != -1; i = hwb->next) {
755 			int spare;
756 
757 			hwb = &s->hw_buf_info[i];
758 #ifdef INVARIANTS
759 			if (fl_pad)
760 				MPASS(hwb->size % sp->pad_boundary == 0);
761 #endif
762 			spare = safe_swz->size - hwb->size;
763 			if (spare >= CL_METADATA_SIZE) {
764 				s->safe_hwidx2 = i;
765 				break;
766 			}
767 		}
768 	}
769 
770 	if (sc->flags & IS_VF)
771 		return (0);
772 
773 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
774 	r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
775 	if (r != v) {
776 		device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
777 		rc = EINVAL;
778 	}
779 
780 	m = v = F_TDDPTAGTCB;
781 	r = t4_read_reg(sc, A_ULP_RX_CTL);
782 	if ((r & m) != v) {
783 		device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
784 		rc = EINVAL;
785 	}
786 
787 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
788 	    F_RESETDDPOFFSET;
789 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
790 	r = t4_read_reg(sc, A_TP_PARA_REG5);
791 	if ((r & m) != v) {
792 		device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
793 		rc = EINVAL;
794 	}
795 
796 	t4_init_tp_params(sc);
797 
798 	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
799 	t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
800 
801 	return (rc);
802 }
803 
804 int
805 t4_create_dma_tag(struct adapter *sc)
806 {
807 	int rc;
808 
809 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
810 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
811 	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
812 	    NULL, &sc->dmat);
813 	if (rc != 0) {
814 		device_printf(sc->dev,
815 		    "failed to create main DMA tag: %d\n", rc);
816 	}
817 
818 	return (rc);
819 }
820 
821 void
822 t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
823     struct sysctl_oid_list *children)
824 {
825 	struct sge_params *sp = &sc->params.sge;
826 
827 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes",
828 	    CTLTYPE_STRING | CTLFLAG_RD, &sc->sge, 0, sysctl_bufsizes, "A",
829 	    "freelist buffer sizes");
830 
831 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD,
832 	    NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)");
833 
834 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD,
835 	    NULL, sp->pad_boundary, "payload pad boundary (bytes)");
836 
837 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD,
838 	    NULL, sp->spg_len, "status page size (bytes)");
839 
840 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
841 	    NULL, cong_drop, "congestion drop setting");
842 
843 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
844 	    NULL, sp->pack_boundary, "payload pack boundary (bytes)");
845 }
846 
847 int
848 t4_destroy_dma_tag(struct adapter *sc)
849 {
850 	if (sc->dmat)
851 		bus_dma_tag_destroy(sc->dmat);
852 
853 	return (0);
854 }
855 
856 /*
857  * Allocate and initialize the firmware event queue and the management queue.
858  *
859  * Returns errno on failure.  Resources allocated up to that point may still be
860  * allocated.  Caller is responsible for cleanup in case this function fails.
861  */
862 int
863 t4_setup_adapter_queues(struct adapter *sc)
864 {
865 	int rc;
866 
867 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
868 
869 	sysctl_ctx_init(&sc->ctx);
870 	sc->flags |= ADAP_SYSCTL_CTX;
871 
872 	/*
873 	 * Firmware event queue
874 	 */
875 	rc = alloc_fwq(sc);
876 	if (rc != 0)
877 		return (rc);
878 
879 	/*
880 	 * Management queue.  This is just a control queue that uses the fwq as
881 	 * its associated iq.
882 	 */
883 	if (!(sc->flags & IS_VF))
884 		rc = alloc_mgmtq(sc);
885 
886 	return (rc);
887 }
888 
889 /*
890  * Idempotent
891  */
892 int
893 t4_teardown_adapter_queues(struct adapter *sc)
894 {
895 
896 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
897 
898 	/* Do this before freeing the queue */
899 	if (sc->flags & ADAP_SYSCTL_CTX) {
900 		sysctl_ctx_free(&sc->ctx);
901 		sc->flags &= ~ADAP_SYSCTL_CTX;
902 	}
903 
904 	free_mgmtq(sc);
905 	free_fwq(sc);
906 
907 	return (0);
908 }
909 
910 static inline int
911 first_vector(struct vi_info *vi)
912 {
913 	struct adapter *sc = vi->pi->adapter;
914 
915 	if (sc->intr_count == 1)
916 		return (0);
917 
918 	return (vi->first_intr);
919 }
920 
921 /*
922  * Given an arbitrary "index," come up with an iq that can be used by other
923  * queues (of this VI) for interrupt forwarding, SGE egress updates, etc.
924  * The iq returned is guaranteed to be something that takes direct interrupts.
925  */
926 static struct sge_iq *
927 vi_intr_iq(struct vi_info *vi, int idx)
928 {
929 	struct adapter *sc = vi->pi->adapter;
930 	struct sge *s = &sc->sge;
931 	struct sge_iq *iq = NULL;
932 	int nintr, i;
933 
934 	if (sc->intr_count == 1)
935 		return (&sc->sge.fwq);
936 
937 	nintr = vi->nintr;
938 	KASSERT(nintr != 0,
939 	    ("%s: vi %p has no exclusive interrupts, total interrupts = %d",
940 	    __func__, vi, sc->intr_count));
941 	i = idx % nintr;
942 
943 	if (vi->flags & INTR_RXQ) {
944 	       	if (i < vi->nrxq) {
945 			iq = &s->rxq[vi->first_rxq + i].iq;
946 			goto done;
947 		}
948 		i -= vi->nrxq;
949 	}
950 #ifdef TCP_OFFLOAD
951 	if (vi->flags & INTR_OFLD_RXQ) {
952 	       	if (i < vi->nofldrxq) {
953 			iq = &s->ofld_rxq[vi->first_ofld_rxq + i].iq;
954 			goto done;
955 		}
956 		i -= vi->nofldrxq;
957 	}
958 #endif
959 	panic("%s: vi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__,
960 	    vi, vi->flags & INTR_ALL, idx, nintr);
961 done:
962 	MPASS(iq != NULL);
963 	KASSERT(iq->flags & IQ_INTR,
964 	    ("%s: iq %p (vi %p, intr_flags 0x%lx, idx %d)", __func__, iq, vi,
965 	    vi->flags & INTR_ALL, idx));
966 	return (iq);
967 }
968 
969 /* Maximum payload that can be delivered with a single iq descriptor */
970 static inline int
971 mtu_to_max_payload(struct adapter *sc, int mtu, const int toe)
972 {
973 	int payload;
974 
975 #ifdef TCP_OFFLOAD
976 	if (toe) {
977 		payload = sc->tt.rx_coalesce ?
978 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)) : mtu;
979 	} else {
980 #endif
981 		/* large enough even when hw VLAN extraction is disabled */
982 		payload = sc->params.sge.fl_pktshift + ETHER_HDR_LEN +
983 		    ETHER_VLAN_ENCAP_LEN + mtu;
984 #ifdef TCP_OFFLOAD
985 	}
986 #endif
987 
988 	return (payload);
989 }
990 
991 int
992 t4_setup_vi_queues(struct vi_info *vi)
993 {
994 	int rc = 0, i, j, intr_idx, iqid;
995 	struct sge_rxq *rxq;
996 	struct sge_txq *txq;
997 	struct sge_wrq *ctrlq;
998 #ifdef TCP_OFFLOAD
999 	struct sge_ofld_rxq *ofld_rxq;
1000 	struct sge_wrq *ofld_txq;
1001 #endif
1002 #ifdef DEV_NETMAP
1003 	int saved_idx;
1004 	struct sge_nm_rxq *nm_rxq;
1005 	struct sge_nm_txq *nm_txq;
1006 #endif
1007 	char name[16];
1008 	struct port_info *pi = vi->pi;
1009 	struct adapter *sc = pi->adapter;
1010 	struct ifnet *ifp = vi->ifp;
1011 	struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev);
1012 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
1013 	int maxp, mtu = ifp->if_mtu;
1014 
1015 	/* Interrupt vector to start from (when using multiple vectors) */
1016 	intr_idx = first_vector(vi);
1017 
1018 #ifdef DEV_NETMAP
1019 	saved_idx = intr_idx;
1020 	if (ifp->if_capabilities & IFCAP_NETMAP) {
1021 
1022 		/* netmap is supported with direct interrupts only. */
1023 		MPASS(vi->flags & INTR_RXQ);
1024 
1025 		/*
1026 		 * We don't have buffers to back the netmap rx queues
1027 		 * right now so we create the queues in a way that
1028 		 * doesn't set off any congestion signal in the chip.
1029 		 */
1030 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq",
1031 		    CTLFLAG_RD, NULL, "rx queues");
1032 		for_each_nm_rxq(vi, i, nm_rxq) {
1033 			rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid);
1034 			if (rc != 0)
1035 				goto done;
1036 			intr_idx++;
1037 		}
1038 
1039 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq",
1040 		    CTLFLAG_RD, NULL, "tx queues");
1041 		for_each_nm_txq(vi, i, nm_txq) {
1042 			iqid = vi->first_nm_rxq + (i % vi->nnmrxq);
1043 			rc = alloc_nm_txq(vi, nm_txq, iqid, i, oid);
1044 			if (rc != 0)
1045 				goto done;
1046 		}
1047 	}
1048 
1049 	/* Normal rx queues and netmap rx queues share the same interrupts. */
1050 	intr_idx = saved_idx;
1051 #endif
1052 
1053 	/*
1054 	 * First pass over all NIC and TOE rx queues:
1055 	 * a) initialize iq and fl
1056 	 * b) allocate queue iff it will take direct interrupts.
1057 	 */
1058 	maxp = mtu_to_max_payload(sc, mtu, 0);
1059 	if (vi->flags & INTR_RXQ) {
1060 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq",
1061 		    CTLFLAG_RD, NULL, "rx queues");
1062 	}
1063 	for_each_rxq(vi, i, rxq) {
1064 
1065 		init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq);
1066 
1067 		snprintf(name, sizeof(name), "%s rxq%d-fl",
1068 		    device_get_nameunit(vi->dev), i);
1069 		init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);
1070 
1071 		if (vi->flags & INTR_RXQ) {
1072 			rxq->iq.flags |= IQ_INTR;
1073 			rc = alloc_rxq(vi, rxq, intr_idx, i, oid);
1074 			if (rc != 0)
1075 				goto done;
1076 			intr_idx++;
1077 		}
1078 	}
1079 #ifdef DEV_NETMAP
1080 	if (ifp->if_capabilities & IFCAP_NETMAP)
1081 		intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq);
1082 #endif
1083 #ifdef TCP_OFFLOAD
1084 	maxp = mtu_to_max_payload(sc, mtu, 1);
1085 	if (vi->flags & INTR_OFLD_RXQ) {
1086 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq",
1087 		    CTLFLAG_RD, NULL,
1088 		    "rx queues for offloaded TCP connections");
1089 	}
1090 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1091 
1092 		init_iq(&ofld_rxq->iq, sc, vi->tmr_idx, vi->pktc_idx,
1093 		    vi->qsize_rxq);
1094 
1095 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
1096 		    device_get_nameunit(vi->dev), i);
1097 		init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);
1098 
1099 		if (vi->flags & INTR_OFLD_RXQ) {
1100 			ofld_rxq->iq.flags |= IQ_INTR;
1101 			rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid);
1102 			if (rc != 0)
1103 				goto done;
1104 			intr_idx++;
1105 		}
1106 	}
1107 #endif
1108 
1109 	/*
1110 	 * Second pass over all NIC and TOE rx queues.  The queues forwarding
1111 	 * their interrupts are allocated now.
1112 	 */
1113 	j = 0;
1114 	if (!(vi->flags & INTR_RXQ)) {
1115 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq",
1116 		    CTLFLAG_RD, NULL, "rx queues");
1117 		for_each_rxq(vi, i, rxq) {
1118 			MPASS(!(rxq->iq.flags & IQ_INTR));
1119 
1120 			intr_idx = vi_intr_iq(vi, j)->abs_id;
1121 
1122 			rc = alloc_rxq(vi, rxq, intr_idx, i, oid);
1123 			if (rc != 0)
1124 				goto done;
1125 			j++;
1126 		}
1127 	}
1128 #ifdef TCP_OFFLOAD
1129 	if (vi->nofldrxq != 0 && !(vi->flags & INTR_OFLD_RXQ)) {
1130 		oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq",
1131 		    CTLFLAG_RD, NULL,
1132 		    "rx queues for offloaded TCP connections");
1133 		for_each_ofld_rxq(vi, i, ofld_rxq) {
1134 			MPASS(!(ofld_rxq->iq.flags & IQ_INTR));
1135 
1136 			intr_idx = vi_intr_iq(vi, j)->abs_id;
1137 
1138 			rc = alloc_ofld_rxq(vi, ofld_rxq, intr_idx, i, oid);
1139 			if (rc != 0)
1140 				goto done;
1141 			j++;
1142 		}
1143 	}
1144 #endif
1145 
1146 	/*
1147 	 * Now the tx queues.  Only one pass needed.
1148 	 */
1149 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
1150 	    NULL, "tx queues");
1151 	j = 0;
1152 	for_each_txq(vi, i, txq) {
1153 		iqid = vi_intr_iq(vi, j)->cntxt_id;
1154 		snprintf(name, sizeof(name), "%s txq%d",
1155 		    device_get_nameunit(vi->dev), i);
1156 		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, iqid,
1157 		    name);
1158 
1159 		rc = alloc_txq(vi, txq, i, oid);
1160 		if (rc != 0)
1161 			goto done;
1162 		j++;
1163 	}
1164 #ifdef TCP_OFFLOAD
1165 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq",
1166 	    CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
1167 	for_each_ofld_txq(vi, i, ofld_txq) {
1168 		struct sysctl_oid *oid2;
1169 
1170 		iqid = vi_intr_iq(vi, j)->cntxt_id;
1171 		snprintf(name, sizeof(name), "%s ofld_txq%d",
1172 		    device_get_nameunit(vi->dev), i);
1173 		init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
1174 		    iqid, name);
1175 
1176 		snprintf(name, sizeof(name), "%d", i);
1177 		oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
1178 		    name, CTLFLAG_RD, NULL, "offload tx queue");
1179 
1180 		rc = alloc_wrq(sc, vi, ofld_txq, oid2);
1181 		if (rc != 0)
1182 			goto done;
1183 		j++;
1184 	}
1185 #endif
1186 
1187 	/*
1188 	 * Finally, the control queue.
1189 	 */
1190 	if (!IS_MAIN_VI(vi) || sc->flags & IS_VF)
1191 		goto done;
1192 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
1193 	    NULL, "ctrl queue");
1194 	ctrlq = &sc->sge.ctrlq[pi->port_id];
1195 	iqid = vi_intr_iq(vi, 0)->cntxt_id;
1196 	snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(vi->dev));
1197 	init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid,
1198 	    name);
1199 	rc = alloc_wrq(sc, vi, ctrlq, oid);
1200 
1201 done:
1202 	if (rc)
1203 		t4_teardown_vi_queues(vi);
1204 
1205 	return (rc);
1206 }
1207 
1208 /*
1209  * Idempotent
1210  */
1211 int
1212 t4_teardown_vi_queues(struct vi_info *vi)
1213 {
1214 	int i;
1215 	struct port_info *pi = vi->pi;
1216 	struct adapter *sc = pi->adapter;
1217 	struct sge_rxq *rxq;
1218 	struct sge_txq *txq;
1219 #ifdef TCP_OFFLOAD
1220 	struct sge_ofld_rxq *ofld_rxq;
1221 	struct sge_wrq *ofld_txq;
1222 #endif
1223 #ifdef DEV_NETMAP
1224 	struct sge_nm_rxq *nm_rxq;
1225 	struct sge_nm_txq *nm_txq;
1226 #endif
1227 
1228 	/* Do this before freeing the queues */
1229 	if (vi->flags & VI_SYSCTL_CTX) {
1230 		sysctl_ctx_free(&vi->ctx);
1231 		vi->flags &= ~VI_SYSCTL_CTX;
1232 	}
1233 
1234 #ifdef DEV_NETMAP
1235 	if (vi->ifp->if_capabilities & IFCAP_NETMAP) {
1236 		for_each_nm_txq(vi, i, nm_txq) {
1237 			free_nm_txq(vi, nm_txq);
1238 		}
1239 
1240 		for_each_nm_rxq(vi, i, nm_rxq) {
1241 			free_nm_rxq(vi, nm_rxq);
1242 		}
1243 	}
1244 #endif
1245 
1246 	/*
1247 	 * Take down all the tx queues first, as they reference the rx queues
1248 	 * (for egress updates, etc.).
1249 	 */
1250 
1251 	if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
1252 		free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
1253 
1254 	for_each_txq(vi, i, txq) {
1255 		free_txq(vi, txq);
1256 	}
1257 #ifdef TCP_OFFLOAD
1258 	for_each_ofld_txq(vi, i, ofld_txq) {
1259 		free_wrq(sc, ofld_txq);
1260 	}
1261 #endif
1262 
1263 	/*
1264 	 * Then take down the rx queues that forward their interrupts, as they
1265 	 * reference other rx queues.
1266 	 */
1267 
1268 	for_each_rxq(vi, i, rxq) {
1269 		if ((rxq->iq.flags & IQ_INTR) == 0)
1270 			free_rxq(vi, rxq);
1271 	}
1272 #ifdef TCP_OFFLOAD
1273 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1274 		if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
1275 			free_ofld_rxq(vi, ofld_rxq);
1276 	}
1277 #endif
1278 
1279 	/*
1280 	 * Then take down the rx queues that take direct interrupts.
1281 	 */
1282 
1283 	for_each_rxq(vi, i, rxq) {
1284 		if (rxq->iq.flags & IQ_INTR)
1285 			free_rxq(vi, rxq);
1286 	}
1287 #ifdef TCP_OFFLOAD
1288 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1289 		if (ofld_rxq->iq.flags & IQ_INTR)
1290 			free_ofld_rxq(vi, ofld_rxq);
1291 	}
1292 #endif
1293 
1294 	return (0);
1295 }
1296 
1297 /*
1298  * Deals with errors and the firmware event queue.  All data rx queues forward
1299  * their interrupt to the firmware event queue.
1300  */
1301 void
1302 t4_intr_all(void *arg)
1303 {
1304 	struct adapter *sc = arg;
1305 	struct sge_iq *fwq = &sc->sge.fwq;
1306 
1307 	t4_intr_err(arg);
1308 	if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
1309 		service_iq(fwq, 0);
1310 		atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
1311 	}
1312 }
1313 
1314 /* Deals with error interrupts */
1315 void
1316 t4_intr_err(void *arg)
1317 {
1318 	struct adapter *sc = arg;
1319 
1320 	t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
1321 	t4_slow_intr_handler(sc);
1322 }
1323 
1324 void
1325 t4_intr_evt(void *arg)
1326 {
1327 	struct sge_iq *iq = arg;
1328 
1329 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1330 		service_iq(iq, 0);
1331 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1332 	}
1333 }
1334 
1335 void
1336 t4_intr(void *arg)
1337 {
1338 	struct sge_iq *iq = arg;
1339 
1340 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1341 		service_iq(iq, 0);
1342 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1343 	}
1344 }
1345 
1346 void
1347 t4_vi_intr(void *arg)
1348 {
1349 	struct irq *irq = arg;
1350 
1351 #ifdef DEV_NETMAP
1352 	if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) {
1353 		t4_nm_intr(irq->nm_rxq);
1354 		atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON);
1355 	}
1356 #endif
1357 	if (irq->rxq != NULL)
1358 		t4_intr(irq->rxq);
1359 }
1360 
1361 /*
1362  * Deals with anything and everything on the given ingress queue.
1363  */
1364 static int
1365 service_iq(struct sge_iq *iq, int budget)
1366 {
1367 	struct sge_iq *q;
1368 	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
1369 	struct sge_fl *fl;			/* Use iff IQ_HAS_FL */
1370 	struct adapter *sc = iq->adapter;
1371 	struct iq_desc *d = &iq->desc[iq->cidx];
1372 	int ndescs = 0, limit;
1373 	int rsp_type, refill;
1374 	uint32_t lq;
1375 	uint16_t fl_hw_cidx;
1376 	struct mbuf *m0;
1377 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1378 #if defined(INET) || defined(INET6)
1379 	const struct timeval lro_timeout = {0, sc->lro_timeout};
1380 #endif
1381 
1382 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1383 
1384 	limit = budget ? budget : iq->qsize / 16;
1385 
1386 	if (iq->flags & IQ_HAS_FL) {
1387 		fl = &rxq->fl;
1388 		fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
1389 	} else {
1390 		fl = NULL;
1391 		fl_hw_cidx = 0;			/* to silence gcc warning */
1392 	}
1393 
1394 	/*
1395 	 * We always come back and check the descriptor ring for new indirect
1396 	 * interrupts and other responses after running a single handler.
1397 	 */
1398 	for (;;) {
1399 		while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1400 
1401 			rmb();
1402 
1403 			refill = 0;
1404 			m0 = NULL;
1405 			rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1406 			lq = be32toh(d->rsp.pldbuflen_qid);
1407 
1408 			switch (rsp_type) {
1409 			case X_RSPD_TYPE_FLBUF:
1410 
1411 				KASSERT(iq->flags & IQ_HAS_FL,
1412 				    ("%s: data for an iq (%p) with no freelist",
1413 				    __func__, iq));
1414 
1415 				m0 = get_fl_payload(sc, fl, lq);
1416 				if (__predict_false(m0 == NULL))
1417 					goto process_iql;
1418 				refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
1419 #ifdef T4_PKT_TIMESTAMP
1420 				/*
1421 				 * 60 bit timestamp for the payload is
1422 				 * *(uint64_t *)m0->m_pktdat.  Note that it is
1423 				 * in the leading free-space in the mbuf.  The
1424 				 * kernel can clobber it during a pullup,
1425 				 * m_copymdata, etc.  You need to make sure that
1426 				 * the mbuf reaches you unmolested if you care
1427 				 * about the timestamp.
1428 				 */
1429 				*(uint64_t *)m0->m_pktdat =
1430 				    be64toh(ctrl->u.last_flit) &
1431 				    0xfffffffffffffff;
1432 #endif
1433 
1434 				/* fall through */
1435 
1436 			case X_RSPD_TYPE_CPL:
1437 				KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1438 				    ("%s: bad opcode %02x.", __func__,
1439 				    d->rss.opcode));
1440 				t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
1441 				break;
1442 
1443 			case X_RSPD_TYPE_INTR:
1444 
1445 				/*
1446 				 * Interrupts should be forwarded only to queues
1447 				 * that are not forwarding their interrupts.
1448 				 * This means service_iq can recurse but only 1
1449 				 * level deep.
1450 				 */
1451 				KASSERT(budget == 0,
1452 				    ("%s: budget %u, rsp_type %u", __func__,
1453 				    budget, rsp_type));
1454 
1455 				/*
1456 				 * There are 1K interrupt-capable queues (qids 0
1457 				 * through 1023).  A response type indicating a
1458 				 * forwarded interrupt with a qid >= 1K is an
1459 				 * iWARP async notification.
1460 				 */
1461 				if (lq >= 1024) {
1462                                         t4_an_handler(iq, &d->rsp);
1463                                         break;
1464                                 }
1465 
1466 				q = sc->sge.iqmap[lq - sc->sge.iq_start -
1467 				    sc->sge.iq_base];
1468 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
1469 				    IQS_BUSY)) {
1470 					if (service_iq(q, q->qsize / 16) == 0) {
1471 						atomic_cmpset_int(&q->state,
1472 						    IQS_BUSY, IQS_IDLE);
1473 					} else {
1474 						STAILQ_INSERT_TAIL(&iql, q,
1475 						    link);
1476 					}
1477 				}
1478 				break;
1479 
1480 			default:
1481 				KASSERT(0,
1482 				    ("%s: illegal response type %d on iq %p",
1483 				    __func__, rsp_type, iq));
1484 				log(LOG_ERR,
1485 				    "%s: illegal response type %d on iq %p",
1486 				    device_get_nameunit(sc->dev), rsp_type, iq);
1487 				break;
1488 			}
1489 
1490 			d++;
1491 			if (__predict_false(++iq->cidx == iq->sidx)) {
1492 				iq->cidx = 0;
1493 				iq->gen ^= F_RSPD_GEN;
1494 				d = &iq->desc[0];
1495 			}
1496 			if (__predict_false(++ndescs == limit)) {
1497 				t4_write_reg(sc, sc->sge_gts_reg,
1498 				    V_CIDXINC(ndescs) |
1499 				    V_INGRESSQID(iq->cntxt_id) |
1500 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1501 				ndescs = 0;
1502 
1503 #if defined(INET) || defined(INET6)
1504 				if (iq->flags & IQ_LRO_ENABLED &&
1505 				    sc->lro_timeout != 0) {
1506 					tcp_lro_flush_inactive(&rxq->lro,
1507 					    &lro_timeout);
1508 				}
1509 #endif
1510 
1511 				if (budget) {
1512 					if (iq->flags & IQ_HAS_FL) {
1513 						FL_LOCK(fl);
1514 						refill_fl(sc, fl, 32);
1515 						FL_UNLOCK(fl);
1516 					}
1517 					return (EINPROGRESS);
1518 				}
1519 			}
1520 			if (refill) {
1521 				FL_LOCK(fl);
1522 				refill_fl(sc, fl, 32);
1523 				FL_UNLOCK(fl);
1524 				fl_hw_cidx = fl->hw_cidx;
1525 			}
1526 		}
1527 
1528 process_iql:
1529 		if (STAILQ_EMPTY(&iql))
1530 			break;
1531 
1532 		/*
1533 		 * Process the head only, and send it to the back of the list if
1534 		 * it's still not done.
1535 		 */
1536 		q = STAILQ_FIRST(&iql);
1537 		STAILQ_REMOVE_HEAD(&iql, link);
1538 		if (service_iq(q, q->qsize / 8) == 0)
1539 			atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
1540 		else
1541 			STAILQ_INSERT_TAIL(&iql, q, link);
1542 	}
1543 
1544 #if defined(INET) || defined(INET6)
1545 	if (iq->flags & IQ_LRO_ENABLED) {
1546 		struct lro_ctrl *lro = &rxq->lro;
1547 
1548 		tcp_lro_flush_all(lro);
1549 	}
1550 #endif
1551 
1552 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1553 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1554 
1555 	if (iq->flags & IQ_HAS_FL) {
1556 		int starved;
1557 
1558 		FL_LOCK(fl);
1559 		starved = refill_fl(sc, fl, 64);
1560 		FL_UNLOCK(fl);
1561 		if (__predict_false(starved != 0))
1562 			add_fl_to_sfl(sc, fl);
1563 	}
1564 
1565 	return (0);
1566 }
1567 
1568 static inline int
1569 cl_has_metadata(struct sge_fl *fl, struct cluster_layout *cll)
1570 {
1571 	int rc = fl->flags & FL_BUF_PACKING || cll->region1 > 0;
1572 
1573 	if (rc)
1574 		MPASS(cll->region3 >= CL_METADATA_SIZE);
1575 
1576 	return (rc);
1577 }
1578 
1579 static inline struct cluster_metadata *
1580 cl_metadata(struct adapter *sc, struct sge_fl *fl, struct cluster_layout *cll,
1581     caddr_t cl)
1582 {
1583 
1584 	if (cl_has_metadata(fl, cll)) {
1585 		struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
1586 
1587 		return ((struct cluster_metadata *)(cl + swz->size) - 1);
1588 	}
1589 	return (NULL);
1590 }
1591 
1592 static void
1593 rxb_free(struct mbuf *m, void *arg1, void *arg2)
1594 {
1595 	uma_zone_t zone = arg1;
1596 	caddr_t cl = arg2;
1597 
1598 	uma_zfree(zone, cl);
1599 	counter_u64_add(extfree_rels, 1);
1600 }
1601 
1602 /*
1603  * The mbuf returned by this function could be allocated from zone_mbuf or
1604  * constructed in spare room in the cluster.
1605  *
1606  * The mbuf carries the payload in one of these ways
1607  * a) frame inside the mbuf (mbuf from zone_mbuf)
1608  * b) m_cljset (for clusters without metadata) zone_mbuf
1609  * c) m_extaddref (cluster with metadata) inline mbuf
1610  * d) m_extaddref (cluster with metadata) zone_mbuf
1611  */
1612 static struct mbuf *
1613 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1614     int remaining)
1615 {
1616 	struct mbuf *m;
1617 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1618 	struct cluster_layout *cll = &sd->cll;
1619 	struct sw_zone_info *swz = &sc->sge.sw_zone_info[cll->zidx];
1620 	struct hw_buf_info *hwb = &sc->sge.hw_buf_info[cll->hwidx];
1621 	struct cluster_metadata *clm = cl_metadata(sc, fl, cll, sd->cl);
1622 	int len, blen;
1623 	caddr_t payload;
1624 
1625 	blen = hwb->size - fl->rx_offset;	/* max possible in this buf */
1626 	len = min(remaining, blen);
1627 	payload = sd->cl + cll->region1 + fl->rx_offset;
1628 	if (fl->flags & FL_BUF_PACKING) {
1629 		const u_int l = fr_offset + len;
1630 		const u_int pad = roundup2(l, fl->buf_boundary) - l;
1631 
1632 		if (fl->rx_offset + len + pad < hwb->size)
1633 			blen = len + pad;
1634 		MPASS(fl->rx_offset + blen <= hwb->size);
1635 	} else {
1636 		MPASS(fl->rx_offset == 0);	/* not packing */
1637 	}
1638 
1639 
1640 	if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
1641 
1642 		/*
1643 		 * Copy payload into a freshly allocated mbuf.
1644 		 */
1645 
1646 		m = fr_offset == 0 ?
1647 		    m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA);
1648 		if (m == NULL)
1649 			return (NULL);
1650 		fl->mbuf_allocated++;
1651 #ifdef T4_PKT_TIMESTAMP
1652 		/* Leave room for a timestamp */
1653 		m->m_data += 8;
1654 #endif
1655 		/* copy data to mbuf */
1656 		bcopy(payload, mtod(m, caddr_t), len);
1657 
1658 	} else if (sd->nmbuf * MSIZE < cll->region1) {
1659 
1660 		/*
1661 		 * There's spare room in the cluster for an mbuf.  Create one
1662 		 * and associate it with the payload that's in the cluster.
1663 		 */
1664 
1665 		MPASS(clm != NULL);
1666 		m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE);
1667 		/* No bzero required */
1668 		if (m_init(m, M_NOWAIT, MT_DATA,
1669 		    fr_offset == 0 ? M_PKTHDR | M_NOFREE : M_NOFREE))
1670 			return (NULL);
1671 		fl->mbuf_inlined++;
1672 		m_extaddref(m, payload, blen, &clm->refcount, rxb_free,
1673 		    swz->zone, sd->cl);
1674 		if (sd->nmbuf++ == 0)
1675 			counter_u64_add(extfree_refs, 1);
1676 
1677 	} else {
1678 
1679 		/*
1680 		 * Grab an mbuf from zone_mbuf and associate it with the
1681 		 * payload in the cluster.
1682 		 */
1683 
1684 		m = fr_offset == 0 ?
1685 		    m_gethdr(M_NOWAIT, MT_DATA) : m_get(M_NOWAIT, MT_DATA);
1686 		if (m == NULL)
1687 			return (NULL);
1688 		fl->mbuf_allocated++;
1689 		if (clm != NULL) {
1690 			m_extaddref(m, payload, blen, &clm->refcount,
1691 			    rxb_free, swz->zone, sd->cl);
1692 			if (sd->nmbuf++ == 0)
1693 				counter_u64_add(extfree_refs, 1);
1694 		} else {
1695 			m_cljset(m, sd->cl, swz->type);
1696 			sd->cl = NULL;	/* consumed, not a recycle candidate */
1697 		}
1698 	}
1699 	if (fr_offset == 0)
1700 		m->m_pkthdr.len = remaining;
1701 	m->m_len = len;
1702 
1703 	if (fl->flags & FL_BUF_PACKING) {
1704 		fl->rx_offset += blen;
1705 		MPASS(fl->rx_offset <= hwb->size);
1706 		if (fl->rx_offset < hwb->size)
1707 			return (m);	/* without advancing the cidx */
1708 	}
1709 
1710 	if (__predict_false(++fl->cidx % 8 == 0)) {
1711 		uint16_t cidx = fl->cidx / 8;
1712 
1713 		if (__predict_false(cidx == fl->sidx))
1714 			fl->cidx = cidx = 0;
1715 		fl->hw_cidx = cidx;
1716 	}
1717 	fl->rx_offset = 0;
1718 
1719 	return (m);
1720 }
1721 
1722 static struct mbuf *
1723 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf)
1724 {
1725 	struct mbuf *m0, *m, **pnext;
1726 	u_int remaining;
1727 	const u_int total = G_RSPD_LEN(len_newbuf);
1728 
1729 	if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1730 		M_ASSERTPKTHDR(fl->m0);
1731 		MPASS(fl->m0->m_pkthdr.len == total);
1732 		MPASS(fl->remaining < total);
1733 
1734 		m0 = fl->m0;
1735 		pnext = fl->pnext;
1736 		remaining = fl->remaining;
1737 		fl->flags &= ~FL_BUF_RESUME;
1738 		goto get_segment;
1739 	}
1740 
1741 	if (fl->rx_offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1742 		fl->rx_offset = 0;
1743 		if (__predict_false(++fl->cidx % 8 == 0)) {
1744 			uint16_t cidx = fl->cidx / 8;
1745 
1746 			if (__predict_false(cidx == fl->sidx))
1747 				fl->cidx = cidx = 0;
1748 			fl->hw_cidx = cidx;
1749 		}
1750 	}
1751 
1752 	/*
1753 	 * Payload starts at rx_offset in the current hw buffer.  Its length is
1754 	 * 'len' and it may span multiple hw buffers.
1755 	 */
1756 
1757 	m0 = get_scatter_segment(sc, fl, 0, total);
1758 	if (m0 == NULL)
1759 		return (NULL);
1760 	remaining = total - m0->m_len;
1761 	pnext = &m0->m_next;
1762 	while (remaining > 0) {
1763 get_segment:
1764 		MPASS(fl->rx_offset == 0);
1765 		m = get_scatter_segment(sc, fl, total - remaining, remaining);
1766 		if (__predict_false(m == NULL)) {
1767 			fl->m0 = m0;
1768 			fl->pnext = pnext;
1769 			fl->remaining = remaining;
1770 			fl->flags |= FL_BUF_RESUME;
1771 			return (NULL);
1772 		}
1773 		*pnext = m;
1774 		pnext = &m->m_next;
1775 		remaining -= m->m_len;
1776 	}
1777 	*pnext = NULL;
1778 
1779 	M_ASSERTPKTHDR(m0);
1780 	return (m0);
1781 }
1782 
1783 static int
1784 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
1785 {
1786 	struct sge_rxq *rxq = iq_to_rxq(iq);
1787 	struct ifnet *ifp = rxq->ifp;
1788 	struct adapter *sc = iq->adapter;
1789 	const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
1790 #if defined(INET) || defined(INET6)
1791 	struct lro_ctrl *lro = &rxq->lro;
1792 #endif
1793 	static const int sw_hashtype[4][2] = {
1794 		{M_HASHTYPE_NONE, M_HASHTYPE_NONE},
1795 		{M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6},
1796 		{M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6},
1797 		{M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6},
1798 	};
1799 
1800 	KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__,
1801 	    rss->opcode));
1802 
1803 	m0->m_pkthdr.len -= sc->params.sge.fl_pktshift;
1804 	m0->m_len -= sc->params.sge.fl_pktshift;
1805 	m0->m_data += sc->params.sge.fl_pktshift;
1806 
1807 	m0->m_pkthdr.rcvif = ifp;
1808 	M_HASHTYPE_SET(m0, sw_hashtype[rss->hash_type][rss->ipv6]);
1809 	m0->m_pkthdr.flowid = be32toh(rss->hash_val);
1810 
1811 	if (cpl->csum_calc && !cpl->err_vec) {
1812 		if (ifp->if_capenable & IFCAP_RXCSUM &&
1813 		    cpl->l2info & htobe32(F_RXF_IP)) {
1814 			m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED |
1815 			    CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1816 			rxq->rxcsum++;
1817 		} else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 &&
1818 		    cpl->l2info & htobe32(F_RXF_IP6)) {
1819 			m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 |
1820 			    CSUM_PSEUDO_HDR);
1821 			rxq->rxcsum++;
1822 		}
1823 
1824 		if (__predict_false(cpl->ip_frag))
1825 			m0->m_pkthdr.csum_data = be16toh(cpl->csum);
1826 		else
1827 			m0->m_pkthdr.csum_data = 0xffff;
1828 	}
1829 
1830 	if (cpl->vlan_ex) {
1831 		m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
1832 		m0->m_flags |= M_VLANTAG;
1833 		rxq->vlan_extraction++;
1834 	}
1835 
1836 #if defined(INET) || defined(INET6)
1837 	if (iq->flags & IQ_LRO_ENABLED &&
1838 	    tcp_lro_rx(lro, m0, 0) == 0) {
1839 		/* queued for LRO */
1840 	} else
1841 #endif
1842 	ifp->if_input(ifp, m0);
1843 
1844 	return (0);
1845 }
1846 
1847 /*
1848  * Must drain the wrq or make sure that someone else will.
1849  */
1850 static void
1851 wrq_tx_drain(void *arg, int n)
1852 {
1853 	struct sge_wrq *wrq = arg;
1854 	struct sge_eq *eq = &wrq->eq;
1855 
1856 	EQ_LOCK(eq);
1857 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
1858 		drain_wrq_wr_list(wrq->adapter, wrq);
1859 	EQ_UNLOCK(eq);
1860 }
1861 
1862 static void
1863 drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq)
1864 {
1865 	struct sge_eq *eq = &wrq->eq;
1866 	u_int available, dbdiff;	/* # of hardware descriptors */
1867 	u_int n;
1868 	struct wrqe *wr;
1869 	struct fw_eth_tx_pkt_wr *dst;	/* any fw WR struct will do */
1870 
1871 	EQ_LOCK_ASSERT_OWNED(eq);
1872 	MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
1873 	wr = STAILQ_FIRST(&wrq->wr_list);
1874 	MPASS(wr != NULL);	/* Must be called with something useful to do */
1875 	MPASS(eq->pidx == eq->dbidx);
1876 	dbdiff = 0;
1877 
1878 	do {
1879 		eq->cidx = read_hw_cidx(eq);
1880 		if (eq->pidx == eq->cidx)
1881 			available = eq->sidx - 1;
1882 		else
1883 			available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
1884 
1885 		MPASS(wr->wrq == wrq);
1886 		n = howmany(wr->wr_len, EQ_ESIZE);
1887 		if (available < n)
1888 			break;
1889 
1890 		dst = (void *)&eq->desc[eq->pidx];
1891 		if (__predict_true(eq->sidx - eq->pidx > n)) {
1892 			/* Won't wrap, won't end exactly at the status page. */
1893 			bcopy(&wr->wr[0], dst, wr->wr_len);
1894 			eq->pidx += n;
1895 		} else {
1896 			int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE;
1897 
1898 			bcopy(&wr->wr[0], dst, first_portion);
1899 			if (wr->wr_len > first_portion) {
1900 				bcopy(&wr->wr[first_portion], &eq->desc[0],
1901 				    wr->wr_len - first_portion);
1902 			}
1903 			eq->pidx = n - (eq->sidx - eq->pidx);
1904 		}
1905 
1906 		if (available < eq->sidx / 4 &&
1907 		    atomic_cmpset_int(&eq->equiq, 0, 1)) {
1908 			dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
1909 			    F_FW_WR_EQUEQ);
1910 			eq->equeqidx = eq->pidx;
1911 		} else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
1912 			dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
1913 			eq->equeqidx = eq->pidx;
1914 		}
1915 
1916 		dbdiff += n;
1917 		if (dbdiff >= 16) {
1918 			ring_eq_db(sc, eq, dbdiff);
1919 			dbdiff = 0;
1920 		}
1921 
1922 		STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1923 		free_wrqe(wr);
1924 		MPASS(wrq->nwr_pending > 0);
1925 		wrq->nwr_pending--;
1926 		MPASS(wrq->ndesc_needed >= n);
1927 		wrq->ndesc_needed -= n;
1928 	} while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL);
1929 
1930 	if (dbdiff)
1931 		ring_eq_db(sc, eq, dbdiff);
1932 }
1933 
1934 /*
1935  * Doesn't fail.  Holds on to work requests it can't send right away.
1936  */
1937 void
1938 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
1939 {
1940 #ifdef INVARIANTS
1941 	struct sge_eq *eq = &wrq->eq;
1942 #endif
1943 
1944 	EQ_LOCK_ASSERT_OWNED(eq);
1945 	MPASS(wr != NULL);
1946 	MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN);
1947 	MPASS((wr->wr_len & 0x7) == 0);
1948 
1949 	STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
1950 	wrq->nwr_pending++;
1951 	wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE);
1952 
1953 	if (!TAILQ_EMPTY(&wrq->incomplete_wrs))
1954 		return;	/* commit_wrq_wr will drain wr_list as well. */
1955 
1956 	drain_wrq_wr_list(sc, wrq);
1957 
1958 	/* Doorbell must have caught up to the pidx. */
1959 	MPASS(eq->pidx == eq->dbidx);
1960 }
1961 
1962 void
1963 t4_update_fl_bufsize(struct ifnet *ifp)
1964 {
1965 	struct vi_info *vi = ifp->if_softc;
1966 	struct adapter *sc = vi->pi->adapter;
1967 	struct sge_rxq *rxq;
1968 #ifdef TCP_OFFLOAD
1969 	struct sge_ofld_rxq *ofld_rxq;
1970 #endif
1971 	struct sge_fl *fl;
1972 	int i, maxp, mtu = ifp->if_mtu;
1973 
1974 	maxp = mtu_to_max_payload(sc, mtu, 0);
1975 	for_each_rxq(vi, i, rxq) {
1976 		fl = &rxq->fl;
1977 
1978 		FL_LOCK(fl);
1979 		find_best_refill_source(sc, fl, maxp);
1980 		FL_UNLOCK(fl);
1981 	}
1982 #ifdef TCP_OFFLOAD
1983 	maxp = mtu_to_max_payload(sc, mtu, 1);
1984 	for_each_ofld_rxq(vi, i, ofld_rxq) {
1985 		fl = &ofld_rxq->fl;
1986 
1987 		FL_LOCK(fl);
1988 		find_best_refill_source(sc, fl, maxp);
1989 		FL_UNLOCK(fl);
1990 	}
1991 #endif
1992 }
1993 
1994 static inline int
1995 mbuf_nsegs(struct mbuf *m)
1996 {
1997 
1998 	M_ASSERTPKTHDR(m);
1999 	KASSERT(m->m_pkthdr.l5hlen > 0,
2000 	    ("%s: mbuf %p missing information on # of segments.", __func__, m));
2001 
2002 	return (m->m_pkthdr.l5hlen);
2003 }
2004 
2005 static inline void
2006 set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
2007 {
2008 
2009 	M_ASSERTPKTHDR(m);
2010 	m->m_pkthdr.l5hlen = nsegs;
2011 }
2012 
2013 static inline int
2014 mbuf_len16(struct mbuf *m)
2015 {
2016 	int n;
2017 
2018 	M_ASSERTPKTHDR(m);
2019 	n = m->m_pkthdr.PH_loc.eight[0];
2020 	MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2021 
2022 	return (n);
2023 }
2024 
2025 static inline void
2026 set_mbuf_len16(struct mbuf *m, uint8_t len16)
2027 {
2028 
2029 	M_ASSERTPKTHDR(m);
2030 	m->m_pkthdr.PH_loc.eight[0] = len16;
2031 }
2032 
2033 static inline int
2034 needs_tso(struct mbuf *m)
2035 {
2036 
2037 	M_ASSERTPKTHDR(m);
2038 
2039 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2040 		KASSERT(m->m_pkthdr.tso_segsz > 0,
2041 		    ("%s: TSO requested in mbuf %p but MSS not provided",
2042 		    __func__, m));
2043 		return (1);
2044 	}
2045 
2046 	return (0);
2047 }
2048 
2049 static inline int
2050 needs_l3_csum(struct mbuf *m)
2051 {
2052 
2053 	M_ASSERTPKTHDR(m);
2054 
2055 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO))
2056 		return (1);
2057 	return (0);
2058 }
2059 
2060 static inline int
2061 needs_l4_csum(struct mbuf *m)
2062 {
2063 
2064 	M_ASSERTPKTHDR(m);
2065 
2066 	if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
2067 	    CSUM_TCP_IPV6 | CSUM_TSO))
2068 		return (1);
2069 	return (0);
2070 }
2071 
2072 static inline int
2073 needs_vlan_insertion(struct mbuf *m)
2074 {
2075 
2076 	M_ASSERTPKTHDR(m);
2077 
2078 	if (m->m_flags & M_VLANTAG) {
2079 		KASSERT(m->m_pkthdr.ether_vtag != 0,
2080 		    ("%s: HWVLAN requested in mbuf %p but tag not provided",
2081 		    __func__, m));
2082 		return (1);
2083 	}
2084 	return (0);
2085 }
2086 
2087 static void *
2088 m_advance(struct mbuf **pm, int *poffset, int len)
2089 {
2090 	struct mbuf *m = *pm;
2091 	int offset = *poffset;
2092 	uintptr_t p = 0;
2093 
2094 	MPASS(len > 0);
2095 
2096 	for (;;) {
2097 		if (offset + len < m->m_len) {
2098 			offset += len;
2099 			p = mtod(m, uintptr_t) + offset;
2100 			break;
2101 		}
2102 		len -= m->m_len - offset;
2103 		m = m->m_next;
2104 		offset = 0;
2105 		MPASS(m != NULL);
2106 	}
2107 	*poffset = offset;
2108 	*pm = m;
2109 	return ((void *)p);
2110 }
2111 
2112 static inline int
2113 same_paddr(char *a, char *b)
2114 {
2115 
2116 	if (a == b)
2117 		return (1);
2118 	else if (a != NULL && b != NULL) {
2119 		vm_offset_t x = (vm_offset_t)a;
2120 		vm_offset_t y = (vm_offset_t)b;
2121 
2122 		if ((x & PAGE_MASK) == (y & PAGE_MASK) &&
2123 		    pmap_kextract(x) == pmap_kextract(y))
2124 			return (1);
2125 	}
2126 
2127 	return (0);
2128 }
2129 
2130 /*
2131  * Can deal with empty mbufs in the chain that have m_len = 0, but the chain
2132  * must have at least one mbuf that's not empty.
2133  */
2134 static inline int
2135 count_mbuf_nsegs(struct mbuf *m)
2136 {
2137 	char *prev_end, *start;
2138 	int len, nsegs;
2139 
2140 	MPASS(m != NULL);
2141 
2142 	nsegs = 0;
2143 	prev_end = NULL;
2144 	for (; m; m = m->m_next) {
2145 
2146 		len = m->m_len;
2147 		if (__predict_false(len == 0))
2148 			continue;
2149 		start = mtod(m, char *);
2150 
2151 		nsegs += sglist_count(start, len);
2152 		if (same_paddr(prev_end, start))
2153 			nsegs--;
2154 		prev_end = start + len;
2155 	}
2156 
2157 	MPASS(nsegs > 0);
2158 	return (nsegs);
2159 }
2160 
2161 /*
2162  * Analyze the mbuf to determine its tx needs.  The mbuf passed in may change:
2163  * a) caller can assume it's been freed if this function returns with an error.
2164  * b) it may get defragged up if the gather list is too long for the hardware.
2165  */
2166 int
2167 parse_pkt(struct adapter *sc, struct mbuf **mp)
2168 {
2169 	struct mbuf *m0 = *mp, *m;
2170 	int rc, nsegs, defragged = 0, offset;
2171 	struct ether_header *eh;
2172 	void *l3hdr;
2173 #if defined(INET) || defined(INET6)
2174 	struct tcphdr *tcp;
2175 #endif
2176 	uint16_t eh_type;
2177 
2178 	M_ASSERTPKTHDR(m0);
2179 	if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) {
2180 		rc = EINVAL;
2181 fail:
2182 		m_freem(m0);
2183 		*mp = NULL;
2184 		return (rc);
2185 	}
2186 restart:
2187 	/*
2188 	 * First count the number of gather list segments in the payload.
2189 	 * Defrag the mbuf if nsegs exceeds the hardware limit.
2190 	 */
2191 	M_ASSERTPKTHDR(m0);
2192 	MPASS(m0->m_pkthdr.len > 0);
2193 	nsegs = count_mbuf_nsegs(m0);
2194 	if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) {
2195 		if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) {
2196 			rc = EFBIG;
2197 			goto fail;
2198 		}
2199 		*mp = m0 = m;	/* update caller's copy after defrag */
2200 		goto restart;
2201 	}
2202 
2203 	if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) {
2204 		m0 = m_pullup(m0, m0->m_pkthdr.len);
2205 		if (m0 == NULL) {
2206 			/* Should have left well enough alone. */
2207 			rc = EFBIG;
2208 			goto fail;
2209 		}
2210 		*mp = m0;	/* update caller's copy after pullup */
2211 		goto restart;
2212 	}
2213 	set_mbuf_nsegs(m0, nsegs);
2214 	if (sc->flags & IS_VF)
2215 		set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0)));
2216 	else
2217 		set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0)));
2218 
2219 	if (!needs_tso(m0) &&
2220 	    !(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0))))
2221 		return (0);
2222 
2223 	m = m0;
2224 	eh = mtod(m, struct ether_header *);
2225 	eh_type = ntohs(eh->ether_type);
2226 	if (eh_type == ETHERTYPE_VLAN) {
2227 		struct ether_vlan_header *evh = (void *)eh;
2228 
2229 		eh_type = ntohs(evh->evl_proto);
2230 		m0->m_pkthdr.l2hlen = sizeof(*evh);
2231 	} else
2232 		m0->m_pkthdr.l2hlen = sizeof(*eh);
2233 
2234 	offset = 0;
2235 	l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
2236 
2237 	switch (eh_type) {
2238 #ifdef INET6
2239 	case ETHERTYPE_IPV6:
2240 	{
2241 		struct ip6_hdr *ip6 = l3hdr;
2242 
2243 		MPASS(!needs_tso(m0) || ip6->ip6_nxt == IPPROTO_TCP);
2244 
2245 		m0->m_pkthdr.l3hlen = sizeof(*ip6);
2246 		break;
2247 	}
2248 #endif
2249 #ifdef INET
2250 	case ETHERTYPE_IP:
2251 	{
2252 		struct ip *ip = l3hdr;
2253 
2254 		m0->m_pkthdr.l3hlen = ip->ip_hl * 4;
2255 		break;
2256 	}
2257 #endif
2258 	default:
2259 		panic("%s: ethertype 0x%04x unknown.  if_cxgbe must be compiled"
2260 		    " with the same INET/INET6 options as the kernel.",
2261 		    __func__, eh_type);
2262 	}
2263 
2264 #if defined(INET) || defined(INET6)
2265 	if (needs_tso(m0)) {
2266 		tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen);
2267 		m0->m_pkthdr.l4hlen = tcp->th_off * 4;
2268 	}
2269 #endif
2270 	MPASS(m0 == *mp);
2271 	return (0);
2272 }
2273 
2274 void *
2275 start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
2276 {
2277 	struct sge_eq *eq = &wrq->eq;
2278 	struct adapter *sc = wrq->adapter;
2279 	int ndesc, available;
2280 	struct wrqe *wr;
2281 	void *w;
2282 
2283 	MPASS(len16 > 0);
2284 	ndesc = howmany(len16, EQ_ESIZE / 16);
2285 	MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
2286 
2287 	EQ_LOCK(eq);
2288 
2289 	if (!STAILQ_EMPTY(&wrq->wr_list))
2290 		drain_wrq_wr_list(sc, wrq);
2291 
2292 	if (!STAILQ_EMPTY(&wrq->wr_list)) {
2293 slowpath:
2294 		EQ_UNLOCK(eq);
2295 		wr = alloc_wrqe(len16 * 16, wrq);
2296 		if (__predict_false(wr == NULL))
2297 			return (NULL);
2298 		cookie->pidx = -1;
2299 		cookie->ndesc = ndesc;
2300 		return (&wr->wr);
2301 	}
2302 
2303 	eq->cidx = read_hw_cidx(eq);
2304 	if (eq->pidx == eq->cidx)
2305 		available = eq->sidx - 1;
2306 	else
2307 		available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2308 	if (available < ndesc)
2309 		goto slowpath;
2310 
2311 	cookie->pidx = eq->pidx;
2312 	cookie->ndesc = ndesc;
2313 	TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link);
2314 
2315 	w = &eq->desc[eq->pidx];
2316 	IDXINCR(eq->pidx, ndesc, eq->sidx);
2317 	if (__predict_false(eq->pidx < ndesc - 1)) {
2318 		w = &wrq->ss[0];
2319 		wrq->ss_pidx = cookie->pidx;
2320 		wrq->ss_len = len16 * 16;
2321 	}
2322 
2323 	EQ_UNLOCK(eq);
2324 
2325 	return (w);
2326 }
2327 
2328 void
2329 commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
2330 {
2331 	struct sge_eq *eq = &wrq->eq;
2332 	struct adapter *sc = wrq->adapter;
2333 	int ndesc, pidx;
2334 	struct wrq_cookie *prev, *next;
2335 
2336 	if (cookie->pidx == -1) {
2337 		struct wrqe *wr = __containerof(w, struct wrqe, wr);
2338 
2339 		t4_wrq_tx(sc, wr);
2340 		return;
2341 	}
2342 
2343 	ndesc = cookie->ndesc;	/* Can be more than SGE_MAX_WR_NDESC here. */
2344 	pidx = cookie->pidx;
2345 	MPASS(pidx >= 0 && pidx < eq->sidx);
2346 	if (__predict_false(w == &wrq->ss[0])) {
2347 		int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE;
2348 
2349 		MPASS(wrq->ss_len > n);	/* WR had better wrap around. */
2350 		bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n);
2351 		bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n);
2352 		wrq->tx_wrs_ss++;
2353 	} else
2354 		wrq->tx_wrs_direct++;
2355 
2356 	EQ_LOCK(eq);
2357 	prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link);
2358 	next = TAILQ_NEXT(cookie, link);
2359 	if (prev == NULL) {
2360 		MPASS(pidx == eq->dbidx);
2361 		if (next == NULL || ndesc >= 16)
2362 			ring_eq_db(wrq->adapter, eq, ndesc);
2363 		else {
2364 			MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
2365 			next->pidx = pidx;
2366 			next->ndesc += ndesc;
2367 		}
2368 	} else {
2369 		MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc);
2370 		prev->ndesc += ndesc;
2371 	}
2372 	TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link);
2373 
2374 	if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2375 		drain_wrq_wr_list(sc, wrq);
2376 
2377 #ifdef INVARIANTS
2378 	if (TAILQ_EMPTY(&wrq->incomplete_wrs)) {
2379 		/* Doorbell must have caught up to the pidx. */
2380 		MPASS(wrq->eq.pidx == wrq->eq.dbidx);
2381 	}
2382 #endif
2383 	EQ_UNLOCK(eq);
2384 }
2385 
2386 static u_int
2387 can_resume_eth_tx(struct mp_ring *r)
2388 {
2389 	struct sge_eq *eq = r->cookie;
2390 
2391 	return (total_available_tx_desc(eq) > eq->sidx / 8);
2392 }
2393 
2394 static inline int
2395 cannot_use_txpkts(struct mbuf *m)
2396 {
2397 	/* maybe put a GL limit too, to avoid silliness? */
2398 
2399 	return (needs_tso(m));
2400 }
2401 
2402 /*
2403  * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to
2404  * be consumed.  Return the actual number consumed.  0 indicates a stall.
2405  */
2406 static u_int
2407 eth_tx(struct mp_ring *r, u_int cidx, u_int pidx)
2408 {
2409 	struct sge_txq *txq = r->cookie;
2410 	struct sge_eq *eq = &txq->eq;
2411 	struct ifnet *ifp = txq->ifp;
2412 	struct vi_info *vi = ifp->if_softc;
2413 	struct port_info *pi = vi->pi;
2414 	struct adapter *sc = pi->adapter;
2415 	u_int total, remaining;		/* # of packets */
2416 	u_int available, dbdiff;	/* # of hardware descriptors */
2417 	u_int n, next_cidx;
2418 	struct mbuf *m0, *tail;
2419 	struct txpkts txp;
2420 	struct fw_eth_tx_pkts_wr *wr;	/* any fw WR struct will do */
2421 
2422 	remaining = IDXDIFF(pidx, cidx, r->size);
2423 	MPASS(remaining > 0);	/* Must not be called without work to do. */
2424 	total = 0;
2425 
2426 	TXQ_LOCK(txq);
2427 	if (__predict_false((eq->flags & EQ_ENABLED) == 0)) {
2428 		while (cidx != pidx) {
2429 			m0 = r->items[cidx];
2430 			m_freem(m0);
2431 			if (++cidx == r->size)
2432 				cidx = 0;
2433 		}
2434 		reclaim_tx_descs(txq, 2048);
2435 		total = remaining;
2436 		goto done;
2437 	}
2438 
2439 	/* How many hardware descriptors do we have readily available. */
2440 	if (eq->pidx == eq->cidx)
2441 		available = eq->sidx - 1;
2442 	else
2443 		available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2444 	dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx);
2445 
2446 	while (remaining > 0) {
2447 
2448 		m0 = r->items[cidx];
2449 		M_ASSERTPKTHDR(m0);
2450 		MPASS(m0->m_nextpkt == NULL);
2451 
2452 		if (available < SGE_MAX_WR_NDESC) {
2453 			available += reclaim_tx_descs(txq, 64);
2454 			if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16))
2455 				break;	/* out of descriptors */
2456 		}
2457 
2458 		next_cidx = cidx + 1;
2459 		if (__predict_false(next_cidx == r->size))
2460 			next_cidx = 0;
2461 
2462 		wr = (void *)&eq->desc[eq->pidx];
2463 		if (sc->flags & IS_VF) {
2464 			total++;
2465 			remaining--;
2466 			ETHER_BPF_MTAP(ifp, m0);
2467 			n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0,
2468 			    available);
2469 		} else if (remaining > 1 &&
2470 		    try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) {
2471 
2472 			/* pkts at cidx, next_cidx should both be in txp. */
2473 			MPASS(txp.npkt == 2);
2474 			tail = r->items[next_cidx];
2475 			MPASS(tail->m_nextpkt == NULL);
2476 			ETHER_BPF_MTAP(ifp, m0);
2477 			ETHER_BPF_MTAP(ifp, tail);
2478 			m0->m_nextpkt = tail;
2479 
2480 			if (__predict_false(++next_cidx == r->size))
2481 				next_cidx = 0;
2482 
2483 			while (next_cidx != pidx) {
2484 				if (add_to_txpkts(r->items[next_cidx], &txp,
2485 				    available) != 0)
2486 					break;
2487 				tail->m_nextpkt = r->items[next_cidx];
2488 				tail = tail->m_nextpkt;
2489 				ETHER_BPF_MTAP(ifp, tail);
2490 				if (__predict_false(++next_cidx == r->size))
2491 					next_cidx = 0;
2492 			}
2493 
2494 			n = write_txpkts_wr(txq, wr, m0, &txp, available);
2495 			total += txp.npkt;
2496 			remaining -= txp.npkt;
2497 		} else {
2498 			total++;
2499 			remaining--;
2500 			ETHER_BPF_MTAP(ifp, m0);
2501 			n = write_txpkt_wr(txq, (void *)wr, m0, available);
2502 		}
2503 		MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC);
2504 
2505 		available -= n;
2506 		dbdiff += n;
2507 		IDXINCR(eq->pidx, n, eq->sidx);
2508 
2509 		if (total_available_tx_desc(eq) < eq->sidx / 4 &&
2510 		    atomic_cmpset_int(&eq->equiq, 0, 1)) {
2511 			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
2512 			    F_FW_WR_EQUEQ);
2513 			eq->equeqidx = eq->pidx;
2514 		} else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
2515 			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
2516 			eq->equeqidx = eq->pidx;
2517 		}
2518 
2519 		if (dbdiff >= 16 && remaining >= 4) {
2520 			ring_eq_db(sc, eq, dbdiff);
2521 			available += reclaim_tx_descs(txq, 4 * dbdiff);
2522 			dbdiff = 0;
2523 		}
2524 
2525 		cidx = next_cidx;
2526 	}
2527 	if (dbdiff != 0) {
2528 		ring_eq_db(sc, eq, dbdiff);
2529 		reclaim_tx_descs(txq, 32);
2530 	}
2531 done:
2532 	TXQ_UNLOCK(txq);
2533 
2534 	return (total);
2535 }
2536 
2537 static inline void
2538 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
2539     int qsize)
2540 {
2541 
2542 	KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
2543 	    ("%s: bad tmr_idx %d", __func__, tmr_idx));
2544 	KASSERT(pktc_idx < SGE_NCOUNTERS,	/* -ve is ok, means don't use */
2545 	    ("%s: bad pktc_idx %d", __func__, pktc_idx));
2546 
2547 	iq->flags = 0;
2548 	iq->adapter = sc;
2549 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
2550 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
2551 	if (pktc_idx >= 0) {
2552 		iq->intr_params |= F_QINTR_CNT_EN;
2553 		iq->intr_pktc_idx = pktc_idx;
2554 	}
2555 	iq->qsize = roundup2(qsize, 16);	/* See FW_IQ_CMD/iqsize */
2556 	iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE;
2557 }
2558 
2559 static inline void
2560 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
2561 {
2562 
2563 	fl->qsize = qsize;
2564 	fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
2565 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
2566 	if (sc->flags & BUF_PACKING_OK &&
2567 	    ((!is_t4(sc) && buffer_packing) ||	/* T5+: enabled unless 0 */
2568 	    (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */
2569 		fl->flags |= FL_BUF_PACKING;
2570 	find_best_refill_source(sc, fl, maxp);
2571 	find_safe_refill_source(sc, fl);
2572 }
2573 
2574 static inline void
2575 init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize,
2576     uint8_t tx_chan, uint16_t iqid, char *name)
2577 {
2578 	KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
2579 
2580 	eq->flags = eqtype & EQ_TYPEMASK;
2581 	eq->tx_chan = tx_chan;
2582 	eq->iqid = iqid;
2583 	eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
2584 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
2585 }
2586 
2587 static int
2588 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
2589     bus_dmamap_t *map, bus_addr_t *pa, void **va)
2590 {
2591 	int rc;
2592 
2593 	rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
2594 	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
2595 	if (rc != 0) {
2596 		device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc);
2597 		goto done;
2598 	}
2599 
2600 	rc = bus_dmamem_alloc(*tag, va,
2601 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
2602 	if (rc != 0) {
2603 		device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc);
2604 		goto done;
2605 	}
2606 
2607 	rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
2608 	if (rc != 0) {
2609 		device_printf(sc->dev, "cannot load DMA map: %d\n", rc);
2610 		goto done;
2611 	}
2612 done:
2613 	if (rc)
2614 		free_ring(sc, *tag, *map, *pa, *va);
2615 
2616 	return (rc);
2617 }
2618 
2619 static int
2620 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
2621     bus_addr_t pa, void *va)
2622 {
2623 	if (pa)
2624 		bus_dmamap_unload(tag, map);
2625 	if (va)
2626 		bus_dmamem_free(tag, va, map);
2627 	if (tag)
2628 		bus_dma_tag_destroy(tag);
2629 
2630 	return (0);
2631 }
2632 
2633 /*
2634  * Allocates the ring for an ingress queue and an optional freelist.  If the
2635  * freelist is specified it will be allocated and then associated with the
2636  * ingress queue.
2637  *
2638  * Returns errno on failure.  Resources allocated up to that point may still be
2639  * allocated.  Caller is responsible for cleanup in case this function fails.
2640  *
2641  * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
2642  * the intr_idx specifies the vector, starting from 0.  Otherwise it specifies
2643  * the abs_id of the ingress queue to which its interrupts should be forwarded.
2644  */
2645 static int
2646 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,
2647     int intr_idx, int cong)
2648 {
2649 	int rc, i, cntxt_id;
2650 	size_t len;
2651 	struct fw_iq_cmd c;
2652 	struct port_info *pi = vi->pi;
2653 	struct adapter *sc = iq->adapter;
2654 	struct sge_params *sp = &sc->params.sge;
2655 	__be32 v = 0;
2656 
2657 	len = iq->qsize * IQ_ESIZE;
2658 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
2659 	    (void **)&iq->desc);
2660 	if (rc != 0)
2661 		return (rc);
2662 
2663 	bzero(&c, sizeof(c));
2664 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
2665 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
2666 	    V_FW_IQ_CMD_VFN(0));
2667 
2668 	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
2669 	    FW_LEN16(c));
2670 
2671 	/* Special handling for firmware event queue */
2672 	if (iq == &sc->sge.fwq)
2673 		v |= F_FW_IQ_CMD_IQASYNCH;
2674 
2675 	if (iq->flags & IQ_INTR) {
2676 		KASSERT(intr_idx < sc->intr_count,
2677 		    ("%s: invalid direct intr_idx %d", __func__, intr_idx));
2678 	} else
2679 		v |= F_FW_IQ_CMD_IQANDST;
2680 	v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
2681 
2682 	c.type_to_iqandstindex = htobe32(v |
2683 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
2684 	    V_FW_IQ_CMD_VIID(vi->viid) |
2685 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
2686 	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
2687 	    F_FW_IQ_CMD_IQGTSMODE |
2688 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
2689 	    V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
2690 	c.iqsize = htobe16(iq->qsize);
2691 	c.iqaddr = htobe64(iq->ba);
2692 	if (cong >= 0)
2693 		c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
2694 
2695 	if (fl) {
2696 		mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
2697 
2698 		len = fl->qsize * EQ_ESIZE;
2699 		rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
2700 		    &fl->ba, (void **)&fl->desc);
2701 		if (rc)
2702 			return (rc);
2703 
2704 		/* Allocate space for one software descriptor per buffer. */
2705 		rc = alloc_fl_sdesc(fl);
2706 		if (rc != 0) {
2707 			device_printf(sc->dev,
2708 			    "failed to setup fl software descriptors: %d\n",
2709 			    rc);
2710 			return (rc);
2711 		}
2712 
2713 		if (fl->flags & FL_BUF_PACKING) {
2714 			fl->lowat = roundup2(sp->fl_starve_threshold2, 8);
2715 			fl->buf_boundary = sp->pack_boundary;
2716 		} else {
2717 			fl->lowat = roundup2(sp->fl_starve_threshold, 8);
2718 			fl->buf_boundary = 16;
2719 		}
2720 		if (fl_pad && fl->buf_boundary < sp->pad_boundary)
2721 			fl->buf_boundary = sp->pad_boundary;
2722 
2723 		c.iqns_to_fl0congen |=
2724 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
2725 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
2726 			(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
2727 			(fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
2728 			    0));
2729 		if (cong >= 0) {
2730 			c.iqns_to_fl0congen |=
2731 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
2732 				    F_FW_IQ_CMD_FL0CONGCIF |
2733 				    F_FW_IQ_CMD_FL0CONGEN);
2734 		}
2735 		c.fl0dcaen_to_fl0cidxfthresh =
2736 		    htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ?
2737 			X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) |
2738 			V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ?
2739 			X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
2740 		c.fl0size = htobe16(fl->qsize);
2741 		c.fl0addr = htobe64(fl->ba);
2742 	}
2743 
2744 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2745 	if (rc != 0) {
2746 		device_printf(sc->dev,
2747 		    "failed to create ingress queue: %d\n", rc);
2748 		return (rc);
2749 	}
2750 
2751 	iq->cidx = 0;
2752 	iq->gen = F_RSPD_GEN;
2753 	iq->intr_next = iq->intr_params;
2754 	iq->cntxt_id = be16toh(c.iqid);
2755 	iq->abs_id = be16toh(c.physiqid);
2756 	iq->flags |= IQ_ALLOCATED;
2757 
2758 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
2759 	if (cntxt_id >= sc->sge.niq) {
2760 		panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
2761 		    cntxt_id, sc->sge.niq - 1);
2762 	}
2763 	sc->sge.iqmap[cntxt_id] = iq;
2764 
2765 	if (fl) {
2766 		u_int qid;
2767 
2768 		iq->flags |= IQ_HAS_FL;
2769 		fl->cntxt_id = be16toh(c.fl0id);
2770 		fl->pidx = fl->cidx = 0;
2771 
2772 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
2773 		if (cntxt_id >= sc->sge.neq) {
2774 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
2775 			    __func__, cntxt_id, sc->sge.neq - 1);
2776 		}
2777 		sc->sge.eqmap[cntxt_id] = (void *)fl;
2778 
2779 		qid = fl->cntxt_id;
2780 		if (isset(&sc->doorbells, DOORBELL_UDB)) {
2781 			uint32_t s_qpp = sc->params.sge.eq_s_qpp;
2782 			uint32_t mask = (1 << s_qpp) - 1;
2783 			volatile uint8_t *udb;
2784 
2785 			udb = sc->udbs_base + UDBS_DB_OFFSET;
2786 			udb += (qid >> s_qpp) << PAGE_SHIFT;
2787 			qid &= mask;
2788 			if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
2789 				udb += qid << UDBS_SEG_SHIFT;
2790 				qid = 0;
2791 			}
2792 			fl->udb = (volatile void *)udb;
2793 		}
2794 		fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
2795 
2796 		FL_LOCK(fl);
2797 		/* Enough to make sure the SGE doesn't think it's starved */
2798 		refill_fl(sc, fl, fl->lowat);
2799 		FL_UNLOCK(fl);
2800 	}
2801 
2802 	if (is_t5(sc) && !(sc->flags & IS_VF) && cong >= 0) {
2803 		uint32_t param, val;
2804 
2805 		param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2806 		    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
2807 		    V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
2808 		if (cong == 0)
2809 			val = 1 << 19;
2810 		else {
2811 			val = 2 << 19;
2812 			for (i = 0; i < 4; i++) {
2813 				if (cong & (1 << i))
2814 					val |= 1 << (i << 2);
2815 			}
2816 		}
2817 
2818 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2819 		if (rc != 0) {
2820 			/* report error but carry on */
2821 			device_printf(sc->dev,
2822 			    "failed to set congestion manager context for "
2823 			    "ingress queue %d: %d\n", iq->cntxt_id, rc);
2824 		}
2825 	}
2826 
2827 	/* Enable IQ interrupts */
2828 	atomic_store_rel_int(&iq->state, IQS_IDLE);
2829 	t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) |
2830 	    V_INGRESSQID(iq->cntxt_id));
2831 
2832 	return (0);
2833 }
2834 
2835 static int
2836 free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)
2837 {
2838 	int rc;
2839 	struct adapter *sc = iq->adapter;
2840 	device_t dev;
2841 
2842 	if (sc == NULL)
2843 		return (0);	/* nothing to do */
2844 
2845 	dev = vi ? vi->dev : sc->dev;
2846 
2847 	if (iq->flags & IQ_ALLOCATED) {
2848 		rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
2849 		    FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
2850 		    fl ? fl->cntxt_id : 0xffff, 0xffff);
2851 		if (rc != 0) {
2852 			device_printf(dev,
2853 			    "failed to free queue %p: %d\n", iq, rc);
2854 			return (rc);
2855 		}
2856 		iq->flags &= ~IQ_ALLOCATED;
2857 	}
2858 
2859 	free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
2860 
2861 	bzero(iq, sizeof(*iq));
2862 
2863 	if (fl) {
2864 		free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba,
2865 		    fl->desc);
2866 
2867 		if (fl->sdesc)
2868 			free_fl_sdesc(sc, fl);
2869 
2870 		if (mtx_initialized(&fl->fl_lock))
2871 			mtx_destroy(&fl->fl_lock);
2872 
2873 		bzero(fl, sizeof(*fl));
2874 	}
2875 
2876 	return (0);
2877 }
2878 
2879 static void
2880 add_fl_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
2881     struct sge_fl *fl)
2882 {
2883 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2884 
2885 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
2886 	    "freelist");
2887 	children = SYSCTL_CHILDREN(oid);
2888 
2889 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
2890 	    CTLTYPE_INT | CTLFLAG_RD, &fl->cntxt_id, 0, sysctl_uint16, "I",
2891 	    "SGE context id of the freelist");
2892 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL,
2893 	    fl_pad ? 1 : 0, "padding enabled");
2894 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL,
2895 	    fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled");
2896 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
2897 	    0, "consumer index");
2898 	if (fl->flags & FL_BUF_PACKING) {
2899 		SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset",
2900 		    CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
2901 	}
2902 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
2903 	    0, "producer index");
2904 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_allocated",
2905 	    CTLFLAG_RD, &fl->mbuf_allocated, "# of mbuf allocated");
2906 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "mbuf_inlined",
2907 	    CTLFLAG_RD, &fl->mbuf_inlined, "# of mbuf inlined in clusters");
2908 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated",
2909 	    CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
2910 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled",
2911 	    CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
2912 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled",
2913 	    CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
2914 }
2915 
2916 static int
2917 alloc_fwq(struct adapter *sc)
2918 {
2919 	int rc, intr_idx;
2920 	struct sge_iq *fwq = &sc->sge.fwq;
2921 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
2922 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2923 
2924 	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE);
2925 	fwq->flags |= IQ_INTR;	/* always */
2926 	if (sc->flags & IS_VF)
2927 		intr_idx = 0;
2928 	else {
2929 		intr_idx = sc->intr_count > 1 ? 1 : 0;
2930 		fwq->set_tcb_rpl = t4_filter_rpl;
2931 		fwq->l2t_write_rpl = do_l2t_write_rpl;
2932 	}
2933 	rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1);
2934 	if (rc != 0) {
2935 		device_printf(sc->dev,
2936 		    "failed to create firmware event queue: %d\n", rc);
2937 		return (rc);
2938 	}
2939 
2940 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD,
2941 	    NULL, "firmware event queue");
2942 	children = SYSCTL_CHILDREN(oid);
2943 
2944 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id",
2945 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I",
2946 	    "absolute id of the queue");
2947 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id",
2948 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I",
2949 	    "SGE context id of the queue");
2950 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
2951 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
2952 	    "consumer index");
2953 
2954 	return (0);
2955 }
2956 
2957 static int
2958 free_fwq(struct adapter *sc)
2959 {
2960 	return free_iq_fl(NULL, &sc->sge.fwq, NULL);
2961 }
2962 
2963 static int
2964 alloc_mgmtq(struct adapter *sc)
2965 {
2966 	int rc;
2967 	struct sge_wrq *mgmtq = &sc->sge.mgmtq;
2968 	char name[16];
2969 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
2970 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2971 
2972 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD,
2973 	    NULL, "management queue");
2974 
2975 	snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev));
2976 	init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
2977 	    sc->sge.fwq.cntxt_id, name);
2978 	rc = alloc_wrq(sc, NULL, mgmtq, oid);
2979 	if (rc != 0) {
2980 		device_printf(sc->dev,
2981 		    "failed to create management queue: %d\n", rc);
2982 		return (rc);
2983 	}
2984 
2985 	return (0);
2986 }
2987 
2988 static int
2989 free_mgmtq(struct adapter *sc)
2990 {
2991 
2992 	return free_wrq(sc, &sc->sge.mgmtq);
2993 }
2994 
2995 int
2996 tnl_cong(struct port_info *pi, int drop)
2997 {
2998 
2999 	if (drop == -1)
3000 		return (-1);
3001 	else if (drop == 1)
3002 		return (0);
3003 	else
3004 		return (pi->rx_chan_map);
3005 }
3006 
3007 static int
3008 alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx,
3009     struct sysctl_oid *oid)
3010 {
3011 	int rc;
3012 	struct adapter *sc = vi->pi->adapter;
3013 	struct sysctl_oid_list *children;
3014 	char name[16];
3015 
3016 	rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx,
3017 	    tnl_cong(vi->pi, cong_drop));
3018 	if (rc != 0)
3019 		return (rc);
3020 
3021 	if (idx == 0)
3022 		sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id;
3023 	else
3024 		KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id,
3025 		    ("iq_base mismatch"));
3026 	KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF,
3027 	    ("PF with non-zero iq_base"));
3028 
3029 	/*
3030 	 * The freelist is just barely above the starvation threshold right now,
3031 	 * fill it up a bit more.
3032 	 */
3033 	FL_LOCK(&rxq->fl);
3034 	refill_fl(sc, &rxq->fl, 128);
3035 	FL_UNLOCK(&rxq->fl);
3036 
3037 #if defined(INET) || defined(INET6)
3038 	rc = tcp_lro_init(&rxq->lro);
3039 	if (rc != 0)
3040 		return (rc);
3041 	rxq->lro.ifp = vi->ifp; /* also indicates LRO init'ed */
3042 
3043 	if (vi->ifp->if_capenable & IFCAP_LRO)
3044 		rxq->iq.flags |= IQ_LRO_ENABLED;
3045 #endif
3046 	rxq->ifp = vi->ifp;
3047 
3048 	children = SYSCTL_CHILDREN(oid);
3049 
3050 	snprintf(name, sizeof(name), "%d", idx);
3051 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3052 	    NULL, "rx queue");
3053 	children = SYSCTL_CHILDREN(oid);
3054 
3055 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id",
3056 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I",
3057 	    "absolute id of the queue");
3058 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id",
3059 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cntxt_id, 0, sysctl_uint16, "I",
3060 	    "SGE context id of the queue");
3061 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx",
3062 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cidx, 0, sysctl_uint16, "I",
3063 	    "consumer index");
3064 #if defined(INET) || defined(INET6)
3065 	SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
3066 	    &rxq->lro.lro_queued, 0, NULL);
3067 	SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
3068 	    &rxq->lro.lro_flushed, 0, NULL);
3069 #endif
3070 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
3071 	    &rxq->rxcsum, "# of times hardware assisted with checksum");
3072 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction",
3073 	    CTLFLAG_RD, &rxq->vlan_extraction,
3074 	    "# of times hardware extracted 802.1Q tag");
3075 
3076 	add_fl_sysctls(&vi->ctx, oid, &rxq->fl);
3077 
3078 	return (rc);
3079 }
3080 
3081 static int
3082 free_rxq(struct vi_info *vi, struct sge_rxq *rxq)
3083 {
3084 	int rc;
3085 
3086 #if defined(INET) || defined(INET6)
3087 	if (rxq->lro.ifp) {
3088 		tcp_lro_free(&rxq->lro);
3089 		rxq->lro.ifp = NULL;
3090 	}
3091 #endif
3092 
3093 	rc = free_iq_fl(vi, &rxq->iq, &rxq->fl);
3094 	if (rc == 0)
3095 		bzero(rxq, sizeof(*rxq));
3096 
3097 	return (rc);
3098 }
3099 
3100 #ifdef TCP_OFFLOAD
3101 static int
3102 alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq,
3103     int intr_idx, int idx, struct sysctl_oid *oid)
3104 {
3105 	int rc;
3106 	struct sysctl_oid_list *children;
3107 	char name[16];
3108 
3109 	rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
3110 	    vi->pi->rx_chan_map);
3111 	if (rc != 0)
3112 		return (rc);
3113 
3114 	children = SYSCTL_CHILDREN(oid);
3115 
3116 	snprintf(name, sizeof(name), "%d", idx);
3117 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3118 	    NULL, "rx queue");
3119 	children = SYSCTL_CHILDREN(oid);
3120 
3121 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "abs_id",
3122 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16,
3123 	    "I", "absolute id of the queue");
3124 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cntxt_id",
3125 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16,
3126 	    "I", "SGE context id of the queue");
3127 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx",
3128 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I",
3129 	    "consumer index");
3130 
3131 	add_fl_sysctls(&vi->ctx, oid, &ofld_rxq->fl);
3132 
3133 	return (rc);
3134 }
3135 
3136 static int
3137 free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
3138 {
3139 	int rc;
3140 
3141 	rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl);
3142 	if (rc == 0)
3143 		bzero(ofld_rxq, sizeof(*ofld_rxq));
3144 
3145 	return (rc);
3146 }
3147 #endif
3148 
3149 #ifdef DEV_NETMAP
3150 static int
3151 alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx,
3152     int idx, struct sysctl_oid *oid)
3153 {
3154 	int rc;
3155 	struct sysctl_oid_list *children;
3156 	struct sysctl_ctx_list *ctx;
3157 	char name[16];
3158 	size_t len;
3159 	struct adapter *sc = vi->pi->adapter;
3160 	struct netmap_adapter *na = NA(vi->ifp);
3161 
3162 	MPASS(na != NULL);
3163 
3164 	len = vi->qsize_rxq * IQ_ESIZE;
3165 	rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,
3166 	    &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);
3167 	if (rc != 0)
3168 		return (rc);
3169 
3170 	len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len;
3171 	rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,
3172 	    &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);
3173 	if (rc != 0)
3174 		return (rc);
3175 
3176 	nm_rxq->vi = vi;
3177 	nm_rxq->nid = idx;
3178 	nm_rxq->iq_cidx = 0;
3179 	nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE;
3180 	nm_rxq->iq_gen = F_RSPD_GEN;
3181 	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
3182 	nm_rxq->fl_sidx = na->num_rx_desc;
3183 	nm_rxq->intr_idx = intr_idx;
3184 
3185 	ctx = &vi->ctx;
3186 	children = SYSCTL_CHILDREN(oid);
3187 
3188 	snprintf(name, sizeof(name), "%d", idx);
3189 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL,
3190 	    "rx queue");
3191 	children = SYSCTL_CHILDREN(oid);
3192 
3193 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id",
3194 	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16,
3195 	    "I", "absolute id of the queue");
3196 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
3197 	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16,
3198 	    "I", "SGE context id of the queue");
3199 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
3200 	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I",
3201 	    "consumer index");
3202 
3203 	children = SYSCTL_CHILDREN(oid);
3204 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
3205 	    "freelist");
3206 	children = SYSCTL_CHILDREN(oid);
3207 
3208 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
3209 	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16,
3210 	    "I", "SGE context id of the freelist");
3211 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
3212 	    &nm_rxq->fl_cidx, 0, "consumer index");
3213 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
3214 	    &nm_rxq->fl_pidx, 0, "producer index");
3215 
3216 	return (rc);
3217 }
3218 
3219 
3220 static int
3221 free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)
3222 {
3223 	struct adapter *sc = vi->pi->adapter;
3224 
3225 	free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,
3226 	    nm_rxq->iq_desc);
3227 	free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,
3228 	    nm_rxq->fl_desc);
3229 
3230 	return (0);
3231 }
3232 
3233 static int
3234 alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx,
3235     struct sysctl_oid *oid)
3236 {
3237 	int rc;
3238 	size_t len;
3239 	struct port_info *pi = vi->pi;
3240 	struct adapter *sc = pi->adapter;
3241 	struct netmap_adapter *na = NA(vi->ifp);
3242 	char name[16];
3243 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3244 
3245 	len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len;
3246 	rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map,
3247 	    &nm_txq->ba, (void **)&nm_txq->desc);
3248 	if (rc)
3249 		return (rc);
3250 
3251 	nm_txq->pidx = nm_txq->cidx = 0;
3252 	nm_txq->sidx = na->num_tx_desc;
3253 	nm_txq->nid = idx;
3254 	nm_txq->iqidx = iqidx;
3255 	nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3256 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_VF_VLD(1) |
3257 	    V_TXPKT_VF(vi->viid));
3258 
3259 	snprintf(name, sizeof(name), "%d", idx);
3260 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3261 	    NULL, "netmap tx queue");
3262 	children = SYSCTL_CHILDREN(oid);
3263 
3264 	SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3265 	    &nm_txq->cntxt_id, 0, "SGE context id of the queue");
3266 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx",
3267 	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I",
3268 	    "consumer index");
3269 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx",
3270 	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I",
3271 	    "producer index");
3272 
3273 	return (rc);
3274 }
3275 
3276 static int
3277 free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq)
3278 {
3279 	struct adapter *sc = vi->pi->adapter;
3280 
3281 	free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba,
3282 	    nm_txq->desc);
3283 
3284 	return (0);
3285 }
3286 #endif
3287 
3288 static int
3289 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
3290 {
3291 	int rc, cntxt_id;
3292 	struct fw_eq_ctrl_cmd c;
3293 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
3294 
3295 	bzero(&c, sizeof(c));
3296 
3297 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
3298 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
3299 	    V_FW_EQ_CTRL_CMD_VFN(0));
3300 	c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
3301 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
3302 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
3303 	c.physeqid_pkd = htobe32(0);
3304 	c.fetchszm_to_iqid =
3305 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
3306 		V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
3307 		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
3308 	c.dcaen_to_eqsize =
3309 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3310 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3311 		V_FW_EQ_CTRL_CMD_EQSIZE(qsize));
3312 	c.eqaddr = htobe64(eq->ba);
3313 
3314 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3315 	if (rc != 0) {
3316 		device_printf(sc->dev,
3317 		    "failed to create control queue %d: %d\n", eq->tx_chan, rc);
3318 		return (rc);
3319 	}
3320 	eq->flags |= EQ_ALLOCATED;
3321 
3322 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
3323 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
3324 	if (cntxt_id >= sc->sge.neq)
3325 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
3326 		cntxt_id, sc->sge.neq - 1);
3327 	sc->sge.eqmap[cntxt_id] = eq;
3328 
3329 	return (rc);
3330 }
3331 
3332 static int
3333 eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
3334 {
3335 	int rc, cntxt_id;
3336 	struct fw_eq_eth_cmd c;
3337 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
3338 
3339 	bzero(&c, sizeof(c));
3340 
3341 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
3342 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
3343 	    V_FW_EQ_ETH_CMD_VFN(0));
3344 	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
3345 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
3346 	c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
3347 	    F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
3348 	c.fetchszm_to_iqid =
3349 	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
3350 		V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
3351 		V_FW_EQ_ETH_CMD_IQID(eq->iqid));
3352 	c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3353 	    V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3354 	    V_FW_EQ_ETH_CMD_EQSIZE(qsize));
3355 	c.eqaddr = htobe64(eq->ba);
3356 
3357 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3358 	if (rc != 0) {
3359 		device_printf(vi->dev,
3360 		    "failed to create Ethernet egress queue: %d\n", rc);
3361 		return (rc);
3362 	}
3363 	eq->flags |= EQ_ALLOCATED;
3364 
3365 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
3366 	eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
3367 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
3368 	if (cntxt_id >= sc->sge.neq)
3369 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
3370 		cntxt_id, sc->sge.neq - 1);
3371 	sc->sge.eqmap[cntxt_id] = eq;
3372 
3373 	return (rc);
3374 }
3375 
3376 #ifdef TCP_OFFLOAD
3377 static int
3378 ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
3379 {
3380 	int rc, cntxt_id;
3381 	struct fw_eq_ofld_cmd c;
3382 	int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
3383 
3384 	bzero(&c, sizeof(c));
3385 
3386 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
3387 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
3388 	    V_FW_EQ_OFLD_CMD_VFN(0));
3389 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
3390 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
3391 	c.fetchszm_to_iqid =
3392 		htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
3393 		    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
3394 		    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
3395 	c.dcaen_to_eqsize =
3396 	    htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
3397 		V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
3398 		V_FW_EQ_OFLD_CMD_EQSIZE(qsize));
3399 	c.eqaddr = htobe64(eq->ba);
3400 
3401 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3402 	if (rc != 0) {
3403 		device_printf(vi->dev,
3404 		    "failed to create egress queue for TCP offload: %d\n", rc);
3405 		return (rc);
3406 	}
3407 	eq->flags |= EQ_ALLOCATED;
3408 
3409 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
3410 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
3411 	if (cntxt_id >= sc->sge.neq)
3412 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
3413 		cntxt_id, sc->sge.neq - 1);
3414 	sc->sge.eqmap[cntxt_id] = eq;
3415 
3416 	return (rc);
3417 }
3418 #endif
3419 
3420 static int
3421 alloc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
3422 {
3423 	int rc, qsize;
3424 	size_t len;
3425 
3426 	mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
3427 
3428 	qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
3429 	len = qsize * EQ_ESIZE;
3430 	rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
3431 	    &eq->ba, (void **)&eq->desc);
3432 	if (rc)
3433 		return (rc);
3434 
3435 	eq->pidx = eq->cidx = 0;
3436 	eq->equeqidx = eq->dbidx = 0;
3437 	eq->doorbells = sc->doorbells;
3438 
3439 	switch (eq->flags & EQ_TYPEMASK) {
3440 	case EQ_CTRL:
3441 		rc = ctrl_eq_alloc(sc, eq);
3442 		break;
3443 
3444 	case EQ_ETH:
3445 		rc = eth_eq_alloc(sc, vi, eq);
3446 		break;
3447 
3448 #ifdef TCP_OFFLOAD
3449 	case EQ_OFLD:
3450 		rc = ofld_eq_alloc(sc, vi, eq);
3451 		break;
3452 #endif
3453 
3454 	default:
3455 		panic("%s: invalid eq type %d.", __func__,
3456 		    eq->flags & EQ_TYPEMASK);
3457 	}
3458 	if (rc != 0) {
3459 		device_printf(sc->dev,
3460 		    "failed to allocate egress queue(%d): %d\n",
3461 		    eq->flags & EQ_TYPEMASK, rc);
3462 	}
3463 
3464 	if (isset(&eq->doorbells, DOORBELL_UDB) ||
3465 	    isset(&eq->doorbells, DOORBELL_UDBWC) ||
3466 	    isset(&eq->doorbells, DOORBELL_WCWR)) {
3467 		uint32_t s_qpp = sc->params.sge.eq_s_qpp;
3468 		uint32_t mask = (1 << s_qpp) - 1;
3469 		volatile uint8_t *udb;
3470 
3471 		udb = sc->udbs_base + UDBS_DB_OFFSET;
3472 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;	/* pg offset */
3473 		eq->udb_qid = eq->cntxt_id & mask;		/* id in page */
3474 		if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
3475 	    		clrbit(&eq->doorbells, DOORBELL_WCWR);
3476 		else {
3477 			udb += eq->udb_qid << UDBS_SEG_SHIFT;	/* seg offset */
3478 			eq->udb_qid = 0;
3479 		}
3480 		eq->udb = (volatile void *)udb;
3481 	}
3482 
3483 	return (rc);
3484 }
3485 
3486 static int
3487 free_eq(struct adapter *sc, struct sge_eq *eq)
3488 {
3489 	int rc;
3490 
3491 	if (eq->flags & EQ_ALLOCATED) {
3492 		switch (eq->flags & EQ_TYPEMASK) {
3493 		case EQ_CTRL:
3494 			rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
3495 			    eq->cntxt_id);
3496 			break;
3497 
3498 		case EQ_ETH:
3499 			rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
3500 			    eq->cntxt_id);
3501 			break;
3502 
3503 #ifdef TCP_OFFLOAD
3504 		case EQ_OFLD:
3505 			rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
3506 			    eq->cntxt_id);
3507 			break;
3508 #endif
3509 
3510 		default:
3511 			panic("%s: invalid eq type %d.", __func__,
3512 			    eq->flags & EQ_TYPEMASK);
3513 		}
3514 		if (rc != 0) {
3515 			device_printf(sc->dev,
3516 			    "failed to free egress queue (%d): %d\n",
3517 			    eq->flags & EQ_TYPEMASK, rc);
3518 			return (rc);
3519 		}
3520 		eq->flags &= ~EQ_ALLOCATED;
3521 	}
3522 
3523 	free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
3524 
3525 	if (mtx_initialized(&eq->eq_lock))
3526 		mtx_destroy(&eq->eq_lock);
3527 
3528 	bzero(eq, sizeof(*eq));
3529 	return (0);
3530 }
3531 
3532 static int
3533 alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq,
3534     struct sysctl_oid *oid)
3535 {
3536 	int rc;
3537 	struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx;
3538 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3539 
3540 	rc = alloc_eq(sc, vi, &wrq->eq);
3541 	if (rc)
3542 		return (rc);
3543 
3544 	wrq->adapter = sc;
3545 	TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq);
3546 	TAILQ_INIT(&wrq->incomplete_wrs);
3547 	STAILQ_INIT(&wrq->wr_list);
3548 	wrq->nwr_pending = 0;
3549 	wrq->ndesc_needed = 0;
3550 
3551 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3552 	    &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
3553 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
3554 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
3555 	    "consumer index");
3556 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
3557 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
3558 	    "producer index");
3559 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD,
3560 	    &wrq->tx_wrs_direct, "# of work requests (direct)");
3561 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD,
3562 	    &wrq->tx_wrs_copied, "# of work requests (copied)");
3563 
3564 	return (rc);
3565 }
3566 
3567 static int
3568 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
3569 {
3570 	int rc;
3571 
3572 	rc = free_eq(sc, &wrq->eq);
3573 	if (rc)
3574 		return (rc);
3575 
3576 	bzero(wrq, sizeof(*wrq));
3577 	return (0);
3578 }
3579 
3580 static int
3581 alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx,
3582     struct sysctl_oid *oid)
3583 {
3584 	int rc;
3585 	struct port_info *pi = vi->pi;
3586 	struct adapter *sc = pi->adapter;
3587 	struct sge_eq *eq = &txq->eq;
3588 	char name[16];
3589 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
3590 
3591 	rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx,
3592 	    M_CXGBE, M_WAITOK);
3593 	if (rc != 0) {
3594 		device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc);
3595 		return (rc);
3596 	}
3597 
3598 	rc = alloc_eq(sc, vi, eq);
3599 	if (rc != 0) {
3600 		mp_ring_free(txq->r);
3601 		txq->r = NULL;
3602 		return (rc);
3603 	}
3604 
3605 	/* Can't fail after this point. */
3606 
3607 	if (idx == 0)
3608 		sc->sge.eq_base = eq->abs_id - eq->cntxt_id;
3609 	else
3610 		KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id,
3611 		    ("eq_base mismatch"));
3612 	KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF,
3613 	    ("PF with non-zero eq_base"));
3614 
3615 	TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
3616 	txq->ifp = vi->ifp;
3617 	txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
3618 	if (sc->flags & IS_VF)
3619 		txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
3620 		    V_TXPKT_INTF(pi->tx_chan));
3621 	else
3622 		txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3623 		    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_VF_VLD(1) |
3624 		    V_TXPKT_VF(vi->viid));
3625 	txq->tc_idx = -1;
3626 	txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
3627 	    M_ZERO | M_WAITOK);
3628 
3629 	snprintf(name, sizeof(name), "%d", idx);
3630 	oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
3631 	    NULL, "tx queue");
3632 	children = SYSCTL_CHILDREN(oid);
3633 
3634 	SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
3635 	    &eq->abs_id, 0, "absolute id of the queue");
3636 	SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3637 	    &eq->cntxt_id, 0, "SGE context id of the queue");
3638 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx",
3639 	    CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I",
3640 	    "consumer index");
3641 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx",
3642 	    CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I",
3643 	    "producer index");
3644 
3645 	SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc",
3646 	    CTLTYPE_INT | CTLFLAG_RW, vi, idx, sysctl_tc, "I",
3647 	    "traffic class (-1 means none)");
3648 
3649 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
3650 	    &txq->txcsum, "# of times hardware assisted with checksum");
3651 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion",
3652 	    CTLFLAG_RD, &txq->vlan_insertion,
3653 	    "# of times hardware inserted 802.1Q tag");
3654 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
3655 	    &txq->tso_wrs, "# of TSO work requests");
3656 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
3657 	    &txq->imm_wrs, "# of work requests with immediate data");
3658 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
3659 	    &txq->sgl_wrs, "# of work requests with direct SGL");
3660 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
3661 	    &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
3662 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs",
3663 	    CTLFLAG_RD, &txq->txpkts0_wrs,
3664 	    "# of txpkts (type 0) work requests");
3665 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs",
3666 	    CTLFLAG_RD, &txq->txpkts1_wrs,
3667 	    "# of txpkts (type 1) work requests");
3668 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts",
3669 	    CTLFLAG_RD, &txq->txpkts0_pkts,
3670 	    "# of frames tx'd using type0 txpkts work requests");
3671 	SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts",
3672 	    CTLFLAG_RD, &txq->txpkts1_pkts,
3673 	    "# of frames tx'd using type1 txpkts work requests");
3674 
3675 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues",
3676 	    CTLFLAG_RD, &txq->r->enqueues,
3677 	    "# of enqueues to the mp_ring for this queue");
3678 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops",
3679 	    CTLFLAG_RD, &txq->r->drops,
3680 	    "# of drops in the mp_ring for this queue");
3681 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts",
3682 	    CTLFLAG_RD, &txq->r->starts,
3683 	    "# of normal consumer starts in the mp_ring for this queue");
3684 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_stalls",
3685 	    CTLFLAG_RD, &txq->r->stalls,
3686 	    "# of consumer stalls in the mp_ring for this queue");
3687 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_restarts",
3688 	    CTLFLAG_RD, &txq->r->restarts,
3689 	    "# of consumer restarts in the mp_ring for this queue");
3690 	SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_abdications",
3691 	    CTLFLAG_RD, &txq->r->abdications,
3692 	    "# of consumer abdications in the mp_ring for this queue");
3693 
3694 	return (0);
3695 }
3696 
3697 static int
3698 free_txq(struct vi_info *vi, struct sge_txq *txq)
3699 {
3700 	int rc;
3701 	struct adapter *sc = vi->pi->adapter;
3702 	struct sge_eq *eq = &txq->eq;
3703 
3704 	rc = free_eq(sc, eq);
3705 	if (rc)
3706 		return (rc);
3707 
3708 	sglist_free(txq->gl);
3709 	free(txq->sdesc, M_CXGBE);
3710 	mp_ring_free(txq->r);
3711 
3712 	bzero(txq, sizeof(*txq));
3713 	return (0);
3714 }
3715 
3716 static void
3717 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3718 {
3719 	bus_addr_t *ba = arg;
3720 
3721 	KASSERT(nseg == 1,
3722 	    ("%s meant for single segment mappings only.", __func__));
3723 
3724 	*ba = error ? 0 : segs->ds_addr;
3725 }
3726 
3727 static inline void
3728 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3729 {
3730 	uint32_t n, v;
3731 
3732 	n = IDXDIFF(fl->pidx / 8, fl->dbidx, fl->sidx);
3733 	MPASS(n > 0);
3734 
3735 	wmb();
3736 	v = fl->dbval | V_PIDX(n);
3737 	if (fl->udb)
3738 		*fl->udb = htole32(v);
3739 	else
3740 		t4_write_reg(sc, sc->sge_kdoorbell_reg, v);
3741 	IDXINCR(fl->dbidx, n, fl->sidx);
3742 }
3743 
3744 /*
3745  * Fills up the freelist by allocating up to 'n' buffers.  Buffers that are
3746  * recycled do not count towards this allocation budget.
3747  *
3748  * Returns non-zero to indicate that this freelist should be added to the list
3749  * of starving freelists.
3750  */
3751 static int
3752 refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
3753 {
3754 	__be64 *d;
3755 	struct fl_sdesc *sd;
3756 	uintptr_t pa;
3757 	caddr_t cl;
3758 	struct cluster_layout *cll;
3759 	struct sw_zone_info *swz;
3760 	struct cluster_metadata *clm;
3761 	uint16_t max_pidx;
3762 	uint16_t hw_cidx = fl->hw_cidx;		/* stable snapshot */
3763 
3764 	FL_LOCK_ASSERT_OWNED(fl);
3765 
3766 	/*
3767 	 * We always stop at the beginning of the hardware descriptor that's just
3768 	 * before the one with the hw cidx.  This is to avoid hw pidx = hw cidx,
3769 	 * which would mean an empty freelist to the chip.
3770 	 */
3771 	max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
3772 	if (fl->pidx == max_pidx * 8)
3773 		return (0);
3774 
3775 	d = &fl->desc[fl->pidx];
3776 	sd = &fl->sdesc[fl->pidx];
3777 	cll = &fl->cll_def;	/* default layout */
3778 	swz = &sc->sge.sw_zone_info[cll->zidx];
3779 
3780 	while (n > 0) {
3781 
3782 		if (sd->cl != NULL) {
3783 
3784 			if (sd->nmbuf == 0) {
3785 				/*
3786 				 * Fast recycle without involving any atomics on
3787 				 * the cluster's metadata (if the cluster has
3788 				 * metadata).  This happens when all frames
3789 				 * received in the cluster were small enough to
3790 				 * fit within a single mbuf each.
3791 				 */
3792 				fl->cl_fast_recycled++;
3793 #ifdef INVARIANTS
3794 				clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
3795 				if (clm != NULL)
3796 					MPASS(clm->refcount == 1);
3797 #endif
3798 				goto recycled_fast;
3799 			}
3800 
3801 			/*
3802 			 * Cluster is guaranteed to have metadata.  Clusters
3803 			 * without metadata always take the fast recycle path
3804 			 * when they're recycled.
3805 			 */
3806 			clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
3807 			MPASS(clm != NULL);
3808 
3809 			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
3810 				fl->cl_recycled++;
3811 				counter_u64_add(extfree_rels, 1);
3812 				goto recycled;
3813 			}
3814 			sd->cl = NULL;	/* gave up my reference */
3815 		}
3816 		MPASS(sd->cl == NULL);
3817 alloc:
3818 		cl = uma_zalloc(swz->zone, M_NOWAIT);
3819 		if (__predict_false(cl == NULL)) {
3820 			if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 ||
3821 			    fl->cll_def.zidx == fl->cll_alt.zidx)
3822 				break;
3823 
3824 			/* fall back to the safe zone */
3825 			cll = &fl->cll_alt;
3826 			swz = &sc->sge.sw_zone_info[cll->zidx];
3827 			goto alloc;
3828 		}
3829 		fl->cl_allocated++;
3830 		n--;
3831 
3832 		pa = pmap_kextract((vm_offset_t)cl);
3833 		pa += cll->region1;
3834 		sd->cl = cl;
3835 		sd->cll = *cll;
3836 		*d = htobe64(pa | cll->hwidx);
3837 		clm = cl_metadata(sc, fl, cll, cl);
3838 		if (clm != NULL) {
3839 recycled:
3840 #ifdef INVARIANTS
3841 			clm->sd = sd;
3842 #endif
3843 			clm->refcount = 1;
3844 		}
3845 		sd->nmbuf = 0;
3846 recycled_fast:
3847 		d++;
3848 		sd++;
3849 		if (__predict_false(++fl->pidx % 8 == 0)) {
3850 			uint16_t pidx = fl->pidx / 8;
3851 
3852 			if (__predict_false(pidx == fl->sidx)) {
3853 				fl->pidx = 0;
3854 				pidx = 0;
3855 				sd = fl->sdesc;
3856 				d = fl->desc;
3857 			}
3858 			if (pidx == max_pidx)
3859 				break;
3860 
3861 			if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
3862 				ring_fl_db(sc, fl);
3863 		}
3864 	}
3865 
3866 	if (fl->pidx / 8 != fl->dbidx)
3867 		ring_fl_db(sc, fl);
3868 
3869 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
3870 }
3871 
3872 /*
3873  * Attempt to refill all starving freelists.
3874  */
3875 static void
3876 refill_sfl(void *arg)
3877 {
3878 	struct adapter *sc = arg;
3879 	struct sge_fl *fl, *fl_temp;
3880 
3881 	mtx_assert(&sc->sfl_lock, MA_OWNED);
3882 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
3883 		FL_LOCK(fl);
3884 		refill_fl(sc, fl, 64);
3885 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
3886 			TAILQ_REMOVE(&sc->sfl, fl, link);
3887 			fl->flags &= ~FL_STARVING;
3888 		}
3889 		FL_UNLOCK(fl);
3890 	}
3891 
3892 	if (!TAILQ_EMPTY(&sc->sfl))
3893 		callout_schedule(&sc->sfl_callout, hz / 5);
3894 }
3895 
3896 static int
3897 alloc_fl_sdesc(struct sge_fl *fl)
3898 {
3899 
3900 	fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE,
3901 	    M_ZERO | M_WAITOK);
3902 
3903 	return (0);
3904 }
3905 
3906 static void
3907 free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
3908 {
3909 	struct fl_sdesc *sd;
3910 	struct cluster_metadata *clm;
3911 	struct cluster_layout *cll;
3912 	int i;
3913 
3914 	sd = fl->sdesc;
3915 	for (i = 0; i < fl->sidx * 8; i++, sd++) {
3916 		if (sd->cl == NULL)
3917 			continue;
3918 
3919 		cll = &sd->cll;
3920 		clm = cl_metadata(sc, fl, cll, sd->cl);
3921 		if (sd->nmbuf == 0)
3922 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
3923 		else if (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1) {
3924 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
3925 			counter_u64_add(extfree_rels, 1);
3926 		}
3927 		sd->cl = NULL;
3928 	}
3929 
3930 	free(fl->sdesc, M_CXGBE);
3931 	fl->sdesc = NULL;
3932 }
3933 
3934 static inline void
3935 get_pkt_gl(struct mbuf *m, struct sglist *gl)
3936 {
3937 	int rc;
3938 
3939 	M_ASSERTPKTHDR(m);
3940 
3941 	sglist_reset(gl);
3942 	rc = sglist_append_mbuf(gl, m);
3943 	if (__predict_false(rc != 0)) {
3944 		panic("%s: mbuf %p (%d segs) was vetted earlier but now fails "
3945 		    "with %d.", __func__, m, mbuf_nsegs(m), rc);
3946 	}
3947 
3948 	KASSERT(gl->sg_nseg == mbuf_nsegs(m),
3949 	    ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m,
3950 	    mbuf_nsegs(m), gl->sg_nseg));
3951 	KASSERT(gl->sg_nseg > 0 &&
3952 	    gl->sg_nseg <= (needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS),
3953 	    ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__,
3954 		gl->sg_nseg, needs_tso(m) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS));
3955 }
3956 
3957 /*
3958  * len16 for a txpkt WR with a GL.  Includes the firmware work request header.
3959  */
3960 static inline u_int
3961 txpkt_len16(u_int nsegs, u_int tso)
3962 {
3963 	u_int n;
3964 
3965 	MPASS(nsegs > 0);
3966 
3967 	nsegs--; /* first segment is part of ulptx_sgl */
3968 	n = sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core) +
3969 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
3970 	if (tso)
3971 		n += sizeof(struct cpl_tx_pkt_lso_core);
3972 
3973 	return (howmany(n, 16));
3974 }
3975 
3976 /*
3977  * len16 for a txpkt_vm WR with a GL.  Includes the firmware work
3978  * request header.
3979  */
3980 static inline u_int
3981 txpkt_vm_len16(u_int nsegs, u_int tso)
3982 {
3983 	u_int n;
3984 
3985 	MPASS(nsegs > 0);
3986 
3987 	nsegs--; /* first segment is part of ulptx_sgl */
3988 	n = sizeof(struct fw_eth_tx_pkt_vm_wr) +
3989 	    sizeof(struct cpl_tx_pkt_core) +
3990 	    sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
3991 	if (tso)
3992 		n += sizeof(struct cpl_tx_pkt_lso_core);
3993 
3994 	return (howmany(n, 16));
3995 }
3996 
3997 /*
3998  * len16 for a txpkts type 0 WR with a GL.  Does not include the firmware work
3999  * request header.
4000  */
4001 static inline u_int
4002 txpkts0_len16(u_int nsegs)
4003 {
4004 	u_int n;
4005 
4006 	MPASS(nsegs > 0);
4007 
4008 	nsegs--; /* first segment is part of ulptx_sgl */
4009 	n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) +
4010 	    sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) +
4011 	    8 * ((3 * nsegs) / 2 + (nsegs & 1));
4012 
4013 	return (howmany(n, 16));
4014 }
4015 
4016 /*
4017  * len16 for a txpkts type 1 WR with a GL.  Does not include the firmware work
4018  * request header.
4019  */
4020 static inline u_int
4021 txpkts1_len16(void)
4022 {
4023 	u_int n;
4024 
4025 	n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl);
4026 
4027 	return (howmany(n, 16));
4028 }
4029 
4030 static inline u_int
4031 imm_payload(u_int ndesc)
4032 {
4033 	u_int n;
4034 
4035 	n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) -
4036 	    sizeof(struct cpl_tx_pkt_core);
4037 
4038 	return (n);
4039 }
4040 
4041 /*
4042  * Write a VM txpkt WR for this packet to the hardware descriptors, update the
4043  * software descriptor, and advance the pidx.  It is guaranteed that enough
4044  * descriptors are available.
4045  *
4046  * The return value is the # of hardware descriptors used.
4047  */
4048 static u_int
4049 write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq,
4050     struct fw_eth_tx_pkt_vm_wr *wr, struct mbuf *m0, u_int available)
4051 {
4052 	struct sge_eq *eq = &txq->eq;
4053 	struct tx_sdesc *txsd;
4054 	struct cpl_tx_pkt_core *cpl;
4055 	uint32_t ctrl;	/* used in many unrelated places */
4056 	uint64_t ctrl1;
4057 	int csum_type, len16, ndesc, pktlen, nsegs;
4058 	caddr_t dst;
4059 
4060 	TXQ_LOCK_ASSERT_OWNED(txq);
4061 	M_ASSERTPKTHDR(m0);
4062 	MPASS(available > 0 && available < eq->sidx);
4063 
4064 	len16 = mbuf_len16(m0);
4065 	nsegs = mbuf_nsegs(m0);
4066 	pktlen = m0->m_pkthdr.len;
4067 	ctrl = sizeof(struct cpl_tx_pkt_core);
4068 	if (needs_tso(m0))
4069 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
4070 	ndesc = howmany(len16, EQ_ESIZE / 16);
4071 	MPASS(ndesc <= available);
4072 
4073 	/* Firmware work request header */
4074 	MPASS(wr == (void *)&eq->desc[eq->pidx]);
4075 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
4076 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
4077 
4078 	ctrl = V_FW_WR_LEN16(len16);
4079 	wr->equiq_to_len16 = htobe32(ctrl);
4080 	wr->r3[0] = 0;
4081 	wr->r3[1] = 0;
4082 
4083 	/*
4084 	 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci.
4085 	 * vlantci is ignored unless the ethtype is 0x8100, so it's
4086 	 * simpler to always copy it rather than making it
4087 	 * conditional.  Also, it seems that we do not have to set
4088 	 * vlantci or fake the ethtype when doing VLAN tag insertion.
4089 	 */
4090 	m_copydata(m0, 0, sizeof(struct ether_header) + 2, wr->ethmacdst);
4091 
4092 	csum_type = -1;
4093 	if (needs_tso(m0)) {
4094 		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
4095 
4096 		KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
4097 		    m0->m_pkthdr.l4hlen > 0,
4098 		    ("%s: mbuf %p needs TSO but missing header lengths",
4099 			__func__, m0));
4100 
4101 		ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
4102 		    F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2)
4103 		    | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
4104 		if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header))
4105 			ctrl |= V_LSO_ETHHDR_LEN(1);
4106 		if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
4107 			ctrl |= F_LSO_IPV6;
4108 
4109 		lso->lso_ctrl = htobe32(ctrl);
4110 		lso->ipid_ofst = htobe16(0);
4111 		lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
4112 		lso->seqno_offset = htobe32(0);
4113 		lso->len = htobe32(pktlen);
4114 
4115 		if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
4116 			csum_type = TX_CSUM_TCPIP6;
4117 		else
4118 			csum_type = TX_CSUM_TCPIP;
4119 
4120 		cpl = (void *)(lso + 1);
4121 
4122 		txq->tso_wrs++;
4123 	} else {
4124 		if (m0->m_pkthdr.csum_flags & CSUM_IP_TCP)
4125 			csum_type = TX_CSUM_TCPIP;
4126 		else if (m0->m_pkthdr.csum_flags & CSUM_IP_UDP)
4127 			csum_type = TX_CSUM_UDPIP;
4128 		else if (m0->m_pkthdr.csum_flags & CSUM_IP6_TCP)
4129 			csum_type = TX_CSUM_TCPIP6;
4130 		else if (m0->m_pkthdr.csum_flags & CSUM_IP6_UDP)
4131 			csum_type = TX_CSUM_UDPIP6;
4132 #if defined(INET)
4133 		else if (m0->m_pkthdr.csum_flags & CSUM_IP) {
4134 			/*
4135 			 * XXX: The firmware appears to stomp on the
4136 			 * fragment/flags field of the IP header when
4137 			 * using TX_CSUM_IP.  Fall back to doing
4138 			 * software checksums.
4139 			 */
4140 			u_short *sump;
4141 			struct mbuf *m;
4142 			int offset;
4143 
4144 			m = m0;
4145 			offset = 0;
4146 			sump = m_advance(&m, &offset, m0->m_pkthdr.l2hlen +
4147 			    offsetof(struct ip, ip_sum));
4148 			*sump = in_cksum_skip(m0, m0->m_pkthdr.l2hlen +
4149 			    m0->m_pkthdr.l3hlen, m0->m_pkthdr.l2hlen);
4150 			m0->m_pkthdr.csum_flags &= ~CSUM_IP;
4151 		}
4152 #endif
4153 
4154 		cpl = (void *)(wr + 1);
4155 	}
4156 
4157 	/* Checksum offload */
4158 	ctrl1 = 0;
4159 	if (needs_l3_csum(m0) == 0)
4160 		ctrl1 |= F_TXPKT_IPCSUM_DIS;
4161 	if (csum_type >= 0) {
4162 		KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0,
4163 	    ("%s: mbuf %p needs checksum offload but missing header lengths",
4164 			__func__, m0));
4165 
4166 		if (chip_id(sc) <= CHELSIO_T5) {
4167 			ctrl1 |= V_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen -
4168 			    ETHER_HDR_LEN);
4169 		} else {
4170 			ctrl1 |= V_T6_TXPKT_ETHHDR_LEN(m0->m_pkthdr.l2hlen -
4171 			    ETHER_HDR_LEN);
4172 		}
4173 		ctrl1 |= V_TXPKT_IPHDR_LEN(m0->m_pkthdr.l3hlen);
4174 		ctrl1 |= V_TXPKT_CSUM_TYPE(csum_type);
4175 	} else
4176 		ctrl1 |= F_TXPKT_L4CSUM_DIS;
4177 	if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4178 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4179 		txq->txcsum++;	/* some hardware assistance provided */
4180 
4181 	/* VLAN tag insertion */
4182 	if (needs_vlan_insertion(m0)) {
4183 		ctrl1 |= F_TXPKT_VLAN_VLD |
4184 		    V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
4185 		txq->vlan_insertion++;
4186 	}
4187 
4188 	/* CPL header */
4189 	cpl->ctrl0 = txq->cpl_ctrl0;
4190 	cpl->pack = 0;
4191 	cpl->len = htobe16(pktlen);
4192 	cpl->ctrl1 = htobe64(ctrl1);
4193 
4194 	/* SGL */
4195 	dst = (void *)(cpl + 1);
4196 
4197 	/*
4198 	 * A packet using TSO will use up an entire descriptor for the
4199 	 * firmware work request header, LSO CPL, and TX_PKT_XT CPL.
4200 	 * If this descriptor is the last descriptor in the ring, wrap
4201 	 * around to the front of the ring explicitly for the start of
4202 	 * the sgl.
4203 	 */
4204 	if (dst == (void *)&eq->desc[eq->sidx]) {
4205 		dst = (void *)&eq->desc[0];
4206 		write_gl_to_txd(txq, m0, &dst, 0);
4207 	} else
4208 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
4209 	txq->sgl_wrs++;
4210 
4211 	txq->txpkt_wrs++;
4212 
4213 	txsd = &txq->sdesc[eq->pidx];
4214 	txsd->m = m0;
4215 	txsd->desc_used = ndesc;
4216 
4217 	return (ndesc);
4218 }
4219 
4220 /*
4221  * Write a txpkt WR for this packet to the hardware descriptors, update the
4222  * software descriptor, and advance the pidx.  It is guaranteed that enough
4223  * descriptors are available.
4224  *
4225  * The return value is the # of hardware descriptors used.
4226  */
4227 static u_int
4228 write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr,
4229     struct mbuf *m0, u_int available)
4230 {
4231 	struct sge_eq *eq = &txq->eq;
4232 	struct tx_sdesc *txsd;
4233 	struct cpl_tx_pkt_core *cpl;
4234 	uint32_t ctrl;	/* used in many unrelated places */
4235 	uint64_t ctrl1;
4236 	int len16, ndesc, pktlen, nsegs;
4237 	caddr_t dst;
4238 
4239 	TXQ_LOCK_ASSERT_OWNED(txq);
4240 	M_ASSERTPKTHDR(m0);
4241 	MPASS(available > 0 && available < eq->sidx);
4242 
4243 	len16 = mbuf_len16(m0);
4244 	nsegs = mbuf_nsegs(m0);
4245 	pktlen = m0->m_pkthdr.len;
4246 	ctrl = sizeof(struct cpl_tx_pkt_core);
4247 	if (needs_tso(m0))
4248 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
4249 	else if (pktlen <= imm_payload(2) && available >= 2) {
4250 		/* Immediate data.  Recalculate len16 and set nsegs to 0. */
4251 		ctrl += pktlen;
4252 		len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) +
4253 		    sizeof(struct cpl_tx_pkt_core) + pktlen, 16);
4254 		nsegs = 0;
4255 	}
4256 	ndesc = howmany(len16, EQ_ESIZE / 16);
4257 	MPASS(ndesc <= available);
4258 
4259 	/* Firmware work request header */
4260 	MPASS(wr == (void *)&eq->desc[eq->pidx]);
4261 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
4262 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
4263 
4264 	ctrl = V_FW_WR_LEN16(len16);
4265 	wr->equiq_to_len16 = htobe32(ctrl);
4266 	wr->r3 = 0;
4267 
4268 	if (needs_tso(m0)) {
4269 		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
4270 
4271 		KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
4272 		    m0->m_pkthdr.l4hlen > 0,
4273 		    ("%s: mbuf %p needs TSO but missing header lengths",
4274 			__func__, m0));
4275 
4276 		ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
4277 		    F_LSO_LAST_SLICE | V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2)
4278 		    | V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
4279 		if (m0->m_pkthdr.l2hlen == sizeof(struct ether_vlan_header))
4280 			ctrl |= V_LSO_ETHHDR_LEN(1);
4281 		if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
4282 			ctrl |= F_LSO_IPV6;
4283 
4284 		lso->lso_ctrl = htobe32(ctrl);
4285 		lso->ipid_ofst = htobe16(0);
4286 		lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
4287 		lso->seqno_offset = htobe32(0);
4288 		lso->len = htobe32(pktlen);
4289 
4290 		cpl = (void *)(lso + 1);
4291 
4292 		txq->tso_wrs++;
4293 	} else
4294 		cpl = (void *)(wr + 1);
4295 
4296 	/* Checksum offload */
4297 	ctrl1 = 0;
4298 	if (needs_l3_csum(m0) == 0)
4299 		ctrl1 |= F_TXPKT_IPCSUM_DIS;
4300 	if (needs_l4_csum(m0) == 0)
4301 		ctrl1 |= F_TXPKT_L4CSUM_DIS;
4302 	if (m0->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4303 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4304 		txq->txcsum++;	/* some hardware assistance provided */
4305 
4306 	/* VLAN tag insertion */
4307 	if (needs_vlan_insertion(m0)) {
4308 		ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
4309 		txq->vlan_insertion++;
4310 	}
4311 
4312 	/* CPL header */
4313 	cpl->ctrl0 = txq->cpl_ctrl0;
4314 	cpl->pack = 0;
4315 	cpl->len = htobe16(pktlen);
4316 	cpl->ctrl1 = htobe64(ctrl1);
4317 
4318 	/* SGL */
4319 	dst = (void *)(cpl + 1);
4320 	if (nsegs > 0) {
4321 
4322 		write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
4323 		txq->sgl_wrs++;
4324 	} else {
4325 		struct mbuf *m;
4326 
4327 		for (m = m0; m != NULL; m = m->m_next) {
4328 			copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
4329 #ifdef INVARIANTS
4330 			pktlen -= m->m_len;
4331 #endif
4332 		}
4333 #ifdef INVARIANTS
4334 		KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
4335 #endif
4336 		txq->imm_wrs++;
4337 	}
4338 
4339 	txq->txpkt_wrs++;
4340 
4341 	txsd = &txq->sdesc[eq->pidx];
4342 	txsd->m = m0;
4343 	txsd->desc_used = ndesc;
4344 
4345 	return (ndesc);
4346 }
4347 
4348 static int
4349 try_txpkts(struct mbuf *m, struct mbuf *n, struct txpkts *txp, u_int available)
4350 {
4351 	u_int needed, nsegs1, nsegs2, l1, l2;
4352 
4353 	if (cannot_use_txpkts(m) || cannot_use_txpkts(n))
4354 		return (1);
4355 
4356 	nsegs1 = mbuf_nsegs(m);
4357 	nsegs2 = mbuf_nsegs(n);
4358 	if (nsegs1 + nsegs2 == 2) {
4359 		txp->wr_type = 1;
4360 		l1 = l2 = txpkts1_len16();
4361 	} else {
4362 		txp->wr_type = 0;
4363 		l1 = txpkts0_len16(nsegs1);
4364 		l2 = txpkts0_len16(nsegs2);
4365 	}
4366 	txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + l1 + l2;
4367 	needed = howmany(txp->len16, EQ_ESIZE / 16);
4368 	if (needed > SGE_MAX_WR_NDESC || needed > available)
4369 		return (1);
4370 
4371 	txp->plen = m->m_pkthdr.len + n->m_pkthdr.len;
4372 	if (txp->plen > 65535)
4373 		return (1);
4374 
4375 	txp->npkt = 2;
4376 	set_mbuf_len16(m, l1);
4377 	set_mbuf_len16(n, l2);
4378 
4379 	return (0);
4380 }
4381 
4382 static int
4383 add_to_txpkts(struct mbuf *m, struct txpkts *txp, u_int available)
4384 {
4385 	u_int plen, len16, needed, nsegs;
4386 
4387 	MPASS(txp->wr_type == 0 || txp->wr_type == 1);
4388 
4389 	nsegs = mbuf_nsegs(m);
4390 	if (needs_tso(m) || (txp->wr_type == 1 && nsegs != 1))
4391 		return (1);
4392 
4393 	plen = txp->plen + m->m_pkthdr.len;
4394 	if (plen > 65535)
4395 		return (1);
4396 
4397 	if (txp->wr_type == 0)
4398 		len16 = txpkts0_len16(nsegs);
4399 	else
4400 		len16 = txpkts1_len16();
4401 	needed = howmany(txp->len16 + len16, EQ_ESIZE / 16);
4402 	if (needed > SGE_MAX_WR_NDESC || needed > available)
4403 		return (1);
4404 
4405 	txp->npkt++;
4406 	txp->plen = plen;
4407 	txp->len16 += len16;
4408 	set_mbuf_len16(m, len16);
4409 
4410 	return (0);
4411 }
4412 
4413 /*
4414  * Write a txpkts WR for the packets in txp to the hardware descriptors, update
4415  * the software descriptor, and advance the pidx.  It is guaranteed that enough
4416  * descriptors are available.
4417  *
4418  * The return value is the # of hardware descriptors used.
4419  */
4420 static u_int
4421 write_txpkts_wr(struct sge_txq *txq, struct fw_eth_tx_pkts_wr *wr,
4422     struct mbuf *m0, const struct txpkts *txp, u_int available)
4423 {
4424 	struct sge_eq *eq = &txq->eq;
4425 	struct tx_sdesc *txsd;
4426 	struct cpl_tx_pkt_core *cpl;
4427 	uint32_t ctrl;
4428 	uint64_t ctrl1;
4429 	int ndesc, checkwrap;
4430 	struct mbuf *m;
4431 	void *flitp;
4432 
4433 	TXQ_LOCK_ASSERT_OWNED(txq);
4434 	MPASS(txp->npkt > 0);
4435 	MPASS(txp->plen < 65536);
4436 	MPASS(m0 != NULL);
4437 	MPASS(m0->m_nextpkt != NULL);
4438 	MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
4439 	MPASS(available > 0 && available < eq->sidx);
4440 
4441 	ndesc = howmany(txp->len16, EQ_ESIZE / 16);
4442 	MPASS(ndesc <= available);
4443 
4444 	MPASS(wr == (void *)&eq->desc[eq->pidx]);
4445 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
4446 	ctrl = V_FW_WR_LEN16(txp->len16);
4447 	wr->equiq_to_len16 = htobe32(ctrl);
4448 	wr->plen = htobe16(txp->plen);
4449 	wr->npkt = txp->npkt;
4450 	wr->r3 = 0;
4451 	wr->type = txp->wr_type;
4452 	flitp = wr + 1;
4453 
4454 	/*
4455 	 * At this point we are 16B into a hardware descriptor.  If checkwrap is
4456 	 * set then we know the WR is going to wrap around somewhere.  We'll
4457 	 * check for that at appropriate points.
4458 	 */
4459 	checkwrap = eq->sidx - ndesc < eq->pidx;
4460 	for (m = m0; m != NULL; m = m->m_nextpkt) {
4461 		if (txp->wr_type == 0) {
4462 			struct ulp_txpkt *ulpmc;
4463 			struct ulptx_idata *ulpsc;
4464 
4465 			/* ULP master command */
4466 			ulpmc = flitp;
4467 			ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
4468 			    V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid));
4469 			ulpmc->len = htobe32(mbuf_len16(m));
4470 
4471 			/* ULP subcommand */
4472 			ulpsc = (void *)(ulpmc + 1);
4473 			ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
4474 			    F_ULP_TX_SC_MORE);
4475 			ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
4476 
4477 			cpl = (void *)(ulpsc + 1);
4478 			if (checkwrap &&
4479 			    (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx])
4480 				cpl = (void *)&eq->desc[0];
4481 			txq->txpkts0_pkts += txp->npkt;
4482 			txq->txpkts0_wrs++;
4483 		} else {
4484 			cpl = flitp;
4485 			txq->txpkts1_pkts += txp->npkt;
4486 			txq->txpkts1_wrs++;
4487 		}
4488 
4489 		/* Checksum offload */
4490 		ctrl1 = 0;
4491 		if (needs_l3_csum(m) == 0)
4492 			ctrl1 |= F_TXPKT_IPCSUM_DIS;
4493 		if (needs_l4_csum(m) == 0)
4494 			ctrl1 |= F_TXPKT_L4CSUM_DIS;
4495 		if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
4496 		    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
4497 			txq->txcsum++;	/* some hardware assistance provided */
4498 
4499 		/* VLAN tag insertion */
4500 		if (needs_vlan_insertion(m)) {
4501 			ctrl1 |= F_TXPKT_VLAN_VLD |
4502 			    V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
4503 			txq->vlan_insertion++;
4504 		}
4505 
4506 		/* CPL header */
4507 		cpl->ctrl0 = txq->cpl_ctrl0;
4508 		cpl->pack = 0;
4509 		cpl->len = htobe16(m->m_pkthdr.len);
4510 		cpl->ctrl1 = htobe64(ctrl1);
4511 
4512 		flitp = cpl + 1;
4513 		if (checkwrap &&
4514 		    (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
4515 			flitp = (void *)&eq->desc[0];
4516 
4517 		write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap);
4518 
4519 	}
4520 
4521 	txsd = &txq->sdesc[eq->pidx];
4522 	txsd->m = m0;
4523 	txsd->desc_used = ndesc;
4524 
4525 	return (ndesc);
4526 }
4527 
4528 /*
4529  * If the SGL ends on an address that is not 16 byte aligned, this function will
4530  * add a 0 filled flit at the end.
4531  */
4532 static void
4533 write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap)
4534 {
4535 	struct sge_eq *eq = &txq->eq;
4536 	struct sglist *gl = txq->gl;
4537 	struct sglist_seg *seg;
4538 	__be64 *flitp, *wrap;
4539 	struct ulptx_sgl *usgl;
4540 	int i, nflits, nsegs;
4541 
4542 	KASSERT(((uintptr_t)(*to) & 0xf) == 0,
4543 	    ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
4544 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
4545 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
4546 
4547 	get_pkt_gl(m, gl);
4548 	nsegs = gl->sg_nseg;
4549 	MPASS(nsegs > 0);
4550 
4551 	nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
4552 	flitp = (__be64 *)(*to);
4553 	wrap = (__be64 *)(&eq->desc[eq->sidx]);
4554 	seg = &gl->sg_segs[0];
4555 	usgl = (void *)flitp;
4556 
4557 	/*
4558 	 * We start at a 16 byte boundary somewhere inside the tx descriptor
4559 	 * ring, so we're at least 16 bytes away from the status page.  There is
4560 	 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
4561 	 */
4562 
4563 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
4564 	    V_ULPTX_NSGE(nsegs));
4565 	usgl->len0 = htobe32(seg->ss_len);
4566 	usgl->addr0 = htobe64(seg->ss_paddr);
4567 	seg++;
4568 
4569 	if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) {
4570 
4571 		/* Won't wrap around at all */
4572 
4573 		for (i = 0; i < nsegs - 1; i++, seg++) {
4574 			usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
4575 			usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
4576 		}
4577 		if (i & 1)
4578 			usgl->sge[i / 2].len[1] = htobe32(0);
4579 		flitp += nflits;
4580 	} else {
4581 
4582 		/* Will wrap somewhere in the rest of the SGL */
4583 
4584 		/* 2 flits already written, write the rest flit by flit */
4585 		flitp = (void *)(usgl + 1);
4586 		for (i = 0; i < nflits - 2; i++) {
4587 			if (flitp == wrap)
4588 				flitp = (void *)eq->desc;
4589 			*flitp++ = get_flit(seg, nsegs - 1, i);
4590 		}
4591 	}
4592 
4593 	if (nflits & 1) {
4594 		MPASS(((uintptr_t)flitp) & 0xf);
4595 		*flitp++ = 0;
4596 	}
4597 
4598 	MPASS((((uintptr_t)flitp) & 0xf) == 0);
4599 	if (__predict_false(flitp == wrap))
4600 		*to = (void *)eq->desc;
4601 	else
4602 		*to = (void *)flitp;
4603 }
4604 
4605 static inline void
4606 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
4607 {
4608 
4609 	MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
4610 	MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
4611 
4612 	if (__predict_true((uintptr_t)(*to) + len <=
4613 	    (uintptr_t)&eq->desc[eq->sidx])) {
4614 		bcopy(from, *to, len);
4615 		(*to) += len;
4616 	} else {
4617 		int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
4618 
4619 		bcopy(from, *to, portion);
4620 		from += portion;
4621 		portion = len - portion;	/* remaining */
4622 		bcopy(from, (void *)eq->desc, portion);
4623 		(*to) = (caddr_t)eq->desc + portion;
4624 	}
4625 }
4626 
4627 static inline void
4628 ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n)
4629 {
4630 	u_int db;
4631 
4632 	MPASS(n > 0);
4633 
4634 	db = eq->doorbells;
4635 	if (n > 1)
4636 		clrbit(&db, DOORBELL_WCWR);
4637 	wmb();
4638 
4639 	switch (ffs(db) - 1) {
4640 	case DOORBELL_UDB:
4641 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
4642 		break;
4643 
4644 	case DOORBELL_WCWR: {
4645 		volatile uint64_t *dst, *src;
4646 		int i;
4647 
4648 		/*
4649 		 * Queues whose 128B doorbell segment fits in the page do not
4650 		 * use relative qid (udb_qid is always 0).  Only queues with
4651 		 * doorbell segments can do WCWR.
4652 		 */
4653 		KASSERT(eq->udb_qid == 0 && n == 1,
4654 		    ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
4655 		    __func__, eq->doorbells, n, eq->dbidx, eq));
4656 
4657 		dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
4658 		    UDBS_DB_OFFSET);
4659 		i = eq->dbidx;
4660 		src = (void *)&eq->desc[i];
4661 		while (src != (void *)&eq->desc[i + 1])
4662 			*dst++ = *src++;
4663 		wmb();
4664 		break;
4665 	}
4666 
4667 	case DOORBELL_UDBWC:
4668 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
4669 		wmb();
4670 		break;
4671 
4672 	case DOORBELL_KDB:
4673 		t4_write_reg(sc, sc->sge_kdoorbell_reg,
4674 		    V_QID(eq->cntxt_id) | V_PIDX(n));
4675 		break;
4676 	}
4677 
4678 	IDXINCR(eq->dbidx, n, eq->sidx);
4679 }
4680 
4681 static inline u_int
4682 reclaimable_tx_desc(struct sge_eq *eq)
4683 {
4684 	uint16_t hw_cidx;
4685 
4686 	hw_cidx = read_hw_cidx(eq);
4687 	return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx));
4688 }
4689 
4690 static inline u_int
4691 total_available_tx_desc(struct sge_eq *eq)
4692 {
4693 	uint16_t hw_cidx, pidx;
4694 
4695 	hw_cidx = read_hw_cidx(eq);
4696 	pidx = eq->pidx;
4697 
4698 	if (pidx == hw_cidx)
4699 		return (eq->sidx - 1);
4700 	else
4701 		return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1);
4702 }
4703 
4704 static inline uint16_t
4705 read_hw_cidx(struct sge_eq *eq)
4706 {
4707 	struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4708 	uint16_t cidx = spg->cidx;	/* stable snapshot */
4709 
4710 	return (be16toh(cidx));
4711 }
4712 
4713 /*
4714  * Reclaim 'n' descriptors approximately.
4715  */
4716 static u_int
4717 reclaim_tx_descs(struct sge_txq *txq, u_int n)
4718 {
4719 	struct tx_sdesc *txsd;
4720 	struct sge_eq *eq = &txq->eq;
4721 	u_int can_reclaim, reclaimed;
4722 
4723 	TXQ_LOCK_ASSERT_OWNED(txq);
4724 	MPASS(n > 0);
4725 
4726 	reclaimed = 0;
4727 	can_reclaim = reclaimable_tx_desc(eq);
4728 	while (can_reclaim && reclaimed < n) {
4729 		int ndesc;
4730 		struct mbuf *m, *nextpkt;
4731 
4732 		txsd = &txq->sdesc[eq->cidx];
4733 		ndesc = txsd->desc_used;
4734 
4735 		/* Firmware doesn't return "partial" credits. */
4736 		KASSERT(can_reclaim >= ndesc,
4737 		    ("%s: unexpected number of credits: %d, %d",
4738 		    __func__, can_reclaim, ndesc));
4739 
4740 		for (m = txsd->m; m != NULL; m = nextpkt) {
4741 			nextpkt = m->m_nextpkt;
4742 			m->m_nextpkt = NULL;
4743 			m_freem(m);
4744 		}
4745 		reclaimed += ndesc;
4746 		can_reclaim -= ndesc;
4747 		IDXINCR(eq->cidx, ndesc, eq->sidx);
4748 	}
4749 
4750 	return (reclaimed);
4751 }
4752 
4753 static void
4754 tx_reclaim(void *arg, int n)
4755 {
4756 	struct sge_txq *txq = arg;
4757 	struct sge_eq *eq = &txq->eq;
4758 
4759 	do {
4760 		if (TXQ_TRYLOCK(txq) == 0)
4761 			break;
4762 		n = reclaim_tx_descs(txq, 32);
4763 		if (eq->cidx == eq->pidx)
4764 			eq->equeqidx = eq->pidx;
4765 		TXQ_UNLOCK(txq);
4766 	} while (n > 0);
4767 }
4768 
4769 static __be64
4770 get_flit(struct sglist_seg *segs, int nsegs, int idx)
4771 {
4772 	int i = (idx / 3) * 2;
4773 
4774 	switch (idx % 3) {
4775 	case 0: {
4776 		__be64 rc;
4777 
4778 		rc = htobe32(segs[i].ss_len);
4779 		if (i + 1 < nsegs)
4780 			rc |= (uint64_t)htobe32(segs[i + 1].ss_len) << 32;
4781 
4782 		return (rc);
4783 	}
4784 	case 1:
4785 		return (htobe64(segs[i].ss_paddr));
4786 	case 2:
4787 		return (htobe64(segs[i + 1].ss_paddr));
4788 	}
4789 
4790 	return (0);
4791 }
4792 
4793 static void
4794 find_best_refill_source(struct adapter *sc, struct sge_fl *fl, int maxp)
4795 {
4796 	int8_t zidx, hwidx, idx;
4797 	uint16_t region1, region3;
4798 	int spare, spare_needed, n;
4799 	struct sw_zone_info *swz;
4800 	struct hw_buf_info *hwb, *hwb_list = &sc->sge.hw_buf_info[0];
4801 
4802 	/*
4803 	 * Buffer Packing: Look for PAGE_SIZE or larger zone which has a bufsize
4804 	 * large enough for the max payload and cluster metadata.  Otherwise
4805 	 * settle for the largest bufsize that leaves enough room in the cluster
4806 	 * for metadata.
4807 	 *
4808 	 * Without buffer packing: Look for the smallest zone which has a
4809 	 * bufsize large enough for the max payload.  Settle for the largest
4810 	 * bufsize available if there's nothing big enough for max payload.
4811 	 */
4812 	spare_needed = fl->flags & FL_BUF_PACKING ? CL_METADATA_SIZE : 0;
4813 	swz = &sc->sge.sw_zone_info[0];
4814 	hwidx = -1;
4815 	for (zidx = 0; zidx < SW_ZONE_SIZES; zidx++, swz++) {
4816 		if (swz->size > largest_rx_cluster) {
4817 			if (__predict_true(hwidx != -1))
4818 				break;
4819 
4820 			/*
4821 			 * This is a misconfiguration.  largest_rx_cluster is
4822 			 * preventing us from finding a refill source.  See
4823 			 * dev.t5nex.<n>.buffer_sizes to figure out why.
4824 			 */
4825 			device_printf(sc->dev, "largest_rx_cluster=%u leaves no"
4826 			    " refill source for fl %p (dma %u).  Ignored.\n",
4827 			    largest_rx_cluster, fl, maxp);
4828 		}
4829 		for (idx = swz->head_hwidx; idx != -1; idx = hwb->next) {
4830 			hwb = &hwb_list[idx];
4831 			spare = swz->size - hwb->size;
4832 			if (spare < spare_needed)
4833 				continue;
4834 
4835 			hwidx = idx;		/* best option so far */
4836 			if (hwb->size >= maxp) {
4837 
4838 				if ((fl->flags & FL_BUF_PACKING) == 0)
4839 					goto done; /* stop looking (not packing) */
4840 
4841 				if (swz->size >= safest_rx_cluster)
4842 					goto done; /* stop looking (packing) */
4843 			}
4844 			break;		/* keep looking, next zone */
4845 		}
4846 	}
4847 done:
4848 	/* A usable hwidx has been located. */
4849 	MPASS(hwidx != -1);
4850 	hwb = &hwb_list[hwidx];
4851 	zidx = hwb->zidx;
4852 	swz = &sc->sge.sw_zone_info[zidx];
4853 	region1 = 0;
4854 	region3 = swz->size - hwb->size;
4855 
4856 	/*
4857 	 * Stay within this zone and see if there is a better match when mbuf
4858 	 * inlining is allowed.  Remember that the hwidx's are sorted in
4859 	 * decreasing order of size (so in increasing order of spare area).
4860 	 */
4861 	for (idx = hwidx; idx != -1; idx = hwb->next) {
4862 		hwb = &hwb_list[idx];
4863 		spare = swz->size - hwb->size;
4864 
4865 		if (allow_mbufs_in_cluster == 0 || hwb->size < maxp)
4866 			break;
4867 
4868 		/*
4869 		 * Do not inline mbufs if doing so would violate the pad/pack
4870 		 * boundary alignment requirement.
4871 		 */
4872 		if (fl_pad && (MSIZE % sc->params.sge.pad_boundary) != 0)
4873 			continue;
4874 		if (fl->flags & FL_BUF_PACKING &&
4875 		    (MSIZE % sc->params.sge.pack_boundary) != 0)
4876 			continue;
4877 
4878 		if (spare < CL_METADATA_SIZE + MSIZE)
4879 			continue;
4880 		n = (spare - CL_METADATA_SIZE) / MSIZE;
4881 		if (n > howmany(hwb->size, maxp))
4882 			break;
4883 
4884 		hwidx = idx;
4885 		if (fl->flags & FL_BUF_PACKING) {
4886 			region1 = n * MSIZE;
4887 			region3 = spare - region1;
4888 		} else {
4889 			region1 = MSIZE;
4890 			region3 = spare - region1;
4891 			break;
4892 		}
4893 	}
4894 
4895 	KASSERT(zidx >= 0 && zidx < SW_ZONE_SIZES,
4896 	    ("%s: bad zone %d for fl %p, maxp %d", __func__, zidx, fl, maxp));
4897 	KASSERT(hwidx >= 0 && hwidx <= SGE_FLBUF_SIZES,
4898 	    ("%s: bad hwidx %d for fl %p, maxp %d", __func__, hwidx, fl, maxp));
4899 	KASSERT(region1 + sc->sge.hw_buf_info[hwidx].size + region3 ==
4900 	    sc->sge.sw_zone_info[zidx].size,
4901 	    ("%s: bad buffer layout for fl %p, maxp %d. "
4902 		"cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
4903 		sc->sge.sw_zone_info[zidx].size, region1,
4904 		sc->sge.hw_buf_info[hwidx].size, region3));
4905 	if (fl->flags & FL_BUF_PACKING || region1 > 0) {
4906 		KASSERT(region3 >= CL_METADATA_SIZE,
4907 		    ("%s: no room for metadata.  fl %p, maxp %d; "
4908 		    "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
4909 		    sc->sge.sw_zone_info[zidx].size, region1,
4910 		    sc->sge.hw_buf_info[hwidx].size, region3));
4911 		KASSERT(region1 % MSIZE == 0,
4912 		    ("%s: bad mbuf region for fl %p, maxp %d. "
4913 		    "cl %d; r1 %d, payload %d, r3 %d", __func__, fl, maxp,
4914 		    sc->sge.sw_zone_info[zidx].size, region1,
4915 		    sc->sge.hw_buf_info[hwidx].size, region3));
4916 	}
4917 
4918 	fl->cll_def.zidx = zidx;
4919 	fl->cll_def.hwidx = hwidx;
4920 	fl->cll_def.region1 = region1;
4921 	fl->cll_def.region3 = region3;
4922 }
4923 
4924 static void
4925 find_safe_refill_source(struct adapter *sc, struct sge_fl *fl)
4926 {
4927 	struct sge *s = &sc->sge;
4928 	struct hw_buf_info *hwb;
4929 	struct sw_zone_info *swz;
4930 	int spare;
4931 	int8_t hwidx;
4932 
4933 	if (fl->flags & FL_BUF_PACKING)
4934 		hwidx = s->safe_hwidx2;	/* with room for metadata */
4935 	else if (allow_mbufs_in_cluster && s->safe_hwidx2 != -1) {
4936 		hwidx = s->safe_hwidx2;
4937 		hwb = &s->hw_buf_info[hwidx];
4938 		swz = &s->sw_zone_info[hwb->zidx];
4939 		spare = swz->size - hwb->size;
4940 
4941 		/* no good if there isn't room for an mbuf as well */
4942 		if (spare < CL_METADATA_SIZE + MSIZE)
4943 			hwidx = s->safe_hwidx1;
4944 	} else
4945 		hwidx = s->safe_hwidx1;
4946 
4947 	if (hwidx == -1) {
4948 		/* No fallback source */
4949 		fl->cll_alt.hwidx = -1;
4950 		fl->cll_alt.zidx = -1;
4951 
4952 		return;
4953 	}
4954 
4955 	hwb = &s->hw_buf_info[hwidx];
4956 	swz = &s->sw_zone_info[hwb->zidx];
4957 	spare = swz->size - hwb->size;
4958 	fl->cll_alt.hwidx = hwidx;
4959 	fl->cll_alt.zidx = hwb->zidx;
4960 	if (allow_mbufs_in_cluster &&
4961 	    (fl_pad == 0 || (MSIZE % sc->params.sge.pad_boundary) == 0))
4962 		fl->cll_alt.region1 = ((spare - CL_METADATA_SIZE) / MSIZE) * MSIZE;
4963 	else
4964 		fl->cll_alt.region1 = 0;
4965 	fl->cll_alt.region3 = spare - fl->cll_alt.region1;
4966 }
4967 
4968 static void
4969 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
4970 {
4971 	mtx_lock(&sc->sfl_lock);
4972 	FL_LOCK(fl);
4973 	if ((fl->flags & FL_DOOMED) == 0) {
4974 		fl->flags |= FL_STARVING;
4975 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
4976 		callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
4977 	}
4978 	FL_UNLOCK(fl);
4979 	mtx_unlock(&sc->sfl_lock);
4980 }
4981 
4982 static void
4983 handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
4984 {
4985 	struct sge_wrq *wrq = (void *)eq;
4986 
4987 	atomic_readandclear_int(&eq->equiq);
4988 	taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
4989 }
4990 
4991 static void
4992 handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
4993 {
4994 	struct sge_txq *txq = (void *)eq;
4995 
4996 	MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH);
4997 
4998 	atomic_readandclear_int(&eq->equiq);
4999 	mp_ring_check_drainage(txq->r, 0);
5000 	taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
5001 }
5002 
5003 static int
5004 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
5005     struct mbuf *m)
5006 {
5007 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
5008 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
5009 	struct adapter *sc = iq->adapter;
5010 	struct sge *s = &sc->sge;
5011 	struct sge_eq *eq;
5012 	static void (*h[])(struct adapter *, struct sge_eq *) = {NULL,
5013 		&handle_wrq_egr_update, &handle_eth_egr_update,
5014 		&handle_wrq_egr_update};
5015 
5016 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5017 	    rss->opcode));
5018 
5019 	eq = s->eqmap[qid - s->eq_start - s->eq_base];
5020 	(*h[eq->flags & EQ_TYPEMASK])(sc, eq);
5021 
5022 	return (0);
5023 }
5024 
5025 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
5026 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
5027     offsetof(struct cpl_fw6_msg, data));
5028 
5029 static int
5030 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5031 {
5032 	struct adapter *sc = iq->adapter;
5033 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
5034 
5035 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5036 	    rss->opcode));
5037 
5038 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
5039 		const struct rss_header *rss2;
5040 
5041 		rss2 = (const struct rss_header *)&cpl->data[0];
5042 		return (t4_cpl_handler[rss2->opcode](iq, rss2, m));
5043 	}
5044 
5045 	return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0]));
5046 }
5047 
5048 /**
5049  *	t4_handle_wrerr_rpl - process a FW work request error message
5050  *	@adap: the adapter
5051  *	@rpl: start of the FW message
5052  */
5053 static int
5054 t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl)
5055 {
5056 	u8 opcode = *(const u8 *)rpl;
5057 	const struct fw_error_cmd *e = (const void *)rpl;
5058 	unsigned int i;
5059 
5060 	if (opcode != FW_ERROR_CMD) {
5061 		log(LOG_ERR,
5062 		    "%s: Received WRERR_RPL message with opcode %#x\n",
5063 		    device_get_nameunit(adap->dev), opcode);
5064 		return (EINVAL);
5065 	}
5066 	log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev),
5067 	    G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" :
5068 	    "non-fatal");
5069 	switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) {
5070 	case FW_ERROR_TYPE_EXCEPTION:
5071 		log(LOG_ERR, "exception info:\n");
5072 		for (i = 0; i < nitems(e->u.exception.info); i++)
5073 			log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ",
5074 			    be32toh(e->u.exception.info[i]));
5075 		log(LOG_ERR, "\n");
5076 		break;
5077 	case FW_ERROR_TYPE_HWMODULE:
5078 		log(LOG_ERR, "HW module regaddr %08x regval %08x\n",
5079 		    be32toh(e->u.hwmodule.regaddr),
5080 		    be32toh(e->u.hwmodule.regval));
5081 		break;
5082 	case FW_ERROR_TYPE_WR:
5083 		log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n",
5084 		    be16toh(e->u.wr.cidx),
5085 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)),
5086 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)),
5087 		    be32toh(e->u.wr.eqid));
5088 		for (i = 0; i < nitems(e->u.wr.wrhdr); i++)
5089 			log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ",
5090 			    e->u.wr.wrhdr[i]);
5091 		log(LOG_ERR, "\n");
5092 		break;
5093 	case FW_ERROR_TYPE_ACL:
5094 		log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s",
5095 		    be16toh(e->u.acl.cidx),
5096 		    G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)),
5097 		    G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)),
5098 		    be32toh(e->u.acl.eqid),
5099 		    G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" :
5100 		    "MAC");
5101 		for (i = 0; i < nitems(e->u.acl.val); i++)
5102 			log(LOG_ERR, " %02x", e->u.acl.val[i]);
5103 		log(LOG_ERR, "\n");
5104 		break;
5105 	default:
5106 		log(LOG_ERR, "type %#x\n",
5107 		    G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type)));
5108 		return (EINVAL);
5109 	}
5110 	return (0);
5111 }
5112 
5113 static int
5114 sysctl_uint16(SYSCTL_HANDLER_ARGS)
5115 {
5116 	uint16_t *id = arg1;
5117 	int i = *id;
5118 
5119 	return sysctl_handle_int(oidp, &i, 0, req);
5120 }
5121 
5122 static int
5123 sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
5124 {
5125 	struct sge *s = arg1;
5126 	struct hw_buf_info *hwb = &s->hw_buf_info[0];
5127 	struct sw_zone_info *swz = &s->sw_zone_info[0];
5128 	int i, rc;
5129 	struct sbuf sb;
5130 	char c;
5131 
5132 	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5133 	for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
5134 		if (hwb->zidx >= 0 && swz[hwb->zidx].size <= largest_rx_cluster)
5135 			c = '*';
5136 		else
5137 			c = '\0';
5138 
5139 		sbuf_printf(&sb, "%u%c ", hwb->size, c);
5140 	}
5141 	sbuf_trim(&sb);
5142 	sbuf_finish(&sb);
5143 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5144 	sbuf_delete(&sb);
5145 	return (rc);
5146 }
5147 
5148 static int
5149 sysctl_tc(SYSCTL_HANDLER_ARGS)
5150 {
5151 	struct vi_info *vi = arg1;
5152 	struct port_info *pi;
5153 	struct adapter *sc;
5154 	struct sge_txq *txq;
5155 	struct tx_sched_class *tc;
5156 	int qidx = arg2, rc, tc_idx;
5157 	uint32_t fw_queue, fw_class;
5158 
5159 	MPASS(qidx >= 0 && qidx < vi->ntxq);
5160 	pi = vi->pi;
5161 	sc = pi->adapter;
5162 	txq = &sc->sge.txq[vi->first_txq + qidx];
5163 
5164 	tc_idx = txq->tc_idx;
5165 	rc = sysctl_handle_int(oidp, &tc_idx, 0, req);
5166 	if (rc != 0 || req->newptr == NULL)
5167 		return (rc);
5168 
5169 	/* Note that -1 is legitimate input (it means unbind). */
5170 	if (tc_idx < -1 || tc_idx >= sc->chip_params->nsched_cls)
5171 		return (EINVAL);
5172 
5173 	rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4stc");
5174 	if (rc)
5175 		return (rc);
5176 
5177 	if (tc_idx == txq->tc_idx) {
5178 		rc = 0;		/* No change, nothing to do. */
5179 		goto done;
5180 	}
5181 
5182 	fw_queue = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
5183 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH) |
5184 	    V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id);
5185 
5186 	if (tc_idx == -1)
5187 		fw_class = 0xffffffff;	/* Unbind. */
5188 	else {
5189 		/*
5190 		 * Bind to a different class.  Ethernet txq's are only allowed
5191 		 * to bind to cl-rl mode-class for now.  XXX: too restrictive.
5192 		 */
5193 		tc = &pi->tc[tc_idx];
5194 		if (tc->flags & TX_SC_OK &&
5195 		    tc->params.level == SCHED_CLASS_LEVEL_CL_RL &&
5196 		    tc->params.mode == SCHED_CLASS_MODE_CLASS) {
5197 			/* Ok to proceed. */
5198 			fw_class = tc_idx;
5199 		} else {
5200 			rc = tc->flags & TX_SC_OK ? EBUSY : ENXIO;
5201 			goto done;
5202 		}
5203 	}
5204 
5205 	rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, &fw_class);
5206 	if (rc == 0) {
5207 		if (txq->tc_idx != -1) {
5208 			tc = &pi->tc[txq->tc_idx];
5209 			MPASS(tc->refcount > 0);
5210 			tc->refcount--;
5211 		}
5212 		if (tc_idx != -1) {
5213 			tc = &pi->tc[tc_idx];
5214 			tc->refcount++;
5215 		}
5216 		txq->tc_idx = tc_idx;
5217 	}
5218 done:
5219 	end_synchronized_op(sc, 0);
5220 	return (rc);
5221 }
5222