xref: /freebsd/sys/dev/cxgbe/t4_sge.c (revision 03836978bec158bdc0ecee7a4198962f91ce8298)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/types.h>
35 #include <sys/mbuf.h>
36 #include <sys/socket.h>
37 #include <sys/kernel.h>
38 #include <sys/kdb.h>
39 #include <sys/malloc.h>
40 #include <sys/queue.h>
41 #include <sys/taskqueue.h>
42 #include <sys/sysctl.h>
43 #include <sys/smp.h>
44 #include <net/bpf.h>
45 #include <net/ethernet.h>
46 #include <net/if.h>
47 #include <net/if_vlan_var.h>
48 #include <netinet/in.h>
49 #include <netinet/ip.h>
50 #include <netinet/ip6.h>
51 #include <netinet/tcp.h>
52 
53 #include "common/common.h"
54 #include "common/t4_regs.h"
55 #include "common/t4_regs_values.h"
56 #include "common/t4_msg.h"
57 
58 struct fl_buf_info {
59 	int size;
60 	int type;
61 	uma_zone_t zone;
62 };
63 
64 /* Filled up by t4_sge_modload */
65 static struct fl_buf_info fl_buf_info[FL_BUF_SIZES];
66 
67 #define FL_BUF_SIZE(x)	(fl_buf_info[x].size)
68 #define FL_BUF_TYPE(x)	(fl_buf_info[x].type)
69 #define FL_BUF_ZONE(x)	(fl_buf_info[x].zone)
70 
71 #ifdef T4_PKT_TIMESTAMP
72 #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
73 #else
74 #define RX_COPY_THRESHOLD MINCLSIZE
75 #endif
76 
77 /*
78  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
79  * 0-7 are valid values.
80  */
81 static int fl_pktshift = 2;
82 TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
83 
84 /*
85  * Pad ethernet payload up to this boundary.
86  * -1: driver should figure out a good value.
87  *  Any power of 2, from 32 to 4096 (both inclusive) is a valid value.
88  */
89 static int fl_pad = -1;
90 TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
91 
92 /*
93  * Status page length.
94  * -1: driver should figure out a good value.
95  *  64 or 128 are the only other valid values.
96  */
97 static int spg_len = -1;
98 TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
99 
100 /*
101  * Congestion drops.
102  * -1: no congestion feedback (not recommended).
103  *  0: backpressure the channel instead of dropping packets right away.
104  *  1: no backpressure, drop packets for the congested queue immediately.
105  */
106 static int cong_drop = 0;
107 TUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop);
108 
109 /* Used to track coalesced tx work request */
110 struct txpkts {
111 	uint64_t *flitp;	/* ptr to flit where next pkt should start */
112 	uint8_t npkt;		/* # of packets in this work request */
113 	uint8_t nflits;		/* # of flits used by this work request */
114 	uint16_t plen;		/* total payload (sum of all packets) */
115 };
116 
117 /* A packet's SGL.  This + m_pkthdr has all info needed for tx */
118 struct sgl {
119 	int nsegs;		/* # of segments in the SGL, 0 means imm. tx */
120 	int nflits;		/* # of flits needed for the SGL */
121 	bus_dma_segment_t seg[TX_SGL_SEGS];
122 };
123 
124 static int service_iq(struct sge_iq *, int);
125 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t,
126     int *);
127 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
128 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
129     int);
130 static inline void init_fl(struct sge_fl *, int, int, char *);
131 static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
132     char *);
133 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
134     bus_addr_t *, void **);
135 static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
136     void *);
137 static int alloc_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *,
138     int, int);
139 static int free_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *);
140 static int alloc_fwq(struct adapter *);
141 static int free_fwq(struct adapter *);
142 static int alloc_mgmtq(struct adapter *);
143 static int free_mgmtq(struct adapter *);
144 static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int,
145     struct sysctl_oid *);
146 static int free_rxq(struct port_info *, struct sge_rxq *);
147 #ifdef TCP_OFFLOAD
148 static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
149     struct sysctl_oid *);
150 static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
151 #endif
152 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
153 static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
154 #ifdef TCP_OFFLOAD
155 static int ofld_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
156 #endif
157 static int alloc_eq(struct adapter *, struct port_info *, struct sge_eq *);
158 static int free_eq(struct adapter *, struct sge_eq *);
159 static int alloc_wrq(struct adapter *, struct port_info *, struct sge_wrq *,
160     struct sysctl_oid *);
161 static int free_wrq(struct adapter *, struct sge_wrq *);
162 static int alloc_txq(struct port_info *, struct sge_txq *, int,
163     struct sysctl_oid *);
164 static int free_txq(struct port_info *, struct sge_txq *);
165 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
166 static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
167 static inline void iq_next(struct sge_iq *);
168 static inline void ring_fl_db(struct adapter *, struct sge_fl *);
169 static int refill_fl(struct adapter *, struct sge_fl *, int);
170 static void refill_sfl(void *);
171 static int alloc_fl_sdesc(struct sge_fl *);
172 static void free_fl_sdesc(struct sge_fl *);
173 static void set_fl_tag_idx(struct sge_fl *, int);
174 static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
175 
176 static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
177 static int free_pkt_sgl(struct sge_txq *, struct sgl *);
178 static int write_txpkt_wr(struct port_info *, struct sge_txq *, struct mbuf *,
179     struct sgl *);
180 static int add_to_txpkts(struct port_info *, struct sge_txq *, struct txpkts *,
181     struct mbuf *, struct sgl *);
182 static void write_txpkts_wr(struct sge_txq *, struct txpkts *);
183 static inline void write_ulp_cpl_sgl(struct port_info *, struct sge_txq *,
184     struct txpkts *, struct mbuf *, struct sgl *);
185 static int write_sgl_to_txd(struct sge_eq *, struct sgl *, caddr_t *);
186 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
187 static inline void ring_eq_db(struct adapter *, struct sge_eq *);
188 static inline int reclaimable(struct sge_eq *);
189 static int reclaim_tx_descs(struct sge_txq *, int, int);
190 static void write_eqflush_wr(struct sge_eq *);
191 static __be64 get_flit(bus_dma_segment_t *, int, int);
192 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
193     struct mbuf *);
194 static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
195     struct mbuf *);
196 
197 static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
198 
199 #if defined(__i386__) || defined(__amd64__)
200 extern u_int cpu_clflush_line_size;
201 #endif
202 
203 /*
204  * Called on MOD_LOAD.  Fills up fl_buf_info[] and validates/calculates the SGE
205  * tunables.
206  */
207 void
208 t4_sge_modload(void)
209 {
210 	int i;
211 	int bufsize[FL_BUF_SIZES] = {
212 		MCLBYTES,
213 #if MJUMPAGESIZE != MCLBYTES
214 		MJUMPAGESIZE,
215 #endif
216 		MJUM9BYTES,
217 		MJUM16BYTES
218 	};
219 
220 	for (i = 0; i < FL_BUF_SIZES; i++) {
221 		FL_BUF_SIZE(i) = bufsize[i];
222 		FL_BUF_TYPE(i) = m_gettype(bufsize[i]);
223 		FL_BUF_ZONE(i) = m_getzone(bufsize[i]);
224 	}
225 
226 	if (fl_pktshift < 0 || fl_pktshift > 7) {
227 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
228 		    " using 2 instead.\n", fl_pktshift);
229 		fl_pktshift = 2;
230 	}
231 
232 	if (fl_pad < 32 || fl_pad > 4096 || !powerof2(fl_pad)) {
233 		int pad;
234 
235 #if defined(__i386__) || defined(__amd64__)
236 		pad = max(cpu_clflush_line_size, 32);
237 #else
238 		pad = max(CACHE_LINE_SIZE, 32);
239 #endif
240 		pad = min(pad, 4096);
241 
242 		if (fl_pad != -1) {
243 			printf("Invalid hw.cxgbe.fl_pad value (%d),"
244 			    " using %d instead.\n", fl_pad, pad);
245 		}
246 		fl_pad = pad;
247 	}
248 
249 	if (spg_len != 64 && spg_len != 128) {
250 		int len;
251 
252 #if defined(__i386__) || defined(__amd64__)
253 		len = cpu_clflush_line_size > 64 ? 128 : 64;
254 #else
255 		len = 64;
256 #endif
257 		if (spg_len != -1) {
258 			printf("Invalid hw.cxgbe.spg_len value (%d),"
259 			    " using %d instead.\n", spg_len, len);
260 		}
261 		spg_len = len;
262 	}
263 
264 	if (cong_drop < -1 || cong_drop > 1) {
265 		printf("Invalid hw.cxgbe.cong_drop value (%d),"
266 		    " using 0 instead.\n", cong_drop);
267 		cong_drop = 0;
268 	}
269 }
270 
271 void
272 t4_init_sge_cpl_handlers(struct adapter *sc)
273 {
274 
275 	t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_msg);
276 	t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_msg);
277 	t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
278 	t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
279 
280 	t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
281 }
282 
283 /*
284  * adap->params.vpd.cclk must be set up before this is called.
285  */
286 void
287 t4_tweak_chip_settings(struct adapter *sc)
288 {
289 	int i;
290 	uint32_t v, m;
291 	int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
292 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
293 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
294 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
295 
296 	KASSERT(sc->flags & MASTER_PF,
297 	    ("%s: trying to change chip settings when not master.", __func__));
298 
299 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE |
300 	    V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE;
301 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
302 	    V_INGPADBOUNDARY(ilog2(fl_pad) - 5) |
303 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
304 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
305 
306 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
307 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
308 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
309 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
310 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
311 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
312 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
313 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
314 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
315 
316 	for (i = 0; i < FL_BUF_SIZES; i++) {
317 		t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
318 		    FL_BUF_SIZE(i));
319 	}
320 
321 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
322 	    V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
323 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
324 
325 	KASSERT(intr_timer[0] <= timer_max,
326 	    ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
327 	    timer_max));
328 	for (i = 1; i < nitems(intr_timer); i++) {
329 		KASSERT(intr_timer[i] >= intr_timer[i - 1],
330 		    ("%s: timers not listed in increasing order (%d)",
331 		    __func__, i));
332 
333 		while (intr_timer[i] > timer_max) {
334 			if (i == nitems(intr_timer) - 1) {
335 				intr_timer[i] = timer_max;
336 				break;
337 			}
338 			intr_timer[i] += intr_timer[i - 1];
339 			intr_timer[i] /= 2;
340 		}
341 	}
342 
343 	v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
344 	    V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
345 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
346 	v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
347 	    V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
348 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
349 	v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
350 	    V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
351 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
352 
353 	if (cong_drop == 0) {
354 		m = F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 |
355 		    F_TUNNELCNGDROP3;
356 		t4_set_reg_field(sc, A_TP_PARA_REG3, m, 0);
357 	}
358 
359 	/* 4K, 16K, 64K, 256K DDP "page sizes" */
360 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
361 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
362 
363 	m = v = F_TDDPTAGTCB;
364 	t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
365 
366 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
367 	    F_RESETDDPOFFSET;
368 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
369 	t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
370 }
371 
372 /*
373  * XXX: driver really should be able to deal with unexpected settings.
374  */
375 int
376 t4_read_chip_settings(struct adapter *sc)
377 {
378 	struct sge *s = &sc->sge;
379 	int i, rc = 0;
380 	uint32_t m, v, r;
381 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
382 
383 	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE |
384 	    V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE;
385 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
386 	    V_INGPADBOUNDARY(ilog2(fl_pad) - 5) |
387 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
388 	r = t4_read_reg(sc, A_SGE_CONTROL);
389 	if ((r & m) != v) {
390 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
391 		rc = EINVAL;
392 	}
393 
394 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
395 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
396 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
397 	    V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
398 	    V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
399 	    V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
400 	    V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
401 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
402 	r = t4_read_reg(sc, A_SGE_HOST_PAGE_SIZE);
403 	if (r != v) {
404 		device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
405 		rc = EINVAL;
406 	}
407 
408 	for (i = 0; i < FL_BUF_SIZES; i++) {
409 		v = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i));
410 		if (v != FL_BUF_SIZE(i)) {
411 			device_printf(sc->dev,
412 			    "invalid SGE_FL_BUFFER_SIZE[%d](0x%x)\n", i, v);
413 			rc = EINVAL;
414 		}
415 	}
416 
417 	r = t4_read_reg(sc, A_SGE_INGRESS_RX_THRESHOLD);
418 	s->counter_val[0] = G_THRESHOLD_0(r);
419 	s->counter_val[1] = G_THRESHOLD_1(r);
420 	s->counter_val[2] = G_THRESHOLD_2(r);
421 	s->counter_val[3] = G_THRESHOLD_3(r);
422 
423 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_0_AND_1);
424 	s->timer_val[0] = G_TIMERVALUE0(r) / core_ticks_per_usec(sc);
425 	s->timer_val[1] = G_TIMERVALUE1(r) / core_ticks_per_usec(sc);
426 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_2_AND_3);
427 	s->timer_val[2] = G_TIMERVALUE2(r) / core_ticks_per_usec(sc);
428 	s->timer_val[3] = G_TIMERVALUE3(r) / core_ticks_per_usec(sc);
429 	r = t4_read_reg(sc, A_SGE_TIMER_VALUE_4_AND_5);
430 	s->timer_val[4] = G_TIMERVALUE4(r) / core_ticks_per_usec(sc);
431 	s->timer_val[5] = G_TIMERVALUE5(r) / core_ticks_per_usec(sc);
432 
433 	if (cong_drop == 0) {
434 		m = F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 |
435 		    F_TUNNELCNGDROP3;
436 		r = t4_read_reg(sc, A_TP_PARA_REG3);
437 		if (r & m) {
438 			device_printf(sc->dev,
439 			    "invalid TP_PARA_REG3(0x%x)\n", r);
440 			rc = EINVAL;
441 		}
442 	}
443 
444 	v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
445 	r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
446 	if (r != v) {
447 		device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
448 		rc = EINVAL;
449 	}
450 
451 	m = v = F_TDDPTAGTCB;
452 	r = t4_read_reg(sc, A_ULP_RX_CTL);
453 	if ((r & m) != v) {
454 		device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
455 		rc = EINVAL;
456 	}
457 
458 	m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
459 	    F_RESETDDPOFFSET;
460 	v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
461 	r = t4_read_reg(sc, A_TP_PARA_REG5);
462 	if ((r & m) != v) {
463 		device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
464 		rc = EINVAL;
465 	}
466 
467 	r = t4_read_reg(sc, A_SGE_CONM_CTRL);
468 	s->fl_starve_threshold = G_EGRTHRESHOLD(r) * 2 + 1;
469 
470 	if (is_t5(sc)) {
471 		r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
472 		r >>= S_QUEUESPERPAGEPF0 +
473 		    (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
474 		s->s_qpp = r & M_QUEUESPERPAGEPF0;
475 	}
476 
477 	r = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
478 	sc->params.tp.tre = G_TIMERRESOLUTION(r);
479 	sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(r);
480 
481 	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
482 	t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
483 
484 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
485 	    A_TP_VLAN_PRI_MAP);
486 
487 	return (rc);
488 }
489 
490 int
491 t4_create_dma_tag(struct adapter *sc)
492 {
493 	int rc;
494 
495 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
496 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
497 	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
498 	    NULL, &sc->dmat);
499 	if (rc != 0) {
500 		device_printf(sc->dev,
501 		    "failed to create main DMA tag: %d\n", rc);
502 	}
503 
504 	return (rc);
505 }
506 
507 int
508 t4_destroy_dma_tag(struct adapter *sc)
509 {
510 	if (sc->dmat)
511 		bus_dma_tag_destroy(sc->dmat);
512 
513 	return (0);
514 }
515 
516 /*
517  * Allocate and initialize the firmware event queue and the management queue.
518  *
519  * Returns errno on failure.  Resources allocated up to that point may still be
520  * allocated.  Caller is responsible for cleanup in case this function fails.
521  */
522 int
523 t4_setup_adapter_queues(struct adapter *sc)
524 {
525 	int rc;
526 
527 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
528 
529 	sysctl_ctx_init(&sc->ctx);
530 	sc->flags |= ADAP_SYSCTL_CTX;
531 
532 	/*
533 	 * Firmware event queue
534 	 */
535 	rc = alloc_fwq(sc);
536 	if (rc != 0)
537 		return (rc);
538 
539 	/*
540 	 * Management queue.  This is just a control queue that uses the fwq as
541 	 * its associated iq.
542 	 */
543 	rc = alloc_mgmtq(sc);
544 
545 	return (rc);
546 }
547 
548 /*
549  * Idempotent
550  */
551 int
552 t4_teardown_adapter_queues(struct adapter *sc)
553 {
554 
555 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
556 
557 	/* Do this before freeing the queue */
558 	if (sc->flags & ADAP_SYSCTL_CTX) {
559 		sysctl_ctx_free(&sc->ctx);
560 		sc->flags &= ~ADAP_SYSCTL_CTX;
561 	}
562 
563 	free_mgmtq(sc);
564 	free_fwq(sc);
565 
566 	return (0);
567 }
568 
569 static inline int
570 first_vector(struct port_info *pi)
571 {
572 	struct adapter *sc = pi->adapter;
573 	int rc = T4_EXTRA_INTR, i;
574 
575 	if (sc->intr_count == 1)
576 		return (0);
577 
578 	for_each_port(sc, i) {
579 		struct port_info *p = sc->port[i];
580 
581 		if (i == pi->port_id)
582 			break;
583 
584 #ifdef TCP_OFFLOAD
585 		if (sc->flags & INTR_DIRECT)
586 			rc += p->nrxq + p->nofldrxq;
587 		else
588 			rc += max(p->nrxq, p->nofldrxq);
589 #else
590 		/*
591 		 * Not compiled with offload support and intr_count > 1.  Only
592 		 * NIC queues exist and they'd better be taking direct
593 		 * interrupts.
594 		 */
595 		KASSERT(sc->flags & INTR_DIRECT,
596 		    ("%s: intr_count %d, !INTR_DIRECT", __func__,
597 		    sc->intr_count));
598 
599 		rc += p->nrxq;
600 #endif
601 	}
602 
603 	return (rc);
604 }
605 
606 /*
607  * Given an arbitrary "index," come up with an iq that can be used by other
608  * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
609  * The iq returned is guaranteed to be something that takes direct interrupts.
610  */
611 static struct sge_iq *
612 port_intr_iq(struct port_info *pi, int idx)
613 {
614 	struct adapter *sc = pi->adapter;
615 	struct sge *s = &sc->sge;
616 	struct sge_iq *iq = NULL;
617 
618 	if (sc->intr_count == 1)
619 		return (&sc->sge.fwq);
620 
621 #ifdef TCP_OFFLOAD
622 	if (sc->flags & INTR_DIRECT) {
623 		idx %= pi->nrxq + pi->nofldrxq;
624 
625 		if (idx >= pi->nrxq) {
626 			idx -= pi->nrxq;
627 			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
628 		} else
629 			iq = &s->rxq[pi->first_rxq + idx].iq;
630 
631 	} else {
632 		idx %= max(pi->nrxq, pi->nofldrxq);
633 
634 		if (pi->nrxq >= pi->nofldrxq)
635 			iq = &s->rxq[pi->first_rxq + idx].iq;
636 		else
637 			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
638 	}
639 #else
640 	/*
641 	 * Not compiled with offload support and intr_count > 1.  Only NIC
642 	 * queues exist and they'd better be taking direct interrupts.
643 	 */
644 	KASSERT(sc->flags & INTR_DIRECT,
645 	    ("%s: intr_count %d, !INTR_DIRECT", __func__, sc->intr_count));
646 
647 	idx %= pi->nrxq;
648 	iq = &s->rxq[pi->first_rxq + idx].iq;
649 #endif
650 
651 	KASSERT(iq->flags & IQ_INTR, ("%s: EDOOFUS", __func__));
652 	return (iq);
653 }
654 
655 static inline int
656 mtu_to_bufsize(int mtu)
657 {
658 	int bufsize;
659 
660 	/* large enough for a frame even when VLAN extraction is disabled */
661 	bufsize = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + mtu;
662 	bufsize = roundup2(bufsize + fl_pktshift, fl_pad);
663 
664 	return (bufsize);
665 }
666 
667 int
668 t4_setup_port_queues(struct port_info *pi)
669 {
670 	int rc = 0, i, j, intr_idx, iqid;
671 	struct sge_rxq *rxq;
672 	struct sge_txq *txq;
673 	struct sge_wrq *ctrlq;
674 #ifdef TCP_OFFLOAD
675 	struct sge_ofld_rxq *ofld_rxq;
676 	struct sge_wrq *ofld_txq;
677 	struct sysctl_oid *oid2 = NULL;
678 #endif
679 	char name[16];
680 	struct adapter *sc = pi->adapter;
681 	struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
682 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
683 	int bufsize = mtu_to_bufsize(pi->ifp->if_mtu);
684 
685 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
686 	    NULL, "rx queues");
687 
688 #ifdef TCP_OFFLOAD
689 	if (is_offload(sc)) {
690 		oid2 = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
691 		    CTLFLAG_RD, NULL,
692 		    "rx queues for offloaded TCP connections");
693 	}
694 #endif
695 
696 	/* Interrupt vector to start from (when using multiple vectors) */
697 	intr_idx = first_vector(pi);
698 
699 	/*
700 	 * First pass over all rx queues (NIC and TOE):
701 	 * a) initialize iq and fl
702 	 * b) allocate queue iff it will take direct interrupts.
703 	 */
704 	for_each_rxq(pi, i, rxq) {
705 
706 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
707 		    RX_IQ_ESIZE);
708 
709 		snprintf(name, sizeof(name), "%s rxq%d-fl",
710 		    device_get_nameunit(pi->dev), i);
711 		init_fl(&rxq->fl, pi->qsize_rxq / 8, bufsize, name);
712 
713 		if (sc->flags & INTR_DIRECT
714 #ifdef TCP_OFFLOAD
715 		    || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
716 #endif
717 		   ) {
718 			rxq->iq.flags |= IQ_INTR;
719 			rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
720 			if (rc != 0)
721 				goto done;
722 			intr_idx++;
723 		}
724 	}
725 
726 #ifdef TCP_OFFLOAD
727 	for_each_ofld_rxq(pi, i, ofld_rxq) {
728 
729 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
730 		    pi->qsize_rxq, RX_IQ_ESIZE);
731 
732 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
733 		    device_get_nameunit(pi->dev), i);
734 		init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, OFLD_BUF_SIZE, name);
735 
736 		if (sc->flags & INTR_DIRECT ||
737 		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
738 			ofld_rxq->iq.flags |= IQ_INTR;
739 			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
740 			if (rc != 0)
741 				goto done;
742 			intr_idx++;
743 		}
744 	}
745 #endif
746 
747 	/*
748 	 * Second pass over all rx queues (NIC and TOE).  The queues forwarding
749 	 * their interrupts are allocated now.
750 	 */
751 	j = 0;
752 	for_each_rxq(pi, i, rxq) {
753 		if (rxq->iq.flags & IQ_INTR)
754 			continue;
755 
756 		intr_idx = port_intr_iq(pi, j)->abs_id;
757 
758 		rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
759 		if (rc != 0)
760 			goto done;
761 		j++;
762 	}
763 
764 #ifdef TCP_OFFLOAD
765 	for_each_ofld_rxq(pi, i, ofld_rxq) {
766 		if (ofld_rxq->iq.flags & IQ_INTR)
767 			continue;
768 
769 		intr_idx = port_intr_iq(pi, j)->abs_id;
770 
771 		rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
772 		if (rc != 0)
773 			goto done;
774 		j++;
775 	}
776 #endif
777 
778 	/*
779 	 * Now the tx queues.  Only one pass needed.
780 	 */
781 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
782 	    NULL, "tx queues");
783 	j = 0;
784 	for_each_txq(pi, i, txq) {
785 		uint16_t iqid;
786 
787 		iqid = port_intr_iq(pi, j)->cntxt_id;
788 
789 		snprintf(name, sizeof(name), "%s txq%d",
790 		    device_get_nameunit(pi->dev), i);
791 		init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
792 		    name);
793 
794 		rc = alloc_txq(pi, txq, i, oid);
795 		if (rc != 0)
796 			goto done;
797 		j++;
798 	}
799 
800 #ifdef TCP_OFFLOAD
801 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
802 	    CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
803 	for_each_ofld_txq(pi, i, ofld_txq) {
804 		uint16_t iqid;
805 
806 		iqid = port_intr_iq(pi, j)->cntxt_id;
807 
808 		snprintf(name, sizeof(name), "%s ofld_txq%d",
809 		    device_get_nameunit(pi->dev), i);
810 		init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
811 		    iqid, name);
812 
813 		snprintf(name, sizeof(name), "%d", i);
814 		oid2 = SYSCTL_ADD_NODE(&pi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
815 		    name, CTLFLAG_RD, NULL, "offload tx queue");
816 
817 		rc = alloc_wrq(sc, pi, ofld_txq, oid2);
818 		if (rc != 0)
819 			goto done;
820 		j++;
821 	}
822 #endif
823 
824 	/*
825 	 * Finally, the control queue.
826 	 */
827 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
828 	    NULL, "ctrl queue");
829 	ctrlq = &sc->sge.ctrlq[pi->port_id];
830 	iqid = port_intr_iq(pi, 0)->cntxt_id;
831 	snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(pi->dev));
832 	init_eq(&ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid, name);
833 	rc = alloc_wrq(sc, pi, ctrlq, oid);
834 
835 done:
836 	if (rc)
837 		t4_teardown_port_queues(pi);
838 
839 	return (rc);
840 }
841 
842 /*
843  * Idempotent
844  */
845 int
846 t4_teardown_port_queues(struct port_info *pi)
847 {
848 	int i;
849 	struct adapter *sc = pi->adapter;
850 	struct sge_rxq *rxq;
851 	struct sge_txq *txq;
852 #ifdef TCP_OFFLOAD
853 	struct sge_ofld_rxq *ofld_rxq;
854 	struct sge_wrq *ofld_txq;
855 #endif
856 
857 	/* Do this before freeing the queues */
858 	if (pi->flags & PORT_SYSCTL_CTX) {
859 		sysctl_ctx_free(&pi->ctx);
860 		pi->flags &= ~PORT_SYSCTL_CTX;
861 	}
862 
863 	/*
864 	 * Take down all the tx queues first, as they reference the rx queues
865 	 * (for egress updates, etc.).
866 	 */
867 
868 	free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
869 
870 	for_each_txq(pi, i, txq) {
871 		free_txq(pi, txq);
872 	}
873 
874 #ifdef TCP_OFFLOAD
875 	for_each_ofld_txq(pi, i, ofld_txq) {
876 		free_wrq(sc, ofld_txq);
877 	}
878 #endif
879 
880 	/*
881 	 * Then take down the rx queues that forward their interrupts, as they
882 	 * reference other rx queues.
883 	 */
884 
885 	for_each_rxq(pi, i, rxq) {
886 		if ((rxq->iq.flags & IQ_INTR) == 0)
887 			free_rxq(pi, rxq);
888 	}
889 
890 #ifdef TCP_OFFLOAD
891 	for_each_ofld_rxq(pi, i, ofld_rxq) {
892 		if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
893 			free_ofld_rxq(pi, ofld_rxq);
894 	}
895 #endif
896 
897 	/*
898 	 * Then take down the rx queues that take direct interrupts.
899 	 */
900 
901 	for_each_rxq(pi, i, rxq) {
902 		if (rxq->iq.flags & IQ_INTR)
903 			free_rxq(pi, rxq);
904 	}
905 
906 #ifdef TCP_OFFLOAD
907 	for_each_ofld_rxq(pi, i, ofld_rxq) {
908 		if (ofld_rxq->iq.flags & IQ_INTR)
909 			free_ofld_rxq(pi, ofld_rxq);
910 	}
911 #endif
912 
913 	return (0);
914 }
915 
916 /*
917  * Deals with errors and the firmware event queue.  All data rx queues forward
918  * their interrupt to the firmware event queue.
919  */
920 void
921 t4_intr_all(void *arg)
922 {
923 	struct adapter *sc = arg;
924 	struct sge_iq *fwq = &sc->sge.fwq;
925 
926 	t4_intr_err(arg);
927 	if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
928 		service_iq(fwq, 0);
929 		atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
930 	}
931 }
932 
933 /* Deals with error interrupts */
934 void
935 t4_intr_err(void *arg)
936 {
937 	struct adapter *sc = arg;
938 
939 	t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
940 	t4_slow_intr_handler(sc);
941 }
942 
943 void
944 t4_intr_evt(void *arg)
945 {
946 	struct sge_iq *iq = arg;
947 
948 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
949 		service_iq(iq, 0);
950 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
951 	}
952 }
953 
954 void
955 t4_intr(void *arg)
956 {
957 	struct sge_iq *iq = arg;
958 
959 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
960 		service_iq(iq, 0);
961 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
962 	}
963 }
964 
965 /*
966  * Deals with anything and everything on the given ingress queue.
967  */
968 static int
969 service_iq(struct sge_iq *iq, int budget)
970 {
971 	struct sge_iq *q;
972 	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
973 	struct sge_fl *fl = &rxq->fl;		/* Use iff IQ_HAS_FL */
974 	struct adapter *sc = iq->adapter;
975 	struct rsp_ctrl *ctrl;
976 	const struct rss_header *rss;
977 	int ndescs = 0, limit, fl_bufs_used = 0;
978 	int rsp_type;
979 	uint32_t lq;
980 	struct mbuf *m0;
981 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
982 
983 	limit = budget ? budget : iq->qsize / 8;
984 
985 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
986 
987 	/*
988 	 * We always come back and check the descriptor ring for new indirect
989 	 * interrupts and other responses after running a single handler.
990 	 */
991 	for (;;) {
992 		while (is_new_response(iq, &ctrl)) {
993 
994 			rmb();
995 
996 			m0 = NULL;
997 			rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
998 			lq = be32toh(ctrl->pldbuflen_qid);
999 			rss = (const void *)iq->cdesc;
1000 
1001 			switch (rsp_type) {
1002 			case X_RSPD_TYPE_FLBUF:
1003 
1004 				KASSERT(iq->flags & IQ_HAS_FL,
1005 				    ("%s: data for an iq (%p) with no freelist",
1006 				    __func__, iq));
1007 
1008 				m0 = get_fl_payload(sc, fl, lq, &fl_bufs_used);
1009 #ifdef T4_PKT_TIMESTAMP
1010 				/*
1011 				 * 60 bit timestamp for the payload is
1012 				 * *(uint64_t *)m0->m_pktdat.  Note that it is
1013 				 * in the leading free-space in the mbuf.  The
1014 				 * kernel can clobber it during a pullup,
1015 				 * m_copymdata, etc.  You need to make sure that
1016 				 * the mbuf reaches you unmolested if you care
1017 				 * about the timestamp.
1018 				 */
1019 				*(uint64_t *)m0->m_pktdat =
1020 				    be64toh(ctrl->u.last_flit) &
1021 				    0xfffffffffffffff;
1022 #endif
1023 
1024 				/* fall through */
1025 
1026 			case X_RSPD_TYPE_CPL:
1027 				KASSERT(rss->opcode < NUM_CPL_CMDS,
1028 				    ("%s: bad opcode %02x.", __func__,
1029 				    rss->opcode));
1030 				sc->cpl_handler[rss->opcode](iq, rss, m0);
1031 				break;
1032 
1033 			case X_RSPD_TYPE_INTR:
1034 
1035 				/*
1036 				 * Interrupts should be forwarded only to queues
1037 				 * that are not forwarding their interrupts.
1038 				 * This means service_iq can recurse but only 1
1039 				 * level deep.
1040 				 */
1041 				KASSERT(budget == 0,
1042 				    ("%s: budget %u, rsp_type %u", __func__,
1043 				    budget, rsp_type));
1044 
1045 				q = sc->sge.iqmap[lq - sc->sge.iq_start];
1046 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
1047 				    IQS_BUSY)) {
1048 					if (service_iq(q, q->qsize / 8) == 0) {
1049 						atomic_cmpset_int(&q->state,
1050 						    IQS_BUSY, IQS_IDLE);
1051 					} else {
1052 						STAILQ_INSERT_TAIL(&iql, q,
1053 						    link);
1054 					}
1055 				}
1056 				break;
1057 
1058 			default:
1059 				sc->an_handler(iq, ctrl);
1060 				break;
1061 			}
1062 
1063 			iq_next(iq);
1064 			if (++ndescs == limit) {
1065 				t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
1066 				    V_CIDXINC(ndescs) |
1067 				    V_INGRESSQID(iq->cntxt_id) |
1068 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1069 				ndescs = 0;
1070 
1071 				if (fl_bufs_used > 0) {
1072 					FL_LOCK(fl);
1073 					fl->needed += fl_bufs_used;
1074 					refill_fl(sc, fl, fl->cap / 8);
1075 					FL_UNLOCK(fl);
1076 					fl_bufs_used = 0;
1077 				}
1078 
1079 				if (budget)
1080 					return (EINPROGRESS);
1081 			}
1082 		}
1083 
1084 		if (STAILQ_EMPTY(&iql))
1085 			break;
1086 
1087 		/*
1088 		 * Process the head only, and send it to the back of the list if
1089 		 * it's still not done.
1090 		 */
1091 		q = STAILQ_FIRST(&iql);
1092 		STAILQ_REMOVE_HEAD(&iql, link);
1093 		if (service_iq(q, q->qsize / 8) == 0)
1094 			atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
1095 		else
1096 			STAILQ_INSERT_TAIL(&iql, q, link);
1097 	}
1098 
1099 #if defined(INET) || defined(INET6)
1100 	if (iq->flags & IQ_LRO_ENABLED) {
1101 		struct lro_ctrl *lro = &rxq->lro;
1102 		struct lro_entry *l;
1103 
1104 		while (!SLIST_EMPTY(&lro->lro_active)) {
1105 			l = SLIST_FIRST(&lro->lro_active);
1106 			SLIST_REMOVE_HEAD(&lro->lro_active, next);
1107 			tcp_lro_flush(lro, l);
1108 		}
1109 	}
1110 #endif
1111 
1112 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
1113 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1114 
1115 	if (iq->flags & IQ_HAS_FL) {
1116 		int starved;
1117 
1118 		FL_LOCK(fl);
1119 		fl->needed += fl_bufs_used;
1120 		starved = refill_fl(sc, fl, fl->cap / 4);
1121 		FL_UNLOCK(fl);
1122 		if (__predict_false(starved != 0))
1123 			add_fl_to_sfl(sc, fl);
1124 	}
1125 
1126 	return (0);
1127 }
1128 
1129 static struct mbuf *
1130 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
1131     int *fl_bufs_used)
1132 {
1133 	struct mbuf *m0, *m;
1134 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1135 	unsigned int nbuf, len;
1136 
1137 	/*
1138 	 * No assertion for the fl lock because we don't need it.  This routine
1139 	 * is called only from the rx interrupt handler and it only updates
1140 	 * fl->cidx.  (Contrast that with fl->pidx/fl->needed which could be
1141 	 * updated in the rx interrupt handler or the starvation helper routine.
1142 	 * That's why code that manipulates fl->pidx/fl->needed needs the fl
1143 	 * lock but this routine does not).
1144 	 */
1145 
1146 	if (__predict_false((len_newbuf & F_RSPD_NEWBUF) == 0))
1147 		panic("%s: cannot handle packed frames", __func__);
1148 	len = G_RSPD_LEN(len_newbuf);
1149 
1150 	m0 = sd->m;
1151 	sd->m = NULL;	/* consumed */
1152 
1153 	bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
1154 	m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR);
1155 #ifdef T4_PKT_TIMESTAMP
1156 	/* Leave room for a timestamp */
1157 	m0->m_data += 8;
1158 #endif
1159 
1160 	if (len < RX_COPY_THRESHOLD) {
1161 		/* copy data to mbuf, buffer will be recycled */
1162 		bcopy(sd->cl, mtod(m0, caddr_t), len);
1163 		m0->m_len = len;
1164 	} else {
1165 		bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
1166 		m_cljset(m0, sd->cl, FL_BUF_TYPE(sd->tag_idx));
1167 		sd->cl = NULL;	/* consumed */
1168 		m0->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
1169 	}
1170 	m0->m_pkthdr.len = len;
1171 
1172 	sd++;
1173 	if (__predict_false(++fl->cidx == fl->cap)) {
1174 		sd = fl->sdesc;
1175 		fl->cidx = 0;
1176 	}
1177 
1178 	m = m0;
1179 	len -= m->m_len;
1180 	nbuf = 1;	/* # of fl buffers used */
1181 
1182 	while (len > 0) {
1183 		m->m_next = sd->m;
1184 		sd->m = NULL;	/* consumed */
1185 		m = m->m_next;
1186 
1187 		bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
1188 		    BUS_DMASYNC_POSTREAD);
1189 
1190 		m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
1191 		if (len <= MLEN) {
1192 			bcopy(sd->cl, mtod(m, caddr_t), len);
1193 			m->m_len = len;
1194 		} else {
1195 			bus_dmamap_unload(fl->tag[sd->tag_idx],
1196 			    sd->map);
1197 			m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
1198 			sd->cl = NULL;	/* consumed */
1199 			m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
1200 		}
1201 
1202 		sd++;
1203 		if (__predict_false(++fl->cidx == fl->cap)) {
1204 			sd = fl->sdesc;
1205 			fl->cidx = 0;
1206 		}
1207 
1208 		len -= m->m_len;
1209 		nbuf++;
1210 	}
1211 
1212 	(*fl_bufs_used) += nbuf;
1213 
1214 	return (m0);
1215 }
1216 
1217 static int
1218 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
1219 {
1220 	struct sge_rxq *rxq = iq_to_rxq(iq);
1221 	struct ifnet *ifp = rxq->ifp;
1222 	const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
1223 #if defined(INET) || defined(INET6)
1224 	struct lro_ctrl *lro = &rxq->lro;
1225 #endif
1226 
1227 	KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__,
1228 	    rss->opcode));
1229 
1230 	m0->m_pkthdr.len -= fl_pktshift;
1231 	m0->m_len -= fl_pktshift;
1232 	m0->m_data += fl_pktshift;
1233 
1234 	m0->m_pkthdr.rcvif = ifp;
1235 	m0->m_flags |= M_FLOWID;
1236 	m0->m_pkthdr.flowid = rss->hash_val;
1237 
1238 	if (cpl->csum_calc && !cpl->err_vec) {
1239 		if (ifp->if_capenable & IFCAP_RXCSUM &&
1240 		    cpl->l2info & htobe32(F_RXF_IP)) {
1241 			m0->m_pkthdr.csum_flags = (CSUM_IP_CHECKED |
1242 			    CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1243 			rxq->rxcsum++;
1244 		} else if (ifp->if_capenable & IFCAP_RXCSUM_IPV6 &&
1245 		    cpl->l2info & htobe32(F_RXF_IP6)) {
1246 			m0->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 |
1247 			    CSUM_PSEUDO_HDR);
1248 			rxq->rxcsum++;
1249 		}
1250 
1251 		if (__predict_false(cpl->ip_frag))
1252 			m0->m_pkthdr.csum_data = be16toh(cpl->csum);
1253 		else
1254 			m0->m_pkthdr.csum_data = 0xffff;
1255 	}
1256 
1257 	if (cpl->vlan_ex) {
1258 		m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
1259 		m0->m_flags |= M_VLANTAG;
1260 		rxq->vlan_extraction++;
1261 	}
1262 
1263 #if defined(INET) || defined(INET6)
1264 	if (cpl->l2info & htobe32(F_RXF_LRO) &&
1265 	    iq->flags & IQ_LRO_ENABLED &&
1266 	    tcp_lro_rx(lro, m0, 0) == 0) {
1267 		/* queued for LRO */
1268 	} else
1269 #endif
1270 	ifp->if_input(ifp, m0);
1271 
1272 	return (0);
1273 }
1274 
1275 /*
1276  * Doesn't fail.  Holds on to work requests it can't send right away.
1277  */
1278 void
1279 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
1280 {
1281 	struct sge_eq *eq = &wrq->eq;
1282 	int can_reclaim;
1283 	caddr_t dst;
1284 
1285 	TXQ_LOCK_ASSERT_OWNED(wrq);
1286 #ifdef TCP_OFFLOAD
1287 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1288 	    (eq->flags & EQ_TYPEMASK) == EQ_CTRL,
1289 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1290 #else
1291 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL,
1292 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1293 #endif
1294 
1295 	if (__predict_true(wr != NULL))
1296 		STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
1297 
1298 	can_reclaim = reclaimable(eq);
1299 	if (__predict_false(eq->flags & EQ_STALLED)) {
1300 		if (can_reclaim < tx_resume_threshold(eq))
1301 			return;
1302 		eq->flags &= ~EQ_STALLED;
1303 		eq->unstalled++;
1304 	}
1305 	eq->cidx += can_reclaim;
1306 	eq->avail += can_reclaim;
1307 	if (__predict_false(eq->cidx >= eq->cap))
1308 		eq->cidx -= eq->cap;
1309 
1310 	while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
1311 		int ndesc;
1312 
1313 		if (__predict_false(wr->wr_len < 0 ||
1314 		    wr->wr_len > SGE_MAX_WR_LEN || (wr->wr_len & 0x7))) {
1315 
1316 #ifdef INVARIANTS
1317 			panic("%s: work request with length %d", __func__,
1318 			    wr->wr_len);
1319 #endif
1320 #ifdef KDB
1321 			kdb_backtrace();
1322 #endif
1323 			log(LOG_ERR, "%s: %s work request with length %d",
1324 			    device_get_nameunit(sc->dev), __func__, wr->wr_len);
1325 			STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1326 			free_wrqe(wr);
1327 			continue;
1328 		}
1329 
1330 		ndesc = howmany(wr->wr_len, EQ_ESIZE);
1331 		if (eq->avail < ndesc) {
1332 			wrq->no_desc++;
1333 			break;
1334 		}
1335 
1336 		dst = (void *)&eq->desc[eq->pidx];
1337 		copy_to_txd(eq, wrtod(wr), &dst, wr->wr_len);
1338 
1339 		eq->pidx += ndesc;
1340 		eq->avail -= ndesc;
1341 		if (__predict_false(eq->pidx >= eq->cap))
1342 			eq->pidx -= eq->cap;
1343 
1344 		eq->pending += ndesc;
1345 		if (eq->pending > 16)
1346 			ring_eq_db(sc, eq);
1347 
1348 		wrq->tx_wrs++;
1349 		STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
1350 		free_wrqe(wr);
1351 
1352 		if (eq->avail < 8) {
1353 			can_reclaim = reclaimable(eq);
1354 			eq->cidx += can_reclaim;
1355 			eq->avail += can_reclaim;
1356 			if (__predict_false(eq->cidx >= eq->cap))
1357 				eq->cidx -= eq->cap;
1358 		}
1359 	}
1360 
1361 	if (eq->pending)
1362 		ring_eq_db(sc, eq);
1363 
1364 	if (wr != NULL) {
1365 		eq->flags |= EQ_STALLED;
1366 		if (callout_pending(&eq->tx_callout) == 0)
1367 			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1368 	}
1369 }
1370 
1371 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1372 #define TXPKTS_PKT_HDR ((\
1373     sizeof(struct ulp_txpkt) + \
1374     sizeof(struct ulptx_idata) + \
1375     sizeof(struct cpl_tx_pkt_core) \
1376     ) / 8)
1377 
1378 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1379 #define TXPKTS_WR_HDR (\
1380     sizeof(struct fw_eth_tx_pkts_wr) / 8 + \
1381     TXPKTS_PKT_HDR)
1382 
1383 /* Header of a tx WR, before SGL of first packet (in flits) */
1384 #define TXPKT_WR_HDR ((\
1385     sizeof(struct fw_eth_tx_pkt_wr) + \
1386     sizeof(struct cpl_tx_pkt_core) \
1387     ) / 8 )
1388 
1389 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1390 #define TXPKT_LSO_WR_HDR ((\
1391     sizeof(struct fw_eth_tx_pkt_wr) + \
1392     sizeof(struct cpl_tx_pkt_lso_core) + \
1393     sizeof(struct cpl_tx_pkt_core) \
1394     ) / 8 )
1395 
1396 int
1397 t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
1398 {
1399 	struct port_info *pi = (void *)ifp->if_softc;
1400 	struct adapter *sc = pi->adapter;
1401 	struct sge_eq *eq = &txq->eq;
1402 	struct buf_ring *br = txq->br;
1403 	struct mbuf *next;
1404 	int rc, coalescing, can_reclaim;
1405 	struct txpkts txpkts;
1406 	struct sgl sgl;
1407 
1408 	TXQ_LOCK_ASSERT_OWNED(txq);
1409 	KASSERT(m, ("%s: called with nothing to do.", __func__));
1410 	KASSERT((eq->flags & EQ_TYPEMASK) == EQ_ETH,
1411 	    ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
1412 
1413 	prefetch(&eq->desc[eq->pidx]);
1414 	prefetch(&txq->sdesc[eq->pidx]);
1415 
1416 	txpkts.npkt = 0;/* indicates there's nothing in txpkts */
1417 	coalescing = 0;
1418 
1419 	can_reclaim = reclaimable(eq);
1420 	if (__predict_false(eq->flags & EQ_STALLED)) {
1421 		if (can_reclaim < tx_resume_threshold(eq)) {
1422 			txq->m = m;
1423 			return (0);
1424 		}
1425 		eq->flags &= ~EQ_STALLED;
1426 		eq->unstalled++;
1427 	}
1428 
1429 	if (__predict_false(eq->flags & EQ_DOOMED)) {
1430 		m_freem(m);
1431 		while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1432 			m_freem(m);
1433 		return (ENETDOWN);
1434 	}
1435 
1436 	if (eq->avail < 8 && can_reclaim)
1437 		reclaim_tx_descs(txq, can_reclaim, 32);
1438 
1439 	for (; m; m = next ? next : drbr_dequeue(ifp, br)) {
1440 
1441 		if (eq->avail < 8)
1442 			break;
1443 
1444 		next = m->m_nextpkt;
1445 		m->m_nextpkt = NULL;
1446 
1447 		if (next || buf_ring_peek(br))
1448 			coalescing = 1;
1449 
1450 		rc = get_pkt_sgl(txq, &m, &sgl, coalescing);
1451 		if (rc != 0) {
1452 			if (rc == ENOMEM) {
1453 
1454 				/* Short of resources, suspend tx */
1455 
1456 				m->m_nextpkt = next;
1457 				break;
1458 			}
1459 
1460 			/*
1461 			 * Unrecoverable error for this packet, throw it away
1462 			 * and move on to the next.  get_pkt_sgl may already
1463 			 * have freed m (it will be NULL in that case and the
1464 			 * m_freem here is still safe).
1465 			 */
1466 
1467 			m_freem(m);
1468 			continue;
1469 		}
1470 
1471 		if (coalescing &&
1472 		    add_to_txpkts(pi, txq, &txpkts, m, &sgl) == 0) {
1473 
1474 			/* Successfully absorbed into txpkts */
1475 
1476 			write_ulp_cpl_sgl(pi, txq, &txpkts, m, &sgl);
1477 			goto doorbell;
1478 		}
1479 
1480 		/*
1481 		 * We weren't coalescing to begin with, or current frame could
1482 		 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1483 		 * given to it can't be coalesced).  Either way there should be
1484 		 * nothing in txpkts.
1485 		 */
1486 		KASSERT(txpkts.npkt == 0,
1487 		    ("%s: txpkts not empty: %d", __func__, txpkts.npkt));
1488 
1489 		/* We're sending out individual packets now */
1490 		coalescing = 0;
1491 
1492 		if (eq->avail < 8)
1493 			reclaim_tx_descs(txq, 0, 8);
1494 		rc = write_txpkt_wr(pi, txq, m, &sgl);
1495 		if (rc != 0) {
1496 
1497 			/* Short of hardware descriptors, suspend tx */
1498 
1499 			/*
1500 			 * This is an unlikely but expensive failure.  We've
1501 			 * done all the hard work (DMA mappings etc.) and now we
1502 			 * can't send out the packet.  What's worse, we have to
1503 			 * spend even more time freeing up everything in sgl.
1504 			 */
1505 			txq->no_desc++;
1506 			free_pkt_sgl(txq, &sgl);
1507 
1508 			m->m_nextpkt = next;
1509 			break;
1510 		}
1511 
1512 		ETHER_BPF_MTAP(ifp, m);
1513 		if (sgl.nsegs == 0)
1514 			m_freem(m);
1515 doorbell:
1516 		if (eq->pending >= 64)
1517 		    ring_eq_db(sc, eq);
1518 
1519 		can_reclaim = reclaimable(eq);
1520 		if (can_reclaim >= 32)
1521 			reclaim_tx_descs(txq, can_reclaim, 64);
1522 	}
1523 
1524 	if (txpkts.npkt > 0)
1525 		write_txpkts_wr(txq, &txpkts);
1526 
1527 	/*
1528 	 * m not NULL means there was an error but we haven't thrown it away.
1529 	 * This can happen when we're short of tx descriptors (no_desc) or maybe
1530 	 * even DMA maps (no_dmamap).  Either way, a credit flush and reclaim
1531 	 * will get things going again.
1532 	 */
1533 	if (m && !(eq->flags & EQ_CRFLUSHED)) {
1534 		struct tx_sdesc *txsd = &txq->sdesc[eq->pidx];
1535 
1536 		/*
1537 		 * If EQ_CRFLUSHED is not set then we know we have at least one
1538 		 * available descriptor because any WR that reduces eq->avail to
1539 		 * 0 also sets EQ_CRFLUSHED.
1540 		 */
1541 		KASSERT(eq->avail > 0, ("%s: no space for eqflush.", __func__));
1542 
1543 		txsd->desc_used = 1;
1544 		txsd->credits = 0;
1545 		write_eqflush_wr(eq);
1546 	}
1547 	txq->m = m;
1548 
1549 	if (eq->pending)
1550 		ring_eq_db(sc, eq);
1551 
1552 	reclaim_tx_descs(txq, 0, 128);
1553 
1554 	if (eq->flags & EQ_STALLED && callout_pending(&eq->tx_callout) == 0)
1555 		callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1556 
1557 	return (0);
1558 }
1559 
1560 void
1561 t4_update_fl_bufsize(struct ifnet *ifp)
1562 {
1563 	struct port_info *pi = ifp->if_softc;
1564 	struct sge_rxq *rxq;
1565 	struct sge_fl *fl;
1566 	int i, bufsize = mtu_to_bufsize(ifp->if_mtu);
1567 
1568 	for_each_rxq(pi, i, rxq) {
1569 		fl = &rxq->fl;
1570 
1571 		FL_LOCK(fl);
1572 		set_fl_tag_idx(fl, bufsize);
1573 		FL_UNLOCK(fl);
1574 	}
1575 }
1576 
1577 int
1578 can_resume_tx(struct sge_eq *eq)
1579 {
1580 	return (reclaimable(eq) >= tx_resume_threshold(eq));
1581 }
1582 
1583 static inline void
1584 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
1585     int qsize, int esize)
1586 {
1587 	KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
1588 	    ("%s: bad tmr_idx %d", __func__, tmr_idx));
1589 	KASSERT(pktc_idx < SGE_NCOUNTERS,	/* -ve is ok, means don't use */
1590 	    ("%s: bad pktc_idx %d", __func__, pktc_idx));
1591 
1592 	iq->flags = 0;
1593 	iq->adapter = sc;
1594 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1595 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1596 	if (pktc_idx >= 0) {
1597 		iq->intr_params |= F_QINTR_CNT_EN;
1598 		iq->intr_pktc_idx = pktc_idx;
1599 	}
1600 	iq->qsize = roundup2(qsize, 16);	/* See FW_IQ_CMD/iqsize */
1601 	iq->esize = max(esize, 16);		/* See FW_IQ_CMD/iqesize */
1602 }
1603 
1604 static inline void
1605 init_fl(struct sge_fl *fl, int qsize, int bufsize, char *name)
1606 {
1607 	fl->qsize = qsize;
1608 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
1609 	set_fl_tag_idx(fl, bufsize);
1610 }
1611 
1612 static inline void
1613 init_eq(struct sge_eq *eq, int eqtype, int qsize, uint8_t tx_chan,
1614     uint16_t iqid, char *name)
1615 {
1616 	KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan));
1617 	KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
1618 
1619 	eq->flags = eqtype & EQ_TYPEMASK;
1620 	eq->tx_chan = tx_chan;
1621 	eq->iqid = iqid;
1622 	eq->qsize = qsize;
1623 	strlcpy(eq->lockname, name, sizeof(eq->lockname));
1624 
1625 	TASK_INIT(&eq->tx_task, 0, t4_tx_task, eq);
1626 	callout_init(&eq->tx_callout, CALLOUT_MPSAFE);
1627 }
1628 
1629 static int
1630 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
1631     bus_dmamap_t *map, bus_addr_t *pa, void **va)
1632 {
1633 	int rc;
1634 
1635 	rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
1636 	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
1637 	if (rc != 0) {
1638 		device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc);
1639 		goto done;
1640 	}
1641 
1642 	rc = bus_dmamem_alloc(*tag, va,
1643 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
1644 	if (rc != 0) {
1645 		device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc);
1646 		goto done;
1647 	}
1648 
1649 	rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
1650 	if (rc != 0) {
1651 		device_printf(sc->dev, "cannot load DMA map: %d\n", rc);
1652 		goto done;
1653 	}
1654 done:
1655 	if (rc)
1656 		free_ring(sc, *tag, *map, *pa, *va);
1657 
1658 	return (rc);
1659 }
1660 
1661 static int
1662 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
1663     bus_addr_t pa, void *va)
1664 {
1665 	if (pa)
1666 		bus_dmamap_unload(tag, map);
1667 	if (va)
1668 		bus_dmamem_free(tag, va, map);
1669 	if (tag)
1670 		bus_dma_tag_destroy(tag);
1671 
1672 	return (0);
1673 }
1674 
1675 /*
1676  * Allocates the ring for an ingress queue and an optional freelist.  If the
1677  * freelist is specified it will be allocated and then associated with the
1678  * ingress queue.
1679  *
1680  * Returns errno on failure.  Resources allocated up to that point may still be
1681  * allocated.  Caller is responsible for cleanup in case this function fails.
1682  *
1683  * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1684  * the intr_idx specifies the vector, starting from 0.  Otherwise it specifies
1685  * the abs_id of the ingress queue to which its interrupts should be forwarded.
1686  */
1687 static int
1688 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1689     int intr_idx, int cong)
1690 {
1691 	int rc, i, cntxt_id;
1692 	size_t len;
1693 	struct fw_iq_cmd c;
1694 	struct adapter *sc = iq->adapter;
1695 	__be32 v = 0;
1696 
1697 	len = iq->qsize * iq->esize;
1698 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
1699 	    (void **)&iq->desc);
1700 	if (rc != 0)
1701 		return (rc);
1702 
1703 	bzero(&c, sizeof(c));
1704 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1705 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1706 	    V_FW_IQ_CMD_VFN(0));
1707 
1708 	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1709 	    FW_LEN16(c));
1710 
1711 	/* Special handling for firmware event queue */
1712 	if (iq == &sc->sge.fwq)
1713 		v |= F_FW_IQ_CMD_IQASYNCH;
1714 
1715 	if (iq->flags & IQ_INTR) {
1716 		KASSERT(intr_idx < sc->intr_count,
1717 		    ("%s: invalid direct intr_idx %d", __func__, intr_idx));
1718 	} else
1719 		v |= F_FW_IQ_CMD_IQANDST;
1720 	v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1721 
1722 	c.type_to_iqandstindex = htobe32(v |
1723 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1724 	    V_FW_IQ_CMD_VIID(pi->viid) |
1725 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1726 	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1727 	    F_FW_IQ_CMD_IQGTSMODE |
1728 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1729 	    V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1730 	c.iqsize = htobe16(iq->qsize);
1731 	c.iqaddr = htobe64(iq->ba);
1732 	if (cong >= 0)
1733 		c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
1734 
1735 	if (fl) {
1736 		mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
1737 
1738 		for (i = 0; i < FL_BUF_SIZES; i++) {
1739 
1740 			/*
1741 			 * A freelist buffer must be 16 byte aligned as the SGE
1742 			 * uses the low 4 bits of the bus addr to figure out the
1743 			 * buffer size.
1744 			 */
1745 			rc = bus_dma_tag_create(sc->dmat, 16, 0,
1746 			    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1747 			    FL_BUF_SIZE(i), 1, FL_BUF_SIZE(i), BUS_DMA_ALLOCNOW,
1748 			    NULL, NULL, &fl->tag[i]);
1749 			if (rc != 0) {
1750 				device_printf(sc->dev,
1751 				    "failed to create fl DMA tag[%d]: %d\n",
1752 				    i, rc);
1753 				return (rc);
1754 			}
1755 		}
1756 		len = fl->qsize * RX_FL_ESIZE;
1757 		rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
1758 		    &fl->ba, (void **)&fl->desc);
1759 		if (rc)
1760 			return (rc);
1761 
1762 		/* Allocate space for one software descriptor per buffer. */
1763 		fl->cap = (fl->qsize - spg_len / RX_FL_ESIZE) * 8;
1764 		FL_LOCK(fl);
1765 		rc = alloc_fl_sdesc(fl);
1766 		FL_UNLOCK(fl);
1767 		if (rc != 0) {
1768 			device_printf(sc->dev,
1769 			    "failed to setup fl software descriptors: %d\n",
1770 			    rc);
1771 			return (rc);
1772 		}
1773 		fl->needed = fl->cap;
1774 		fl->lowat = roundup2(sc->sge.fl_starve_threshold, 8);
1775 
1776 		c.iqns_to_fl0congen |=
1777 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1778 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
1779 			F_FW_IQ_CMD_FL0PADEN);
1780 		if (cong >= 0) {
1781 			c.iqns_to_fl0congen |=
1782 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1783 				    F_FW_IQ_CMD_FL0CONGCIF |
1784 				    F_FW_IQ_CMD_FL0CONGEN);
1785 		}
1786 		c.fl0dcaen_to_fl0cidxfthresh =
1787 		    htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
1788 			V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
1789 		c.fl0size = htobe16(fl->qsize);
1790 		c.fl0addr = htobe64(fl->ba);
1791 	}
1792 
1793 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
1794 	if (rc != 0) {
1795 		device_printf(sc->dev,
1796 		    "failed to create ingress queue: %d\n", rc);
1797 		return (rc);
1798 	}
1799 
1800 	iq->cdesc = iq->desc;
1801 	iq->cidx = 0;
1802 	iq->gen = 1;
1803 	iq->intr_next = iq->intr_params;
1804 	iq->cntxt_id = be16toh(c.iqid);
1805 	iq->abs_id = be16toh(c.physiqid);
1806 	iq->flags |= IQ_ALLOCATED;
1807 
1808 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1809 	if (cntxt_id >= sc->sge.niq) {
1810 		panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1811 		    cntxt_id, sc->sge.niq - 1);
1812 	}
1813 	sc->sge.iqmap[cntxt_id] = iq;
1814 
1815 	if (fl) {
1816 		fl->cntxt_id = be16toh(c.fl0id);
1817 		fl->pidx = fl->cidx = 0;
1818 
1819 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1820 		if (cntxt_id >= sc->sge.neq) {
1821 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1822 			    __func__, cntxt_id, sc->sge.neq - 1);
1823 		}
1824 		sc->sge.eqmap[cntxt_id] = (void *)fl;
1825 
1826 		FL_LOCK(fl);
1827 		/* Enough to make sure the SGE doesn't think it's starved */
1828 		refill_fl(sc, fl, fl->lowat);
1829 		FL_UNLOCK(fl);
1830 
1831 		iq->flags |= IQ_HAS_FL;
1832 	}
1833 
1834 	/* Enable IQ interrupts */
1835 	atomic_store_rel_int(&iq->state, IQS_IDLE);
1836 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1837 	    V_INGRESSQID(iq->cntxt_id));
1838 
1839 	return (0);
1840 }
1841 
1842 static int
1843 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1844 {
1845 	int i, rc;
1846 	struct adapter *sc = iq->adapter;
1847 	device_t dev;
1848 
1849 	if (sc == NULL)
1850 		return (0);	/* nothing to do */
1851 
1852 	dev = pi ? pi->dev : sc->dev;
1853 
1854 	if (iq->flags & IQ_ALLOCATED) {
1855 		rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1856 		    FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1857 		    fl ? fl->cntxt_id : 0xffff, 0xffff);
1858 		if (rc != 0) {
1859 			device_printf(dev,
1860 			    "failed to free queue %p: %d\n", iq, rc);
1861 			return (rc);
1862 		}
1863 		iq->flags &= ~IQ_ALLOCATED;
1864 	}
1865 
1866 	free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
1867 
1868 	bzero(iq, sizeof(*iq));
1869 
1870 	if (fl) {
1871 		free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba,
1872 		    fl->desc);
1873 
1874 		if (fl->sdesc) {
1875 			FL_LOCK(fl);
1876 			free_fl_sdesc(fl);
1877 			FL_UNLOCK(fl);
1878 		}
1879 
1880 		if (mtx_initialized(&fl->fl_lock))
1881 			mtx_destroy(&fl->fl_lock);
1882 
1883 		for (i = 0; i < FL_BUF_SIZES; i++) {
1884 			if (fl->tag[i])
1885 				bus_dma_tag_destroy(fl->tag[i]);
1886 		}
1887 
1888 		bzero(fl, sizeof(*fl));
1889 	}
1890 
1891 	return (0);
1892 }
1893 
1894 static int
1895 alloc_fwq(struct adapter *sc)
1896 {
1897 	int rc, intr_idx;
1898 	struct sge_iq *fwq = &sc->sge.fwq;
1899 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
1900 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
1901 
1902 	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1903 	fwq->flags |= IQ_INTR;	/* always */
1904 	intr_idx = sc->intr_count > 1 ? 1 : 0;
1905 	rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1906 	if (rc != 0) {
1907 		device_printf(sc->dev,
1908 		    "failed to create firmware event queue: %d\n", rc);
1909 		return (rc);
1910 	}
1911 
1912 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD,
1913 	    NULL, "firmware event queue");
1914 	children = SYSCTL_CHILDREN(oid);
1915 
1916 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id",
1917 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I",
1918 	    "absolute id of the queue");
1919 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id",
1920 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I",
1921 	    "SGE context id of the queue");
1922 	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
1923 	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
1924 	    "consumer index");
1925 
1926 	return (0);
1927 }
1928 
1929 static int
1930 free_fwq(struct adapter *sc)
1931 {
1932 	return free_iq_fl(NULL, &sc->sge.fwq, NULL);
1933 }
1934 
1935 static int
1936 alloc_mgmtq(struct adapter *sc)
1937 {
1938 	int rc;
1939 	struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1940 	char name[16];
1941 	struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
1942 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
1943 
1944 	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD,
1945 	    NULL, "management queue");
1946 
1947 	snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev));
1948 	init_eq(&mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1949 	    sc->sge.fwq.cntxt_id, name);
1950 	rc = alloc_wrq(sc, NULL, mgmtq, oid);
1951 	if (rc != 0) {
1952 		device_printf(sc->dev,
1953 		    "failed to create management queue: %d\n", rc);
1954 		return (rc);
1955 	}
1956 
1957 	return (0);
1958 }
1959 
1960 static int
1961 free_mgmtq(struct adapter *sc)
1962 {
1963 
1964 	return free_wrq(sc, &sc->sge.mgmtq);
1965 }
1966 
1967 static inline int
1968 tnl_cong(struct port_info *pi)
1969 {
1970 
1971 	if (cong_drop == -1)
1972 		return (-1);
1973 	else if (cong_drop == 1)
1974 		return (0);
1975 	else
1976 		return (1 << pi->tx_chan);
1977 }
1978 
1979 static int
1980 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx,
1981     struct sysctl_oid *oid)
1982 {
1983 	int rc;
1984 	struct sysctl_oid_list *children;
1985 	char name[16];
1986 
1987 	rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, tnl_cong(pi));
1988 	if (rc != 0)
1989 		return (rc);
1990 
1991 	FL_LOCK(&rxq->fl);
1992 	refill_fl(pi->adapter, &rxq->fl, rxq->fl.needed / 8);
1993 	FL_UNLOCK(&rxq->fl);
1994 
1995 #if defined(INET) || defined(INET6)
1996 	rc = tcp_lro_init(&rxq->lro);
1997 	if (rc != 0)
1998 		return (rc);
1999 	rxq->lro.ifp = pi->ifp; /* also indicates LRO init'ed */
2000 
2001 	if (pi->ifp->if_capenable & IFCAP_LRO)
2002 		rxq->iq.flags |= IQ_LRO_ENABLED;
2003 #endif
2004 	rxq->ifp = pi->ifp;
2005 
2006 	children = SYSCTL_CHILDREN(oid);
2007 
2008 	snprintf(name, sizeof(name), "%d", idx);
2009 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
2010 	    NULL, "rx queue");
2011 	children = SYSCTL_CHILDREN(oid);
2012 
2013 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
2014 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I",
2015 	    "absolute id of the queue");
2016 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
2017 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cntxt_id, 0, sysctl_uint16, "I",
2018 	    "SGE context id of the queue");
2019 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
2020 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.cidx, 0, sysctl_uint16, "I",
2021 	    "consumer index");
2022 #if defined(INET) || defined(INET6)
2023 	SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
2024 	    &rxq->lro.lro_queued, 0, NULL);
2025 	SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
2026 	    &rxq->lro.lro_flushed, 0, NULL);
2027 #endif
2028 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
2029 	    &rxq->rxcsum, "# of times hardware assisted with checksum");
2030 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_extraction",
2031 	    CTLFLAG_RD, &rxq->vlan_extraction,
2032 	    "# of times hardware extracted 802.1Q tag");
2033 
2034 	children = SYSCTL_CHILDREN(oid);
2035 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "fl", CTLFLAG_RD,
2036 	    NULL, "freelist");
2037 	children = SYSCTL_CHILDREN(oid);
2038 
2039 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
2040 	    CTLTYPE_INT | CTLFLAG_RD, &rxq->fl.cntxt_id, 0, sysctl_uint16, "I",
2041 	    "SGE context id of the queue");
2042 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
2043 	    &rxq->fl.cidx, 0, "consumer index");
2044 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
2045 	    &rxq->fl.pidx, 0, "producer index");
2046 
2047 	return (rc);
2048 }
2049 
2050 static int
2051 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
2052 {
2053 	int rc;
2054 
2055 #if defined(INET) || defined(INET6)
2056 	if (rxq->lro.ifp) {
2057 		tcp_lro_free(&rxq->lro);
2058 		rxq->lro.ifp = NULL;
2059 	}
2060 #endif
2061 
2062 	rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
2063 	if (rc == 0)
2064 		bzero(rxq, sizeof(*rxq));
2065 
2066 	return (rc);
2067 }
2068 
2069 #ifdef TCP_OFFLOAD
2070 static int
2071 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
2072     int intr_idx, int idx, struct sysctl_oid *oid)
2073 {
2074 	int rc;
2075 	struct sysctl_oid_list *children;
2076 	char name[16];
2077 
2078 	rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
2079 	    1 << pi->tx_chan);
2080 	if (rc != 0)
2081 		return (rc);
2082 
2083 	children = SYSCTL_CHILDREN(oid);
2084 
2085 	snprintf(name, sizeof(name), "%d", idx);
2086 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
2087 	    NULL, "rx queue");
2088 	children = SYSCTL_CHILDREN(oid);
2089 
2090 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
2091 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16,
2092 	    "I", "absolute id of the queue");
2093 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
2094 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16,
2095 	    "I", "SGE context id of the queue");
2096 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
2097 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I",
2098 	    "consumer index");
2099 
2100 	children = SYSCTL_CHILDREN(oid);
2101 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "fl", CTLFLAG_RD,
2102 	    NULL, "freelist");
2103 	children = SYSCTL_CHILDREN(oid);
2104 
2105 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
2106 	    CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->fl.cntxt_id, 0, sysctl_uint16,
2107 	    "I", "SGE context id of the queue");
2108 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
2109 	    &ofld_rxq->fl.cidx, 0, "consumer index");
2110 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
2111 	    &ofld_rxq->fl.pidx, 0, "producer index");
2112 
2113 	return (rc);
2114 }
2115 
2116 static int
2117 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
2118 {
2119 	int rc;
2120 
2121 	rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
2122 	if (rc == 0)
2123 		bzero(ofld_rxq, sizeof(*ofld_rxq));
2124 
2125 	return (rc);
2126 }
2127 #endif
2128 
2129 static int
2130 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
2131 {
2132 	int rc, cntxt_id;
2133 	struct fw_eq_ctrl_cmd c;
2134 
2135 	bzero(&c, sizeof(c));
2136 
2137 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
2138 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
2139 	    V_FW_EQ_CTRL_CMD_VFN(0));
2140 	c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
2141 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
2142 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* XXX */
2143 	c.physeqid_pkd = htobe32(0);
2144 	c.fetchszm_to_iqid =
2145 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
2146 		V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
2147 		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
2148 	c.dcaen_to_eqsize =
2149 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2150 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2151 		V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2152 		V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
2153 	c.eqaddr = htobe64(eq->ba);
2154 
2155 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2156 	if (rc != 0) {
2157 		device_printf(sc->dev,
2158 		    "failed to create control queue %d: %d\n", eq->tx_chan, rc);
2159 		return (rc);
2160 	}
2161 	eq->flags |= EQ_ALLOCATED;
2162 
2163 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
2164 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
2165 	if (cntxt_id >= sc->sge.neq)
2166 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
2167 		cntxt_id, sc->sge.neq - 1);
2168 	sc->sge.eqmap[cntxt_id] = eq;
2169 
2170 	return (rc);
2171 }
2172 
2173 static int
2174 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2175 {
2176 	int rc, cntxt_id;
2177 	struct fw_eq_eth_cmd c;
2178 
2179 	bzero(&c, sizeof(c));
2180 
2181 	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
2182 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
2183 	    V_FW_EQ_ETH_CMD_VFN(0));
2184 	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
2185 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
2186 	c.viid_pkd = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
2187 	c.fetchszm_to_iqid =
2188 	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
2189 		V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
2190 		V_FW_EQ_ETH_CMD_IQID(eq->iqid));
2191 	c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2192 		      V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2193 		      V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2194 		      V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
2195 	c.eqaddr = htobe64(eq->ba);
2196 
2197 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2198 	if (rc != 0) {
2199 		device_printf(pi->dev,
2200 		    "failed to create Ethernet egress queue: %d\n", rc);
2201 		return (rc);
2202 	}
2203 	eq->flags |= EQ_ALLOCATED;
2204 
2205 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
2206 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
2207 	if (cntxt_id >= sc->sge.neq)
2208 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
2209 		cntxt_id, sc->sge.neq - 1);
2210 	sc->sge.eqmap[cntxt_id] = eq;
2211 
2212 	return (rc);
2213 }
2214 
2215 #ifdef TCP_OFFLOAD
2216 static int
2217 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2218 {
2219 	int rc, cntxt_id;
2220 	struct fw_eq_ofld_cmd c;
2221 
2222 	bzero(&c, sizeof(c));
2223 
2224 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
2225 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
2226 	    V_FW_EQ_OFLD_CMD_VFN(0));
2227 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
2228 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
2229 	c.fetchszm_to_iqid =
2230 		htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
2231 		    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
2232 		    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
2233 	c.dcaen_to_eqsize =
2234 	    htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
2235 		V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
2236 		V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
2237 		V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
2238 	c.eqaddr = htobe64(eq->ba);
2239 
2240 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
2241 	if (rc != 0) {
2242 		device_printf(pi->dev,
2243 		    "failed to create egress queue for TCP offload: %d\n", rc);
2244 		return (rc);
2245 	}
2246 	eq->flags |= EQ_ALLOCATED;
2247 
2248 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
2249 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
2250 	if (cntxt_id >= sc->sge.neq)
2251 	    panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
2252 		cntxt_id, sc->sge.neq - 1);
2253 	sc->sge.eqmap[cntxt_id] = eq;
2254 
2255 	return (rc);
2256 }
2257 #endif
2258 
2259 static int
2260 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
2261 {
2262 	int rc;
2263 	size_t len;
2264 
2265 	mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
2266 
2267 	len = eq->qsize * EQ_ESIZE;
2268 	rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
2269 	    &eq->ba, (void **)&eq->desc);
2270 	if (rc)
2271 		return (rc);
2272 
2273 	eq->cap = eq->qsize - spg_len / EQ_ESIZE;
2274 	eq->spg = (void *)&eq->desc[eq->cap];
2275 	eq->avail = eq->cap - 1;	/* one less to avoid cidx = pidx */
2276 	eq->pidx = eq->cidx = 0;
2277 	eq->doorbells = sc->doorbells;
2278 
2279 	switch (eq->flags & EQ_TYPEMASK) {
2280 	case EQ_CTRL:
2281 		rc = ctrl_eq_alloc(sc, eq);
2282 		break;
2283 
2284 	case EQ_ETH:
2285 		rc = eth_eq_alloc(sc, pi, eq);
2286 		break;
2287 
2288 #ifdef TCP_OFFLOAD
2289 	case EQ_OFLD:
2290 		rc = ofld_eq_alloc(sc, pi, eq);
2291 		break;
2292 #endif
2293 
2294 	default:
2295 		panic("%s: invalid eq type %d.", __func__,
2296 		    eq->flags & EQ_TYPEMASK);
2297 	}
2298 	if (rc != 0) {
2299 		device_printf(sc->dev,
2300 		    "failed to allocate egress queue(%d): %d",
2301 		    eq->flags & EQ_TYPEMASK, rc);
2302 	}
2303 
2304 	eq->tx_callout.c_cpu = eq->cntxt_id % mp_ncpus;
2305 
2306 	if (isset(&eq->doorbells, DOORBELL_UDB) ||
2307 	    isset(&eq->doorbells, DOORBELL_UDBWC) ||
2308 	    isset(&eq->doorbells, DOORBELL_WCWR)) {
2309 		uint32_t s_qpp = sc->sge.s_qpp;
2310 		uint32_t mask = (1 << s_qpp) - 1;
2311 		volatile uint8_t *udb;
2312 
2313 		udb = sc->udbs_base + UDBS_DB_OFFSET;
2314 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;	/* pg offset */
2315 		eq->udb_qid = eq->cntxt_id & mask;		/* id in page */
2316 		if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
2317 	    		clrbit(&eq->doorbells, DOORBELL_WCWR);
2318 		else {
2319 			udb += eq->udb_qid << UDBS_SEG_SHIFT;	/* seg offset */
2320 			eq->udb_qid = 0;
2321 		}
2322 		eq->udb = (volatile void *)udb;
2323 	}
2324 
2325 	return (rc);
2326 }
2327 
2328 static int
2329 free_eq(struct adapter *sc, struct sge_eq *eq)
2330 {
2331 	int rc;
2332 
2333 	if (eq->flags & EQ_ALLOCATED) {
2334 		switch (eq->flags & EQ_TYPEMASK) {
2335 		case EQ_CTRL:
2336 			rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
2337 			    eq->cntxt_id);
2338 			break;
2339 
2340 		case EQ_ETH:
2341 			rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
2342 			    eq->cntxt_id);
2343 			break;
2344 
2345 #ifdef TCP_OFFLOAD
2346 		case EQ_OFLD:
2347 			rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
2348 			    eq->cntxt_id);
2349 			break;
2350 #endif
2351 
2352 		default:
2353 			panic("%s: invalid eq type %d.", __func__,
2354 			    eq->flags & EQ_TYPEMASK);
2355 		}
2356 		if (rc != 0) {
2357 			device_printf(sc->dev,
2358 			    "failed to free egress queue (%d): %d\n",
2359 			    eq->flags & EQ_TYPEMASK, rc);
2360 			return (rc);
2361 		}
2362 		eq->flags &= ~EQ_ALLOCATED;
2363 	}
2364 
2365 	free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
2366 
2367 	if (mtx_initialized(&eq->eq_lock))
2368 		mtx_destroy(&eq->eq_lock);
2369 
2370 	bzero(eq, sizeof(*eq));
2371 	return (0);
2372 }
2373 
2374 static int
2375 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
2376     struct sysctl_oid *oid)
2377 {
2378 	int rc;
2379 	struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx;
2380 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2381 
2382 	rc = alloc_eq(sc, pi, &wrq->eq);
2383 	if (rc)
2384 		return (rc);
2385 
2386 	wrq->adapter = sc;
2387 	STAILQ_INIT(&wrq->wr_list);
2388 
2389 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
2390 	    &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
2391 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
2392 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
2393 	    "consumer index");
2394 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
2395 	    CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
2396 	    "producer index");
2397 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs", CTLFLAG_RD,
2398 	    &wrq->tx_wrs, "# of work requests");
2399 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
2400 	    &wrq->no_desc, 0,
2401 	    "# of times queue ran out of hardware descriptors");
2402 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
2403 	    &wrq->eq.unstalled, 0, "# of times queue recovered after stall");
2404 
2405 
2406 	return (rc);
2407 }
2408 
2409 static int
2410 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
2411 {
2412 	int rc;
2413 
2414 	rc = free_eq(sc, &wrq->eq);
2415 	if (rc)
2416 		return (rc);
2417 
2418 	bzero(wrq, sizeof(*wrq));
2419 	return (0);
2420 }
2421 
2422 static int
2423 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx,
2424     struct sysctl_oid *oid)
2425 {
2426 	int rc;
2427 	struct adapter *sc = pi->adapter;
2428 	struct sge_eq *eq = &txq->eq;
2429 	char name[16];
2430 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
2431 
2432 	rc = alloc_eq(sc, pi, eq);
2433 	if (rc)
2434 		return (rc);
2435 
2436 	txq->ifp = pi->ifp;
2437 
2438 	txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE,
2439 	    M_ZERO | M_WAITOK);
2440 	txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock);
2441 
2442 	rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR,
2443 	    BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS,
2444 	    BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &txq->tx_tag);
2445 	if (rc != 0) {
2446 		device_printf(sc->dev,
2447 		    "failed to create tx DMA tag: %d\n", rc);
2448 		return (rc);
2449 	}
2450 
2451 	/*
2452 	 * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE
2453 	 * limit for any WR).  txq->no_dmamap events shouldn't occur if maps is
2454 	 * sized for the worst case.
2455 	 */
2456 	rc = t4_alloc_tx_maps(&txq->txmaps, txq->tx_tag, eq->qsize * 10 / 8,
2457 	    M_WAITOK);
2458 	if (rc != 0) {
2459 		device_printf(sc->dev, "failed to setup tx DMA maps: %d\n", rc);
2460 		return (rc);
2461 	}
2462 
2463 	snprintf(name, sizeof(name), "%d", idx);
2464 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
2465 	    NULL, "tx queue");
2466 	children = SYSCTL_CHILDREN(oid);
2467 
2468 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
2469 	    &eq->cntxt_id, 0, "SGE context id of the queue");
2470 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
2471 	    CTLTYPE_INT | CTLFLAG_RD, &eq->cidx, 0, sysctl_uint16, "I",
2472 	    "consumer index");
2473 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "pidx",
2474 	    CTLTYPE_INT | CTLFLAG_RD, &eq->pidx, 0, sysctl_uint16, "I",
2475 	    "producer index");
2476 
2477 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
2478 	    &txq->txcsum, "# of times hardware assisted with checksum");
2479 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "vlan_insertion",
2480 	    CTLFLAG_RD, &txq->vlan_insertion,
2481 	    "# of times hardware inserted 802.1Q tag");
2482 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
2483 	    &txq->tso_wrs, "# of TSO work requests");
2484 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
2485 	    &txq->imm_wrs, "# of work requests with immediate data");
2486 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
2487 	    &txq->sgl_wrs, "# of work requests with direct SGL");
2488 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
2489 	    &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
2490 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_wrs", CTLFLAG_RD,
2491 	    &txq->txpkts_wrs, "# of txpkts work requests (multiple pkts/WR)");
2492 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_pkts", CTLFLAG_RD,
2493 	    &txq->txpkts_pkts, "# of frames tx'd using txpkts work requests");
2494 
2495 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "br_drops", CTLFLAG_RD,
2496 	    &txq->br->br_drops, "# of drops in the buf_ring for this queue");
2497 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_dmamap", CTLFLAG_RD,
2498 	    &txq->no_dmamap, 0, "# of times txq ran out of DMA maps");
2499 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
2500 	    &txq->no_desc, 0, "# of times txq ran out of hardware descriptors");
2501 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "egr_update", CTLFLAG_RD,
2502 	    &eq->egr_update, 0, "egress update notifications from the SGE");
2503 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
2504 	    &eq->unstalled, 0, "# of times txq recovered after stall");
2505 
2506 	return (rc);
2507 }
2508 
2509 static int
2510 free_txq(struct port_info *pi, struct sge_txq *txq)
2511 {
2512 	int rc;
2513 	struct adapter *sc = pi->adapter;
2514 	struct sge_eq *eq = &txq->eq;
2515 
2516 	rc = free_eq(sc, eq);
2517 	if (rc)
2518 		return (rc);
2519 
2520 	free(txq->sdesc, M_CXGBE);
2521 
2522 	if (txq->txmaps.maps)
2523 		t4_free_tx_maps(&txq->txmaps, txq->tx_tag);
2524 
2525 	buf_ring_free(txq->br, M_CXGBE);
2526 
2527 	if (txq->tx_tag)
2528 		bus_dma_tag_destroy(txq->tx_tag);
2529 
2530 	bzero(txq, sizeof(*txq));
2531 	return (0);
2532 }
2533 
2534 static void
2535 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2536 {
2537 	bus_addr_t *ba = arg;
2538 
2539 	KASSERT(nseg == 1,
2540 	    ("%s meant for single segment mappings only.", __func__));
2541 
2542 	*ba = error ? 0 : segs->ds_addr;
2543 }
2544 
2545 static inline bool
2546 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2547 {
2548 	*ctrl = (void *)((uintptr_t)iq->cdesc +
2549 	    (iq->esize - sizeof(struct rsp_ctrl)));
2550 
2551 	return (((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen);
2552 }
2553 
2554 static inline void
2555 iq_next(struct sge_iq *iq)
2556 {
2557 	iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2558 	if (__predict_false(++iq->cidx == iq->qsize - 1)) {
2559 		iq->cidx = 0;
2560 		iq->gen ^= 1;
2561 		iq->cdesc = iq->desc;
2562 	}
2563 }
2564 
2565 #define FL_HW_IDX(x) ((x) >> 3)
2566 static inline void
2567 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
2568 {
2569 	int ndesc = fl->pending / 8;
2570 	uint32_t v;
2571 
2572 	if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
2573 		ndesc--;	/* hold back one credit */
2574 
2575 	if (ndesc <= 0)
2576 		return;		/* nothing to do */
2577 
2578 	v = F_DBPRIO | V_QID(fl->cntxt_id) | V_PIDX(ndesc);
2579 	if (is_t5(sc))
2580 		v |= F_DBTYPE;
2581 
2582 	wmb();
2583 
2584 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
2585 	fl->pending -= ndesc * 8;
2586 }
2587 
2588 /*
2589  * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2590  *
2591  * Returns non-zero to indicate that it should be added to the list of starving
2592  * freelists.
2593  */
2594 static int
2595 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2596 {
2597 	__be64 *d = &fl->desc[fl->pidx];
2598 	struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2599 	bus_dma_tag_t tag;
2600 	bus_addr_t pa;
2601 	caddr_t cl;
2602 	int rc;
2603 
2604 	FL_LOCK_ASSERT_OWNED(fl);
2605 
2606 	if (nbufs > fl->needed)
2607 		nbufs = fl->needed;
2608 
2609 	while (nbufs--) {
2610 
2611 		if (sd->cl != NULL) {
2612 
2613 			/*
2614 			 * This happens when a frame small enough to fit
2615 			 * entirely in an mbuf was received in cl last time.
2616 			 * We'd held on to cl and can reuse it now.  Note that
2617 			 * we reuse a cluster of the old size if fl->tag_idx is
2618 			 * no longer the same as sd->tag_idx.
2619 			 */
2620 
2621 			KASSERT(*d == sd->ba_tag,
2622 			    ("%s: recyling problem at pidx %d",
2623 			    __func__, fl->pidx));
2624 
2625 			d++;
2626 			goto recycled;
2627 		}
2628 
2629 
2630 		if (fl->tag_idx != sd->tag_idx) {
2631 			bus_dmamap_t map;
2632 			bus_dma_tag_t newtag = fl->tag[fl->tag_idx];
2633 			bus_dma_tag_t oldtag = fl->tag[sd->tag_idx];
2634 
2635 			/*
2636 			 * An MTU change can get us here.  Discard the old map
2637 			 * which was created with the old tag, but only if
2638 			 * we're able to get a new one.
2639 			 */
2640 			rc = bus_dmamap_create(newtag, 0, &map);
2641 			if (rc == 0) {
2642 				bus_dmamap_destroy(oldtag, sd->map);
2643 				sd->map = map;
2644 				sd->tag_idx = fl->tag_idx;
2645 			}
2646 		}
2647 
2648 		tag = fl->tag[sd->tag_idx];
2649 
2650 		cl = m_cljget(NULL, M_NOWAIT, FL_BUF_SIZE(sd->tag_idx));
2651 		if (cl == NULL)
2652 			break;
2653 
2654 		rc = bus_dmamap_load(tag, sd->map, cl, FL_BUF_SIZE(sd->tag_idx),
2655 		    oneseg_dma_callback, &pa, 0);
2656 		if (rc != 0 || pa == 0) {
2657 			fl->dmamap_failed++;
2658 			uma_zfree(FL_BUF_ZONE(sd->tag_idx), cl);
2659 			break;
2660 		}
2661 
2662 		sd->cl = cl;
2663 		*d++ = htobe64(pa | sd->tag_idx);
2664 
2665 #ifdef INVARIANTS
2666 		sd->ba_tag = htobe64(pa | sd->tag_idx);
2667 #endif
2668 
2669 recycled:
2670 		/* sd->m is never recycled, should always be NULL */
2671 		KASSERT(sd->m == NULL, ("%s: stray mbuf", __func__));
2672 
2673 		sd->m = m_gethdr(M_NOWAIT, MT_NOINIT);
2674 		if (sd->m == NULL)
2675 			break;
2676 
2677 		fl->pending++;
2678 		fl->needed--;
2679 		sd++;
2680 		if (++fl->pidx == fl->cap) {
2681 			fl->pidx = 0;
2682 			sd = fl->sdesc;
2683 			d = fl->desc;
2684 		}
2685 	}
2686 
2687 	if (fl->pending >= 8)
2688 		ring_fl_db(sc, fl);
2689 
2690 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2691 }
2692 
2693 /*
2694  * Attempt to refill all starving freelists.
2695  */
2696 static void
2697 refill_sfl(void *arg)
2698 {
2699 	struct adapter *sc = arg;
2700 	struct sge_fl *fl, *fl_temp;
2701 
2702 	mtx_lock(&sc->sfl_lock);
2703 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2704 		FL_LOCK(fl);
2705 		refill_fl(sc, fl, 64);
2706 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2707 			TAILQ_REMOVE(&sc->sfl, fl, link);
2708 			fl->flags &= ~FL_STARVING;
2709 		}
2710 		FL_UNLOCK(fl);
2711 	}
2712 
2713 	if (!TAILQ_EMPTY(&sc->sfl))
2714 		callout_schedule(&sc->sfl_callout, hz / 5);
2715 	mtx_unlock(&sc->sfl_lock);
2716 }
2717 
2718 static int
2719 alloc_fl_sdesc(struct sge_fl *fl)
2720 {
2721 	struct fl_sdesc *sd;
2722 	bus_dma_tag_t tag;
2723 	int i, rc;
2724 
2725 	FL_LOCK_ASSERT_OWNED(fl);
2726 
2727 	fl->sdesc = malloc(fl->cap * sizeof(struct fl_sdesc), M_CXGBE,
2728 	    M_ZERO | M_WAITOK);
2729 
2730 	tag = fl->tag[fl->tag_idx];
2731 	sd = fl->sdesc;
2732 	for (i = 0; i < fl->cap; i++, sd++) {
2733 
2734 		sd->tag_idx = fl->tag_idx;
2735 		rc = bus_dmamap_create(tag, 0, &sd->map);
2736 		if (rc != 0)
2737 			goto failed;
2738 	}
2739 
2740 	return (0);
2741 failed:
2742 	while (--i >= 0) {
2743 		sd--;
2744 		bus_dmamap_destroy(tag, sd->map);
2745 		if (sd->m) {
2746 			m_init(sd->m, NULL, 0, M_NOWAIT, MT_DATA, 0);
2747 			m_free(sd->m);
2748 			sd->m = NULL;
2749 		}
2750 	}
2751 	KASSERT(sd == fl->sdesc, ("%s: EDOOFUS", __func__));
2752 
2753 	free(fl->sdesc, M_CXGBE);
2754 	fl->sdesc = NULL;
2755 
2756 	return (rc);
2757 }
2758 
2759 static void
2760 free_fl_sdesc(struct sge_fl *fl)
2761 {
2762 	struct fl_sdesc *sd;
2763 	int i;
2764 
2765 	FL_LOCK_ASSERT_OWNED(fl);
2766 
2767 	sd = fl->sdesc;
2768 	for (i = 0; i < fl->cap; i++, sd++) {
2769 
2770 		if (sd->m) {
2771 			m_init(sd->m, NULL, 0, M_NOWAIT, MT_DATA, 0);
2772 			m_free(sd->m);
2773 			sd->m = NULL;
2774 		}
2775 
2776 		if (sd->cl) {
2777 			bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
2778 			uma_zfree(FL_BUF_ZONE(sd->tag_idx), sd->cl);
2779 			sd->cl = NULL;
2780 		}
2781 
2782 		bus_dmamap_destroy(fl->tag[sd->tag_idx], sd->map);
2783 	}
2784 
2785 	free(fl->sdesc, M_CXGBE);
2786 	fl->sdesc = NULL;
2787 }
2788 
2789 int
2790 t4_alloc_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag, int count,
2791     int flags)
2792 {
2793 	struct tx_map *txm;
2794 	int i, rc;
2795 
2796 	txmaps->map_total = txmaps->map_avail = count;
2797 	txmaps->map_cidx = txmaps->map_pidx = 0;
2798 
2799 	txmaps->maps = malloc(count * sizeof(struct tx_map), M_CXGBE,
2800 	    M_ZERO | flags);
2801 
2802 	txm = txmaps->maps;
2803 	for (i = 0; i < count; i++, txm++) {
2804 		rc = bus_dmamap_create(tx_tag, 0, &txm->map);
2805 		if (rc != 0)
2806 			goto failed;
2807 	}
2808 
2809 	return (0);
2810 failed:
2811 	while (--i >= 0) {
2812 		txm--;
2813 		bus_dmamap_destroy(tx_tag, txm->map);
2814 	}
2815 	KASSERT(txm == txmaps->maps, ("%s: EDOOFUS", __func__));
2816 
2817 	free(txmaps->maps, M_CXGBE);
2818 	txmaps->maps = NULL;
2819 
2820 	return (rc);
2821 }
2822 
2823 void
2824 t4_free_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag)
2825 {
2826 	struct tx_map *txm;
2827 	int i;
2828 
2829 	txm = txmaps->maps;
2830 	for (i = 0; i < txmaps->map_total; i++, txm++) {
2831 
2832 		if (txm->m) {
2833 			bus_dmamap_unload(tx_tag, txm->map);
2834 			m_freem(txm->m);
2835 			txm->m = NULL;
2836 		}
2837 
2838 		bus_dmamap_destroy(tx_tag, txm->map);
2839 	}
2840 
2841 	free(txmaps->maps, M_CXGBE);
2842 	txmaps->maps = NULL;
2843 }
2844 
2845 /*
2846  * We'll do immediate data tx for non-TSO, but only when not coalescing.  We're
2847  * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2848  * of immediate data.
2849  */
2850 #define IMM_LEN ( \
2851       2 * EQ_ESIZE \
2852     - sizeof(struct fw_eth_tx_pkt_wr) \
2853     - sizeof(struct cpl_tx_pkt_core))
2854 
2855 /*
2856  * Returns non-zero on failure, no need to cleanup anything in that case.
2857  *
2858  * Note 1: We always try to defrag the mbuf if required and return EFBIG only
2859  * if the resulting chain still won't fit in a tx descriptor.
2860  *
2861  * Note 2: We'll pullup the mbuf chain if TSO is requested and the first mbuf
2862  * does not have the TCP header in it.
2863  */
2864 static int
2865 get_pkt_sgl(struct sge_txq *txq, struct mbuf **fp, struct sgl *sgl,
2866     int sgl_only)
2867 {
2868 	struct mbuf *m = *fp;
2869 	struct tx_maps *txmaps;
2870 	struct tx_map *txm;
2871 	int rc, defragged = 0, n;
2872 
2873 	TXQ_LOCK_ASSERT_OWNED(txq);
2874 
2875 	if (m->m_pkthdr.tso_segsz)
2876 		sgl_only = 1;	/* Do not allow immediate data with LSO */
2877 
2878 start:	sgl->nsegs = 0;
2879 
2880 	if (m->m_pkthdr.len <= IMM_LEN && !sgl_only)
2881 		return (0);	/* nsegs = 0 tells caller to use imm. tx */
2882 
2883 	txmaps = &txq->txmaps;
2884 	if (txmaps->map_avail == 0) {
2885 		txq->no_dmamap++;
2886 		return (ENOMEM);
2887 	}
2888 	txm = &txmaps->maps[txmaps->map_pidx];
2889 
2890 	if (m->m_pkthdr.tso_segsz && m->m_len < 50) {
2891 		*fp = m_pullup(m, 50);
2892 		m = *fp;
2893 		if (m == NULL)
2894 			return (ENOBUFS);
2895 	}
2896 
2897 	rc = bus_dmamap_load_mbuf_sg(txq->tx_tag, txm->map, m, sgl->seg,
2898 	    &sgl->nsegs, BUS_DMA_NOWAIT);
2899 	if (rc == EFBIG && defragged == 0) {
2900 		m = m_defrag(m, M_NOWAIT);
2901 		if (m == NULL)
2902 			return (EFBIG);
2903 
2904 		defragged = 1;
2905 		*fp = m;
2906 		goto start;
2907 	}
2908 	if (rc != 0)
2909 		return (rc);
2910 
2911 	txm->m = m;
2912 	txmaps->map_avail--;
2913 	if (++txmaps->map_pidx == txmaps->map_total)
2914 		txmaps->map_pidx = 0;
2915 
2916 	KASSERT(sgl->nsegs > 0 && sgl->nsegs <= TX_SGL_SEGS,
2917 	    ("%s: bad DMA mapping (%d segments)", __func__, sgl->nsegs));
2918 
2919 	/*
2920 	 * Store the # of flits required to hold this frame's SGL in nflits.  An
2921 	 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2922 	 * multiple (len0 + len1, addr0, addr1) tuples.  If addr1 is not used
2923 	 * then len1 must be set to 0.
2924 	 */
2925 	n = sgl->nsegs - 1;
2926 	sgl->nflits = (3 * n) / 2 + (n & 1) + 2;
2927 
2928 	return (0);
2929 }
2930 
2931 
2932 /*
2933  * Releases all the txq resources used up in the specified sgl.
2934  */
2935 static int
2936 free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl)
2937 {
2938 	struct tx_maps *txmaps;
2939 	struct tx_map *txm;
2940 
2941 	TXQ_LOCK_ASSERT_OWNED(txq);
2942 
2943 	if (sgl->nsegs == 0)
2944 		return (0);	/* didn't use any map */
2945 
2946 	txmaps = &txq->txmaps;
2947 
2948 	/* 1 pkt uses exactly 1 map, back it out */
2949 
2950 	txmaps->map_avail++;
2951 	if (txmaps->map_pidx > 0)
2952 		txmaps->map_pidx--;
2953 	else
2954 		txmaps->map_pidx = txmaps->map_total - 1;
2955 
2956 	txm = &txmaps->maps[txmaps->map_pidx];
2957 	bus_dmamap_unload(txq->tx_tag, txm->map);
2958 	txm->m = NULL;
2959 
2960 	return (0);
2961 }
2962 
2963 static int
2964 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, struct mbuf *m,
2965     struct sgl *sgl)
2966 {
2967 	struct sge_eq *eq = &txq->eq;
2968 	struct fw_eth_tx_pkt_wr *wr;
2969 	struct cpl_tx_pkt_core *cpl;
2970 	uint32_t ctrl;	/* used in many unrelated places */
2971 	uint64_t ctrl1;
2972 	int nflits, ndesc, pktlen;
2973 	struct tx_sdesc *txsd;
2974 	caddr_t dst;
2975 
2976 	TXQ_LOCK_ASSERT_OWNED(txq);
2977 
2978 	pktlen = m->m_pkthdr.len;
2979 
2980 	/*
2981 	 * Do we have enough flits to send this frame out?
2982 	 */
2983 	ctrl = sizeof(struct cpl_tx_pkt_core);
2984 	if (m->m_pkthdr.tso_segsz) {
2985 		nflits = TXPKT_LSO_WR_HDR;
2986 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2987 	} else
2988 		nflits = TXPKT_WR_HDR;
2989 	if (sgl->nsegs > 0)
2990 		nflits += sgl->nflits;
2991 	else {
2992 		nflits += howmany(pktlen, 8);
2993 		ctrl += pktlen;
2994 	}
2995 	ndesc = howmany(nflits, 8);
2996 	if (ndesc > eq->avail)
2997 		return (ENOMEM);
2998 
2999 	/* Firmware work request header */
3000 	wr = (void *)&eq->desc[eq->pidx];
3001 	wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
3002 	    V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
3003 	ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
3004 	if (eq->avail == ndesc) {
3005 		if (!(eq->flags & EQ_CRFLUSHED)) {
3006 			ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
3007 			eq->flags |= EQ_CRFLUSHED;
3008 		}
3009 		eq->flags |= EQ_STALLED;
3010 	}
3011 
3012 	wr->equiq_to_len16 = htobe32(ctrl);
3013 	wr->r3 = 0;
3014 
3015 	if (m->m_pkthdr.tso_segsz) {
3016 		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
3017 		struct ether_header *eh;
3018 		void *l3hdr;
3019 #if defined(INET) || defined(INET6)
3020 		struct tcphdr *tcp;
3021 #endif
3022 		uint16_t eh_type;
3023 
3024 		ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
3025 		    F_LSO_LAST_SLICE;
3026 
3027 		eh = mtod(m, struct ether_header *);
3028 		eh_type = ntohs(eh->ether_type);
3029 		if (eh_type == ETHERTYPE_VLAN) {
3030 			struct ether_vlan_header *evh = (void *)eh;
3031 
3032 			ctrl |= V_LSO_ETHHDR_LEN(1);
3033 			l3hdr = evh + 1;
3034 			eh_type = ntohs(evh->evl_proto);
3035 		} else
3036 			l3hdr = eh + 1;
3037 
3038 		switch (eh_type) {
3039 #ifdef INET6
3040 		case ETHERTYPE_IPV6:
3041 		{
3042 			struct ip6_hdr *ip6 = l3hdr;
3043 
3044 			/*
3045 			 * XXX-BZ For now we do not pretend to support
3046 			 * IPv6 extension headers.
3047 			 */
3048 			KASSERT(ip6->ip6_nxt == IPPROTO_TCP, ("%s: CSUM_TSO "
3049 			    "with ip6_nxt != TCP: %u", __func__, ip6->ip6_nxt));
3050 			tcp = (struct tcphdr *)(ip6 + 1);
3051 			ctrl |= F_LSO_IPV6;
3052 			ctrl |= V_LSO_IPHDR_LEN(sizeof(*ip6) >> 2) |
3053 			    V_LSO_TCPHDR_LEN(tcp->th_off);
3054 			break;
3055 		}
3056 #endif
3057 #ifdef INET
3058 		case ETHERTYPE_IP:
3059 		{
3060 			struct ip *ip = l3hdr;
3061 
3062 			tcp = (void *)((uintptr_t)ip + ip->ip_hl * 4);
3063 			ctrl |= V_LSO_IPHDR_LEN(ip->ip_hl) |
3064 			    V_LSO_TCPHDR_LEN(tcp->th_off);
3065 			break;
3066 		}
3067 #endif
3068 		default:
3069 			panic("%s: CSUM_TSO but no supported IP version "
3070 			    "(0x%04x)", __func__, eh_type);
3071 		}
3072 
3073 		lso->lso_ctrl = htobe32(ctrl);
3074 		lso->ipid_ofst = htobe16(0);
3075 		lso->mss = htobe16(m->m_pkthdr.tso_segsz);
3076 		lso->seqno_offset = htobe32(0);
3077 		lso->len = htobe32(pktlen);
3078 
3079 		cpl = (void *)(lso + 1);
3080 
3081 		txq->tso_wrs++;
3082 	} else
3083 		cpl = (void *)(wr + 1);
3084 
3085 	/* Checksum offload */
3086 	ctrl1 = 0;
3087 	if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
3088 		ctrl1 |= F_TXPKT_IPCSUM_DIS;
3089 	if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
3090 	    CSUM_TCP_IPV6 | CSUM_TSO)))
3091 		ctrl1 |= F_TXPKT_L4CSUM_DIS;
3092 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
3093 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
3094 		txq->txcsum++;	/* some hardware assistance provided */
3095 
3096 	/* VLAN tag insertion */
3097 	if (m->m_flags & M_VLANTAG) {
3098 		ctrl1 |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
3099 		txq->vlan_insertion++;
3100 	}
3101 
3102 	/* CPL header */
3103 	cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3104 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3105 	cpl->pack = 0;
3106 	cpl->len = htobe16(pktlen);
3107 	cpl->ctrl1 = htobe64(ctrl1);
3108 
3109 	/* Software descriptor */
3110 	txsd = &txq->sdesc[eq->pidx];
3111 	txsd->desc_used = ndesc;
3112 
3113 	eq->pending += ndesc;
3114 	eq->avail -= ndesc;
3115 	eq->pidx += ndesc;
3116 	if (eq->pidx >= eq->cap)
3117 		eq->pidx -= eq->cap;
3118 
3119 	/* SGL */
3120 	dst = (void *)(cpl + 1);
3121 	if (sgl->nsegs > 0) {
3122 		txsd->credits = 1;
3123 		txq->sgl_wrs++;
3124 		write_sgl_to_txd(eq, sgl, &dst);
3125 	} else {
3126 		txsd->credits = 0;
3127 		txq->imm_wrs++;
3128 		for (; m; m = m->m_next) {
3129 			copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
3130 #ifdef INVARIANTS
3131 			pktlen -= m->m_len;
3132 #endif
3133 		}
3134 #ifdef INVARIANTS
3135 		KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
3136 #endif
3137 
3138 	}
3139 
3140 	txq->txpkt_wrs++;
3141 	return (0);
3142 }
3143 
3144 /*
3145  * Returns 0 to indicate that m has been accepted into a coalesced tx work
3146  * request.  It has either been folded into txpkts or txpkts was flushed and m
3147  * has started a new coalesced work request (as the first frame in a fresh
3148  * txpkts).
3149  *
3150  * Returns non-zero to indicate a failure - caller is responsible for
3151  * transmitting m, if there was anything in txpkts it has been flushed.
3152  */
3153 static int
3154 add_to_txpkts(struct port_info *pi, struct sge_txq *txq, struct txpkts *txpkts,
3155     struct mbuf *m, struct sgl *sgl)
3156 {
3157 	struct sge_eq *eq = &txq->eq;
3158 	int can_coalesce;
3159 	struct tx_sdesc *txsd;
3160 	int flits;
3161 
3162 	TXQ_LOCK_ASSERT_OWNED(txq);
3163 
3164 	KASSERT(sgl->nsegs, ("%s: can't coalesce imm data", __func__));
3165 
3166 	if (txpkts->npkt > 0) {
3167 		flits = TXPKTS_PKT_HDR + sgl->nflits;
3168 		can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
3169 		    txpkts->nflits + flits <= TX_WR_FLITS &&
3170 		    txpkts->nflits + flits <= eq->avail * 8 &&
3171 		    txpkts->plen + m->m_pkthdr.len < 65536;
3172 
3173 		if (can_coalesce) {
3174 			txpkts->npkt++;
3175 			txpkts->nflits += flits;
3176 			txpkts->plen += m->m_pkthdr.len;
3177 
3178 			txsd = &txq->sdesc[eq->pidx];
3179 			txsd->credits++;
3180 
3181 			return (0);
3182 		}
3183 
3184 		/*
3185 		 * Couldn't coalesce m into txpkts.  The first order of business
3186 		 * is to send txpkts on its way.  Then we'll revisit m.
3187 		 */
3188 		write_txpkts_wr(txq, txpkts);
3189 	}
3190 
3191 	/*
3192 	 * Check if we can start a new coalesced tx work request with m as
3193 	 * the first packet in it.
3194 	 */
3195 
3196 	KASSERT(txpkts->npkt == 0, ("%s: txpkts not empty", __func__));
3197 
3198 	flits = TXPKTS_WR_HDR + sgl->nflits;
3199 	can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
3200 	    flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
3201 
3202 	if (can_coalesce == 0)
3203 		return (EINVAL);
3204 
3205 	/*
3206 	 * Start a fresh coalesced tx WR with m as the first frame in it.
3207 	 */
3208 	txpkts->npkt = 1;
3209 	txpkts->nflits = flits;
3210 	txpkts->flitp = &eq->desc[eq->pidx].flit[2];
3211 	txpkts->plen = m->m_pkthdr.len;
3212 
3213 	txsd = &txq->sdesc[eq->pidx];
3214 	txsd->credits = 1;
3215 
3216 	return (0);
3217 }
3218 
3219 /*
3220  * Note that write_txpkts_wr can never run out of hardware descriptors (but
3221  * write_txpkt_wr can).  add_to_txpkts ensures that a frame is accepted for
3222  * coalescing only if sufficient hardware descriptors are available.
3223  */
3224 static void
3225 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
3226 {
3227 	struct sge_eq *eq = &txq->eq;
3228 	struct fw_eth_tx_pkts_wr *wr;
3229 	struct tx_sdesc *txsd;
3230 	uint32_t ctrl;
3231 	int ndesc;
3232 
3233 	TXQ_LOCK_ASSERT_OWNED(txq);
3234 
3235 	ndesc = howmany(txpkts->nflits, 8);
3236 
3237 	wr = (void *)&eq->desc[eq->pidx];
3238 	wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
3239 	ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
3240 	if (eq->avail == ndesc) {
3241 		if (!(eq->flags & EQ_CRFLUSHED)) {
3242 			ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
3243 			eq->flags |= EQ_CRFLUSHED;
3244 		}
3245 		eq->flags |= EQ_STALLED;
3246 	}
3247 	wr->equiq_to_len16 = htobe32(ctrl);
3248 	wr->plen = htobe16(txpkts->plen);
3249 	wr->npkt = txpkts->npkt;
3250 	wr->r3 = wr->type = 0;
3251 
3252 	/* Everything else already written */
3253 
3254 	txsd = &txq->sdesc[eq->pidx];
3255 	txsd->desc_used = ndesc;
3256 
3257 	KASSERT(eq->avail >= ndesc, ("%s: out of descriptors", __func__));
3258 
3259 	eq->pending += ndesc;
3260 	eq->avail -= ndesc;
3261 	eq->pidx += ndesc;
3262 	if (eq->pidx >= eq->cap)
3263 		eq->pidx -= eq->cap;
3264 
3265 	txq->txpkts_pkts += txpkts->npkt;
3266 	txq->txpkts_wrs++;
3267 	txpkts->npkt = 0;	/* emptied */
3268 }
3269 
3270 static inline void
3271 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
3272     struct txpkts *txpkts, struct mbuf *m, struct sgl *sgl)
3273 {
3274 	struct ulp_txpkt *ulpmc;
3275 	struct ulptx_idata *ulpsc;
3276 	struct cpl_tx_pkt_core *cpl;
3277 	struct sge_eq *eq = &txq->eq;
3278 	uintptr_t flitp, start, end;
3279 	uint64_t ctrl;
3280 	caddr_t dst;
3281 
3282 	KASSERT(txpkts->npkt > 0, ("%s: txpkts is empty", __func__));
3283 
3284 	start = (uintptr_t)eq->desc;
3285 	end = (uintptr_t)eq->spg;
3286 
3287 	/* Checksum offload */
3288 	ctrl = 0;
3289 	if (!(m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)))
3290 		ctrl |= F_TXPKT_IPCSUM_DIS;
3291 	if (!(m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 |
3292 	    CSUM_TCP_IPV6 | CSUM_TSO)))
3293 		ctrl |= F_TXPKT_L4CSUM_DIS;
3294 	if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP |
3295 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO))
3296 		txq->txcsum++;	/* some hardware assistance provided */
3297 
3298 	/* VLAN tag insertion */
3299 	if (m->m_flags & M_VLANTAG) {
3300 		ctrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
3301 		txq->vlan_insertion++;
3302 	}
3303 
3304 	/*
3305 	 * The previous packet's SGL must have ended at a 16 byte boundary (this
3306 	 * is required by the firmware/hardware).  It follows that flitp cannot
3307 	 * wrap around between the ULPTX master command and ULPTX subcommand (8
3308 	 * bytes each), and that it can not wrap around in the middle of the
3309 	 * cpl_tx_pkt_core either.
3310 	 */
3311 	flitp = (uintptr_t)txpkts->flitp;
3312 	KASSERT((flitp & 0xf) == 0,
3313 	    ("%s: last SGL did not end at 16 byte boundary: %p",
3314 	    __func__, txpkts->flitp));
3315 
3316 	/* ULP master command */
3317 	ulpmc = (void *)flitp;
3318 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0) |
3319 	    V_ULP_TXPKT_FID(eq->iqid));
3320 	ulpmc->len = htonl(howmany(sizeof(*ulpmc) + sizeof(*ulpsc) +
3321 	    sizeof(*cpl) + 8 * sgl->nflits, 16));
3322 
3323 	/* ULP subcommand */
3324 	ulpsc = (void *)(ulpmc + 1);
3325 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3326 	    F_ULP_TX_SC_MORE);
3327 	ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
3328 
3329 	flitp += sizeof(*ulpmc) + sizeof(*ulpsc);
3330 	if (flitp == end)
3331 		flitp = start;
3332 
3333 	/* CPL_TX_PKT */
3334 	cpl = (void *)flitp;
3335 	cpl->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3336 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3337 	cpl->pack = 0;
3338 	cpl->len = htobe16(m->m_pkthdr.len);
3339 	cpl->ctrl1 = htobe64(ctrl);
3340 
3341 	flitp += sizeof(*cpl);
3342 	if (flitp == end)
3343 		flitp = start;
3344 
3345 	/* SGL for this frame */
3346 	dst = (caddr_t)flitp;
3347 	txpkts->nflits += write_sgl_to_txd(eq, sgl, &dst);
3348 	txpkts->flitp = (void *)dst;
3349 
3350 	KASSERT(((uintptr_t)dst & 0xf) == 0,
3351 	    ("%s: SGL ends at %p (not a 16 byte boundary)", __func__, dst));
3352 }
3353 
3354 /*
3355  * If the SGL ends on an address that is not 16 byte aligned, this function will
3356  * add a 0 filled flit at the end.  It returns 1 in that case.
3357  */
3358 static int
3359 write_sgl_to_txd(struct sge_eq *eq, struct sgl *sgl, caddr_t *to)
3360 {
3361 	__be64 *flitp, *end;
3362 	struct ulptx_sgl *usgl;
3363 	bus_dma_segment_t *seg;
3364 	int i, padded;
3365 
3366 	KASSERT(sgl->nsegs > 0 && sgl->nflits > 0,
3367 	    ("%s: bad SGL - nsegs=%d, nflits=%d",
3368 	    __func__, sgl->nsegs, sgl->nflits));
3369 
3370 	KASSERT(((uintptr_t)(*to) & 0xf) == 0,
3371 	    ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
3372 
3373 	flitp = (__be64 *)(*to);
3374 	end = flitp + sgl->nflits;
3375 	seg = &sgl->seg[0];
3376 	usgl = (void *)flitp;
3377 
3378 	/*
3379 	 * We start at a 16 byte boundary somewhere inside the tx descriptor
3380 	 * ring, so we're at least 16 bytes away from the status page.  There is
3381 	 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
3382 	 */
3383 
3384 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
3385 	    V_ULPTX_NSGE(sgl->nsegs));
3386 	usgl->len0 = htobe32(seg->ds_len);
3387 	usgl->addr0 = htobe64(seg->ds_addr);
3388 	seg++;
3389 
3390 	if ((uintptr_t)end <= (uintptr_t)eq->spg) {
3391 
3392 		/* Won't wrap around at all */
3393 
3394 		for (i = 0; i < sgl->nsegs - 1; i++, seg++) {
3395 			usgl->sge[i / 2].len[i & 1] = htobe32(seg->ds_len);
3396 			usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ds_addr);
3397 		}
3398 		if (i & 1)
3399 			usgl->sge[i / 2].len[1] = htobe32(0);
3400 	} else {
3401 
3402 		/* Will wrap somewhere in the rest of the SGL */
3403 
3404 		/* 2 flits already written, write the rest flit by flit */
3405 		flitp = (void *)(usgl + 1);
3406 		for (i = 0; i < sgl->nflits - 2; i++) {
3407 			if ((uintptr_t)flitp == (uintptr_t)eq->spg)
3408 				flitp = (void *)eq->desc;
3409 			*flitp++ = get_flit(seg, sgl->nsegs - 1, i);
3410 		}
3411 		end = flitp;
3412 	}
3413 
3414 	if ((uintptr_t)end & 0xf) {
3415 		*(uint64_t *)end = 0;
3416 		end++;
3417 		padded = 1;
3418 	} else
3419 		padded = 0;
3420 
3421 	if ((uintptr_t)end == (uintptr_t)eq->spg)
3422 		*to = (void *)eq->desc;
3423 	else
3424 		*to = (void *)end;
3425 
3426 	return (padded);
3427 }
3428 
3429 static inline void
3430 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3431 {
3432 	if (__predict_true((uintptr_t)(*to) + len <= (uintptr_t)eq->spg)) {
3433 		bcopy(from, *to, len);
3434 		(*to) += len;
3435 	} else {
3436 		int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3437 
3438 		bcopy(from, *to, portion);
3439 		from += portion;
3440 		portion = len - portion;	/* remaining */
3441 		bcopy(from, (void *)eq->desc, portion);
3442 		(*to) = (caddr_t)eq->desc + portion;
3443 	}
3444 }
3445 
3446 static inline void
3447 ring_eq_db(struct adapter *sc, struct sge_eq *eq)
3448 {
3449 	u_int db, pending;
3450 
3451 	db = eq->doorbells;
3452 	pending = eq->pending;
3453 	if (pending > 1)
3454 		clrbit(&db, DOORBELL_WCWR);
3455 	eq->pending = 0;
3456 	wmb();
3457 
3458 	switch (ffs(db) - 1) {
3459 	case DOORBELL_UDB:
3460 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
3461 		return;
3462 
3463 	case DOORBELL_WCWR: {
3464 		volatile uint64_t *dst, *src;
3465 		int i;
3466 
3467 		/*
3468 		 * Queues whose 128B doorbell segment fits in the page do not
3469 		 * use relative qid (udb_qid is always 0).  Only queues with
3470 		 * doorbell segments can do WCWR.
3471 		 */
3472 		KASSERT(eq->udb_qid == 0 && pending == 1,
3473 		    ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
3474 		    __func__, eq->doorbells, pending, eq->pidx, eq));
3475 
3476 		dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
3477 		    UDBS_DB_OFFSET);
3478 		i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3479 		src = (void *)&eq->desc[i];
3480 		while (src != (void *)&eq->desc[i + 1])
3481 			*dst++ = *src++;
3482 		wmb();
3483 		return;
3484 	}
3485 
3486 	case DOORBELL_UDBWC:
3487 		*eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(pending));
3488 		wmb();
3489 		return;
3490 
3491 	case DOORBELL_KDB:
3492 		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3493 		    V_QID(eq->cntxt_id) | V_PIDX(pending));
3494 		return;
3495 	}
3496 }
3497 
3498 static inline int
3499 reclaimable(struct sge_eq *eq)
3500 {
3501 	unsigned int cidx;
3502 
3503 	cidx = eq->spg->cidx;	/* stable snapshot */
3504 	cidx = be16toh(cidx);
3505 
3506 	if (cidx >= eq->cidx)
3507 		return (cidx - eq->cidx);
3508 	else
3509 		return (cidx + eq->cap - eq->cidx);
3510 }
3511 
3512 /*
3513  * There are "can_reclaim" tx descriptors ready to be reclaimed.  Reclaim as
3514  * many as possible but stop when there are around "n" mbufs to free.
3515  *
3516  * The actual number reclaimed is provided as the return value.
3517  */
3518 static int
3519 reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
3520 {
3521 	struct tx_sdesc *txsd;
3522 	struct tx_maps *txmaps;
3523 	struct tx_map *txm;
3524 	unsigned int reclaimed, maps;
3525 	struct sge_eq *eq = &txq->eq;
3526 
3527 	TXQ_LOCK_ASSERT_OWNED(txq);
3528 
3529 	if (can_reclaim == 0)
3530 		can_reclaim = reclaimable(eq);
3531 
3532 	maps = reclaimed = 0;
3533 	while (can_reclaim && maps < n) {
3534 		int ndesc;
3535 
3536 		txsd = &txq->sdesc[eq->cidx];
3537 		ndesc = txsd->desc_used;
3538 
3539 		/* Firmware doesn't return "partial" credits. */
3540 		KASSERT(can_reclaim >= ndesc,
3541 		    ("%s: unexpected number of credits: %d, %d",
3542 		    __func__, can_reclaim, ndesc));
3543 
3544 		maps += txsd->credits;
3545 
3546 		reclaimed += ndesc;
3547 		can_reclaim -= ndesc;
3548 
3549 		eq->cidx += ndesc;
3550 		if (__predict_false(eq->cidx >= eq->cap))
3551 			eq->cidx -= eq->cap;
3552 	}
3553 
3554 	txmaps = &txq->txmaps;
3555 	txm = &txmaps->maps[txmaps->map_cidx];
3556 	if (maps)
3557 		prefetch(txm->m);
3558 
3559 	eq->avail += reclaimed;
3560 	KASSERT(eq->avail < eq->cap,	/* avail tops out at (cap - 1) */
3561 	    ("%s: too many descriptors available", __func__));
3562 
3563 	txmaps->map_avail += maps;
3564 	KASSERT(txmaps->map_avail <= txmaps->map_total,
3565 	    ("%s: too many maps available", __func__));
3566 
3567 	while (maps--) {
3568 		struct tx_map *next;
3569 
3570 		next = txm + 1;
3571 		if (__predict_false(txmaps->map_cidx + 1 == txmaps->map_total))
3572 			next = txmaps->maps;
3573 		prefetch(next->m);
3574 
3575 		bus_dmamap_unload(txq->tx_tag, txm->map);
3576 		m_freem(txm->m);
3577 		txm->m = NULL;
3578 
3579 		txm = next;
3580 		if (__predict_false(++txmaps->map_cidx == txmaps->map_total))
3581 			txmaps->map_cidx = 0;
3582 	}
3583 
3584 	return (reclaimed);
3585 }
3586 
3587 static void
3588 write_eqflush_wr(struct sge_eq *eq)
3589 {
3590 	struct fw_eq_flush_wr *wr;
3591 
3592 	EQ_LOCK_ASSERT_OWNED(eq);
3593 	KASSERT(eq->avail > 0, ("%s: no descriptors left.", __func__));
3594 	KASSERT(!(eq->flags & EQ_CRFLUSHED), ("%s: flushed already", __func__));
3595 
3596 	wr = (void *)&eq->desc[eq->pidx];
3597 	bzero(wr, sizeof(*wr));
3598 	wr->opcode = FW_EQ_FLUSH_WR;
3599 	wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(sizeof(*wr) / 16) |
3600 	    F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3601 
3602 	eq->flags |= (EQ_CRFLUSHED | EQ_STALLED);
3603 	eq->pending++;
3604 	eq->avail--;
3605 	if (++eq->pidx == eq->cap)
3606 		eq->pidx = 0;
3607 }
3608 
3609 static __be64
3610 get_flit(bus_dma_segment_t *sgl, int nsegs, int idx)
3611 {
3612 	int i = (idx / 3) * 2;
3613 
3614 	switch (idx % 3) {
3615 	case 0: {
3616 		__be64 rc;
3617 
3618 		rc = htobe32(sgl[i].ds_len);
3619 		if (i + 1 < nsegs)
3620 			rc |= (uint64_t)htobe32(sgl[i + 1].ds_len) << 32;
3621 
3622 		return (rc);
3623 	}
3624 	case 1:
3625 		return htobe64(sgl[i].ds_addr);
3626 	case 2:
3627 		return htobe64(sgl[i + 1].ds_addr);
3628 	}
3629 
3630 	return (0);
3631 }
3632 
3633 static void
3634 set_fl_tag_idx(struct sge_fl *fl, int bufsize)
3635 {
3636 	int i;
3637 
3638 	for (i = 0; i < FL_BUF_SIZES - 1; i++) {
3639 		if (FL_BUF_SIZE(i) >= bufsize)
3640 			break;
3641 	}
3642 
3643 	fl->tag_idx = i;
3644 }
3645 
3646 static void
3647 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
3648 {
3649 	mtx_lock(&sc->sfl_lock);
3650 	FL_LOCK(fl);
3651 	if ((fl->flags & FL_DOOMED) == 0) {
3652 		fl->flags |= FL_STARVING;
3653 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
3654 		callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
3655 	}
3656 	FL_UNLOCK(fl);
3657 	mtx_unlock(&sc->sfl_lock);
3658 }
3659 
3660 static int
3661 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3662     struct mbuf *m)
3663 {
3664 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3665 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3666 	struct adapter *sc = iq->adapter;
3667 	struct sge *s = &sc->sge;
3668 	struct sge_eq *eq;
3669 
3670 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
3671 	    rss->opcode));
3672 
3673 	eq = s->eqmap[qid - s->eq_start];
3674 	EQ_LOCK(eq);
3675 	KASSERT(eq->flags & EQ_CRFLUSHED,
3676 	    ("%s: unsolicited egress update", __func__));
3677 	eq->flags &= ~EQ_CRFLUSHED;
3678 	eq->egr_update++;
3679 
3680 	if (__predict_false(eq->flags & EQ_DOOMED))
3681 		wakeup_one(eq);
3682 	else if (eq->flags & EQ_STALLED && can_resume_tx(eq))
3683 		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
3684 	EQ_UNLOCK(eq);
3685 
3686 	return (0);
3687 }
3688 
3689 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
3690 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
3691     offsetof(struct cpl_fw6_msg, data));
3692 
3693 static int
3694 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
3695 {
3696 	struct adapter *sc = iq->adapter;
3697 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3698 
3699 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
3700 	    rss->opcode));
3701 
3702 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3703 		const struct rss_header *rss2;
3704 
3705 		rss2 = (const struct rss_header *)&cpl->data[0];
3706 		return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3707 	}
3708 
3709 	return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3710 }
3711 
3712 static int
3713 sysctl_uint16(SYSCTL_HANDLER_ARGS)
3714 {
3715 	uint16_t *id = arg1;
3716 	int i = *id;
3717 
3718 	return sysctl_handle_int(oidp, &i, 0, req);
3719 }
3720