xref: /freebsd/sys/dev/cxgbe/t4_main.c (revision a223d3ed90bfe313ce5987d468a25a915d7d1254)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/param.h>
35 #include <sys/conf.h>
36 #include <sys/priv.h>
37 #include <sys/kernel.h>
38 #include <sys/bus.h>
39 #include <sys/module.h>
40 #include <sys/malloc.h>
41 #include <sys/queue.h>
42 #include <sys/taskqueue.h>
43 #include <sys/pciio.h>
44 #include <dev/pci/pcireg.h>
45 #include <dev/pci/pcivar.h>
46 #include <dev/pci/pci_private.h>
47 #include <sys/firmware.h>
48 #include <sys/sbuf.h>
49 #include <sys/smp.h>
50 #include <sys/socket.h>
51 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <net/ethernet.h>
54 #include <net/if.h>
55 #include <net/if_types.h>
56 #include <net/if_dl.h>
57 #include <net/if_vlan_var.h>
58 #if defined(__i386__) || defined(__amd64__)
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #endif
62 
63 #include "common/common.h"
64 #include "common/t4_msg.h"
65 #include "common/t4_regs.h"
66 #include "common/t4_regs_values.h"
67 #include "t4_ioctl.h"
68 #include "t4_l2t.h"
69 
70 /* T4 bus driver interface */
71 static int t4_probe(device_t);
72 static int t4_attach(device_t);
73 static int t4_detach(device_t);
74 static device_method_t t4_methods[] = {
75 	DEVMETHOD(device_probe,		t4_probe),
76 	DEVMETHOD(device_attach,	t4_attach),
77 	DEVMETHOD(device_detach,	t4_detach),
78 
79 	DEVMETHOD_END
80 };
81 static driver_t t4_driver = {
82 	"t4nex",
83 	t4_methods,
84 	sizeof(struct adapter)
85 };
86 
87 
88 /* T4 port (cxgbe) interface */
89 static int cxgbe_probe(device_t);
90 static int cxgbe_attach(device_t);
91 static int cxgbe_detach(device_t);
92 static device_method_t cxgbe_methods[] = {
93 	DEVMETHOD(device_probe,		cxgbe_probe),
94 	DEVMETHOD(device_attach,	cxgbe_attach),
95 	DEVMETHOD(device_detach,	cxgbe_detach),
96 	{ 0, 0 }
97 };
98 static driver_t cxgbe_driver = {
99 	"cxgbe",
100 	cxgbe_methods,
101 	sizeof(struct port_info)
102 };
103 
104 static d_ioctl_t t4_ioctl;
105 static d_open_t t4_open;
106 static d_close_t t4_close;
107 
108 static struct cdevsw t4_cdevsw = {
109        .d_version = D_VERSION,
110        .d_flags = 0,
111        .d_open = t4_open,
112        .d_close = t4_close,
113        .d_ioctl = t4_ioctl,
114        .d_name = "t4nex",
115 };
116 
117 /* T5 bus driver interface */
118 static int t5_probe(device_t);
119 static device_method_t t5_methods[] = {
120 	DEVMETHOD(device_probe,		t5_probe),
121 	DEVMETHOD(device_attach,	t4_attach),
122 	DEVMETHOD(device_detach,	t4_detach),
123 
124 	DEVMETHOD_END
125 };
126 static driver_t t5_driver = {
127 	"t5nex",
128 	t5_methods,
129 	sizeof(struct adapter)
130 };
131 
132 
133 /* T5 port (cxl) interface */
134 static driver_t cxl_driver = {
135 	"cxl",
136 	cxgbe_methods,
137 	sizeof(struct port_info)
138 };
139 
140 static struct cdevsw t5_cdevsw = {
141        .d_version = D_VERSION,
142        .d_flags = 0,
143        .d_open = t4_open,
144        .d_close = t4_close,
145        .d_ioctl = t4_ioctl,
146        .d_name = "t5nex",
147 };
148 
149 /* ifnet + media interface */
150 static void cxgbe_init(void *);
151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
153 static void cxgbe_qflush(struct ifnet *);
154 static int cxgbe_media_change(struct ifnet *);
155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
156 
157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
158 
159 /*
160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
161  * then ADAPTER_LOCK, then t4_uld_list_lock.
162  */
163 static struct sx t4_list_lock;
164 SLIST_HEAD(, adapter) t4_list;
165 #ifdef TCP_OFFLOAD
166 static struct sx t4_uld_list_lock;
167 SLIST_HEAD(, uld_info) t4_uld_list;
168 #endif
169 
170 /*
171  * Tunables.  See tweak_tunables() too.
172  *
173  * Each tunable is set to a default value here if it's known at compile-time.
174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
175  * provide a reasonable default when the driver is loaded.
176  *
177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
178  * T5 are under hw.cxl.
179  */
180 
181 /*
182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
183  */
184 #define NTXQ_10G 16
185 static int t4_ntxq10g = -1;
186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
187 
188 #define NRXQ_10G 8
189 static int t4_nrxq10g = -1;
190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
191 
192 #define NTXQ_1G 4
193 static int t4_ntxq1g = -1;
194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
195 
196 #define NRXQ_1G 2
197 static int t4_nrxq1g = -1;
198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
199 
200 static int t4_rsrv_noflowq = 0;
201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
202 
203 #ifdef TCP_OFFLOAD
204 #define NOFLDTXQ_10G 8
205 static int t4_nofldtxq10g = -1;
206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
207 
208 #define NOFLDRXQ_10G 2
209 static int t4_nofldrxq10g = -1;
210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
211 
212 #define NOFLDTXQ_1G 2
213 static int t4_nofldtxq1g = -1;
214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
215 
216 #define NOFLDRXQ_1G 1
217 static int t4_nofldrxq1g = -1;
218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
219 #endif
220 
221 #ifdef DEV_NETMAP
222 #define NNMTXQ_10G 2
223 static int t4_nnmtxq10g = -1;
224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
225 
226 #define NNMRXQ_10G 2
227 static int t4_nnmrxq10g = -1;
228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
229 
230 #define NNMTXQ_1G 1
231 static int t4_nnmtxq1g = -1;
232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
233 
234 #define NNMRXQ_1G 1
235 static int t4_nnmrxq1g = -1;
236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
237 #endif
238 
239 /*
240  * Holdoff parameters for 10G and 1G ports.
241  */
242 #define TMR_IDX_10G 1
243 static int t4_tmr_idx_10g = TMR_IDX_10G;
244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
245 
246 #define PKTC_IDX_10G (-1)
247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
249 
250 #define TMR_IDX_1G 1
251 static int t4_tmr_idx_1g = TMR_IDX_1G;
252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
253 
254 #define PKTC_IDX_1G (-1)
255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
257 
258 /*
259  * Size (# of entries) of each tx and rx queue.
260  */
261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
263 
264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
266 
267 /*
268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
269  */
270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
272 
273 /*
274  * Configuration file.
275  */
276 #define DEFAULT_CF	"default"
277 #define FLASH_CF	"flash"
278 #define UWIRE_CF	"uwire"
279 #define FPGA_CF		"fpga"
280 static char t4_cfg_file[32] = DEFAULT_CF;
281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
282 
283 /*
284  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
285  * encouraged respectively).
286  */
287 static unsigned int t4_fw_install = 1;
288 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
289 
290 /*
291  * ASIC features that will be used.  Disable the ones you don't want so that the
292  * chip resources aren't wasted on features that will not be used.
293  */
294 static int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
295 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
296 
297 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
298 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
299 
300 static int t4_toecaps_allowed = -1;
301 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
302 
303 static int t4_rdmacaps_allowed = 0;
304 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
305 
306 static int t4_iscsicaps_allowed = 0;
307 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
308 
309 static int t4_fcoecaps_allowed = 0;
310 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
311 
312 static int t5_write_combine = 0;
313 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
314 
315 struct intrs_and_queues {
316 	uint16_t intr_type;	/* INTx, MSI, or MSI-X */
317 	uint16_t nirq;		/* Total # of vectors */
318 	uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
319 	uint16_t intr_flags_1g;	/* Interrupt flags for each 1G port */
320 	uint16_t ntxq10g;	/* # of NIC txq's for each 10G port */
321 	uint16_t nrxq10g;	/* # of NIC rxq's for each 10G port */
322 	uint16_t ntxq1g;	/* # of NIC txq's for each 1G port */
323 	uint16_t nrxq1g;	/* # of NIC rxq's for each 1G port */
324 	uint16_t rsrv_noflowq;	/* Flag whether to reserve queue 0 */
325 #ifdef TCP_OFFLOAD
326 	uint16_t nofldtxq10g;	/* # of TOE txq's for each 10G port */
327 	uint16_t nofldrxq10g;	/* # of TOE rxq's for each 10G port */
328 	uint16_t nofldtxq1g;	/* # of TOE txq's for each 1G port */
329 	uint16_t nofldrxq1g;	/* # of TOE rxq's for each 1G port */
330 #endif
331 #ifdef DEV_NETMAP
332 	uint16_t nnmtxq10g;	/* # of netmap txq's for each 10G port */
333 	uint16_t nnmrxq10g;	/* # of netmap rxq's for each 10G port */
334 	uint16_t nnmtxq1g;	/* # of netmap txq's for each 1G port */
335 	uint16_t nnmrxq1g;	/* # of netmap rxq's for each 1G port */
336 #endif
337 };
338 
339 struct filter_entry {
340         uint32_t valid:1;	/* filter allocated and valid */
341         uint32_t locked:1;	/* filter is administratively locked */
342         uint32_t pending:1;	/* filter action is pending firmware reply */
343 	uint32_t smtidx:8;	/* Source MAC Table index for smac */
344 	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
345 
346         struct t4_filter_specification fs;
347 };
348 
349 static int map_bars_0_and_4(struct adapter *);
350 static int map_bar_2(struct adapter *);
351 static void setup_memwin(struct adapter *);
352 static int validate_mem_range(struct adapter *, uint32_t, int);
353 static int fwmtype_to_hwmtype(int);
354 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
355     uint32_t *);
356 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
357 static uint32_t position_memwin(struct adapter *, int, uint32_t);
358 static int cfg_itype_and_nqueues(struct adapter *, int, int,
359     struct intrs_and_queues *);
360 static int prep_firmware(struct adapter *);
361 static int partition_resources(struct adapter *, const struct firmware *,
362     const char *);
363 static int get_params__pre_init(struct adapter *);
364 static int get_params__post_init(struct adapter *);
365 static int set_params__post_init(struct adapter *);
366 static void t4_set_desc(struct adapter *);
367 static void build_medialist(struct port_info *, struct ifmedia *);
368 static int cxgbe_init_synchronized(struct port_info *);
369 static int cxgbe_uninit_synchronized(struct port_info *);
370 static int setup_intr_handlers(struct adapter *);
371 static void quiesce_eq(struct adapter *, struct sge_eq *);
372 static void quiesce_iq(struct adapter *, struct sge_iq *);
373 static void quiesce_fl(struct adapter *, struct sge_fl *);
374 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
375     driver_intr_t *, void *, char *);
376 static int t4_free_irq(struct adapter *, struct irq *);
377 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
378     unsigned int);
379 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
380 static void cxgbe_tick(void *);
381 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
382 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
383     struct mbuf *);
384 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
385 static int fw_msg_not_handled(struct adapter *, const __be64 *);
386 static int t4_sysctls(struct adapter *);
387 static int cxgbe_sysctls(struct port_info *);
388 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
389 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
390 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
391 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
392 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
393 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
394 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
395 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
396 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
397 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
398 #ifdef SBUF_DRAIN
399 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
400 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
401 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
402 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
403 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
404 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
405 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
406 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
407 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
408 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
409 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
410 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
411 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
412 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
413 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
414 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
415 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
416 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
417 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
418 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
419 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
420 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
421 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
422 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
423 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
424 #endif
425 static inline void txq_start(struct ifnet *, struct sge_txq *);
426 static uint32_t fconf_to_mode(uint32_t);
427 static uint32_t mode_to_fconf(uint32_t);
428 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
429 static int get_filter_mode(struct adapter *, uint32_t *);
430 static int set_filter_mode(struct adapter *, uint32_t);
431 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
432 static int get_filter(struct adapter *, struct t4_filter *);
433 static int set_filter(struct adapter *, struct t4_filter *);
434 static int del_filter(struct adapter *, struct t4_filter *);
435 static void clear_filter(struct filter_entry *);
436 static int set_filter_wr(struct adapter *, int);
437 static int del_filter_wr(struct adapter *, int);
438 static int get_sge_context(struct adapter *, struct t4_sge_context *);
439 static int load_fw(struct adapter *, struct t4_data *);
440 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
441 static int read_i2c(struct adapter *, struct t4_i2c_data *);
442 static int set_sched_class(struct adapter *, struct t4_sched_params *);
443 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
444 #ifdef TCP_OFFLOAD
445 static int toe_capability(struct port_info *, int);
446 #endif
447 static int mod_event(module_t, int, void *);
448 
449 struct {
450 	uint16_t device;
451 	char *desc;
452 } t4_pciids[] = {
453 	{0xa000, "Chelsio Terminator 4 FPGA"},
454 	{0x4400, "Chelsio T440-dbg"},
455 	{0x4401, "Chelsio T420-CR"},
456 	{0x4402, "Chelsio T422-CR"},
457 	{0x4403, "Chelsio T440-CR"},
458 	{0x4404, "Chelsio T420-BCH"},
459 	{0x4405, "Chelsio T440-BCH"},
460 	{0x4406, "Chelsio T440-CH"},
461 	{0x4407, "Chelsio T420-SO"},
462 	{0x4408, "Chelsio T420-CX"},
463 	{0x4409, "Chelsio T420-BT"},
464 	{0x440a, "Chelsio T404-BT"},
465 	{0x440e, "Chelsio T440-LP-CR"},
466 }, t5_pciids[] = {
467 	{0xb000, "Chelsio Terminator 5 FPGA"},
468 	{0x5400, "Chelsio T580-dbg"},
469 	{0x5401,  "Chelsio T520-CR"},		/* 2 x 10G */
470 	{0x5402,  "Chelsio T522-CR"},		/* 2 x 10G, 2 X 1G */
471 	{0x5403,  "Chelsio T540-CR"},		/* 4 x 10G */
472 	{0x5407,  "Chelsio T520-SO"},		/* 2 x 10G, nomem */
473 	{0x5409,  "Chelsio T520-BT"},		/* 2 x 10GBaseT */
474 	{0x540a,  "Chelsio T504-BT"},		/* 4 x 1G */
475 	{0x540d,  "Chelsio T580-CR"},		/* 2 x 40G */
476 	{0x540e,  "Chelsio T540-LP-CR"},	/* 4 x 10G */
477 	{0x5410,  "Chelsio T580-LP-CR"},	/* 2 x 40G */
478 	{0x5411,  "Chelsio T520-LL-CR"},	/* 2 x 10G */
479 	{0x5412,  "Chelsio T560-CR"},		/* 1 x 40G, 2 x 10G */
480 	{0x5414,  "Chelsio T580-LP-SO-CR"},	/* 2 x 40G, nomem */
481 #ifdef notyet
482 	{0x5404,  "Chelsio T520-BCH"},
483 	{0x5405,  "Chelsio T540-BCH"},
484 	{0x5406,  "Chelsio T540-CH"},
485 	{0x5408,  "Chelsio T520-CX"},
486 	{0x540b,  "Chelsio B520-SR"},
487 	{0x540c,  "Chelsio B504-BT"},
488 	{0x540f,  "Chelsio Amsterdam"},
489 	{0x5413,  "Chelsio T580-CHR"},
490 #endif
491 };
492 
493 #ifdef TCP_OFFLOAD
494 /*
495  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
496  * exactly the same for both rxq and ofld_rxq.
497  */
498 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
499 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
500 #endif
501 
502 /* No easy way to include t4_msg.h before adapter.h so we check this way */
503 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
504 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
505 
506 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
507 
508 static int
509 t4_probe(device_t dev)
510 {
511 	int i;
512 	uint16_t v = pci_get_vendor(dev);
513 	uint16_t d = pci_get_device(dev);
514 	uint8_t f = pci_get_function(dev);
515 
516 	if (v != PCI_VENDOR_ID_CHELSIO)
517 		return (ENXIO);
518 
519 	/* Attach only to PF0 of the FPGA */
520 	if (d == 0xa000 && f != 0)
521 		return (ENXIO);
522 
523 	for (i = 0; i < nitems(t4_pciids); i++) {
524 		if (d == t4_pciids[i].device) {
525 			device_set_desc(dev, t4_pciids[i].desc);
526 			return (BUS_PROBE_DEFAULT);
527 		}
528 	}
529 
530 	return (ENXIO);
531 }
532 
533 static int
534 t5_probe(device_t dev)
535 {
536 	int i;
537 	uint16_t v = pci_get_vendor(dev);
538 	uint16_t d = pci_get_device(dev);
539 	uint8_t f = pci_get_function(dev);
540 
541 	if (v != PCI_VENDOR_ID_CHELSIO)
542 		return (ENXIO);
543 
544 	/* Attach only to PF0 of the FPGA */
545 	if (d == 0xb000 && f != 0)
546 		return (ENXIO);
547 
548 	for (i = 0; i < nitems(t5_pciids); i++) {
549 		if (d == t5_pciids[i].device) {
550 			device_set_desc(dev, t5_pciids[i].desc);
551 			return (BUS_PROBE_DEFAULT);
552 		}
553 	}
554 
555 	return (ENXIO);
556 }
557 
558 static int
559 t4_attach(device_t dev)
560 {
561 	struct adapter *sc;
562 	int rc = 0, i, n10g, n1g, rqidx, tqidx;
563 	struct intrs_and_queues iaq;
564 	struct sge *s;
565 #ifdef TCP_OFFLOAD
566 	int ofld_rqidx, ofld_tqidx;
567 #endif
568 #ifdef DEV_NETMAP
569 	int nm_rqidx, nm_tqidx;
570 #endif
571 	const char *pcie_ts;
572 
573 	sc = device_get_softc(dev);
574 	sc->dev = dev;
575 
576 	pci_enable_busmaster(dev);
577 	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
578 		uint32_t v;
579 
580 		pci_set_max_read_req(dev, 4096);
581 		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
582 		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
583 		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
584 	}
585 
586 	sc->traceq = -1;
587 	mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
588 	snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
589 	    device_get_nameunit(dev));
590 
591 	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
592 	    device_get_nameunit(dev));
593 	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
594 	sx_xlock(&t4_list_lock);
595 	SLIST_INSERT_HEAD(&t4_list, sc, link);
596 	sx_xunlock(&t4_list_lock);
597 
598 	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
599 	TAILQ_INIT(&sc->sfl);
600 	callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
601 
602 	rc = map_bars_0_and_4(sc);
603 	if (rc != 0)
604 		goto done; /* error message displayed already */
605 
606 	/*
607 	 * This is the real PF# to which we're attaching.  Works from within PCI
608 	 * passthrough environments too, where pci_get_function() could return a
609 	 * different PF# depending on the passthrough configuration.  We need to
610 	 * use the real PF# in all our communication with the firmware.
611 	 */
612 	sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
613 	sc->mbox = sc->pf;
614 
615 	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
616 	sc->an_handler = an_not_handled;
617 	for (i = 0; i < nitems(sc->cpl_handler); i++)
618 		sc->cpl_handler[i] = cpl_not_handled;
619 	for (i = 0; i < nitems(sc->fw_msg_handler); i++)
620 		sc->fw_msg_handler[i] = fw_msg_not_handled;
621 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
622 	t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
623 	t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
624 	t4_init_sge_cpl_handlers(sc);
625 
626 	/* Prepare the adapter for operation */
627 	rc = -t4_prep_adapter(sc);
628 	if (rc != 0) {
629 		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
630 		goto done;
631 	}
632 
633 	/*
634 	 * Do this really early, with the memory windows set up even before the
635 	 * character device.  The userland tool's register i/o and mem read
636 	 * will work even in "recovery mode".
637 	 */
638 	setup_memwin(sc);
639 	sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
640 	    device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
641 	    device_get_nameunit(dev));
642 	if (sc->cdev == NULL)
643 		device_printf(dev, "failed to create nexus char device.\n");
644 	else
645 		sc->cdev->si_drv1 = sc;
646 
647 	/* Go no further if recovery mode has been requested. */
648 	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
649 		device_printf(dev, "recovery mode.\n");
650 		goto done;
651 	}
652 
653 	/* Prepare the firmware for operation */
654 	rc = prep_firmware(sc);
655 	if (rc != 0)
656 		goto done; /* error message displayed already */
657 
658 	rc = get_params__post_init(sc);
659 	if (rc != 0)
660 		goto done; /* error message displayed already */
661 
662 	rc = set_params__post_init(sc);
663 	if (rc != 0)
664 		goto done; /* error message displayed already */
665 
666 	rc = map_bar_2(sc);
667 	if (rc != 0)
668 		goto done; /* error message displayed already */
669 
670 	rc = t4_create_dma_tag(sc);
671 	if (rc != 0)
672 		goto done; /* error message displayed already */
673 
674 	/*
675 	 * First pass over all the ports - allocate VIs and initialize some
676 	 * basic parameters like mac address, port type, etc.  We also figure
677 	 * out whether a port is 10G or 1G and use that information when
678 	 * calculating how many interrupts to attempt to allocate.
679 	 */
680 	n10g = n1g = 0;
681 	for_each_port(sc, i) {
682 		struct port_info *pi;
683 
684 		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
685 		sc->port[i] = pi;
686 
687 		/* These must be set before t4_port_init */
688 		pi->adapter = sc;
689 		pi->port_id = i;
690 
691 		/* Allocate the vi and initialize parameters like mac addr */
692 		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
693 		if (rc != 0) {
694 			device_printf(dev, "unable to initialize port %d: %d\n",
695 			    i, rc);
696 			free(pi, M_CXGBE);
697 			sc->port[i] = NULL;
698 			goto done;
699 		}
700 		rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
701 		if (rc != 0) {
702 			device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
703 			free(pi, M_CXGBE);
704 			sc->port[i] = NULL;
705 			goto done;
706 		}
707 
708 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
709 		    device_get_nameunit(dev), i);
710 		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
711 		sc->chan_map[pi->tx_chan] = i;
712 
713 		if (is_10G_port(pi) || is_40G_port(pi)) {
714 			n10g++;
715 			pi->tmr_idx = t4_tmr_idx_10g;
716 			pi->pktc_idx = t4_pktc_idx_10g;
717 		} else {
718 			n1g++;
719 			pi->tmr_idx = t4_tmr_idx_1g;
720 			pi->pktc_idx = t4_pktc_idx_1g;
721 		}
722 
723 		pi->xact_addr_filt = -1;
724 		pi->linkdnrc = -1;
725 
726 		pi->qsize_rxq = t4_qsize_rxq;
727 		pi->qsize_txq = t4_qsize_txq;
728 
729 		pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
730 		if (pi->dev == NULL) {
731 			device_printf(dev,
732 			    "failed to add device for port %d.\n", i);
733 			rc = ENXIO;
734 			goto done;
735 		}
736 		device_set_softc(pi->dev, pi);
737 	}
738 
739 	/*
740 	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
741 	 */
742 	rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
743 	if (rc != 0)
744 		goto done; /* error message displayed already */
745 
746 	sc->intr_type = iaq.intr_type;
747 	sc->intr_count = iaq.nirq;
748 
749 	s = &sc->sge;
750 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
751 	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
752 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
753 	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
754 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
755 #ifdef TCP_OFFLOAD
756 	if (is_offload(sc)) {
757 		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
758 		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
759 		s->neq += s->nofldtxq + s->nofldrxq;
760 		s->niq += s->nofldrxq;
761 
762 		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
763 		    M_CXGBE, M_ZERO | M_WAITOK);
764 		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
765 		    M_CXGBE, M_ZERO | M_WAITOK);
766 	}
767 #endif
768 #ifdef DEV_NETMAP
769 	s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
770 	s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
771 	s->neq += s->nnmtxq + s->nnmrxq;
772 	s->niq += s->nnmrxq;
773 
774 	s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
775 	    M_CXGBE, M_ZERO | M_WAITOK);
776 	s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
777 	    M_CXGBE, M_ZERO | M_WAITOK);
778 #endif
779 
780 	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
781 	    M_ZERO | M_WAITOK);
782 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
783 	    M_ZERO | M_WAITOK);
784 	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
785 	    M_ZERO | M_WAITOK);
786 	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
787 	    M_ZERO | M_WAITOK);
788 	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
789 	    M_ZERO | M_WAITOK);
790 
791 	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
792 	    M_ZERO | M_WAITOK);
793 
794 	t4_init_l2t(sc, M_WAITOK);
795 
796 	/*
797 	 * Second pass over the ports.  This time we know the number of rx and
798 	 * tx queues that each port should get.
799 	 */
800 	rqidx = tqidx = 0;
801 #ifdef TCP_OFFLOAD
802 	ofld_rqidx = ofld_tqidx = 0;
803 #endif
804 #ifdef DEV_NETMAP
805 	nm_rqidx = nm_tqidx = 0;
806 #endif
807 	for_each_port(sc, i) {
808 		struct port_info *pi = sc->port[i];
809 
810 		if (pi == NULL)
811 			continue;
812 
813 		pi->first_rxq = rqidx;
814 		pi->first_txq = tqidx;
815 		if (is_10G_port(pi) || is_40G_port(pi)) {
816 			pi->flags |= iaq.intr_flags_10g;
817 			pi->nrxq = iaq.nrxq10g;
818 			pi->ntxq = iaq.ntxq10g;
819 		} else {
820 			pi->flags |= iaq.intr_flags_1g;
821 			pi->nrxq = iaq.nrxq1g;
822 			pi->ntxq = iaq.ntxq1g;
823 		}
824 
825 		if (pi->ntxq > 1)
826 			pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
827 		else
828 			pi->rsrv_noflowq = 0;
829 
830 		rqidx += pi->nrxq;
831 		tqidx += pi->ntxq;
832 #ifdef TCP_OFFLOAD
833 		if (is_offload(sc)) {
834 			pi->first_ofld_rxq = ofld_rqidx;
835 			pi->first_ofld_txq = ofld_tqidx;
836 			if (is_10G_port(pi) || is_40G_port(pi)) {
837 				pi->nofldrxq = iaq.nofldrxq10g;
838 				pi->nofldtxq = iaq.nofldtxq10g;
839 			} else {
840 				pi->nofldrxq = iaq.nofldrxq1g;
841 				pi->nofldtxq = iaq.nofldtxq1g;
842 			}
843 			ofld_rqidx += pi->nofldrxq;
844 			ofld_tqidx += pi->nofldtxq;
845 		}
846 #endif
847 #ifdef DEV_NETMAP
848 		pi->first_nm_rxq = nm_rqidx;
849 		pi->first_nm_txq = nm_tqidx;
850 		if (is_10G_port(pi) || is_40G_port(pi)) {
851 			pi->nnmrxq = iaq.nnmrxq10g;
852 			pi->nnmtxq = iaq.nnmtxq10g;
853 		} else {
854 			pi->nnmrxq = iaq.nnmrxq1g;
855 			pi->nnmtxq = iaq.nnmtxq1g;
856 		}
857 		nm_rqidx += pi->nnmrxq;
858 		nm_tqidx += pi->nnmtxq;
859 #endif
860 	}
861 
862 	rc = setup_intr_handlers(sc);
863 	if (rc != 0) {
864 		device_printf(dev,
865 		    "failed to setup interrupt handlers: %d\n", rc);
866 		goto done;
867 	}
868 
869 	rc = bus_generic_attach(dev);
870 	if (rc != 0) {
871 		device_printf(dev,
872 		    "failed to attach all child ports: %d\n", rc);
873 		goto done;
874 	}
875 
876 	switch (sc->params.pci.speed) {
877 		case 0x1:
878 			pcie_ts = "2.5";
879 			break;
880 		case 0x2:
881 			pcie_ts = "5.0";
882 			break;
883 		case 0x3:
884 			pcie_ts = "8.0";
885 			break;
886 		default:
887 			pcie_ts = "??";
888 			break;
889 	}
890 	device_printf(dev,
891 	    "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
892 	    sc->params.pci.width, pcie_ts, sc->params.pci.speed,
893 	    sc->params.nports, sc->intr_count,
894 	    sc->intr_type == INTR_MSIX ? "MSI-X" :
895 	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
896 	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
897 
898 	t4_set_desc(sc);
899 
900 done:
901 	if (rc != 0 && sc->cdev) {
902 		/* cdev was created and so cxgbetool works; recover that way. */
903 		device_printf(dev,
904 		    "error during attach, adapter is now in recovery mode.\n");
905 		rc = 0;
906 	}
907 
908 	if (rc != 0)
909 		t4_detach(dev);
910 	else
911 		t4_sysctls(sc);
912 
913 	return (rc);
914 }
915 
916 /*
917  * Idempotent
918  */
919 static int
920 t4_detach(device_t dev)
921 {
922 	struct adapter *sc;
923 	struct port_info *pi;
924 	int i, rc;
925 
926 	sc = device_get_softc(dev);
927 
928 	if (sc->flags & FULL_INIT_DONE)
929 		t4_intr_disable(sc);
930 
931 	if (sc->cdev) {
932 		destroy_dev(sc->cdev);
933 		sc->cdev = NULL;
934 	}
935 
936 	rc = bus_generic_detach(dev);
937 	if (rc) {
938 		device_printf(dev,
939 		    "failed to detach child devices: %d\n", rc);
940 		return (rc);
941 	}
942 
943 	for (i = 0; i < sc->intr_count; i++)
944 		t4_free_irq(sc, &sc->irq[i]);
945 
946 	for (i = 0; i < MAX_NPORTS; i++) {
947 		pi = sc->port[i];
948 		if (pi) {
949 			t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
950 			if (pi->dev)
951 				device_delete_child(dev, pi->dev);
952 
953 			mtx_destroy(&pi->pi_lock);
954 			free(pi, M_CXGBE);
955 		}
956 	}
957 
958 	if (sc->flags & FULL_INIT_DONE)
959 		adapter_full_uninit(sc);
960 
961 	if (sc->flags & FW_OK)
962 		t4_fw_bye(sc, sc->mbox);
963 
964 	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
965 		pci_release_msi(dev);
966 
967 	if (sc->regs_res)
968 		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
969 		    sc->regs_res);
970 
971 	if (sc->udbs_res)
972 		bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
973 		    sc->udbs_res);
974 
975 	if (sc->msix_res)
976 		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
977 		    sc->msix_res);
978 
979 	if (sc->l2t)
980 		t4_free_l2t(sc->l2t);
981 
982 #ifdef TCP_OFFLOAD
983 	free(sc->sge.ofld_rxq, M_CXGBE);
984 	free(sc->sge.ofld_txq, M_CXGBE);
985 #endif
986 #ifdef DEV_NETMAP
987 	free(sc->sge.nm_rxq, M_CXGBE);
988 	free(sc->sge.nm_txq, M_CXGBE);
989 #endif
990 	free(sc->irq, M_CXGBE);
991 	free(sc->sge.rxq, M_CXGBE);
992 	free(sc->sge.txq, M_CXGBE);
993 	free(sc->sge.ctrlq, M_CXGBE);
994 	free(sc->sge.iqmap, M_CXGBE);
995 	free(sc->sge.eqmap, M_CXGBE);
996 	free(sc->tids.ftid_tab, M_CXGBE);
997 	t4_destroy_dma_tag(sc);
998 	if (mtx_initialized(&sc->sc_lock)) {
999 		sx_xlock(&t4_list_lock);
1000 		SLIST_REMOVE(&t4_list, sc, adapter, link);
1001 		sx_xunlock(&t4_list_lock);
1002 		mtx_destroy(&sc->sc_lock);
1003 	}
1004 
1005 	if (mtx_initialized(&sc->tids.ftid_lock))
1006 		mtx_destroy(&sc->tids.ftid_lock);
1007 	if (mtx_initialized(&sc->sfl_lock))
1008 		mtx_destroy(&sc->sfl_lock);
1009 	if (mtx_initialized(&sc->ifp_lock))
1010 		mtx_destroy(&sc->ifp_lock);
1011 
1012 	bzero(sc, sizeof(*sc));
1013 
1014 	return (0);
1015 }
1016 
1017 static int
1018 cxgbe_probe(device_t dev)
1019 {
1020 	char buf[128];
1021 	struct port_info *pi = device_get_softc(dev);
1022 
1023 	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1024 	device_set_desc_copy(dev, buf);
1025 
1026 	return (BUS_PROBE_DEFAULT);
1027 }
1028 
1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1030     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1031     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1032 #define T4_CAP_ENABLE (T4_CAP)
1033 
1034 static int
1035 cxgbe_attach(device_t dev)
1036 {
1037 	struct port_info *pi = device_get_softc(dev);
1038 	struct ifnet *ifp;
1039 	char *s;
1040 	int n, o;
1041 
1042 	/* Allocate an ifnet and set it up */
1043 	ifp = if_alloc(IFT_ETHER);
1044 	if (ifp == NULL) {
1045 		device_printf(dev, "Cannot allocate ifnet\n");
1046 		return (ENOMEM);
1047 	}
1048 	pi->ifp = ifp;
1049 	ifp->if_softc = pi;
1050 
1051 	callout_init(&pi->tick, CALLOUT_MPSAFE);
1052 
1053 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1054 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1055 
1056 	ifp->if_init = cxgbe_init;
1057 	ifp->if_ioctl = cxgbe_ioctl;
1058 	ifp->if_transmit = cxgbe_transmit;
1059 	ifp->if_qflush = cxgbe_qflush;
1060 
1061 	ifp->if_capabilities = T4_CAP;
1062 #ifdef TCP_OFFLOAD
1063 	if (is_offload(pi->adapter))
1064 		ifp->if_capabilities |= IFCAP_TOE;
1065 #endif
1066 	ifp->if_capenable = T4_CAP_ENABLE;
1067 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1068 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1069 
1070 	/* Initialize ifmedia for this port */
1071 	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1072 	    cxgbe_media_status);
1073 	build_medialist(pi, &pi->media);
1074 
1075 	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1076 	    EVENTHANDLER_PRI_ANY);
1077 
1078 	ether_ifattach(ifp, pi->hw_addr);
1079 
1080 	n = 128;
1081 	s = malloc(n, M_CXGBE, M_WAITOK);
1082 	o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1083 	MPASS(n > o);
1084 #ifdef TCP_OFFLOAD
1085 	if (is_offload(pi->adapter)) {
1086 		o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1087 		    pi->nofldtxq, pi->nofldrxq);
1088 		MPASS(n > o);
1089 	}
1090 #endif
1091 #ifdef DEV_NETMAP
1092 	o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1093 	    pi->nnmrxq);
1094 	MPASS(n > o);
1095 #endif
1096 	device_printf(dev, "%s\n", s);
1097 	free(s, M_CXGBE);
1098 
1099 #ifdef DEV_NETMAP
1100 	/* nm_media handled here to keep implementation private to this file */
1101 	ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1102 	    cxgbe_media_status);
1103 	build_medialist(pi, &pi->nm_media);
1104 	create_netmap_ifnet(pi);	/* logs errors it something fails */
1105 #endif
1106 	cxgbe_sysctls(pi);
1107 
1108 	return (0);
1109 }
1110 
1111 static int
1112 cxgbe_detach(device_t dev)
1113 {
1114 	struct port_info *pi = device_get_softc(dev);
1115 	struct adapter *sc = pi->adapter;
1116 	struct ifnet *ifp = pi->ifp;
1117 
1118 	/* Tell if_ioctl and if_init that the port is going away */
1119 	ADAPTER_LOCK(sc);
1120 	SET_DOOMED(pi);
1121 	wakeup(&sc->flags);
1122 	while (IS_BUSY(sc))
1123 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1124 	SET_BUSY(sc);
1125 #ifdef INVARIANTS
1126 	sc->last_op = "t4detach";
1127 	sc->last_op_thr = curthread;
1128 #endif
1129 	ADAPTER_UNLOCK(sc);
1130 
1131 	if (pi->flags & HAS_TRACEQ) {
1132 		sc->traceq = -1;	/* cloner should not create ifnet */
1133 		t4_tracer_port_detach(sc);
1134 	}
1135 
1136 	if (pi->vlan_c)
1137 		EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1138 
1139 	PORT_LOCK(pi);
1140 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1141 	callout_stop(&pi->tick);
1142 	PORT_UNLOCK(pi);
1143 	callout_drain(&pi->tick);
1144 
1145 	/* Let detach proceed even if these fail. */
1146 	cxgbe_uninit_synchronized(pi);
1147 	port_full_uninit(pi);
1148 
1149 	ifmedia_removeall(&pi->media);
1150 	ether_ifdetach(pi->ifp);
1151 	if_free(pi->ifp);
1152 
1153 #ifdef DEV_NETMAP
1154 	/* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1155 	destroy_netmap_ifnet(pi);
1156 #endif
1157 
1158 	ADAPTER_LOCK(sc);
1159 	CLR_BUSY(sc);
1160 	wakeup(&sc->flags);
1161 	ADAPTER_UNLOCK(sc);
1162 
1163 	return (0);
1164 }
1165 
1166 static void
1167 cxgbe_init(void *arg)
1168 {
1169 	struct port_info *pi = arg;
1170 	struct adapter *sc = pi->adapter;
1171 
1172 	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1173 		return;
1174 	cxgbe_init_synchronized(pi);
1175 	end_synchronized_op(sc, 0);
1176 }
1177 
1178 static int
1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1180 {
1181 	int rc = 0, mtu, flags;
1182 	struct port_info *pi = ifp->if_softc;
1183 	struct adapter *sc = pi->adapter;
1184 	struct ifreq *ifr = (struct ifreq *)data;
1185 	uint32_t mask;
1186 
1187 	switch (cmd) {
1188 	case SIOCSIFMTU:
1189 		mtu = ifr->ifr_mtu;
1190 		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1191 			return (EINVAL);
1192 
1193 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1194 		if (rc)
1195 			return (rc);
1196 		ifp->if_mtu = mtu;
1197 		if (pi->flags & PORT_INIT_DONE) {
1198 			t4_update_fl_bufsize(ifp);
1199 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1200 				rc = update_mac_settings(ifp, XGMAC_MTU);
1201 		}
1202 		end_synchronized_op(sc, 0);
1203 		break;
1204 
1205 	case SIOCSIFFLAGS:
1206 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1207 		if (rc)
1208 			return (rc);
1209 
1210 		if (ifp->if_flags & IFF_UP) {
1211 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1212 				flags = pi->if_flags;
1213 				if ((ifp->if_flags ^ flags) &
1214 				    (IFF_PROMISC | IFF_ALLMULTI)) {
1215 					rc = update_mac_settings(ifp,
1216 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
1217 				}
1218 			} else
1219 				rc = cxgbe_init_synchronized(pi);
1220 			pi->if_flags = ifp->if_flags;
1221 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1222 			rc = cxgbe_uninit_synchronized(pi);
1223 		end_synchronized_op(sc, 0);
1224 		break;
1225 
1226 	case SIOCADDMULTI:
1227 	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1228 		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1229 		if (rc)
1230 			return (rc);
1231 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1232 			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1233 		end_synchronized_op(sc, LOCK_HELD);
1234 		break;
1235 
1236 	case SIOCSIFCAP:
1237 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1238 		if (rc)
1239 			return (rc);
1240 
1241 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1242 		if (mask & IFCAP_TXCSUM) {
1243 			ifp->if_capenable ^= IFCAP_TXCSUM;
1244 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1245 
1246 			if (IFCAP_TSO4 & ifp->if_capenable &&
1247 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1248 				ifp->if_capenable &= ~IFCAP_TSO4;
1249 				if_printf(ifp,
1250 				    "tso4 disabled due to -txcsum.\n");
1251 			}
1252 		}
1253 		if (mask & IFCAP_TXCSUM_IPV6) {
1254 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1255 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1256 
1257 			if (IFCAP_TSO6 & ifp->if_capenable &&
1258 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1259 				ifp->if_capenable &= ~IFCAP_TSO6;
1260 				if_printf(ifp,
1261 				    "tso6 disabled due to -txcsum6.\n");
1262 			}
1263 		}
1264 		if (mask & IFCAP_RXCSUM)
1265 			ifp->if_capenable ^= IFCAP_RXCSUM;
1266 		if (mask & IFCAP_RXCSUM_IPV6)
1267 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1268 
1269 		/*
1270 		 * Note that we leave CSUM_TSO alone (it is always set).  The
1271 		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1272 		 * sending a TSO request our way, so it's sufficient to toggle
1273 		 * IFCAP_TSOx only.
1274 		 */
1275 		if (mask & IFCAP_TSO4) {
1276 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1277 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1278 				if_printf(ifp, "enable txcsum first.\n");
1279 				rc = EAGAIN;
1280 				goto fail;
1281 			}
1282 			ifp->if_capenable ^= IFCAP_TSO4;
1283 		}
1284 		if (mask & IFCAP_TSO6) {
1285 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1286 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1287 				if_printf(ifp, "enable txcsum6 first.\n");
1288 				rc = EAGAIN;
1289 				goto fail;
1290 			}
1291 			ifp->if_capenable ^= IFCAP_TSO6;
1292 		}
1293 		if (mask & IFCAP_LRO) {
1294 #if defined(INET) || defined(INET6)
1295 			int i;
1296 			struct sge_rxq *rxq;
1297 
1298 			ifp->if_capenable ^= IFCAP_LRO;
1299 			for_each_rxq(pi, i, rxq) {
1300 				if (ifp->if_capenable & IFCAP_LRO)
1301 					rxq->iq.flags |= IQ_LRO_ENABLED;
1302 				else
1303 					rxq->iq.flags &= ~IQ_LRO_ENABLED;
1304 			}
1305 #endif
1306 		}
1307 #ifdef TCP_OFFLOAD
1308 		if (mask & IFCAP_TOE) {
1309 			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1310 
1311 			rc = toe_capability(pi, enable);
1312 			if (rc != 0)
1313 				goto fail;
1314 
1315 			ifp->if_capenable ^= mask;
1316 		}
1317 #endif
1318 		if (mask & IFCAP_VLAN_HWTAGGING) {
1319 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1320 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1321 				rc = update_mac_settings(ifp, XGMAC_VLANEX);
1322 		}
1323 		if (mask & IFCAP_VLAN_MTU) {
1324 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
1325 
1326 			/* Need to find out how to disable auto-mtu-inflation */
1327 		}
1328 		if (mask & IFCAP_VLAN_HWTSO)
1329 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1330 		if (mask & IFCAP_VLAN_HWCSUM)
1331 			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1332 
1333 #ifdef VLAN_CAPABILITIES
1334 		VLAN_CAPABILITIES(ifp);
1335 #endif
1336 fail:
1337 		end_synchronized_op(sc, 0);
1338 		break;
1339 
1340 	case SIOCSIFMEDIA:
1341 	case SIOCGIFMEDIA:
1342 		ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1343 		break;
1344 
1345 	default:
1346 		rc = ether_ioctl(ifp, cmd, data);
1347 	}
1348 
1349 	return (rc);
1350 }
1351 
1352 static int
1353 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1354 {
1355 	struct port_info *pi = ifp->if_softc;
1356 	struct adapter *sc = pi->adapter;
1357 	struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1358 	struct buf_ring *br;
1359 	int rc;
1360 
1361 	M_ASSERTPKTHDR(m);
1362 
1363 	if (__predict_false(pi->link_cfg.link_ok == 0)) {
1364 		m_freem(m);
1365 		return (ENETDOWN);
1366 	}
1367 
1368 	if (m->m_flags & M_FLOWID)
1369 		txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1370 		    + pi->rsrv_noflowq);
1371 	br = txq->br;
1372 
1373 	if (TXQ_TRYLOCK(txq) == 0) {
1374 		struct sge_eq *eq = &txq->eq;
1375 
1376 		/*
1377 		 * It is possible that t4_eth_tx finishes up and releases the
1378 		 * lock between the TRYLOCK above and the drbr_enqueue here.  We
1379 		 * need to make sure that this mbuf doesn't just sit there in
1380 		 * the drbr.
1381 		 */
1382 
1383 		rc = drbr_enqueue(ifp, br, m);
1384 		if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1385 		    !(eq->flags & EQ_DOOMED))
1386 			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1387 		return (rc);
1388 	}
1389 
1390 	/*
1391 	 * txq->m is the mbuf that is held up due to a temporary shortage of
1392 	 * resources and it should be put on the wire first.  Then what's in
1393 	 * drbr and finally the mbuf that was just passed in to us.
1394 	 *
1395 	 * Return code should indicate the fate of the mbuf that was passed in
1396 	 * this time.
1397 	 */
1398 
1399 	TXQ_LOCK_ASSERT_OWNED(txq);
1400 	if (drbr_needs_enqueue(ifp, br) || txq->m) {
1401 
1402 		/* Queued for transmission. */
1403 
1404 		rc = drbr_enqueue(ifp, br, m);
1405 		m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1406 		(void) t4_eth_tx(ifp, txq, m);
1407 		TXQ_UNLOCK(txq);
1408 		return (rc);
1409 	}
1410 
1411 	/* Direct transmission. */
1412 	rc = t4_eth_tx(ifp, txq, m);
1413 	if (rc != 0 && txq->m)
1414 		rc = 0;	/* held, will be transmitted soon (hopefully) */
1415 
1416 	TXQ_UNLOCK(txq);
1417 	return (rc);
1418 }
1419 
1420 static void
1421 cxgbe_qflush(struct ifnet *ifp)
1422 {
1423 	struct port_info *pi = ifp->if_softc;
1424 	struct sge_txq *txq;
1425 	int i;
1426 	struct mbuf *m;
1427 
1428 	/* queues do not exist if !PORT_INIT_DONE. */
1429 	if (pi->flags & PORT_INIT_DONE) {
1430 		for_each_txq(pi, i, txq) {
1431 			TXQ_LOCK(txq);
1432 			m_freem(txq->m);
1433 			txq->m = NULL;
1434 			while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1435 				m_freem(m);
1436 			TXQ_UNLOCK(txq);
1437 		}
1438 	}
1439 	if_qflush(ifp);
1440 }
1441 
1442 static int
1443 cxgbe_media_change(struct ifnet *ifp)
1444 {
1445 	struct port_info *pi = ifp->if_softc;
1446 
1447 	device_printf(pi->dev, "%s unimplemented.\n", __func__);
1448 
1449 	return (EOPNOTSUPP);
1450 }
1451 
1452 static void
1453 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1454 {
1455 	struct port_info *pi = ifp->if_softc;
1456 	struct ifmedia *media = NULL;
1457 	struct ifmedia_entry *cur;
1458 	int speed = pi->link_cfg.speed;
1459 	int data = (pi->port_type << 8) | pi->mod_type;
1460 
1461 	if (ifp == pi->ifp)
1462 		media = &pi->media;
1463 #ifdef DEV_NETMAP
1464 	else if (ifp == pi->nm_ifp)
1465 		media = &pi->nm_media;
1466 #endif
1467 	MPASS(media != NULL);
1468 
1469 	cur = media->ifm_cur;
1470 	if (cur->ifm_data != data) {
1471 		build_medialist(pi, media);
1472 		cur = media->ifm_cur;
1473 	}
1474 
1475 	ifmr->ifm_status = IFM_AVALID;
1476 	if (!pi->link_cfg.link_ok)
1477 		return;
1478 
1479 	ifmr->ifm_status |= IFM_ACTIVE;
1480 
1481 	/* active and current will differ iff current media is autoselect. */
1482 	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1483 		return;
1484 
1485 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1486 	if (speed == SPEED_10000)
1487 		ifmr->ifm_active |= IFM_10G_T;
1488 	else if (speed == SPEED_1000)
1489 		ifmr->ifm_active |= IFM_1000_T;
1490 	else if (speed == SPEED_100)
1491 		ifmr->ifm_active |= IFM_100_TX;
1492 	else if (speed == SPEED_10)
1493 		ifmr->ifm_active |= IFM_10_T;
1494 	else
1495 		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1496 			    speed));
1497 }
1498 
1499 void
1500 t4_fatal_err(struct adapter *sc)
1501 {
1502 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1503 	t4_intr_disable(sc);
1504 	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1505 	    device_get_nameunit(sc->dev));
1506 }
1507 
1508 static int
1509 map_bars_0_and_4(struct adapter *sc)
1510 {
1511 	sc->regs_rid = PCIR_BAR(0);
1512 	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1513 	    &sc->regs_rid, RF_ACTIVE);
1514 	if (sc->regs_res == NULL) {
1515 		device_printf(sc->dev, "cannot map registers.\n");
1516 		return (ENXIO);
1517 	}
1518 	sc->bt = rman_get_bustag(sc->regs_res);
1519 	sc->bh = rman_get_bushandle(sc->regs_res);
1520 	sc->mmio_len = rman_get_size(sc->regs_res);
1521 	setbit(&sc->doorbells, DOORBELL_KDB);
1522 
1523 	sc->msix_rid = PCIR_BAR(4);
1524 	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1525 	    &sc->msix_rid, RF_ACTIVE);
1526 	if (sc->msix_res == NULL) {
1527 		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1528 		return (ENXIO);
1529 	}
1530 
1531 	return (0);
1532 }
1533 
1534 static int
1535 map_bar_2(struct adapter *sc)
1536 {
1537 
1538 	/*
1539 	 * T4: only iWARP driver uses the userspace doorbells.  There is no need
1540 	 * to map it if RDMA is disabled.
1541 	 */
1542 	if (is_t4(sc) && sc->rdmacaps == 0)
1543 		return (0);
1544 
1545 	sc->udbs_rid = PCIR_BAR(2);
1546 	sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1547 	    &sc->udbs_rid, RF_ACTIVE);
1548 	if (sc->udbs_res == NULL) {
1549 		device_printf(sc->dev, "cannot map doorbell BAR.\n");
1550 		return (ENXIO);
1551 	}
1552 	sc->udbs_base = rman_get_virtual(sc->udbs_res);
1553 
1554 	if (is_t5(sc)) {
1555 		setbit(&sc->doorbells, DOORBELL_UDB);
1556 #if defined(__i386__) || defined(__amd64__)
1557 		if (t5_write_combine) {
1558 			int rc;
1559 
1560 			/*
1561 			 * Enable write combining on BAR2.  This is the
1562 			 * userspace doorbell BAR and is split into 128B
1563 			 * (UDBS_SEG_SIZE) doorbell regions, each associated
1564 			 * with an egress queue.  The first 64B has the doorbell
1565 			 * and the second 64B can be used to submit a tx work
1566 			 * request with an implicit doorbell.
1567 			 */
1568 
1569 			rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1570 			    rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1571 			if (rc == 0) {
1572 				clrbit(&sc->doorbells, DOORBELL_UDB);
1573 				setbit(&sc->doorbells, DOORBELL_WCWR);
1574 				setbit(&sc->doorbells, DOORBELL_UDBWC);
1575 			} else {
1576 				device_printf(sc->dev,
1577 				    "couldn't enable write combining: %d\n",
1578 				    rc);
1579 			}
1580 
1581 			t4_write_reg(sc, A_SGE_STAT_CFG,
1582 			    V_STATSOURCE_T5(7) | V_STATMODE(0));
1583 		}
1584 #endif
1585 	}
1586 
1587 	return (0);
1588 }
1589 
1590 static const struct memwin t4_memwin[] = {
1591 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1592 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1593 	{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1594 };
1595 
1596 static const struct memwin t5_memwin[] = {
1597 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1598 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1599 	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1600 };
1601 
1602 static void
1603 setup_memwin(struct adapter *sc)
1604 {
1605 	const struct memwin *mw;
1606 	int i, n;
1607 	uint32_t bar0;
1608 
1609 	if (is_t4(sc)) {
1610 		/*
1611 		 * Read low 32b of bar0 indirectly via the hardware backdoor
1612 		 * mechanism.  Works from within PCI passthrough environments
1613 		 * too, where rman_get_start() can return a different value.  We
1614 		 * need to program the T4 memory window decoders with the actual
1615 		 * addresses that will be coming across the PCIe link.
1616 		 */
1617 		bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1618 		bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1619 
1620 		mw = &t4_memwin[0];
1621 		n = nitems(t4_memwin);
1622 	} else {
1623 		/* T5 uses the relative offset inside the PCIe BAR */
1624 		bar0 = 0;
1625 
1626 		mw = &t5_memwin[0];
1627 		n = nitems(t5_memwin);
1628 	}
1629 
1630 	for (i = 0; i < n; i++, mw++) {
1631 		t4_write_reg(sc,
1632 		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1633 		    (mw->base + bar0) | V_BIR(0) |
1634 		    V_WINDOW(ilog2(mw->aperture) - 10));
1635 	}
1636 
1637 	/* flush */
1638 	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1639 }
1640 
1641 /*
1642  * Verify that the memory range specified by the addr/len pair is valid and lies
1643  * entirely within a single region (EDCx or MCx).
1644  */
1645 static int
1646 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1647 {
1648 	uint32_t em, addr_len, maddr, mlen;
1649 
1650 	/* Memory can only be accessed in naturally aligned 4 byte units */
1651 	if (addr & 3 || len & 3 || len == 0)
1652 		return (EINVAL);
1653 
1654 	/* Enabled memories */
1655 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1656 	if (em & F_EDRAM0_ENABLE) {
1657 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1658 		maddr = G_EDRAM0_BASE(addr_len) << 20;
1659 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
1660 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1661 		    addr + len <= maddr + mlen)
1662 			return (0);
1663 	}
1664 	if (em & F_EDRAM1_ENABLE) {
1665 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1666 		maddr = G_EDRAM1_BASE(addr_len) << 20;
1667 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
1668 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1669 		    addr + len <= maddr + mlen)
1670 			return (0);
1671 	}
1672 	if (em & F_EXT_MEM_ENABLE) {
1673 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1674 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
1675 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1676 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1677 		    addr + len <= maddr + mlen)
1678 			return (0);
1679 	}
1680 	if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1681 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1682 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1683 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1684 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1685 		    addr + len <= maddr + mlen)
1686 			return (0);
1687 	}
1688 
1689 	return (EFAULT);
1690 }
1691 
1692 static int
1693 fwmtype_to_hwmtype(int mtype)
1694 {
1695 
1696 	switch (mtype) {
1697 	case FW_MEMTYPE_EDC0:
1698 		return (MEM_EDC0);
1699 	case FW_MEMTYPE_EDC1:
1700 		return (MEM_EDC1);
1701 	case FW_MEMTYPE_EXTMEM:
1702 		return (MEM_MC0);
1703 	case FW_MEMTYPE_EXTMEM1:
1704 		return (MEM_MC1);
1705 	default:
1706 		panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1707 	}
1708 }
1709 
1710 /*
1711  * Verify that the memory range specified by the memtype/offset/len pair is
1712  * valid and lies entirely within the memtype specified.  The global address of
1713  * the start of the range is returned in addr.
1714  */
1715 static int
1716 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1717     uint32_t *addr)
1718 {
1719 	uint32_t em, addr_len, maddr, mlen;
1720 
1721 	/* Memory can only be accessed in naturally aligned 4 byte units */
1722 	if (off & 3 || len & 3 || len == 0)
1723 		return (EINVAL);
1724 
1725 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1726 	switch (fwmtype_to_hwmtype(mtype)) {
1727 	case MEM_EDC0:
1728 		if (!(em & F_EDRAM0_ENABLE))
1729 			return (EINVAL);
1730 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1731 		maddr = G_EDRAM0_BASE(addr_len) << 20;
1732 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
1733 		break;
1734 	case MEM_EDC1:
1735 		if (!(em & F_EDRAM1_ENABLE))
1736 			return (EINVAL);
1737 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1738 		maddr = G_EDRAM1_BASE(addr_len) << 20;
1739 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
1740 		break;
1741 	case MEM_MC:
1742 		if (!(em & F_EXT_MEM_ENABLE))
1743 			return (EINVAL);
1744 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1745 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
1746 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1747 		break;
1748 	case MEM_MC1:
1749 		if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1750 			return (EINVAL);
1751 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1752 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1753 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1754 		break;
1755 	default:
1756 		return (EINVAL);
1757 	}
1758 
1759 	if (mlen > 0 && off < mlen && off + len <= mlen) {
1760 		*addr = maddr + off;	/* global address */
1761 		return (0);
1762 	}
1763 
1764 	return (EFAULT);
1765 }
1766 
1767 static void
1768 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1769 {
1770 	const struct memwin *mw;
1771 
1772 	if (is_t4(sc)) {
1773 		KASSERT(win >= 0 && win < nitems(t4_memwin),
1774 		    ("%s: incorrect memwin# (%d)", __func__, win));
1775 		mw = &t4_memwin[win];
1776 	} else {
1777 		KASSERT(win >= 0 && win < nitems(t5_memwin),
1778 		    ("%s: incorrect memwin# (%d)", __func__, win));
1779 		mw = &t5_memwin[win];
1780 	}
1781 
1782 	if (base != NULL)
1783 		*base = mw->base;
1784 	if (aperture != NULL)
1785 		*aperture = mw->aperture;
1786 }
1787 
1788 /*
1789  * Positions the memory window such that it can be used to access the specified
1790  * address in the chip's address space.  The return value is the offset of addr
1791  * from the start of the window.
1792  */
1793 static uint32_t
1794 position_memwin(struct adapter *sc, int n, uint32_t addr)
1795 {
1796 	uint32_t start, pf;
1797 	uint32_t reg;
1798 
1799 	KASSERT(n >= 0 && n <= 3,
1800 	    ("%s: invalid window %d.", __func__, n));
1801 	KASSERT((addr & 3) == 0,
1802 	    ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1803 
1804 	if (is_t4(sc)) {
1805 		pf = 0;
1806 		start = addr & ~0xf;	/* start must be 16B aligned */
1807 	} else {
1808 		pf = V_PFNUM(sc->pf);
1809 		start = addr & ~0x7f;	/* start must be 128B aligned */
1810 	}
1811 	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1812 
1813 	t4_write_reg(sc, reg, start | pf);
1814 	t4_read_reg(sc, reg);
1815 
1816 	return (addr - start);
1817 }
1818 
1819 static int
1820 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1821     struct intrs_and_queues *iaq)
1822 {
1823 	int rc, itype, navail, nrxq10g, nrxq1g, n;
1824 	int nofldrxq10g = 0, nofldrxq1g = 0;
1825 	int nnmrxq10g = 0, nnmrxq1g = 0;
1826 
1827 	bzero(iaq, sizeof(*iaq));
1828 
1829 	iaq->ntxq10g = t4_ntxq10g;
1830 	iaq->ntxq1g = t4_ntxq1g;
1831 	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1832 	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1833 	iaq->rsrv_noflowq = t4_rsrv_noflowq;
1834 #ifdef TCP_OFFLOAD
1835 	if (is_offload(sc)) {
1836 		iaq->nofldtxq10g = t4_nofldtxq10g;
1837 		iaq->nofldtxq1g = t4_nofldtxq1g;
1838 		iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1839 		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1840 	}
1841 #endif
1842 #ifdef DEV_NETMAP
1843 	iaq->nnmtxq10g = t4_nnmtxq10g;
1844 	iaq->nnmtxq1g = t4_nnmtxq1g;
1845 	iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1846 	iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1847 #endif
1848 
1849 	for (itype = INTR_MSIX; itype; itype >>= 1) {
1850 
1851 		if ((itype & t4_intr_types) == 0)
1852 			continue;	/* not allowed */
1853 
1854 		if (itype == INTR_MSIX)
1855 			navail = pci_msix_count(sc->dev);
1856 		else if (itype == INTR_MSI)
1857 			navail = pci_msi_count(sc->dev);
1858 		else
1859 			navail = 1;
1860 restart:
1861 		if (navail == 0)
1862 			continue;
1863 
1864 		iaq->intr_type = itype;
1865 		iaq->intr_flags_10g = 0;
1866 		iaq->intr_flags_1g = 0;
1867 
1868 		/*
1869 		 * Best option: an interrupt vector for errors, one for the
1870 		 * firmware event queue, and one for every rxq (NIC, TOE, and
1871 		 * netmap).
1872 		 */
1873 		iaq->nirq = T4_EXTRA_INTR;
1874 		iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1875 		iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1876 		if (iaq->nirq <= navail &&
1877 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
1878 			iaq->intr_flags_10g = INTR_ALL;
1879 			iaq->intr_flags_1g = INTR_ALL;
1880 			goto allocate;
1881 		}
1882 
1883 		/*
1884 		 * Second best option: a vector for errors, one for the firmware
1885 		 * event queue, and vectors for either all the NIC rx queues or
1886 		 * all the TOE rx queues.  The queues that don't get vectors
1887 		 * will forward their interrupts to those that do.
1888 		 *
1889 		 * Note: netmap rx queues cannot be created early and so they
1890 		 * can't be setup to receive forwarded interrupts for others.
1891 		 */
1892 		iaq->nirq = T4_EXTRA_INTR;
1893 		if (nrxq10g >= nofldrxq10g) {
1894 			iaq->intr_flags_10g = INTR_RXQ;
1895 			iaq->nirq += n10g * nrxq10g;
1896 #ifdef DEV_NETMAP
1897 			iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1898 #endif
1899 		} else {
1900 			iaq->intr_flags_10g = INTR_OFLD_RXQ;
1901 			iaq->nirq += n10g * nofldrxq10g;
1902 #ifdef DEV_NETMAP
1903 			iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1904 #endif
1905 		}
1906 		if (nrxq1g >= nofldrxq1g) {
1907 			iaq->intr_flags_1g = INTR_RXQ;
1908 			iaq->nirq += n1g * nrxq1g;
1909 #ifdef DEV_NETMAP
1910 			iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1911 #endif
1912 		} else {
1913 			iaq->intr_flags_1g = INTR_OFLD_RXQ;
1914 			iaq->nirq += n1g * nofldrxq1g;
1915 #ifdef DEV_NETMAP
1916 			iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1917 #endif
1918 		}
1919 		if (iaq->nirq <= navail &&
1920 		    (itype != INTR_MSI || powerof2(iaq->nirq)))
1921 			goto allocate;
1922 
1923 		/*
1924 		 * Next best option: an interrupt vector for errors, one for the
1925 		 * firmware event queue, and at least one per port.  At this
1926 		 * point we know we'll have to downsize nrxq and/or nofldrxq
1927 		 * and/or nnmrxq to fit what's available to us.
1928 		 */
1929 		iaq->nirq = T4_EXTRA_INTR;
1930 		iaq->nirq += n10g + n1g;
1931 		if (iaq->nirq <= navail) {
1932 			int leftover = navail - iaq->nirq;
1933 
1934 			if (n10g > 0) {
1935 				int target = max(nrxq10g, nofldrxq10g);
1936 
1937 				iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1938 				    INTR_RXQ : INTR_OFLD_RXQ;
1939 
1940 				n = 1;
1941 				while (n < target && leftover >= n10g) {
1942 					leftover -= n10g;
1943 					iaq->nirq += n10g;
1944 					n++;
1945 				}
1946 				iaq->nrxq10g = min(n, nrxq10g);
1947 #ifdef TCP_OFFLOAD
1948 				iaq->nofldrxq10g = min(n, nofldrxq10g);
1949 #endif
1950 #ifdef DEV_NETMAP
1951 				iaq->nnmrxq10g = min(n, nnmrxq10g);
1952 #endif
1953 			}
1954 
1955 			if (n1g > 0) {
1956 				int target = max(nrxq1g, nofldrxq1g);
1957 
1958 				iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1959 				    INTR_RXQ : INTR_OFLD_RXQ;
1960 
1961 				n = 1;
1962 				while (n < target && leftover >= n1g) {
1963 					leftover -= n1g;
1964 					iaq->nirq += n1g;
1965 					n++;
1966 				}
1967 				iaq->nrxq1g = min(n, nrxq1g);
1968 #ifdef TCP_OFFLOAD
1969 				iaq->nofldrxq1g = min(n, nofldrxq1g);
1970 #endif
1971 #ifdef DEV_NETMAP
1972 				iaq->nnmrxq1g = min(n, nnmrxq1g);
1973 #endif
1974 			}
1975 
1976 			if (itype != INTR_MSI || powerof2(iaq->nirq))
1977 				goto allocate;
1978 		}
1979 
1980 		/*
1981 		 * Least desirable option: one interrupt vector for everything.
1982 		 */
1983 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1984 		iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
1985 #ifdef TCP_OFFLOAD
1986 		if (is_offload(sc))
1987 			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1988 #endif
1989 #ifdef DEV_NETMAP
1990 		iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
1991 #endif
1992 
1993 allocate:
1994 		navail = iaq->nirq;
1995 		rc = 0;
1996 		if (itype == INTR_MSIX)
1997 			rc = pci_alloc_msix(sc->dev, &navail);
1998 		else if (itype == INTR_MSI)
1999 			rc = pci_alloc_msi(sc->dev, &navail);
2000 
2001 		if (rc == 0) {
2002 			if (navail == iaq->nirq)
2003 				return (0);
2004 
2005 			/*
2006 			 * Didn't get the number requested.  Use whatever number
2007 			 * the kernel is willing to allocate (it's in navail).
2008 			 */
2009 			device_printf(sc->dev, "fewer vectors than requested, "
2010 			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2011 			    itype, iaq->nirq, navail);
2012 			pci_release_msi(sc->dev);
2013 			goto restart;
2014 		}
2015 
2016 		device_printf(sc->dev,
2017 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2018 		    itype, rc, iaq->nirq, navail);
2019 	}
2020 
2021 	device_printf(sc->dev,
2022 	    "failed to find a usable interrupt type.  "
2023 	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2024 	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2025 
2026 	return (ENXIO);
2027 }
2028 
2029 #define FW_VERSION(chip) ( \
2030     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2031     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2032     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2033     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2034 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2035 
2036 struct fw_info {
2037 	uint8_t chip;
2038 	char *kld_name;
2039 	char *fw_mod_name;
2040 	struct fw_hdr fw_hdr;	/* XXX: waste of space, need a sparse struct */
2041 } fw_info[] = {
2042 	{
2043 		.chip = CHELSIO_T4,
2044 		.kld_name = "t4fw_cfg",
2045 		.fw_mod_name = "t4fw",
2046 		.fw_hdr = {
2047 			.chip = FW_HDR_CHIP_T4,
2048 			.fw_ver = htobe32_const(FW_VERSION(T4)),
2049 			.intfver_nic = FW_INTFVER(T4, NIC),
2050 			.intfver_vnic = FW_INTFVER(T4, VNIC),
2051 			.intfver_ofld = FW_INTFVER(T4, OFLD),
2052 			.intfver_ri = FW_INTFVER(T4, RI),
2053 			.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2054 			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
2055 			.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2056 			.intfver_fcoe = FW_INTFVER(T4, FCOE),
2057 		},
2058 	}, {
2059 		.chip = CHELSIO_T5,
2060 		.kld_name = "t5fw_cfg",
2061 		.fw_mod_name = "t5fw",
2062 		.fw_hdr = {
2063 			.chip = FW_HDR_CHIP_T5,
2064 			.fw_ver = htobe32_const(FW_VERSION(T5)),
2065 			.intfver_nic = FW_INTFVER(T5, NIC),
2066 			.intfver_vnic = FW_INTFVER(T5, VNIC),
2067 			.intfver_ofld = FW_INTFVER(T5, OFLD),
2068 			.intfver_ri = FW_INTFVER(T5, RI),
2069 			.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2070 			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
2071 			.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2072 			.intfver_fcoe = FW_INTFVER(T5, FCOE),
2073 		},
2074 	}
2075 };
2076 
2077 static struct fw_info *
2078 find_fw_info(int chip)
2079 {
2080 	int i;
2081 
2082 	for (i = 0; i < nitems(fw_info); i++) {
2083 		if (fw_info[i].chip == chip)
2084 			return (&fw_info[i]);
2085 	}
2086 	return (NULL);
2087 }
2088 
2089 /*
2090  * Is the given firmware API compatible with the one the driver was compiled
2091  * with?
2092  */
2093 static int
2094 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2095 {
2096 
2097 	/* short circuit if it's the exact same firmware version */
2098 	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2099 		return (1);
2100 
2101 	/*
2102 	 * XXX: Is this too conservative?  Perhaps I should limit this to the
2103 	 * features that are supported in the driver.
2104 	 */
2105 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2106 	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2107 	    SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2108 	    SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2109 		return (1);
2110 #undef SAME_INTF
2111 
2112 	return (0);
2113 }
2114 
2115 /*
2116  * The firmware in the KLD is usable, but should it be installed?  This routine
2117  * explains itself in detail if it indicates the KLD firmware should be
2118  * installed.
2119  */
2120 static int
2121 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2122 {
2123 	const char *reason;
2124 
2125 	if (!card_fw_usable) {
2126 		reason = "incompatible or unusable";
2127 		goto install;
2128 	}
2129 
2130 	if (k > c) {
2131 		reason = "older than the version bundled with this driver";
2132 		goto install;
2133 	}
2134 
2135 	if (t4_fw_install == 2 && k != c) {
2136 		reason = "different than the version bundled with this driver";
2137 		goto install;
2138 	}
2139 
2140 	return (0);
2141 
2142 install:
2143 	if (t4_fw_install == 0) {
2144 		device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2145 		    "but the driver is prohibited from installing a different "
2146 		    "firmware on the card.\n",
2147 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2148 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2149 
2150 		return (0);
2151 	}
2152 
2153 	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2154 	    "installing firmware %u.%u.%u.%u on card.\n",
2155 	    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2156 	    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2157 	    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2158 	    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2159 
2160 	return (1);
2161 }
2162 /*
2163  * Establish contact with the firmware and determine if we are the master driver
2164  * or not, and whether we are responsible for chip initialization.
2165  */
2166 static int
2167 prep_firmware(struct adapter *sc)
2168 {
2169 	const struct firmware *fw = NULL, *default_cfg;
2170 	int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2171 	enum dev_state state;
2172 	struct fw_info *fw_info;
2173 	struct fw_hdr *card_fw;		/* fw on the card */
2174 	const struct fw_hdr *kld_fw;	/* fw in the KLD */
2175 	const struct fw_hdr *drv_fw;	/* fw header the driver was compiled
2176 					   against */
2177 
2178 	/* Contact firmware. */
2179 	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2180 	if (rc < 0 || state == DEV_STATE_ERR) {
2181 		rc = -rc;
2182 		device_printf(sc->dev,
2183 		    "failed to connect to the firmware: %d, %d.\n", rc, state);
2184 		return (rc);
2185 	}
2186 	pf = rc;
2187 	if (pf == sc->mbox)
2188 		sc->flags |= MASTER_PF;
2189 	else if (state == DEV_STATE_UNINIT) {
2190 		/*
2191 		 * We didn't get to be the master so we definitely won't be
2192 		 * configuring the chip.  It's a bug if someone else hasn't
2193 		 * configured it already.
2194 		 */
2195 		device_printf(sc->dev, "couldn't be master(%d), "
2196 		    "device not already initialized either(%d).\n", rc, state);
2197 		return (EDOOFUS);
2198 	}
2199 
2200 	/* This is the firmware whose headers the driver was compiled against */
2201 	fw_info = find_fw_info(chip_id(sc));
2202 	if (fw_info == NULL) {
2203 		device_printf(sc->dev,
2204 		    "unable to look up firmware information for chip %d.\n",
2205 		    chip_id(sc));
2206 		return (EINVAL);
2207 	}
2208 	drv_fw = &fw_info->fw_hdr;
2209 
2210 	/*
2211 	 * The firmware KLD contains many modules.  The KLD name is also the
2212 	 * name of the module that contains the default config file.
2213 	 */
2214 	default_cfg = firmware_get(fw_info->kld_name);
2215 
2216 	/* Read the header of the firmware on the card */
2217 	card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2218 	rc = -t4_read_flash(sc, FLASH_FW_START,
2219 	    sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2220 	if (rc == 0)
2221 		card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2222 	else {
2223 		device_printf(sc->dev,
2224 		    "Unable to read card's firmware header: %d\n", rc);
2225 		card_fw_usable = 0;
2226 	}
2227 
2228 	/* This is the firmware in the KLD */
2229 	fw = firmware_get(fw_info->fw_mod_name);
2230 	if (fw != NULL) {
2231 		kld_fw = (const void *)fw->data;
2232 		kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2233 	} else {
2234 		kld_fw = NULL;
2235 		kld_fw_usable = 0;
2236 	}
2237 
2238 	if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2239 	    (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2240 		/*
2241 		 * Common case: the firmware on the card is an exact match and
2242 		 * the KLD is an exact match too, or the KLD is
2243 		 * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2244 		 * here -- use cxgbetool loadfw if you want to reinstall the
2245 		 * same firmware as the one on the card.
2246 		 */
2247 	} else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2248 	    should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2249 	    be32toh(card_fw->fw_ver))) {
2250 
2251 		rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2252 		if (rc != 0) {
2253 			device_printf(sc->dev,
2254 			    "failed to install firmware: %d\n", rc);
2255 			goto done;
2256 		}
2257 
2258 		/* Installed successfully, update the cached header too. */
2259 		memcpy(card_fw, kld_fw, sizeof(*card_fw));
2260 		card_fw_usable = 1;
2261 		need_fw_reset = 0;	/* already reset as part of load_fw */
2262 	}
2263 
2264 	if (!card_fw_usable) {
2265 		uint32_t d, c, k;
2266 
2267 		d = ntohl(drv_fw->fw_ver);
2268 		c = ntohl(card_fw->fw_ver);
2269 		k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2270 
2271 		device_printf(sc->dev, "Cannot find a usable firmware: "
2272 		    "fw_install %d, chip state %d, "
2273 		    "driver compiled with %d.%d.%d.%d, "
2274 		    "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2275 		    t4_fw_install, state,
2276 		    G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2277 		    G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2278 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2279 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2280 		    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2281 		    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2282 		rc = EINVAL;
2283 		goto done;
2284 	}
2285 
2286 	/* We're using whatever's on the card and it's known to be good. */
2287 	sc->params.fw_vers = ntohl(card_fw->fw_ver);
2288 	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2289 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2290 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2291 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2292 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2293 	t4_get_tp_version(sc, &sc->params.tp_vers);
2294 
2295 	/* Reset device */
2296 	if (need_fw_reset &&
2297 	    (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2298 		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2299 		if (rc != ETIMEDOUT && rc != EIO)
2300 			t4_fw_bye(sc, sc->mbox);
2301 		goto done;
2302 	}
2303 	sc->flags |= FW_OK;
2304 
2305 	rc = get_params__pre_init(sc);
2306 	if (rc != 0)
2307 		goto done; /* error message displayed already */
2308 
2309 	/* Partition adapter resources as specified in the config file. */
2310 	if (state == DEV_STATE_UNINIT) {
2311 
2312 		KASSERT(sc->flags & MASTER_PF,
2313 		    ("%s: trying to change chip settings when not master.",
2314 		    __func__));
2315 
2316 		rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2317 		if (rc != 0)
2318 			goto done;	/* error message displayed already */
2319 
2320 		t4_tweak_chip_settings(sc);
2321 
2322 		/* get basic stuff going */
2323 		rc = -t4_fw_initialize(sc, sc->mbox);
2324 		if (rc != 0) {
2325 			device_printf(sc->dev, "fw init failed: %d.\n", rc);
2326 			goto done;
2327 		}
2328 	} else {
2329 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2330 		sc->cfcsum = 0;
2331 	}
2332 
2333 done:
2334 	free(card_fw, M_CXGBE);
2335 	if (fw != NULL)
2336 		firmware_put(fw, FIRMWARE_UNLOAD);
2337 	if (default_cfg != NULL)
2338 		firmware_put(default_cfg, FIRMWARE_UNLOAD);
2339 
2340 	return (rc);
2341 }
2342 
2343 #define FW_PARAM_DEV(param) \
2344 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2345 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2346 #define FW_PARAM_PFVF(param) \
2347 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2348 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2349 
2350 /*
2351  * Partition chip resources for use between various PFs, VFs, etc.
2352  */
2353 static int
2354 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2355     const char *name_prefix)
2356 {
2357 	const struct firmware *cfg = NULL;
2358 	int rc = 0;
2359 	struct fw_caps_config_cmd caps;
2360 	uint32_t mtype, moff, finicsum, cfcsum;
2361 
2362 	/*
2363 	 * Figure out what configuration file to use.  Pick the default config
2364 	 * file for the card if the user hasn't specified one explicitly.
2365 	 */
2366 	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2367 	if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2368 		/* Card specific overrides go here. */
2369 		if (pci_get_device(sc->dev) == 0x440a)
2370 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2371 		if (is_fpga(sc))
2372 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2373 	}
2374 
2375 	/*
2376 	 * We need to load another module if the profile is anything except
2377 	 * "default" or "flash".
2378 	 */
2379 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2380 	    strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2381 		char s[32];
2382 
2383 		snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2384 		cfg = firmware_get(s);
2385 		if (cfg == NULL) {
2386 			if (default_cfg != NULL) {
2387 				device_printf(sc->dev,
2388 				    "unable to load module \"%s\" for "
2389 				    "configuration profile \"%s\", will use "
2390 				    "the default config file instead.\n",
2391 				    s, sc->cfg_file);
2392 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2393 				    "%s", DEFAULT_CF);
2394 			} else {
2395 				device_printf(sc->dev,
2396 				    "unable to load module \"%s\" for "
2397 				    "configuration profile \"%s\", will use "
2398 				    "the config file on the card's flash "
2399 				    "instead.\n", s, sc->cfg_file);
2400 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2401 				    "%s", FLASH_CF);
2402 			}
2403 		}
2404 	}
2405 
2406 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2407 	    default_cfg == NULL) {
2408 		device_printf(sc->dev,
2409 		    "default config file not available, will use the config "
2410 		    "file on the card's flash instead.\n");
2411 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2412 	}
2413 
2414 	if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2415 		u_int cflen, i, n;
2416 		const uint32_t *cfdata;
2417 		uint32_t param, val, addr, off, mw_base, mw_aperture;
2418 
2419 		KASSERT(cfg != NULL || default_cfg != NULL,
2420 		    ("%s: no config to upload", __func__));
2421 
2422 		/*
2423 		 * Ask the firmware where it wants us to upload the config file.
2424 		 */
2425 		param = FW_PARAM_DEV(CF);
2426 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2427 		if (rc != 0) {
2428 			/* No support for config file?  Shouldn't happen. */
2429 			device_printf(sc->dev,
2430 			    "failed to query config file location: %d.\n", rc);
2431 			goto done;
2432 		}
2433 		mtype = G_FW_PARAMS_PARAM_Y(val);
2434 		moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2435 
2436 		/*
2437 		 * XXX: sheer laziness.  We deliberately added 4 bytes of
2438 		 * useless stuffing/comments at the end of the config file so
2439 		 * it's ok to simply throw away the last remaining bytes when
2440 		 * the config file is not an exact multiple of 4.  This also
2441 		 * helps with the validate_mt_off_len check.
2442 		 */
2443 		if (cfg != NULL) {
2444 			cflen = cfg->datasize & ~3;
2445 			cfdata = cfg->data;
2446 		} else {
2447 			cflen = default_cfg->datasize & ~3;
2448 			cfdata = default_cfg->data;
2449 		}
2450 
2451 		if (cflen > FLASH_CFG_MAX_SIZE) {
2452 			device_printf(sc->dev,
2453 			    "config file too long (%d, max allowed is %d).  "
2454 			    "Will try to use the config on the card, if any.\n",
2455 			    cflen, FLASH_CFG_MAX_SIZE);
2456 			goto use_config_on_flash;
2457 		}
2458 
2459 		rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2460 		if (rc != 0) {
2461 			device_printf(sc->dev,
2462 			    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2463 			    "Will try to use the config on the card, if any.\n",
2464 			    __func__, mtype, moff, cflen, rc);
2465 			goto use_config_on_flash;
2466 		}
2467 
2468 		memwin_info(sc, 2, &mw_base, &mw_aperture);
2469 		while (cflen) {
2470 			off = position_memwin(sc, 2, addr);
2471 			n = min(cflen, mw_aperture - off);
2472 			for (i = 0; i < n; i += 4)
2473 				t4_write_reg(sc, mw_base + off + i, *cfdata++);
2474 			cflen -= n;
2475 			addr += n;
2476 		}
2477 	} else {
2478 use_config_on_flash:
2479 		mtype = FW_MEMTYPE_FLASH;
2480 		moff = t4_flash_cfg_addr(sc);
2481 	}
2482 
2483 	bzero(&caps, sizeof(caps));
2484 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2485 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2486 	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2487 	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2488 	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2489 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2490 	if (rc != 0) {
2491 		device_printf(sc->dev,
2492 		    "failed to pre-process config file: %d "
2493 		    "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2494 		goto done;
2495 	}
2496 
2497 	finicsum = be32toh(caps.finicsum);
2498 	cfcsum = be32toh(caps.cfcsum);
2499 	if (finicsum != cfcsum) {
2500 		device_printf(sc->dev,
2501 		    "WARNING: config file checksum mismatch: %08x %08x\n",
2502 		    finicsum, cfcsum);
2503 	}
2504 	sc->cfcsum = cfcsum;
2505 
2506 #define LIMIT_CAPS(x) do { \
2507 	caps.x &= htobe16(t4_##x##_allowed); \
2508 } while (0)
2509 
2510 	/*
2511 	 * Let the firmware know what features will (not) be used so it can tune
2512 	 * things accordingly.
2513 	 */
2514 	LIMIT_CAPS(linkcaps);
2515 	LIMIT_CAPS(niccaps);
2516 	LIMIT_CAPS(toecaps);
2517 	LIMIT_CAPS(rdmacaps);
2518 	LIMIT_CAPS(iscsicaps);
2519 	LIMIT_CAPS(fcoecaps);
2520 #undef LIMIT_CAPS
2521 
2522 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2523 	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2524 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2525 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2526 	if (rc != 0) {
2527 		device_printf(sc->dev,
2528 		    "failed to process config file: %d.\n", rc);
2529 	}
2530 done:
2531 	if (cfg != NULL)
2532 		firmware_put(cfg, FIRMWARE_UNLOAD);
2533 	return (rc);
2534 }
2535 
2536 /*
2537  * Retrieve parameters that are needed (or nice to have) very early.
2538  */
2539 static int
2540 get_params__pre_init(struct adapter *sc)
2541 {
2542 	int rc;
2543 	uint32_t param[2], val[2];
2544 	struct fw_devlog_cmd cmd;
2545 	struct devlog_params *dlog = &sc->params.devlog;
2546 
2547 	param[0] = FW_PARAM_DEV(PORTVEC);
2548 	param[1] = FW_PARAM_DEV(CCLK);
2549 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2550 	if (rc != 0) {
2551 		device_printf(sc->dev,
2552 		    "failed to query parameters (pre_init): %d.\n", rc);
2553 		return (rc);
2554 	}
2555 
2556 	sc->params.portvec = val[0];
2557 	sc->params.nports = bitcount32(val[0]);
2558 	sc->params.vpd.cclk = val[1];
2559 
2560 	/* Read device log parameters. */
2561 	bzero(&cmd, sizeof(cmd));
2562 	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2563 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2564 	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2565 	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2566 	if (rc != 0) {
2567 		device_printf(sc->dev,
2568 		    "failed to get devlog parameters: %d.\n", rc);
2569 		bzero(dlog, sizeof (*dlog));
2570 		rc = 0;	/* devlog isn't critical for device operation */
2571 	} else {
2572 		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2573 		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2574 		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2575 		dlog->size = be32toh(cmd.memsize_devlog);
2576 	}
2577 
2578 	return (rc);
2579 }
2580 
2581 /*
2582  * Retrieve various parameters that are of interest to the driver.  The device
2583  * has been initialized by the firmware at this point.
2584  */
2585 static int
2586 get_params__post_init(struct adapter *sc)
2587 {
2588 	int rc;
2589 	uint32_t param[7], val[7];
2590 	struct fw_caps_config_cmd caps;
2591 
2592 	param[0] = FW_PARAM_PFVF(IQFLINT_START);
2593 	param[1] = FW_PARAM_PFVF(EQ_START);
2594 	param[2] = FW_PARAM_PFVF(FILTER_START);
2595 	param[3] = FW_PARAM_PFVF(FILTER_END);
2596 	param[4] = FW_PARAM_PFVF(L2T_START);
2597 	param[5] = FW_PARAM_PFVF(L2T_END);
2598 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2599 	if (rc != 0) {
2600 		device_printf(sc->dev,
2601 		    "failed to query parameters (post_init): %d.\n", rc);
2602 		return (rc);
2603 	}
2604 
2605 	sc->sge.iq_start = val[0];
2606 	sc->sge.eq_start = val[1];
2607 	sc->tids.ftid_base = val[2];
2608 	sc->tids.nftids = val[3] - val[2] + 1;
2609 	sc->params.ftid_min = val[2];
2610 	sc->params.ftid_max = val[3];
2611 	sc->vres.l2t.start = val[4];
2612 	sc->vres.l2t.size = val[5] - val[4] + 1;
2613 	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2614 	    ("%s: L2 table size (%u) larger than expected (%u)",
2615 	    __func__, sc->vres.l2t.size, L2T_SIZE));
2616 
2617 	/* get capabilites */
2618 	bzero(&caps, sizeof(caps));
2619 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2620 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2621 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2622 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2623 	if (rc != 0) {
2624 		device_printf(sc->dev,
2625 		    "failed to get card capabilities: %d.\n", rc);
2626 		return (rc);
2627 	}
2628 
2629 #define READ_CAPS(x) do { \
2630 	sc->x = htobe16(caps.x); \
2631 } while (0)
2632 	READ_CAPS(linkcaps);
2633 	READ_CAPS(niccaps);
2634 	READ_CAPS(toecaps);
2635 	READ_CAPS(rdmacaps);
2636 	READ_CAPS(iscsicaps);
2637 	READ_CAPS(fcoecaps);
2638 
2639 	if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2640 		param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2641 		param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2642 		param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2643 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2644 		if (rc != 0) {
2645 			device_printf(sc->dev,
2646 			    "failed to query NIC parameters: %d.\n", rc);
2647 			return (rc);
2648 		}
2649 		sc->tids.etid_base = val[0];
2650 		sc->params.etid_min = val[0];
2651 		sc->tids.netids = val[1] - val[0] + 1;
2652 		sc->params.netids = sc->tids.netids;
2653 		sc->params.eo_wr_cred = val[2];
2654 		sc->params.ethoffload = 1;
2655 	}
2656 
2657 	if (sc->toecaps) {
2658 		/* query offload-related parameters */
2659 		param[0] = FW_PARAM_DEV(NTID);
2660 		param[1] = FW_PARAM_PFVF(SERVER_START);
2661 		param[2] = FW_PARAM_PFVF(SERVER_END);
2662 		param[3] = FW_PARAM_PFVF(TDDP_START);
2663 		param[4] = FW_PARAM_PFVF(TDDP_END);
2664 		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2665 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2666 		if (rc != 0) {
2667 			device_printf(sc->dev,
2668 			    "failed to query TOE parameters: %d.\n", rc);
2669 			return (rc);
2670 		}
2671 		sc->tids.ntids = val[0];
2672 		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2673 		sc->tids.stid_base = val[1];
2674 		sc->tids.nstids = val[2] - val[1] + 1;
2675 		sc->vres.ddp.start = val[3];
2676 		sc->vres.ddp.size = val[4] - val[3] + 1;
2677 		sc->params.ofldq_wr_cred = val[5];
2678 		sc->params.offload = 1;
2679 	}
2680 	if (sc->rdmacaps) {
2681 		param[0] = FW_PARAM_PFVF(STAG_START);
2682 		param[1] = FW_PARAM_PFVF(STAG_END);
2683 		param[2] = FW_PARAM_PFVF(RQ_START);
2684 		param[3] = FW_PARAM_PFVF(RQ_END);
2685 		param[4] = FW_PARAM_PFVF(PBL_START);
2686 		param[5] = FW_PARAM_PFVF(PBL_END);
2687 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2688 		if (rc != 0) {
2689 			device_printf(sc->dev,
2690 			    "failed to query RDMA parameters(1): %d.\n", rc);
2691 			return (rc);
2692 		}
2693 		sc->vres.stag.start = val[0];
2694 		sc->vres.stag.size = val[1] - val[0] + 1;
2695 		sc->vres.rq.start = val[2];
2696 		sc->vres.rq.size = val[3] - val[2] + 1;
2697 		sc->vres.pbl.start = val[4];
2698 		sc->vres.pbl.size = val[5] - val[4] + 1;
2699 
2700 		param[0] = FW_PARAM_PFVF(SQRQ_START);
2701 		param[1] = FW_PARAM_PFVF(SQRQ_END);
2702 		param[2] = FW_PARAM_PFVF(CQ_START);
2703 		param[3] = FW_PARAM_PFVF(CQ_END);
2704 		param[4] = FW_PARAM_PFVF(OCQ_START);
2705 		param[5] = FW_PARAM_PFVF(OCQ_END);
2706 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2707 		if (rc != 0) {
2708 			device_printf(sc->dev,
2709 			    "failed to query RDMA parameters(2): %d.\n", rc);
2710 			return (rc);
2711 		}
2712 		sc->vres.qp.start = val[0];
2713 		sc->vres.qp.size = val[1] - val[0] + 1;
2714 		sc->vres.cq.start = val[2];
2715 		sc->vres.cq.size = val[3] - val[2] + 1;
2716 		sc->vres.ocq.start = val[4];
2717 		sc->vres.ocq.size = val[5] - val[4] + 1;
2718 	}
2719 	if (sc->iscsicaps) {
2720 		param[0] = FW_PARAM_PFVF(ISCSI_START);
2721 		param[1] = FW_PARAM_PFVF(ISCSI_END);
2722 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2723 		if (rc != 0) {
2724 			device_printf(sc->dev,
2725 			    "failed to query iSCSI parameters: %d.\n", rc);
2726 			return (rc);
2727 		}
2728 		sc->vres.iscsi.start = val[0];
2729 		sc->vres.iscsi.size = val[1] - val[0] + 1;
2730 	}
2731 
2732 	/*
2733 	 * We've got the params we wanted to query via the firmware.  Now grab
2734 	 * some others directly from the chip.
2735 	 */
2736 	rc = t4_read_chip_settings(sc);
2737 
2738 	return (rc);
2739 }
2740 
2741 static int
2742 set_params__post_init(struct adapter *sc)
2743 {
2744 	uint32_t param, val;
2745 
2746 	/* ask for encapsulated CPLs */
2747 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2748 	val = 1;
2749 	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2750 
2751 	return (0);
2752 }
2753 
2754 #undef FW_PARAM_PFVF
2755 #undef FW_PARAM_DEV
2756 
2757 static void
2758 t4_set_desc(struct adapter *sc)
2759 {
2760 	char buf[128];
2761 	struct adapter_params *p = &sc->params;
2762 
2763 	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2764 	    "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2765 	    chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2766 
2767 	device_set_desc_copy(sc->dev, buf);
2768 }
2769 
2770 static void
2771 build_medialist(struct port_info *pi, struct ifmedia *media)
2772 {
2773 	int data, m;
2774 
2775 	PORT_LOCK(pi);
2776 
2777 	ifmedia_removeall(media);
2778 
2779 	m = IFM_ETHER | IFM_FDX;
2780 	data = (pi->port_type << 8) | pi->mod_type;
2781 
2782 	switch(pi->port_type) {
2783 	case FW_PORT_TYPE_BT_XFI:
2784 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2785 		break;
2786 
2787 	case FW_PORT_TYPE_BT_XAUI:
2788 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2789 		/* fall through */
2790 
2791 	case FW_PORT_TYPE_BT_SGMII:
2792 		ifmedia_add(media, m | IFM_1000_T, data, NULL);
2793 		ifmedia_add(media, m | IFM_100_TX, data, NULL);
2794 		ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2795 		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2796 		break;
2797 
2798 	case FW_PORT_TYPE_CX4:
2799 		ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2800 		ifmedia_set(media, m | IFM_10G_CX4);
2801 		break;
2802 
2803 	case FW_PORT_TYPE_QSFP_10G:
2804 	case FW_PORT_TYPE_SFP:
2805 	case FW_PORT_TYPE_FIBER_XFI:
2806 	case FW_PORT_TYPE_FIBER_XAUI:
2807 		switch (pi->mod_type) {
2808 
2809 		case FW_PORT_MOD_TYPE_LR:
2810 			ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2811 			ifmedia_set(media, m | IFM_10G_LR);
2812 			break;
2813 
2814 		case FW_PORT_MOD_TYPE_SR:
2815 			ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2816 			ifmedia_set(media, m | IFM_10G_SR);
2817 			break;
2818 
2819 		case FW_PORT_MOD_TYPE_LRM:
2820 			ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2821 			ifmedia_set(media, m | IFM_10G_LRM);
2822 			break;
2823 
2824 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2825 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2826 			ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2827 			ifmedia_set(media, m | IFM_10G_TWINAX);
2828 			break;
2829 
2830 		case FW_PORT_MOD_TYPE_NONE:
2831 			m &= ~IFM_FDX;
2832 			ifmedia_add(media, m | IFM_NONE, data, NULL);
2833 			ifmedia_set(media, m | IFM_NONE);
2834 			break;
2835 
2836 		case FW_PORT_MOD_TYPE_NA:
2837 		case FW_PORT_MOD_TYPE_ER:
2838 		default:
2839 			device_printf(pi->dev,
2840 			    "unknown port_type (%d), mod_type (%d)\n",
2841 			    pi->port_type, pi->mod_type);
2842 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2843 			ifmedia_set(media, m | IFM_UNKNOWN);
2844 			break;
2845 		}
2846 		break;
2847 
2848 	case FW_PORT_TYPE_QSFP:
2849 		switch (pi->mod_type) {
2850 
2851 		case FW_PORT_MOD_TYPE_LR:
2852 			ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2853 			ifmedia_set(media, m | IFM_40G_LR4);
2854 			break;
2855 
2856 		case FW_PORT_MOD_TYPE_SR:
2857 			ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2858 			ifmedia_set(media, m | IFM_40G_SR4);
2859 			break;
2860 
2861 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2862 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2863 			ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2864 			ifmedia_set(media, m | IFM_40G_CR4);
2865 			break;
2866 
2867 		case FW_PORT_MOD_TYPE_NONE:
2868 			m &= ~IFM_FDX;
2869 			ifmedia_add(media, m | IFM_NONE, data, NULL);
2870 			ifmedia_set(media, m | IFM_NONE);
2871 			break;
2872 
2873 		default:
2874 			device_printf(pi->dev,
2875 			    "unknown port_type (%d), mod_type (%d)\n",
2876 			    pi->port_type, pi->mod_type);
2877 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2878 			ifmedia_set(media, m | IFM_UNKNOWN);
2879 			break;
2880 		}
2881 		break;
2882 
2883 	default:
2884 		device_printf(pi->dev,
2885 		    "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2886 		    pi->mod_type);
2887 		ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2888 		ifmedia_set(media, m | IFM_UNKNOWN);
2889 		break;
2890 	}
2891 
2892 	PORT_UNLOCK(pi);
2893 }
2894 
2895 #define FW_MAC_EXACT_CHUNK	7
2896 
2897 /*
2898  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2899  * indicates which parameters should be programmed (the rest are left alone).
2900  */
2901 int
2902 update_mac_settings(struct ifnet *ifp, int flags)
2903 {
2904 	int rc = 0;
2905 	struct port_info *pi = ifp->if_softc;
2906 	struct adapter *sc = pi->adapter;
2907 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2908 	uint16_t viid = 0xffff;
2909 	int16_t *xact_addr_filt = NULL;
2910 
2911 	ASSERT_SYNCHRONIZED_OP(sc);
2912 	KASSERT(flags, ("%s: not told what to update.", __func__));
2913 
2914 	if (ifp == pi->ifp) {
2915 		viid = pi->viid;
2916 		xact_addr_filt = &pi->xact_addr_filt;
2917 	}
2918 #ifdef DEV_NETMAP
2919 	else if (ifp == pi->nm_ifp) {
2920 		viid = pi->nm_viid;
2921 		xact_addr_filt = &pi->nm_xact_addr_filt;
2922 	}
2923 #endif
2924 	if (flags & XGMAC_MTU)
2925 		mtu = ifp->if_mtu;
2926 
2927 	if (flags & XGMAC_PROMISC)
2928 		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2929 
2930 	if (flags & XGMAC_ALLMULTI)
2931 		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2932 
2933 	if (flags & XGMAC_VLANEX)
2934 		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2935 
2936 	if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2937 		rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2938 		    1, vlanex, false);
2939 		if (rc) {
2940 			if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2941 			    rc);
2942 			return (rc);
2943 		}
2944 	}
2945 
2946 	if (flags & XGMAC_UCADDR) {
2947 		uint8_t ucaddr[ETHER_ADDR_LEN];
2948 
2949 		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2950 		rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2951 		    true, true);
2952 		if (rc < 0) {
2953 			rc = -rc;
2954 			if_printf(ifp, "change_mac failed: %d\n", rc);
2955 			return (rc);
2956 		} else {
2957 			*xact_addr_filt = rc;
2958 			rc = 0;
2959 		}
2960 	}
2961 
2962 	if (flags & XGMAC_MCADDRS) {
2963 		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2964 		int del = 1;
2965 		uint64_t hash = 0;
2966 		struct ifmultiaddr *ifma;
2967 		int i = 0, j;
2968 
2969 		if_maddr_rlock(ifp);
2970 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2971 			if (ifma->ifma_addr->sa_family != AF_LINK)
2972 				continue;
2973 			mcaddr[i++] =
2974 			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2975 
2976 			if (i == FW_MAC_EXACT_CHUNK) {
2977 				rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2978 				    i, mcaddr, NULL, &hash, 0);
2979 				if (rc < 0) {
2980 					rc = -rc;
2981 					for (j = 0; j < i; j++) {
2982 						if_printf(ifp,
2983 						    "failed to add mc address"
2984 						    " %02x:%02x:%02x:"
2985 						    "%02x:%02x:%02x rc=%d\n",
2986 						    mcaddr[j][0], mcaddr[j][1],
2987 						    mcaddr[j][2], mcaddr[j][3],
2988 						    mcaddr[j][4], mcaddr[j][5],
2989 						    rc);
2990 					}
2991 					goto mcfail;
2992 				}
2993 				del = 0;
2994 				i = 0;
2995 			}
2996 		}
2997 		if (i > 0) {
2998 			rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
2999 			    mcaddr, NULL, &hash, 0);
3000 			if (rc < 0) {
3001 				rc = -rc;
3002 				for (j = 0; j < i; j++) {
3003 					if_printf(ifp,
3004 					    "failed to add mc address"
3005 					    " %02x:%02x:%02x:"
3006 					    "%02x:%02x:%02x rc=%d\n",
3007 					    mcaddr[j][0], mcaddr[j][1],
3008 					    mcaddr[j][2], mcaddr[j][3],
3009 					    mcaddr[j][4], mcaddr[j][5],
3010 					    rc);
3011 				}
3012 				goto mcfail;
3013 			}
3014 		}
3015 
3016 		rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3017 		if (rc != 0)
3018 			if_printf(ifp, "failed to set mc address hash: %d", rc);
3019 mcfail:
3020 		if_maddr_runlock(ifp);
3021 	}
3022 
3023 	return (rc);
3024 }
3025 
3026 int
3027 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3028     char *wmesg)
3029 {
3030 	int rc, pri;
3031 
3032 #ifdef WITNESS
3033 	/* the caller thinks it's ok to sleep, but is it really? */
3034 	if (flags & SLEEP_OK)
3035 		pause("t4slptst", 1);
3036 #endif
3037 
3038 	if (INTR_OK)
3039 		pri = PCATCH;
3040 	else
3041 		pri = 0;
3042 
3043 	ADAPTER_LOCK(sc);
3044 	for (;;) {
3045 
3046 		if (pi && IS_DOOMED(pi)) {
3047 			rc = ENXIO;
3048 			goto done;
3049 		}
3050 
3051 		if (!IS_BUSY(sc)) {
3052 			rc = 0;
3053 			break;
3054 		}
3055 
3056 		if (!(flags & SLEEP_OK)) {
3057 			rc = EBUSY;
3058 			goto done;
3059 		}
3060 
3061 		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3062 			rc = EINTR;
3063 			goto done;
3064 		}
3065 	}
3066 
3067 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3068 	SET_BUSY(sc);
3069 #ifdef INVARIANTS
3070 	sc->last_op = wmesg;
3071 	sc->last_op_thr = curthread;
3072 #endif
3073 
3074 done:
3075 	if (!(flags & HOLD_LOCK) || rc)
3076 		ADAPTER_UNLOCK(sc);
3077 
3078 	return (rc);
3079 }
3080 
3081 void
3082 end_synchronized_op(struct adapter *sc, int flags)
3083 {
3084 
3085 	if (flags & LOCK_HELD)
3086 		ADAPTER_LOCK_ASSERT_OWNED(sc);
3087 	else
3088 		ADAPTER_LOCK(sc);
3089 
3090 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3091 	CLR_BUSY(sc);
3092 	wakeup(&sc->flags);
3093 	ADAPTER_UNLOCK(sc);
3094 }
3095 
3096 static int
3097 cxgbe_init_synchronized(struct port_info *pi)
3098 {
3099 	struct adapter *sc = pi->adapter;
3100 	struct ifnet *ifp = pi->ifp;
3101 	int rc = 0;
3102 
3103 	ASSERT_SYNCHRONIZED_OP(sc);
3104 
3105 	if (isset(&sc->open_device_map, pi->port_id)) {
3106 		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3107 		    ("mismatch between open_device_map and if_drv_flags"));
3108 		return (0);	/* already running */
3109 	}
3110 
3111 	if (!(sc->flags & FULL_INIT_DONE) &&
3112 	    ((rc = adapter_full_init(sc)) != 0))
3113 		return (rc);	/* error message displayed already */
3114 
3115 	if (!(pi->flags & PORT_INIT_DONE) &&
3116 	    ((rc = port_full_init(pi)) != 0))
3117 		return (rc); /* error message displayed already */
3118 
3119 	rc = update_mac_settings(ifp, XGMAC_ALL);
3120 	if (rc)
3121 		goto done;	/* error message displayed already */
3122 
3123 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3124 	if (rc != 0) {
3125 		if_printf(ifp, "enable_vi failed: %d\n", rc);
3126 		goto done;
3127 	}
3128 
3129 	/*
3130 	 * The first iq of the first port to come up is used for tracing.
3131 	 */
3132 	if (sc->traceq < 0) {
3133 		sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3134 		t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3135 		    A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3136 		    V_QUEUENUMBER(sc->traceq));
3137 		pi->flags |= HAS_TRACEQ;
3138 	}
3139 
3140 	/* all ok */
3141 	setbit(&sc->open_device_map, pi->port_id);
3142 	PORT_LOCK(pi);
3143 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
3144 	PORT_UNLOCK(pi);
3145 
3146 	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3147 done:
3148 	if (rc != 0)
3149 		cxgbe_uninit_synchronized(pi);
3150 
3151 	return (rc);
3152 }
3153 
3154 /*
3155  * Idempotent.
3156  */
3157 static int
3158 cxgbe_uninit_synchronized(struct port_info *pi)
3159 {
3160 	struct adapter *sc = pi->adapter;
3161 	struct ifnet *ifp = pi->ifp;
3162 	int rc;
3163 
3164 	ASSERT_SYNCHRONIZED_OP(sc);
3165 
3166 	/*
3167 	 * Disable the VI so that all its data in either direction is discarded
3168 	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3169 	 * tick) intact as the TP can deliver negative advice or data that it's
3170 	 * holding in its RAM (for an offloaded connection) even after the VI is
3171 	 * disabled.
3172 	 */
3173 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3174 	if (rc) {
3175 		if_printf(ifp, "disable_vi failed: %d\n", rc);
3176 		return (rc);
3177 	}
3178 
3179 	clrbit(&sc->open_device_map, pi->port_id);
3180 	PORT_LOCK(pi);
3181 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3182 	PORT_UNLOCK(pi);
3183 
3184 	pi->link_cfg.link_ok = 0;
3185 	pi->link_cfg.speed = 0;
3186 	pi->linkdnrc = -1;
3187 	t4_os_link_changed(sc, pi->port_id, 0, -1);
3188 
3189 	return (0);
3190 }
3191 
3192 /*
3193  * It is ok for this function to fail midway and return right away.  t4_detach
3194  * will walk the entire sc->irq list and clean up whatever is valid.
3195  */
3196 static int
3197 setup_intr_handlers(struct adapter *sc)
3198 {
3199 	int rc, rid, p, q;
3200 	char s[8];
3201 	struct irq *irq;
3202 	struct port_info *pi;
3203 	struct sge_rxq *rxq;
3204 #ifdef TCP_OFFLOAD
3205 	struct sge_ofld_rxq *ofld_rxq;
3206 #endif
3207 #ifdef DEV_NETMAP
3208 	struct sge_nm_rxq *nm_rxq;
3209 #endif
3210 
3211 	/*
3212 	 * Setup interrupts.
3213 	 */
3214 	irq = &sc->irq[0];
3215 	rid = sc->intr_type == INTR_INTX ? 0 : 1;
3216 	if (sc->intr_count == 1)
3217 		return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3218 
3219 	/* Multiple interrupts. */
3220 	KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3221 	    ("%s: too few intr.", __func__));
3222 
3223 	/* The first one is always error intr */
3224 	rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3225 	if (rc != 0)
3226 		return (rc);
3227 	irq++;
3228 	rid++;
3229 
3230 	/* The second one is always the firmware event queue */
3231 	rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3232 	if (rc != 0)
3233 		return (rc);
3234 	irq++;
3235 	rid++;
3236 
3237 	for_each_port(sc, p) {
3238 		pi = sc->port[p];
3239 
3240 		if (pi->flags & INTR_RXQ) {
3241 			for_each_rxq(pi, q, rxq) {
3242 				snprintf(s, sizeof(s), "%d.%d", p, q);
3243 				rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3244 				    s);
3245 				if (rc != 0)
3246 					return (rc);
3247 				irq++;
3248 				rid++;
3249 			}
3250 		}
3251 #ifdef TCP_OFFLOAD
3252 		if (pi->flags & INTR_OFLD_RXQ) {
3253 			for_each_ofld_rxq(pi, q, ofld_rxq) {
3254 				snprintf(s, sizeof(s), "%d,%d", p, q);
3255 				rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3256 				    ofld_rxq, s);
3257 				if (rc != 0)
3258 					return (rc);
3259 				irq++;
3260 				rid++;
3261 			}
3262 		}
3263 #endif
3264 #ifdef DEV_NETMAP
3265 		if (pi->flags & INTR_NM_RXQ) {
3266 			for_each_nm_rxq(pi, q, nm_rxq) {
3267 				snprintf(s, sizeof(s), "%d-%d", p, q);
3268 				rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3269 				    nm_rxq, s);
3270 				if (rc != 0)
3271 					return (rc);
3272 				irq++;
3273 				rid++;
3274 			}
3275 		}
3276 #endif
3277 	}
3278 	MPASS(irq == &sc->irq[sc->intr_count]);
3279 
3280 	return (0);
3281 }
3282 
3283 int
3284 adapter_full_init(struct adapter *sc)
3285 {
3286 	int rc, i;
3287 
3288 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3289 	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3290 	    ("%s: FULL_INIT_DONE already", __func__));
3291 
3292 	/*
3293 	 * queues that belong to the adapter (not any particular port).
3294 	 */
3295 	rc = t4_setup_adapter_queues(sc);
3296 	if (rc != 0)
3297 		goto done;
3298 
3299 	for (i = 0; i < nitems(sc->tq); i++) {
3300 		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3301 		    taskqueue_thread_enqueue, &sc->tq[i]);
3302 		if (sc->tq[i] == NULL) {
3303 			device_printf(sc->dev,
3304 			    "failed to allocate task queue %d\n", i);
3305 			rc = ENOMEM;
3306 			goto done;
3307 		}
3308 		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3309 		    device_get_nameunit(sc->dev), i);
3310 	}
3311 
3312 	t4_intr_enable(sc);
3313 	sc->flags |= FULL_INIT_DONE;
3314 done:
3315 	if (rc != 0)
3316 		adapter_full_uninit(sc);
3317 
3318 	return (rc);
3319 }
3320 
3321 int
3322 adapter_full_uninit(struct adapter *sc)
3323 {
3324 	int i;
3325 
3326 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3327 
3328 	t4_teardown_adapter_queues(sc);
3329 
3330 	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3331 		taskqueue_free(sc->tq[i]);
3332 		sc->tq[i] = NULL;
3333 	}
3334 
3335 	sc->flags &= ~FULL_INIT_DONE;
3336 
3337 	return (0);
3338 }
3339 
3340 int
3341 port_full_init(struct port_info *pi)
3342 {
3343 	struct adapter *sc = pi->adapter;
3344 	struct ifnet *ifp = pi->ifp;
3345 	uint16_t *rss;
3346 	struct sge_rxq *rxq;
3347 	int rc, i, j;
3348 
3349 	ASSERT_SYNCHRONIZED_OP(sc);
3350 	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3351 	    ("%s: PORT_INIT_DONE already", __func__));
3352 
3353 	sysctl_ctx_init(&pi->ctx);
3354 	pi->flags |= PORT_SYSCTL_CTX;
3355 
3356 	/*
3357 	 * Allocate tx/rx/fl queues for this port.
3358 	 */
3359 	rc = t4_setup_port_queues(pi);
3360 	if (rc != 0)
3361 		goto done;	/* error message displayed already */
3362 
3363 	/*
3364 	 * Setup RSS for this port.  Save a copy of the RSS table for later use.
3365 	 */
3366 	rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3367 	for (i = 0; i < pi->rss_size;) {
3368 		for_each_rxq(pi, j, rxq) {
3369 			rss[i++] = rxq->iq.abs_id;
3370 			if (i == pi->rss_size)
3371 				break;
3372 		}
3373 	}
3374 
3375 	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3376 	    pi->rss_size);
3377 	if (rc != 0) {
3378 		if_printf(ifp, "rss_config failed: %d\n", rc);
3379 		goto done;
3380 	}
3381 
3382 	pi->rss = rss;
3383 	pi->flags |= PORT_INIT_DONE;
3384 done:
3385 	if (rc != 0)
3386 		port_full_uninit(pi);
3387 
3388 	return (rc);
3389 }
3390 
3391 /*
3392  * Idempotent.
3393  */
3394 int
3395 port_full_uninit(struct port_info *pi)
3396 {
3397 	struct adapter *sc = pi->adapter;
3398 	int i;
3399 	struct sge_rxq *rxq;
3400 	struct sge_txq *txq;
3401 #ifdef TCP_OFFLOAD
3402 	struct sge_ofld_rxq *ofld_rxq;
3403 	struct sge_wrq *ofld_txq;
3404 #endif
3405 
3406 	if (pi->flags & PORT_INIT_DONE) {
3407 
3408 		/* Need to quiesce queues.  XXX: ctrl queues? */
3409 
3410 		for_each_txq(pi, i, txq) {
3411 			quiesce_eq(sc, &txq->eq);
3412 		}
3413 
3414 #ifdef TCP_OFFLOAD
3415 		for_each_ofld_txq(pi, i, ofld_txq) {
3416 			quiesce_eq(sc, &ofld_txq->eq);
3417 		}
3418 #endif
3419 
3420 		for_each_rxq(pi, i, rxq) {
3421 			quiesce_iq(sc, &rxq->iq);
3422 			quiesce_fl(sc, &rxq->fl);
3423 		}
3424 
3425 #ifdef TCP_OFFLOAD
3426 		for_each_ofld_rxq(pi, i, ofld_rxq) {
3427 			quiesce_iq(sc, &ofld_rxq->iq);
3428 			quiesce_fl(sc, &ofld_rxq->fl);
3429 		}
3430 #endif
3431 		free(pi->rss, M_CXGBE);
3432 	}
3433 
3434 	t4_teardown_port_queues(pi);
3435 	pi->flags &= ~PORT_INIT_DONE;
3436 
3437 	return (0);
3438 }
3439 
3440 static void
3441 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3442 {
3443 	EQ_LOCK(eq);
3444 	eq->flags |= EQ_DOOMED;
3445 
3446 	/*
3447 	 * Wait for the response to a credit flush if one's
3448 	 * pending.
3449 	 */
3450 	while (eq->flags & EQ_CRFLUSHED)
3451 		mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3452 	EQ_UNLOCK(eq);
3453 
3454 	callout_drain(&eq->tx_callout);	/* XXX: iffy */
3455 	pause("callout", 10);		/* Still iffy */
3456 
3457 	taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3458 }
3459 
3460 static void
3461 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3462 {
3463 	(void) sc;	/* unused */
3464 
3465 	/* Synchronize with the interrupt handler */
3466 	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3467 		pause("iqfree", 1);
3468 }
3469 
3470 static void
3471 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3472 {
3473 	mtx_lock(&sc->sfl_lock);
3474 	FL_LOCK(fl);
3475 	fl->flags |= FL_DOOMED;
3476 	FL_UNLOCK(fl);
3477 	mtx_unlock(&sc->sfl_lock);
3478 
3479 	callout_drain(&sc->sfl_callout);
3480 	KASSERT((fl->flags & FL_STARVING) == 0,
3481 	    ("%s: still starving", __func__));
3482 }
3483 
3484 static int
3485 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3486     driver_intr_t *handler, void *arg, char *name)
3487 {
3488 	int rc;
3489 
3490 	irq->rid = rid;
3491 	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3492 	    RF_SHAREABLE | RF_ACTIVE);
3493 	if (irq->res == NULL) {
3494 		device_printf(sc->dev,
3495 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3496 		return (ENOMEM);
3497 	}
3498 
3499 	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3500 	    NULL, handler, arg, &irq->tag);
3501 	if (rc != 0) {
3502 		device_printf(sc->dev,
3503 		    "failed to setup interrupt for rid %d, name %s: %d\n",
3504 		    rid, name, rc);
3505 	} else if (name)
3506 		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3507 
3508 	return (rc);
3509 }
3510 
3511 static int
3512 t4_free_irq(struct adapter *sc, struct irq *irq)
3513 {
3514 	if (irq->tag)
3515 		bus_teardown_intr(sc->dev, irq->res, irq->tag);
3516 	if (irq->res)
3517 		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3518 
3519 	bzero(irq, sizeof(*irq));
3520 
3521 	return (0);
3522 }
3523 
3524 static void
3525 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3526     unsigned int end)
3527 {
3528 	uint32_t *p = (uint32_t *)(buf + start);
3529 
3530 	for ( ; start <= end; start += sizeof(uint32_t))
3531 		*p++ = t4_read_reg(sc, start);
3532 }
3533 
3534 static void
3535 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3536 {
3537 	int i, n;
3538 	const unsigned int *reg_ranges;
3539 	static const unsigned int t4_reg_ranges[] = {
3540 		0x1008, 0x1108,
3541 		0x1180, 0x11b4,
3542 		0x11fc, 0x123c,
3543 		0x1300, 0x173c,
3544 		0x1800, 0x18fc,
3545 		0x3000, 0x30d8,
3546 		0x30e0, 0x5924,
3547 		0x5960, 0x59d4,
3548 		0x5a00, 0x5af8,
3549 		0x6000, 0x6098,
3550 		0x6100, 0x6150,
3551 		0x6200, 0x6208,
3552 		0x6240, 0x6248,
3553 		0x6280, 0x6338,
3554 		0x6370, 0x638c,
3555 		0x6400, 0x643c,
3556 		0x6500, 0x6524,
3557 		0x6a00, 0x6a38,
3558 		0x6a60, 0x6a78,
3559 		0x6b00, 0x6b84,
3560 		0x6bf0, 0x6c84,
3561 		0x6cf0, 0x6d84,
3562 		0x6df0, 0x6e84,
3563 		0x6ef0, 0x6f84,
3564 		0x6ff0, 0x7084,
3565 		0x70f0, 0x7184,
3566 		0x71f0, 0x7284,
3567 		0x72f0, 0x7384,
3568 		0x73f0, 0x7450,
3569 		0x7500, 0x7530,
3570 		0x7600, 0x761c,
3571 		0x7680, 0x76cc,
3572 		0x7700, 0x7798,
3573 		0x77c0, 0x77fc,
3574 		0x7900, 0x79fc,
3575 		0x7b00, 0x7c38,
3576 		0x7d00, 0x7efc,
3577 		0x8dc0, 0x8e1c,
3578 		0x8e30, 0x8e78,
3579 		0x8ea0, 0x8f6c,
3580 		0x8fc0, 0x9074,
3581 		0x90fc, 0x90fc,
3582 		0x9400, 0x9458,
3583 		0x9600, 0x96bc,
3584 		0x9800, 0x9808,
3585 		0x9820, 0x983c,
3586 		0x9850, 0x9864,
3587 		0x9c00, 0x9c6c,
3588 		0x9c80, 0x9cec,
3589 		0x9d00, 0x9d6c,
3590 		0x9d80, 0x9dec,
3591 		0x9e00, 0x9e6c,
3592 		0x9e80, 0x9eec,
3593 		0x9f00, 0x9f6c,
3594 		0x9f80, 0x9fec,
3595 		0xd004, 0xd03c,
3596 		0xdfc0, 0xdfe0,
3597 		0xe000, 0xea7c,
3598 		0xf000, 0x11110,
3599 		0x11118, 0x11190,
3600 		0x19040, 0x1906c,
3601 		0x19078, 0x19080,
3602 		0x1908c, 0x19124,
3603 		0x19150, 0x191b0,
3604 		0x191d0, 0x191e8,
3605 		0x19238, 0x1924c,
3606 		0x193f8, 0x19474,
3607 		0x19490, 0x194f8,
3608 		0x19800, 0x19f30,
3609 		0x1a000, 0x1a06c,
3610 		0x1a0b0, 0x1a120,
3611 		0x1a128, 0x1a138,
3612 		0x1a190, 0x1a1c4,
3613 		0x1a1fc, 0x1a1fc,
3614 		0x1e040, 0x1e04c,
3615 		0x1e284, 0x1e28c,
3616 		0x1e2c0, 0x1e2c0,
3617 		0x1e2e0, 0x1e2e0,
3618 		0x1e300, 0x1e384,
3619 		0x1e3c0, 0x1e3c8,
3620 		0x1e440, 0x1e44c,
3621 		0x1e684, 0x1e68c,
3622 		0x1e6c0, 0x1e6c0,
3623 		0x1e6e0, 0x1e6e0,
3624 		0x1e700, 0x1e784,
3625 		0x1e7c0, 0x1e7c8,
3626 		0x1e840, 0x1e84c,
3627 		0x1ea84, 0x1ea8c,
3628 		0x1eac0, 0x1eac0,
3629 		0x1eae0, 0x1eae0,
3630 		0x1eb00, 0x1eb84,
3631 		0x1ebc0, 0x1ebc8,
3632 		0x1ec40, 0x1ec4c,
3633 		0x1ee84, 0x1ee8c,
3634 		0x1eec0, 0x1eec0,
3635 		0x1eee0, 0x1eee0,
3636 		0x1ef00, 0x1ef84,
3637 		0x1efc0, 0x1efc8,
3638 		0x1f040, 0x1f04c,
3639 		0x1f284, 0x1f28c,
3640 		0x1f2c0, 0x1f2c0,
3641 		0x1f2e0, 0x1f2e0,
3642 		0x1f300, 0x1f384,
3643 		0x1f3c0, 0x1f3c8,
3644 		0x1f440, 0x1f44c,
3645 		0x1f684, 0x1f68c,
3646 		0x1f6c0, 0x1f6c0,
3647 		0x1f6e0, 0x1f6e0,
3648 		0x1f700, 0x1f784,
3649 		0x1f7c0, 0x1f7c8,
3650 		0x1f840, 0x1f84c,
3651 		0x1fa84, 0x1fa8c,
3652 		0x1fac0, 0x1fac0,
3653 		0x1fae0, 0x1fae0,
3654 		0x1fb00, 0x1fb84,
3655 		0x1fbc0, 0x1fbc8,
3656 		0x1fc40, 0x1fc4c,
3657 		0x1fe84, 0x1fe8c,
3658 		0x1fec0, 0x1fec0,
3659 		0x1fee0, 0x1fee0,
3660 		0x1ff00, 0x1ff84,
3661 		0x1ffc0, 0x1ffc8,
3662 		0x20000, 0x2002c,
3663 		0x20100, 0x2013c,
3664 		0x20190, 0x201c8,
3665 		0x20200, 0x20318,
3666 		0x20400, 0x20528,
3667 		0x20540, 0x20614,
3668 		0x21000, 0x21040,
3669 		0x2104c, 0x21060,
3670 		0x210c0, 0x210ec,
3671 		0x21200, 0x21268,
3672 		0x21270, 0x21284,
3673 		0x212fc, 0x21388,
3674 		0x21400, 0x21404,
3675 		0x21500, 0x21518,
3676 		0x2152c, 0x2153c,
3677 		0x21550, 0x21554,
3678 		0x21600, 0x21600,
3679 		0x21608, 0x21628,
3680 		0x21630, 0x2163c,
3681 		0x21700, 0x2171c,
3682 		0x21780, 0x2178c,
3683 		0x21800, 0x21c38,
3684 		0x21c80, 0x21d7c,
3685 		0x21e00, 0x21e04,
3686 		0x22000, 0x2202c,
3687 		0x22100, 0x2213c,
3688 		0x22190, 0x221c8,
3689 		0x22200, 0x22318,
3690 		0x22400, 0x22528,
3691 		0x22540, 0x22614,
3692 		0x23000, 0x23040,
3693 		0x2304c, 0x23060,
3694 		0x230c0, 0x230ec,
3695 		0x23200, 0x23268,
3696 		0x23270, 0x23284,
3697 		0x232fc, 0x23388,
3698 		0x23400, 0x23404,
3699 		0x23500, 0x23518,
3700 		0x2352c, 0x2353c,
3701 		0x23550, 0x23554,
3702 		0x23600, 0x23600,
3703 		0x23608, 0x23628,
3704 		0x23630, 0x2363c,
3705 		0x23700, 0x2371c,
3706 		0x23780, 0x2378c,
3707 		0x23800, 0x23c38,
3708 		0x23c80, 0x23d7c,
3709 		0x23e00, 0x23e04,
3710 		0x24000, 0x2402c,
3711 		0x24100, 0x2413c,
3712 		0x24190, 0x241c8,
3713 		0x24200, 0x24318,
3714 		0x24400, 0x24528,
3715 		0x24540, 0x24614,
3716 		0x25000, 0x25040,
3717 		0x2504c, 0x25060,
3718 		0x250c0, 0x250ec,
3719 		0x25200, 0x25268,
3720 		0x25270, 0x25284,
3721 		0x252fc, 0x25388,
3722 		0x25400, 0x25404,
3723 		0x25500, 0x25518,
3724 		0x2552c, 0x2553c,
3725 		0x25550, 0x25554,
3726 		0x25600, 0x25600,
3727 		0x25608, 0x25628,
3728 		0x25630, 0x2563c,
3729 		0x25700, 0x2571c,
3730 		0x25780, 0x2578c,
3731 		0x25800, 0x25c38,
3732 		0x25c80, 0x25d7c,
3733 		0x25e00, 0x25e04,
3734 		0x26000, 0x2602c,
3735 		0x26100, 0x2613c,
3736 		0x26190, 0x261c8,
3737 		0x26200, 0x26318,
3738 		0x26400, 0x26528,
3739 		0x26540, 0x26614,
3740 		0x27000, 0x27040,
3741 		0x2704c, 0x27060,
3742 		0x270c0, 0x270ec,
3743 		0x27200, 0x27268,
3744 		0x27270, 0x27284,
3745 		0x272fc, 0x27388,
3746 		0x27400, 0x27404,
3747 		0x27500, 0x27518,
3748 		0x2752c, 0x2753c,
3749 		0x27550, 0x27554,
3750 		0x27600, 0x27600,
3751 		0x27608, 0x27628,
3752 		0x27630, 0x2763c,
3753 		0x27700, 0x2771c,
3754 		0x27780, 0x2778c,
3755 		0x27800, 0x27c38,
3756 		0x27c80, 0x27d7c,
3757 		0x27e00, 0x27e04
3758 	};
3759 	static const unsigned int t5_reg_ranges[] = {
3760 		0x1008, 0x1148,
3761 		0x1180, 0x11b4,
3762 		0x11fc, 0x123c,
3763 		0x1280, 0x173c,
3764 		0x1800, 0x18fc,
3765 		0x3000, 0x3028,
3766 		0x3060, 0x30d8,
3767 		0x30e0, 0x30fc,
3768 		0x3140, 0x357c,
3769 		0x35a8, 0x35cc,
3770 		0x35ec, 0x35ec,
3771 		0x3600, 0x5624,
3772 		0x56cc, 0x575c,
3773 		0x580c, 0x5814,
3774 		0x5890, 0x58bc,
3775 		0x5940, 0x59dc,
3776 		0x59fc, 0x5a18,
3777 		0x5a60, 0x5a9c,
3778 		0x5b94, 0x5bfc,
3779 		0x6000, 0x6040,
3780 		0x6058, 0x614c,
3781 		0x7700, 0x7798,
3782 		0x77c0, 0x78fc,
3783 		0x7b00, 0x7c54,
3784 		0x7d00, 0x7efc,
3785 		0x8dc0, 0x8de0,
3786 		0x8df8, 0x8e84,
3787 		0x8ea0, 0x8f84,
3788 		0x8fc0, 0x90f8,
3789 		0x9400, 0x9470,
3790 		0x9600, 0x96f4,
3791 		0x9800, 0x9808,
3792 		0x9820, 0x983c,
3793 		0x9850, 0x9864,
3794 		0x9c00, 0x9c6c,
3795 		0x9c80, 0x9cec,
3796 		0x9d00, 0x9d6c,
3797 		0x9d80, 0x9dec,
3798 		0x9e00, 0x9e6c,
3799 		0x9e80, 0x9eec,
3800 		0x9f00, 0x9f6c,
3801 		0x9f80, 0xa020,
3802 		0xd004, 0xd03c,
3803 		0xdfc0, 0xdfe0,
3804 		0xe000, 0x11088,
3805 		0x1109c, 0x11110,
3806 		0x11118, 0x1117c,
3807 		0x11190, 0x11204,
3808 		0x19040, 0x1906c,
3809 		0x19078, 0x19080,
3810 		0x1908c, 0x19124,
3811 		0x19150, 0x191b0,
3812 		0x191d0, 0x191e8,
3813 		0x19238, 0x19290,
3814 		0x193f8, 0x19474,
3815 		0x19490, 0x194cc,
3816 		0x194f0, 0x194f8,
3817 		0x19c00, 0x19c60,
3818 		0x19c94, 0x19e10,
3819 		0x19e50, 0x19f34,
3820 		0x19f40, 0x19f50,
3821 		0x19f90, 0x19fe4,
3822 		0x1a000, 0x1a06c,
3823 		0x1a0b0, 0x1a120,
3824 		0x1a128, 0x1a138,
3825 		0x1a190, 0x1a1c4,
3826 		0x1a1fc, 0x1a1fc,
3827 		0x1e008, 0x1e00c,
3828 		0x1e040, 0x1e04c,
3829 		0x1e284, 0x1e290,
3830 		0x1e2c0, 0x1e2c0,
3831 		0x1e2e0, 0x1e2e0,
3832 		0x1e300, 0x1e384,
3833 		0x1e3c0, 0x1e3c8,
3834 		0x1e408, 0x1e40c,
3835 		0x1e440, 0x1e44c,
3836 		0x1e684, 0x1e690,
3837 		0x1e6c0, 0x1e6c0,
3838 		0x1e6e0, 0x1e6e0,
3839 		0x1e700, 0x1e784,
3840 		0x1e7c0, 0x1e7c8,
3841 		0x1e808, 0x1e80c,
3842 		0x1e840, 0x1e84c,
3843 		0x1ea84, 0x1ea90,
3844 		0x1eac0, 0x1eac0,
3845 		0x1eae0, 0x1eae0,
3846 		0x1eb00, 0x1eb84,
3847 		0x1ebc0, 0x1ebc8,
3848 		0x1ec08, 0x1ec0c,
3849 		0x1ec40, 0x1ec4c,
3850 		0x1ee84, 0x1ee90,
3851 		0x1eec0, 0x1eec0,
3852 		0x1eee0, 0x1eee0,
3853 		0x1ef00, 0x1ef84,
3854 		0x1efc0, 0x1efc8,
3855 		0x1f008, 0x1f00c,
3856 		0x1f040, 0x1f04c,
3857 		0x1f284, 0x1f290,
3858 		0x1f2c0, 0x1f2c0,
3859 		0x1f2e0, 0x1f2e0,
3860 		0x1f300, 0x1f384,
3861 		0x1f3c0, 0x1f3c8,
3862 		0x1f408, 0x1f40c,
3863 		0x1f440, 0x1f44c,
3864 		0x1f684, 0x1f690,
3865 		0x1f6c0, 0x1f6c0,
3866 		0x1f6e0, 0x1f6e0,
3867 		0x1f700, 0x1f784,
3868 		0x1f7c0, 0x1f7c8,
3869 		0x1f808, 0x1f80c,
3870 		0x1f840, 0x1f84c,
3871 		0x1fa84, 0x1fa90,
3872 		0x1fac0, 0x1fac0,
3873 		0x1fae0, 0x1fae0,
3874 		0x1fb00, 0x1fb84,
3875 		0x1fbc0, 0x1fbc8,
3876 		0x1fc08, 0x1fc0c,
3877 		0x1fc40, 0x1fc4c,
3878 		0x1fe84, 0x1fe90,
3879 		0x1fec0, 0x1fec0,
3880 		0x1fee0, 0x1fee0,
3881 		0x1ff00, 0x1ff84,
3882 		0x1ffc0, 0x1ffc8,
3883 		0x30000, 0x30030,
3884 		0x30100, 0x30144,
3885 		0x30190, 0x301d0,
3886 		0x30200, 0x30318,
3887 		0x30400, 0x3052c,
3888 		0x30540, 0x3061c,
3889 		0x30800, 0x30834,
3890 		0x308c0, 0x30908,
3891 		0x30910, 0x309ac,
3892 		0x30a00, 0x30a2c,
3893 		0x30a44, 0x30a50,
3894 		0x30a74, 0x30c24,
3895 		0x30d00, 0x30d00,
3896 		0x30d08, 0x30d14,
3897 		0x30d1c, 0x30d20,
3898 		0x30d3c, 0x30d50,
3899 		0x31200, 0x3120c,
3900 		0x31220, 0x31220,
3901 		0x31240, 0x31240,
3902 		0x31600, 0x3160c,
3903 		0x31a00, 0x31a1c,
3904 		0x31e00, 0x31e20,
3905 		0x31e38, 0x31e3c,
3906 		0x31e80, 0x31e80,
3907 		0x31e88, 0x31ea8,
3908 		0x31eb0, 0x31eb4,
3909 		0x31ec8, 0x31ed4,
3910 		0x31fb8, 0x32004,
3911 		0x32200, 0x32200,
3912 		0x32208, 0x32240,
3913 		0x32248, 0x32280,
3914 		0x32288, 0x322c0,
3915 		0x322c8, 0x322fc,
3916 		0x32600, 0x32630,
3917 		0x32a00, 0x32abc,
3918 		0x32b00, 0x32b70,
3919 		0x33000, 0x33048,
3920 		0x33060, 0x3309c,
3921 		0x330f0, 0x33148,
3922 		0x33160, 0x3319c,
3923 		0x331f0, 0x332e4,
3924 		0x332f8, 0x333e4,
3925 		0x333f8, 0x33448,
3926 		0x33460, 0x3349c,
3927 		0x334f0, 0x33548,
3928 		0x33560, 0x3359c,
3929 		0x335f0, 0x336e4,
3930 		0x336f8, 0x337e4,
3931 		0x337f8, 0x337fc,
3932 		0x33814, 0x33814,
3933 		0x3382c, 0x3382c,
3934 		0x33880, 0x3388c,
3935 		0x338e8, 0x338ec,
3936 		0x33900, 0x33948,
3937 		0x33960, 0x3399c,
3938 		0x339f0, 0x33ae4,
3939 		0x33af8, 0x33b10,
3940 		0x33b28, 0x33b28,
3941 		0x33b3c, 0x33b50,
3942 		0x33bf0, 0x33c10,
3943 		0x33c28, 0x33c28,
3944 		0x33c3c, 0x33c50,
3945 		0x33cf0, 0x33cfc,
3946 		0x34000, 0x34030,
3947 		0x34100, 0x34144,
3948 		0x34190, 0x341d0,
3949 		0x34200, 0x34318,
3950 		0x34400, 0x3452c,
3951 		0x34540, 0x3461c,
3952 		0x34800, 0x34834,
3953 		0x348c0, 0x34908,
3954 		0x34910, 0x349ac,
3955 		0x34a00, 0x34a2c,
3956 		0x34a44, 0x34a50,
3957 		0x34a74, 0x34c24,
3958 		0x34d00, 0x34d00,
3959 		0x34d08, 0x34d14,
3960 		0x34d1c, 0x34d20,
3961 		0x34d3c, 0x34d50,
3962 		0x35200, 0x3520c,
3963 		0x35220, 0x35220,
3964 		0x35240, 0x35240,
3965 		0x35600, 0x3560c,
3966 		0x35a00, 0x35a1c,
3967 		0x35e00, 0x35e20,
3968 		0x35e38, 0x35e3c,
3969 		0x35e80, 0x35e80,
3970 		0x35e88, 0x35ea8,
3971 		0x35eb0, 0x35eb4,
3972 		0x35ec8, 0x35ed4,
3973 		0x35fb8, 0x36004,
3974 		0x36200, 0x36200,
3975 		0x36208, 0x36240,
3976 		0x36248, 0x36280,
3977 		0x36288, 0x362c0,
3978 		0x362c8, 0x362fc,
3979 		0x36600, 0x36630,
3980 		0x36a00, 0x36abc,
3981 		0x36b00, 0x36b70,
3982 		0x37000, 0x37048,
3983 		0x37060, 0x3709c,
3984 		0x370f0, 0x37148,
3985 		0x37160, 0x3719c,
3986 		0x371f0, 0x372e4,
3987 		0x372f8, 0x373e4,
3988 		0x373f8, 0x37448,
3989 		0x37460, 0x3749c,
3990 		0x374f0, 0x37548,
3991 		0x37560, 0x3759c,
3992 		0x375f0, 0x376e4,
3993 		0x376f8, 0x377e4,
3994 		0x377f8, 0x377fc,
3995 		0x37814, 0x37814,
3996 		0x3782c, 0x3782c,
3997 		0x37880, 0x3788c,
3998 		0x378e8, 0x378ec,
3999 		0x37900, 0x37948,
4000 		0x37960, 0x3799c,
4001 		0x379f0, 0x37ae4,
4002 		0x37af8, 0x37b10,
4003 		0x37b28, 0x37b28,
4004 		0x37b3c, 0x37b50,
4005 		0x37bf0, 0x37c10,
4006 		0x37c28, 0x37c28,
4007 		0x37c3c, 0x37c50,
4008 		0x37cf0, 0x37cfc,
4009 		0x38000, 0x38030,
4010 		0x38100, 0x38144,
4011 		0x38190, 0x381d0,
4012 		0x38200, 0x38318,
4013 		0x38400, 0x3852c,
4014 		0x38540, 0x3861c,
4015 		0x38800, 0x38834,
4016 		0x388c0, 0x38908,
4017 		0x38910, 0x389ac,
4018 		0x38a00, 0x38a2c,
4019 		0x38a44, 0x38a50,
4020 		0x38a74, 0x38c24,
4021 		0x38d00, 0x38d00,
4022 		0x38d08, 0x38d14,
4023 		0x38d1c, 0x38d20,
4024 		0x38d3c, 0x38d50,
4025 		0x39200, 0x3920c,
4026 		0x39220, 0x39220,
4027 		0x39240, 0x39240,
4028 		0x39600, 0x3960c,
4029 		0x39a00, 0x39a1c,
4030 		0x39e00, 0x39e20,
4031 		0x39e38, 0x39e3c,
4032 		0x39e80, 0x39e80,
4033 		0x39e88, 0x39ea8,
4034 		0x39eb0, 0x39eb4,
4035 		0x39ec8, 0x39ed4,
4036 		0x39fb8, 0x3a004,
4037 		0x3a200, 0x3a200,
4038 		0x3a208, 0x3a240,
4039 		0x3a248, 0x3a280,
4040 		0x3a288, 0x3a2c0,
4041 		0x3a2c8, 0x3a2fc,
4042 		0x3a600, 0x3a630,
4043 		0x3aa00, 0x3aabc,
4044 		0x3ab00, 0x3ab70,
4045 		0x3b000, 0x3b048,
4046 		0x3b060, 0x3b09c,
4047 		0x3b0f0, 0x3b148,
4048 		0x3b160, 0x3b19c,
4049 		0x3b1f0, 0x3b2e4,
4050 		0x3b2f8, 0x3b3e4,
4051 		0x3b3f8, 0x3b448,
4052 		0x3b460, 0x3b49c,
4053 		0x3b4f0, 0x3b548,
4054 		0x3b560, 0x3b59c,
4055 		0x3b5f0, 0x3b6e4,
4056 		0x3b6f8, 0x3b7e4,
4057 		0x3b7f8, 0x3b7fc,
4058 		0x3b814, 0x3b814,
4059 		0x3b82c, 0x3b82c,
4060 		0x3b880, 0x3b88c,
4061 		0x3b8e8, 0x3b8ec,
4062 		0x3b900, 0x3b948,
4063 		0x3b960, 0x3b99c,
4064 		0x3b9f0, 0x3bae4,
4065 		0x3baf8, 0x3bb10,
4066 		0x3bb28, 0x3bb28,
4067 		0x3bb3c, 0x3bb50,
4068 		0x3bbf0, 0x3bc10,
4069 		0x3bc28, 0x3bc28,
4070 		0x3bc3c, 0x3bc50,
4071 		0x3bcf0, 0x3bcfc,
4072 		0x3c000, 0x3c030,
4073 		0x3c100, 0x3c144,
4074 		0x3c190, 0x3c1d0,
4075 		0x3c200, 0x3c318,
4076 		0x3c400, 0x3c52c,
4077 		0x3c540, 0x3c61c,
4078 		0x3c800, 0x3c834,
4079 		0x3c8c0, 0x3c908,
4080 		0x3c910, 0x3c9ac,
4081 		0x3ca00, 0x3ca2c,
4082 		0x3ca44, 0x3ca50,
4083 		0x3ca74, 0x3cc24,
4084 		0x3cd00, 0x3cd00,
4085 		0x3cd08, 0x3cd14,
4086 		0x3cd1c, 0x3cd20,
4087 		0x3cd3c, 0x3cd50,
4088 		0x3d200, 0x3d20c,
4089 		0x3d220, 0x3d220,
4090 		0x3d240, 0x3d240,
4091 		0x3d600, 0x3d60c,
4092 		0x3da00, 0x3da1c,
4093 		0x3de00, 0x3de20,
4094 		0x3de38, 0x3de3c,
4095 		0x3de80, 0x3de80,
4096 		0x3de88, 0x3dea8,
4097 		0x3deb0, 0x3deb4,
4098 		0x3dec8, 0x3ded4,
4099 		0x3dfb8, 0x3e004,
4100 		0x3e200, 0x3e200,
4101 		0x3e208, 0x3e240,
4102 		0x3e248, 0x3e280,
4103 		0x3e288, 0x3e2c0,
4104 		0x3e2c8, 0x3e2fc,
4105 		0x3e600, 0x3e630,
4106 		0x3ea00, 0x3eabc,
4107 		0x3eb00, 0x3eb70,
4108 		0x3f000, 0x3f048,
4109 		0x3f060, 0x3f09c,
4110 		0x3f0f0, 0x3f148,
4111 		0x3f160, 0x3f19c,
4112 		0x3f1f0, 0x3f2e4,
4113 		0x3f2f8, 0x3f3e4,
4114 		0x3f3f8, 0x3f448,
4115 		0x3f460, 0x3f49c,
4116 		0x3f4f0, 0x3f548,
4117 		0x3f560, 0x3f59c,
4118 		0x3f5f0, 0x3f6e4,
4119 		0x3f6f8, 0x3f7e4,
4120 		0x3f7f8, 0x3f7fc,
4121 		0x3f814, 0x3f814,
4122 		0x3f82c, 0x3f82c,
4123 		0x3f880, 0x3f88c,
4124 		0x3f8e8, 0x3f8ec,
4125 		0x3f900, 0x3f948,
4126 		0x3f960, 0x3f99c,
4127 		0x3f9f0, 0x3fae4,
4128 		0x3faf8, 0x3fb10,
4129 		0x3fb28, 0x3fb28,
4130 		0x3fb3c, 0x3fb50,
4131 		0x3fbf0, 0x3fc10,
4132 		0x3fc28, 0x3fc28,
4133 		0x3fc3c, 0x3fc50,
4134 		0x3fcf0, 0x3fcfc,
4135 		0x40000, 0x4000c,
4136 		0x40040, 0x40068,
4137 		0x4007c, 0x40144,
4138 		0x40180, 0x4018c,
4139 		0x40200, 0x40298,
4140 		0x402ac, 0x4033c,
4141 		0x403f8, 0x403fc,
4142 		0x41304, 0x413c4,
4143 		0x41400, 0x4141c,
4144 		0x41480, 0x414d0,
4145 		0x44000, 0x44078,
4146 		0x440c0, 0x44278,
4147 		0x442c0, 0x44478,
4148 		0x444c0, 0x44678,
4149 		0x446c0, 0x44878,
4150 		0x448c0, 0x449fc,
4151 		0x45000, 0x45068,
4152 		0x45080, 0x45084,
4153 		0x450a0, 0x450b0,
4154 		0x45200, 0x45268,
4155 		0x45280, 0x45284,
4156 		0x452a0, 0x452b0,
4157 		0x460c0, 0x460e4,
4158 		0x47000, 0x4708c,
4159 		0x47200, 0x47250,
4160 		0x47400, 0x47420,
4161 		0x47600, 0x47618,
4162 		0x47800, 0x47814,
4163 		0x48000, 0x4800c,
4164 		0x48040, 0x48068,
4165 		0x4807c, 0x48144,
4166 		0x48180, 0x4818c,
4167 		0x48200, 0x48298,
4168 		0x482ac, 0x4833c,
4169 		0x483f8, 0x483fc,
4170 		0x49304, 0x493c4,
4171 		0x49400, 0x4941c,
4172 		0x49480, 0x494d0,
4173 		0x4c000, 0x4c078,
4174 		0x4c0c0, 0x4c278,
4175 		0x4c2c0, 0x4c478,
4176 		0x4c4c0, 0x4c678,
4177 		0x4c6c0, 0x4c878,
4178 		0x4c8c0, 0x4c9fc,
4179 		0x4d000, 0x4d068,
4180 		0x4d080, 0x4d084,
4181 		0x4d0a0, 0x4d0b0,
4182 		0x4d200, 0x4d268,
4183 		0x4d280, 0x4d284,
4184 		0x4d2a0, 0x4d2b0,
4185 		0x4e0c0, 0x4e0e4,
4186 		0x4f000, 0x4f08c,
4187 		0x4f200, 0x4f250,
4188 		0x4f400, 0x4f420,
4189 		0x4f600, 0x4f618,
4190 		0x4f800, 0x4f814,
4191 		0x50000, 0x500cc,
4192 		0x50400, 0x50400,
4193 		0x50800, 0x508cc,
4194 		0x50c00, 0x50c00,
4195 		0x51000, 0x5101c,
4196 		0x51300, 0x51308,
4197 	};
4198 
4199 	if (is_t4(sc)) {
4200 		reg_ranges = &t4_reg_ranges[0];
4201 		n = nitems(t4_reg_ranges);
4202 	} else {
4203 		reg_ranges = &t5_reg_ranges[0];
4204 		n = nitems(t5_reg_ranges);
4205 	}
4206 
4207 	regs->version = chip_id(sc) | chip_rev(sc) << 10;
4208 	for (i = 0; i < n; i += 2)
4209 		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4210 }
4211 
4212 static void
4213 cxgbe_tick(void *arg)
4214 {
4215 	struct port_info *pi = arg;
4216 	struct adapter *sc = pi->adapter;
4217 	struct ifnet *ifp = pi->ifp;
4218 	struct sge_txq *txq;
4219 	int i, drops;
4220 	struct port_stats *s = &pi->stats;
4221 
4222 	PORT_LOCK(pi);
4223 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4224 		PORT_UNLOCK(pi);
4225 		return;	/* without scheduling another callout */
4226 	}
4227 
4228 	t4_get_port_stats(sc, pi->tx_chan, s);
4229 
4230 	ifp->if_opackets = s->tx_frames - s->tx_pause;
4231 	ifp->if_ipackets = s->rx_frames - s->rx_pause;
4232 	ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4233 	ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4234 	ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4235 	ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4236 	ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4237 	    s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4238 	    s->rx_trunc3;
4239 	for (i = 0; i < 4; i++) {
4240 		if (pi->rx_chan_map & (1 << i)) {
4241 			uint32_t v;
4242 
4243 			/*
4244 			 * XXX: indirect reads from the same ADDR/DATA pair can
4245 			 * race with each other.
4246 			 */
4247 			t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4248 			    1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4249 			ifp->if_iqdrops += v;
4250 		}
4251 	}
4252 
4253 	drops = s->tx_drop;
4254 	for_each_txq(pi, i, txq)
4255 		drops += txq->br->br_drops;
4256 	ifp->if_oqdrops = drops;
4257 
4258 	ifp->if_oerrors = s->tx_error_frames;
4259 	ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4260 	    s->rx_fcs_err + s->rx_len_err;
4261 
4262 	callout_schedule(&pi->tick, hz);
4263 	PORT_UNLOCK(pi);
4264 }
4265 
4266 static void
4267 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4268 {
4269 	struct ifnet *vlan;
4270 
4271 	if (arg != ifp || ifp->if_type != IFT_ETHER)
4272 		return;
4273 
4274 	vlan = VLAN_DEVAT(ifp, vid);
4275 	VLAN_SETCOOKIE(vlan, ifp);
4276 }
4277 
4278 static int
4279 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4280 {
4281 
4282 #ifdef INVARIANTS
4283 	panic("%s: opcode 0x%02x on iq %p with payload %p",
4284 	    __func__, rss->opcode, iq, m);
4285 #else
4286 	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4287 	    __func__, rss->opcode, iq, m);
4288 	m_freem(m);
4289 #endif
4290 	return (EDOOFUS);
4291 }
4292 
4293 int
4294 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4295 {
4296 	uintptr_t *loc, new;
4297 
4298 	if (opcode >= nitems(sc->cpl_handler))
4299 		return (EINVAL);
4300 
4301 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4302 	loc = (uintptr_t *) &sc->cpl_handler[opcode];
4303 	atomic_store_rel_ptr(loc, new);
4304 
4305 	return (0);
4306 }
4307 
4308 static int
4309 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4310 {
4311 
4312 #ifdef INVARIANTS
4313 	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4314 #else
4315 	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4316 	    __func__, iq, ctrl);
4317 #endif
4318 	return (EDOOFUS);
4319 }
4320 
4321 int
4322 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4323 {
4324 	uintptr_t *loc, new;
4325 
4326 	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4327 	loc = (uintptr_t *) &sc->an_handler;
4328 	atomic_store_rel_ptr(loc, new);
4329 
4330 	return (0);
4331 }
4332 
4333 static int
4334 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4335 {
4336 	const struct cpl_fw6_msg *cpl =
4337 	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
4338 
4339 #ifdef INVARIANTS
4340 	panic("%s: fw_msg type %d", __func__, cpl->type);
4341 #else
4342 	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4343 #endif
4344 	return (EDOOFUS);
4345 }
4346 
4347 int
4348 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4349 {
4350 	uintptr_t *loc, new;
4351 
4352 	if (type >= nitems(sc->fw_msg_handler))
4353 		return (EINVAL);
4354 
4355 	/*
4356 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4357 	 * handler dispatch table.  Reject any attempt to install a handler for
4358 	 * this subtype.
4359 	 */
4360 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4361 		return (EINVAL);
4362 
4363 	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4364 	loc = (uintptr_t *) &sc->fw_msg_handler[type];
4365 	atomic_store_rel_ptr(loc, new);
4366 
4367 	return (0);
4368 }
4369 
4370 static int
4371 t4_sysctls(struct adapter *sc)
4372 {
4373 	struct sysctl_ctx_list *ctx;
4374 	struct sysctl_oid *oid;
4375 	struct sysctl_oid_list *children, *c0;
4376 	static char *caps[] = {
4377 		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
4378 		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"	/* caps[1] niccaps */
4379 		    "\6HASHFILTER\7ETHOFLD",
4380 		"\20\1TOE",				/* caps[2] toecaps */
4381 		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
4382 		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
4383 		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4384 		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4385 		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
4386 		    "\4PO_INITIAOR\5PO_TARGET"
4387 	};
4388 	static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4389 
4390 	ctx = device_get_sysctl_ctx(sc->dev);
4391 
4392 	/*
4393 	 * dev.t4nex.X.
4394 	 */
4395 	oid = device_get_sysctl_tree(sc->dev);
4396 	c0 = children = SYSCTL_CHILDREN(oid);
4397 
4398 	sc->sc_do_rxcopy = 1;
4399 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4400 	    &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4401 
4402 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4403 	    sc->params.nports, "# of ports");
4404 
4405 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4406 	    NULL, chip_rev(sc), "chip hardware revision");
4407 
4408 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4409 	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4410 
4411 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4412 	    CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4413 
4414 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4415 	    sc->cfcsum, "config file checksum");
4416 
4417 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4418 	    CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4419 	    sysctl_bitfield, "A", "available doorbells");
4420 
4421 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4422 	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4423 	    sysctl_bitfield, "A", "available link capabilities");
4424 
4425 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4426 	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4427 	    sysctl_bitfield, "A", "available NIC capabilities");
4428 
4429 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4430 	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4431 	    sysctl_bitfield, "A", "available TCP offload capabilities");
4432 
4433 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4434 	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4435 	    sysctl_bitfield, "A", "available RDMA capabilities");
4436 
4437 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4438 	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4439 	    sysctl_bitfield, "A", "available iSCSI capabilities");
4440 
4441 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4442 	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4443 	    sysctl_bitfield, "A", "available FCoE capabilities");
4444 
4445 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4446 	    sc->params.vpd.cclk, "core clock frequency (in KHz)");
4447 
4448 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4449 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4450 	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4451 	    "interrupt holdoff timer values (us)");
4452 
4453 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4454 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4455 	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4456 	    "interrupt holdoff packet counter values");
4457 
4458 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4459 	    NULL, sc->tids.nftids, "number of filters");
4460 
4461 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4462 	    CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4463 	    "chip temperature (in Celsius)");
4464 
4465 	t4_sge_sysctls(sc, ctx, children);
4466 
4467 	sc->lro_timeout = 100;
4468 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4469 	    &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4470 
4471 #ifdef SBUF_DRAIN
4472 	/*
4473 	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4474 	 */
4475 	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4476 	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4477 	    "logs and miscellaneous information");
4478 	children = SYSCTL_CHILDREN(oid);
4479 
4480 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4481 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4482 	    sysctl_cctrl, "A", "congestion control");
4483 
4484 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4485 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4486 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4487 
4488 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4489 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4490 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4491 
4492 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4493 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4494 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4495 
4496 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4497 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4498 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4499 
4500 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4501 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4502 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4503 
4504 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4505 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4506 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4507 
4508 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4509 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4510 	    sysctl_cim_la, "A", "CIM logic analyzer");
4511 
4512 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4513 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4514 	    sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4515 
4516 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4517 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4518 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4519 
4520 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4521 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4522 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4523 
4524 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4525 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4526 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4527 
4528 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4529 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4530 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4531 
4532 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4533 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4534 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4535 
4536 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4537 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4538 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4539 
4540 	if (is_t5(sc)) {
4541 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4542 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4543 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4544 
4545 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4546 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4547 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4548 	}
4549 
4550 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4551 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4552 	    sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4553 
4554 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4555 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4556 	    sysctl_cim_qcfg, "A", "CIM queue configuration");
4557 
4558 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4559 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4560 	    sysctl_cpl_stats, "A", "CPL statistics");
4561 
4562 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4563 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4564 	    sysctl_ddp_stats, "A", "DDP statistics");
4565 
4566 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4567 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4568 	    sysctl_devlog, "A", "firmware's device log");
4569 
4570 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4571 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4572 	    sysctl_fcoe_stats, "A", "FCoE statistics");
4573 
4574 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4575 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4576 	    sysctl_hw_sched, "A", "hardware scheduler ");
4577 
4578 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4579 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580 	    sysctl_l2t, "A", "hardware L2 table");
4581 
4582 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4583 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584 	    sysctl_lb_stats, "A", "loopback statistics");
4585 
4586 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4587 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588 	    sysctl_meminfo, "A", "memory regions");
4589 
4590 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4591 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4592 	    sysctl_mps_tcam, "A", "MPS TCAM entries");
4593 
4594 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4595 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4596 	    sysctl_path_mtus, "A", "path MTUs");
4597 
4598 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4599 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4600 	    sysctl_pm_stats, "A", "PM statistics");
4601 
4602 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4603 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4604 	    sysctl_rdma_stats, "A", "RDMA statistics");
4605 
4606 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4607 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4608 	    sysctl_tcp_stats, "A", "TCP statistics");
4609 
4610 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4611 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612 	    sysctl_tids, "A", "TID information");
4613 
4614 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4615 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616 	    sysctl_tp_err_stats, "A", "TP error statistics");
4617 
4618 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4619 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620 	    sysctl_tp_la, "A", "TP logic analyzer");
4621 
4622 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4623 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624 	    sysctl_tx_rate, "A", "Tx rate");
4625 
4626 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4627 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4628 	    sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4629 
4630 	if (is_t5(sc)) {
4631 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4632 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4633 		    sysctl_wcwr_stats, "A", "write combined work requests");
4634 	}
4635 #endif
4636 
4637 #ifdef TCP_OFFLOAD
4638 	if (is_offload(sc)) {
4639 		/*
4640 		 * dev.t4nex.X.toe.
4641 		 */
4642 		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4643 		    NULL, "TOE parameters");
4644 		children = SYSCTL_CHILDREN(oid);
4645 
4646 		sc->tt.sndbuf = 256 * 1024;
4647 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4648 		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
4649 
4650 		sc->tt.ddp = 0;
4651 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4652 		    &sc->tt.ddp, 0, "DDP allowed");
4653 
4654 		sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4655 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4656 		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
4657 
4658 		sc->tt.ddp_thres =
4659 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4660 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4661 		    &sc->tt.ddp_thres, 0, "DDP threshold");
4662 
4663 		sc->tt.rx_coalesce = 1;
4664 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4665 		    CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4666 	}
4667 #endif
4668 
4669 
4670 	return (0);
4671 }
4672 
4673 static int
4674 cxgbe_sysctls(struct port_info *pi)
4675 {
4676 	struct sysctl_ctx_list *ctx;
4677 	struct sysctl_oid *oid;
4678 	struct sysctl_oid_list *children;
4679 	struct adapter *sc = pi->adapter;
4680 
4681 	ctx = device_get_sysctl_ctx(pi->dev);
4682 
4683 	/*
4684 	 * dev.cxgbe.X.
4685 	 */
4686 	oid = device_get_sysctl_tree(pi->dev);
4687 	children = SYSCTL_CHILDREN(oid);
4688 
4689 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4690 	   CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4691 	if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4692 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4693 		    CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4694 		    "PHY temperature (in Celsius)");
4695 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4696 		    CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4697 		    "PHY firmware version");
4698 	}
4699 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4700 	    &pi->nrxq, 0, "# of rx queues");
4701 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4702 	    &pi->ntxq, 0, "# of tx queues");
4703 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4704 	    &pi->first_rxq, 0, "index of first rx queue");
4705 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4706 	    &pi->first_txq, 0, "index of first tx queue");
4707 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4708 	    CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4709 	    "Reserve queue 0 for non-flowid packets");
4710 
4711 #ifdef TCP_OFFLOAD
4712 	if (is_offload(sc)) {
4713 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4714 		    &pi->nofldrxq, 0,
4715 		    "# of rx queues for offloaded TCP connections");
4716 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4717 		    &pi->nofldtxq, 0,
4718 		    "# of tx queues for offloaded TCP connections");
4719 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4720 		    CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4721 		    "index of first TOE rx queue");
4722 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4723 		    CTLFLAG_RD, &pi->first_ofld_txq, 0,
4724 		    "index of first TOE tx queue");
4725 	}
4726 #endif
4727 #ifdef DEV_NETMAP
4728 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4729 	    &pi->nnmrxq, 0, "# of rx queues for netmap");
4730 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4731 	    &pi->nnmtxq, 0, "# of tx queues for netmap");
4732 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4733 	    CTLFLAG_RD, &pi->first_nm_rxq, 0,
4734 	    "index of first netmap rx queue");
4735 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4736 	    CTLFLAG_RD, &pi->first_nm_txq, 0,
4737 	    "index of first netmap tx queue");
4738 #endif
4739 
4740 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4741 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4742 	    "holdoff timer index");
4743 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4744 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4745 	    "holdoff packet counter index");
4746 
4747 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4748 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4749 	    "rx queue size");
4750 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4751 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4752 	    "tx queue size");
4753 
4754 	/*
4755 	 * dev.cxgbe.X.stats.
4756 	 */
4757 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4758 	    NULL, "port statistics");
4759 	children = SYSCTL_CHILDREN(oid);
4760 
4761 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4762 	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4763 	    CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4764 	    sysctl_handle_t4_reg64, "QU", desc)
4765 
4766 	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4767 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4768 	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4769 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4770 	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4771 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4772 	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4773 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4774 	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4775 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4776 	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4777 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4778 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4779 	    "# of tx frames in this range",
4780 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4781 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4782 	    "# of tx frames in this range",
4783 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4784 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4785 	    "# of tx frames in this range",
4786 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4787 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4788 	    "# of tx frames in this range",
4789 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4790 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4791 	    "# of tx frames in this range",
4792 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4793 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4794 	    "# of tx frames in this range",
4795 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4796 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4797 	    "# of tx frames in this range",
4798 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4799 	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4800 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4801 	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4802 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4803 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4804 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4805 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4806 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4807 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4808 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4809 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4810 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4811 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4812 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4813 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4814 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4815 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4816 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4817 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4818 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4819 
4820 	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4821 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4822 	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4823 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4824 	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4825 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4826 	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4827 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4828 	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4829 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4830 	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4831 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4832 	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4833 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4834 	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4835 	    "# of frames received with bad FCS",
4836 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4837 	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4838 	    "# of frames received with length error",
4839 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4840 	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4841 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4842 	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4843 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4844 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4845 	    "# of rx frames in this range",
4846 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4847 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4848 	    "# of rx frames in this range",
4849 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4850 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4851 	    "# of rx frames in this range",
4852 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4853 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4854 	    "# of rx frames in this range",
4855 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4856 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4857 	    "# of rx frames in this range",
4858 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4859 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4860 	    "# of rx frames in this range",
4861 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4862 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4863 	    "# of rx frames in this range",
4864 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4865 	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4866 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4867 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4868 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4869 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4870 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4871 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4872 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4873 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4874 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4875 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4876 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4877 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4878 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4879 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4880 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4881 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4882 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4883 
4884 #undef SYSCTL_ADD_T4_REG64
4885 
4886 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4887 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4888 	    &pi->stats.name, desc)
4889 
4890 	/* We get these from port_stats and they may be stale by upto 1s */
4891 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4892 	    "# drops due to buffer-group 0 overflows");
4893 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4894 	    "# drops due to buffer-group 1 overflows");
4895 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4896 	    "# drops due to buffer-group 2 overflows");
4897 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4898 	    "# drops due to buffer-group 3 overflows");
4899 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4900 	    "# of buffer-group 0 truncated packets");
4901 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4902 	    "# of buffer-group 1 truncated packets");
4903 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4904 	    "# of buffer-group 2 truncated packets");
4905 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4906 	    "# of buffer-group 3 truncated packets");
4907 
4908 #undef SYSCTL_ADD_T4_PORTSTAT
4909 
4910 	return (0);
4911 }
4912 
4913 static int
4914 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4915 {
4916 	int rc, *i;
4917 	struct sbuf sb;
4918 
4919 	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4920 	for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4921 		sbuf_printf(&sb, "%d ", *i);
4922 	sbuf_trim(&sb);
4923 	sbuf_finish(&sb);
4924 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4925 	sbuf_delete(&sb);
4926 	return (rc);
4927 }
4928 
4929 static int
4930 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4931 {
4932 	int rc;
4933 	struct sbuf *sb;
4934 
4935 	rc = sysctl_wire_old_buffer(req, 0);
4936 	if (rc != 0)
4937 		return(rc);
4938 
4939 	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4940 	if (sb == NULL)
4941 		return (ENOMEM);
4942 
4943 	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4944 	rc = sbuf_finish(sb);
4945 	sbuf_delete(sb);
4946 
4947 	return (rc);
4948 }
4949 
4950 static int
4951 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4952 {
4953 	struct port_info *pi = arg1;
4954 	int op = arg2;
4955 	struct adapter *sc = pi->adapter;
4956 	u_int v;
4957 	int rc;
4958 
4959 	rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4960 	if (rc)
4961 		return (rc);
4962 	/* XXX: magic numbers */
4963 	rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4964 	    &v);
4965 	end_synchronized_op(sc, 0);
4966 	if (rc)
4967 		return (rc);
4968 	if (op == 0)
4969 		v /= 256;
4970 
4971 	rc = sysctl_handle_int(oidp, &v, 0, req);
4972 	return (rc);
4973 }
4974 
4975 static int
4976 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
4977 {
4978 	struct port_info *pi = arg1;
4979 	int rc, val;
4980 
4981 	val = pi->rsrv_noflowq;
4982 	rc = sysctl_handle_int(oidp, &val, 0, req);
4983 	if (rc != 0 || req->newptr == NULL)
4984 		return (rc);
4985 
4986 	if ((val >= 1) && (pi->ntxq > 1))
4987 		pi->rsrv_noflowq = 1;
4988 	else
4989 		pi->rsrv_noflowq = 0;
4990 
4991 	return (rc);
4992 }
4993 
4994 static int
4995 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
4996 {
4997 	struct port_info *pi = arg1;
4998 	struct adapter *sc = pi->adapter;
4999 	int idx, rc, i;
5000 	struct sge_rxq *rxq;
5001 #ifdef TCP_OFFLOAD
5002 	struct sge_ofld_rxq *ofld_rxq;
5003 #endif
5004 	uint8_t v;
5005 
5006 	idx = pi->tmr_idx;
5007 
5008 	rc = sysctl_handle_int(oidp, &idx, 0, req);
5009 	if (rc != 0 || req->newptr == NULL)
5010 		return (rc);
5011 
5012 	if (idx < 0 || idx >= SGE_NTIMERS)
5013 		return (EINVAL);
5014 
5015 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5016 	    "t4tmr");
5017 	if (rc)
5018 		return (rc);
5019 
5020 	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5021 	for_each_rxq(pi, i, rxq) {
5022 #ifdef atomic_store_rel_8
5023 		atomic_store_rel_8(&rxq->iq.intr_params, v);
5024 #else
5025 		rxq->iq.intr_params = v;
5026 #endif
5027 	}
5028 #ifdef TCP_OFFLOAD
5029 	for_each_ofld_rxq(pi, i, ofld_rxq) {
5030 #ifdef atomic_store_rel_8
5031 		atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5032 #else
5033 		ofld_rxq->iq.intr_params = v;
5034 #endif
5035 	}
5036 #endif
5037 	pi->tmr_idx = idx;
5038 
5039 	end_synchronized_op(sc, LOCK_HELD);
5040 	return (0);
5041 }
5042 
5043 static int
5044 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5045 {
5046 	struct port_info *pi = arg1;
5047 	struct adapter *sc = pi->adapter;
5048 	int idx, rc;
5049 
5050 	idx = pi->pktc_idx;
5051 
5052 	rc = sysctl_handle_int(oidp, &idx, 0, req);
5053 	if (rc != 0 || req->newptr == NULL)
5054 		return (rc);
5055 
5056 	if (idx < -1 || idx >= SGE_NCOUNTERS)
5057 		return (EINVAL);
5058 
5059 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5060 	    "t4pktc");
5061 	if (rc)
5062 		return (rc);
5063 
5064 	if (pi->flags & PORT_INIT_DONE)
5065 		rc = EBUSY; /* cannot be changed once the queues are created */
5066 	else
5067 		pi->pktc_idx = idx;
5068 
5069 	end_synchronized_op(sc, LOCK_HELD);
5070 	return (rc);
5071 }
5072 
5073 static int
5074 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5075 {
5076 	struct port_info *pi = arg1;
5077 	struct adapter *sc = pi->adapter;
5078 	int qsize, rc;
5079 
5080 	qsize = pi->qsize_rxq;
5081 
5082 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
5083 	if (rc != 0 || req->newptr == NULL)
5084 		return (rc);
5085 
5086 	if (qsize < 128 || (qsize & 7))
5087 		return (EINVAL);
5088 
5089 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5090 	    "t4rxqs");
5091 	if (rc)
5092 		return (rc);
5093 
5094 	if (pi->flags & PORT_INIT_DONE)
5095 		rc = EBUSY; /* cannot be changed once the queues are created */
5096 	else
5097 		pi->qsize_rxq = qsize;
5098 
5099 	end_synchronized_op(sc, LOCK_HELD);
5100 	return (rc);
5101 }
5102 
5103 static int
5104 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5105 {
5106 	struct port_info *pi = arg1;
5107 	struct adapter *sc = pi->adapter;
5108 	int qsize, rc;
5109 
5110 	qsize = pi->qsize_txq;
5111 
5112 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
5113 	if (rc != 0 || req->newptr == NULL)
5114 		return (rc);
5115 
5116 	/* bufring size must be powerof2 */
5117 	if (qsize < 128 || !powerof2(qsize))
5118 		return (EINVAL);
5119 
5120 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5121 	    "t4txqs");
5122 	if (rc)
5123 		return (rc);
5124 
5125 	if (pi->flags & PORT_INIT_DONE)
5126 		rc = EBUSY; /* cannot be changed once the queues are created */
5127 	else
5128 		pi->qsize_txq = qsize;
5129 
5130 	end_synchronized_op(sc, LOCK_HELD);
5131 	return (rc);
5132 }
5133 
5134 static int
5135 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5136 {
5137 	struct adapter *sc = arg1;
5138 	int reg = arg2;
5139 	uint64_t val;
5140 
5141 	val = t4_read_reg64(sc, reg);
5142 
5143 	return (sysctl_handle_64(oidp, &val, 0, req));
5144 }
5145 
5146 static int
5147 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5148 {
5149 	struct adapter *sc = arg1;
5150 	int rc, t;
5151 	uint32_t param, val;
5152 
5153 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5154 	if (rc)
5155 		return (rc);
5156 	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5157 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5158 	    V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5159 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5160 	end_synchronized_op(sc, 0);
5161 	if (rc)
5162 		return (rc);
5163 
5164 	/* unknown is returned as 0 but we display -1 in that case */
5165 	t = val == 0 ? -1 : val;
5166 
5167 	rc = sysctl_handle_int(oidp, &t, 0, req);
5168 	return (rc);
5169 }
5170 
5171 #ifdef SBUF_DRAIN
5172 static int
5173 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5174 {
5175 	struct adapter *sc = arg1;
5176 	struct sbuf *sb;
5177 	int rc, i;
5178 	uint16_t incr[NMTUS][NCCTRL_WIN];
5179 	static const char *dec_fac[] = {
5180 		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5181 		"0.9375"
5182 	};
5183 
5184 	rc = sysctl_wire_old_buffer(req, 0);
5185 	if (rc != 0)
5186 		return (rc);
5187 
5188 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5189 	if (sb == NULL)
5190 		return (ENOMEM);
5191 
5192 	t4_read_cong_tbl(sc, incr);
5193 
5194 	for (i = 0; i < NCCTRL_WIN; ++i) {
5195 		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5196 		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5197 		    incr[5][i], incr[6][i], incr[7][i]);
5198 		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5199 		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5200 		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5201 		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5202 	}
5203 
5204 	rc = sbuf_finish(sb);
5205 	sbuf_delete(sb);
5206 
5207 	return (rc);
5208 }
5209 
5210 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5211 	"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",	/* ibq's */
5212 	"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI",	/* obq's */
5213 	"SGE0-RX", "SGE1-RX"	/* additional obq's (T5 onwards) */
5214 };
5215 
5216 static int
5217 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5218 {
5219 	struct adapter *sc = arg1;
5220 	struct sbuf *sb;
5221 	int rc, i, n, qid = arg2;
5222 	uint32_t *buf, *p;
5223 	char *qtype;
5224 	u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5225 
5226 	KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5227 	    ("%s: bad qid %d\n", __func__, qid));
5228 
5229 	if (qid < CIM_NUM_IBQ) {
5230 		/* inbound queue */
5231 		qtype = "IBQ";
5232 		n = 4 * CIM_IBQ_SIZE;
5233 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5234 		rc = t4_read_cim_ibq(sc, qid, buf, n);
5235 	} else {
5236 		/* outbound queue */
5237 		qtype = "OBQ";
5238 		qid -= CIM_NUM_IBQ;
5239 		n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5240 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5241 		rc = t4_read_cim_obq(sc, qid, buf, n);
5242 	}
5243 
5244 	if (rc < 0) {
5245 		rc = -rc;
5246 		goto done;
5247 	}
5248 	n = rc * sizeof(uint32_t);	/* rc has # of words actually read */
5249 
5250 	rc = sysctl_wire_old_buffer(req, 0);
5251 	if (rc != 0)
5252 		goto done;
5253 
5254 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5255 	if (sb == NULL) {
5256 		rc = ENOMEM;
5257 		goto done;
5258 	}
5259 
5260 	sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5261 	for (i = 0, p = buf; i < n; i += 16, p += 4)
5262 		sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5263 		    p[2], p[3]);
5264 
5265 	rc = sbuf_finish(sb);
5266 	sbuf_delete(sb);
5267 done:
5268 	free(buf, M_CXGBE);
5269 	return (rc);
5270 }
5271 
5272 static int
5273 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5274 {
5275 	struct adapter *sc = arg1;
5276 	u_int cfg;
5277 	struct sbuf *sb;
5278 	uint32_t *buf, *p;
5279 	int rc;
5280 
5281 	rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5282 	if (rc != 0)
5283 		return (rc);
5284 
5285 	rc = sysctl_wire_old_buffer(req, 0);
5286 	if (rc != 0)
5287 		return (rc);
5288 
5289 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5290 	if (sb == NULL)
5291 		return (ENOMEM);
5292 
5293 	buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5294 	    M_ZERO | M_WAITOK);
5295 
5296 	rc = -t4_cim_read_la(sc, buf, NULL);
5297 	if (rc != 0)
5298 		goto done;
5299 
5300 	sbuf_printf(sb, "Status   Data      PC%s",
5301 	    cfg & F_UPDBGLACAPTPCONLY ? "" :
5302 	    "     LS0Stat  LS0Addr             LS0Data");
5303 
5304 	KASSERT((sc->params.cim_la_size & 7) == 0,
5305 	    ("%s: p will walk off the end of buf", __func__));
5306 
5307 	for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5308 		if (cfg & F_UPDBGLACAPTPCONLY) {
5309 			sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5310 			    p[6], p[7]);
5311 			sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5312 			    (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5313 			    p[4] & 0xff, p[5] >> 8);
5314 			sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5315 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5316 			    p[1] & 0xf, p[2] >> 4);
5317 		} else {
5318 			sbuf_printf(sb,
5319 			    "\n  %02x   %x%07x %x%07x %08x %08x "
5320 			    "%08x%08x%08x%08x",
5321 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5322 			    p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5323 			    p[6], p[7]);
5324 		}
5325 	}
5326 
5327 	rc = sbuf_finish(sb);
5328 	sbuf_delete(sb);
5329 done:
5330 	free(buf, M_CXGBE);
5331 	return (rc);
5332 }
5333 
5334 static int
5335 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5336 {
5337 	struct adapter *sc = arg1;
5338 	u_int i;
5339 	struct sbuf *sb;
5340 	uint32_t *buf, *p;
5341 	int rc;
5342 
5343 	rc = sysctl_wire_old_buffer(req, 0);
5344 	if (rc != 0)
5345 		return (rc);
5346 
5347 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5348 	if (sb == NULL)
5349 		return (ENOMEM);
5350 
5351 	buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5352 	    M_ZERO | M_WAITOK);
5353 
5354 	t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5355 	p = buf;
5356 
5357 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5358 		sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5359 		    p[1], p[0]);
5360 	}
5361 
5362 	sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5363 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5364 		sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5365 		    (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5366 		    (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5367 		    (p[1] >> 2) | ((p[2] & 3) << 30),
5368 		    (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5369 		    p[0] & 1);
5370 	}
5371 
5372 	rc = sbuf_finish(sb);
5373 	sbuf_delete(sb);
5374 	free(buf, M_CXGBE);
5375 	return (rc);
5376 }
5377 
5378 static int
5379 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5380 {
5381 	struct adapter *sc = arg1;
5382 	u_int i;
5383 	struct sbuf *sb;
5384 	uint32_t *buf, *p;
5385 	int rc;
5386 
5387 	rc = sysctl_wire_old_buffer(req, 0);
5388 	if (rc != 0)
5389 		return (rc);
5390 
5391 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5392 	if (sb == NULL)
5393 		return (ENOMEM);
5394 
5395 	buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5396 	    M_ZERO | M_WAITOK);
5397 
5398 	t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5399 	p = buf;
5400 
5401 	sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5402 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5403 		sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5404 		    (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5405 		    p[4], p[3], p[2], p[1], p[0]);
5406 	}
5407 
5408 	sbuf_printf(sb, "\n\nCntl ID               Data");
5409 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5410 		sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5411 		    (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5412 	}
5413 
5414 	rc = sbuf_finish(sb);
5415 	sbuf_delete(sb);
5416 	free(buf, M_CXGBE);
5417 	return (rc);
5418 }
5419 
5420 static int
5421 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5422 {
5423 	struct adapter *sc = arg1;
5424 	struct sbuf *sb;
5425 	int rc, i;
5426 	uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5427 	uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5428 	uint16_t thres[CIM_NUM_IBQ];
5429 	uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5430 	uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5431 	u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5432 
5433 	if (is_t4(sc)) {
5434 		cim_num_obq = CIM_NUM_OBQ;
5435 		ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5436 		obq_rdaddr = A_UP_OBQ_0_REALADDR;
5437 	} else {
5438 		cim_num_obq = CIM_NUM_OBQ_T5;
5439 		ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5440 		obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5441 	}
5442 	nq = CIM_NUM_IBQ + cim_num_obq;
5443 
5444 	rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5445 	if (rc == 0)
5446 		rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5447 	if (rc != 0)
5448 		return (rc);
5449 
5450 	t4_read_cimq_cfg(sc, base, size, thres);
5451 
5452 	rc = sysctl_wire_old_buffer(req, 0);
5453 	if (rc != 0)
5454 		return (rc);
5455 
5456 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5457 	if (sb == NULL)
5458 		return (ENOMEM);
5459 
5460 	sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5461 
5462 	for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5463 		sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5464 		    qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5465 		    G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5466 		    G_QUEREMFLITS(p[2]) * 16);
5467 	for ( ; i < nq; i++, p += 4, wr += 2)
5468 		sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5469 		    base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5470 		    wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5471 		    G_QUEREMFLITS(p[2]) * 16);
5472 
5473 	rc = sbuf_finish(sb);
5474 	sbuf_delete(sb);
5475 
5476 	return (rc);
5477 }
5478 
5479 static int
5480 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5481 {
5482 	struct adapter *sc = arg1;
5483 	struct sbuf *sb;
5484 	int rc;
5485 	struct tp_cpl_stats stats;
5486 
5487 	rc = sysctl_wire_old_buffer(req, 0);
5488 	if (rc != 0)
5489 		return (rc);
5490 
5491 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5492 	if (sb == NULL)
5493 		return (ENOMEM);
5494 
5495 	t4_tp_get_cpl_stats(sc, &stats);
5496 
5497 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5498 	    "channel 3\n");
5499 	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5500 		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5501 	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5502 		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5503 
5504 	rc = sbuf_finish(sb);
5505 	sbuf_delete(sb);
5506 
5507 	return (rc);
5508 }
5509 
5510 static int
5511 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5512 {
5513 	struct adapter *sc = arg1;
5514 	struct sbuf *sb;
5515 	int rc;
5516 	struct tp_usm_stats stats;
5517 
5518 	rc = sysctl_wire_old_buffer(req, 0);
5519 	if (rc != 0)
5520 		return(rc);
5521 
5522 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5523 	if (sb == NULL)
5524 		return (ENOMEM);
5525 
5526 	t4_get_usm_stats(sc, &stats);
5527 
5528 	sbuf_printf(sb, "Frames: %u\n", stats.frames);
5529 	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5530 	sbuf_printf(sb, "Drops:  %u", stats.drops);
5531 
5532 	rc = sbuf_finish(sb);
5533 	sbuf_delete(sb);
5534 
5535 	return (rc);
5536 }
5537 
5538 const char *devlog_level_strings[] = {
5539 	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
5540 	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
5541 	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
5542 	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
5543 	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
5544 	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
5545 };
5546 
5547 const char *devlog_facility_strings[] = {
5548 	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
5549 	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
5550 	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
5551 	[FW_DEVLOG_FACILITY_RES]	= "RES",
5552 	[FW_DEVLOG_FACILITY_HW]		= "HW",
5553 	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
5554 	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
5555 	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
5556 	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
5557 	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
5558 	[FW_DEVLOG_FACILITY_VI]		= "VI",
5559 	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
5560 	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
5561 	[FW_DEVLOG_FACILITY_TM]		= "TM",
5562 	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
5563 	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
5564 	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
5565 	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
5566 	[FW_DEVLOG_FACILITY_RI]		= "RI",
5567 	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
5568 	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
5569 	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
5570 	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
5571 };
5572 
5573 static int
5574 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5575 {
5576 	struct adapter *sc = arg1;
5577 	struct devlog_params *dparams = &sc->params.devlog;
5578 	struct fw_devlog_e *buf, *e;
5579 	int i, j, rc, nentries, first = 0, m;
5580 	struct sbuf *sb;
5581 	uint64_t ftstamp = UINT64_MAX;
5582 
5583 	if (dparams->start == 0) {
5584 		dparams->memtype = FW_MEMTYPE_EDC0;
5585 		dparams->start = 0x84000;
5586 		dparams->size = 32768;
5587 	}
5588 
5589 	nentries = dparams->size / sizeof(struct fw_devlog_e);
5590 
5591 	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5592 	if (buf == NULL)
5593 		return (ENOMEM);
5594 
5595 	m = fwmtype_to_hwmtype(dparams->memtype);
5596 	rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5597 	if (rc != 0)
5598 		goto done;
5599 
5600 	for (i = 0; i < nentries; i++) {
5601 		e = &buf[i];
5602 
5603 		if (e->timestamp == 0)
5604 			break;	/* end */
5605 
5606 		e->timestamp = be64toh(e->timestamp);
5607 		e->seqno = be32toh(e->seqno);
5608 		for (j = 0; j < 8; j++)
5609 			e->params[j] = be32toh(e->params[j]);
5610 
5611 		if (e->timestamp < ftstamp) {
5612 			ftstamp = e->timestamp;
5613 			first = i;
5614 		}
5615 	}
5616 
5617 	if (buf[first].timestamp == 0)
5618 		goto done;	/* nothing in the log */
5619 
5620 	rc = sysctl_wire_old_buffer(req, 0);
5621 	if (rc != 0)
5622 		goto done;
5623 
5624 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5625 	if (sb == NULL) {
5626 		rc = ENOMEM;
5627 		goto done;
5628 	}
5629 	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5630 	    "Seq#", "Tstamp", "Level", "Facility", "Message");
5631 
5632 	i = first;
5633 	do {
5634 		e = &buf[i];
5635 		if (e->timestamp == 0)
5636 			break;	/* end */
5637 
5638 		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5639 		    e->seqno, e->timestamp,
5640 		    (e->level < nitems(devlog_level_strings) ?
5641 			devlog_level_strings[e->level] : "UNKNOWN"),
5642 		    (e->facility < nitems(devlog_facility_strings) ?
5643 			devlog_facility_strings[e->facility] : "UNKNOWN"));
5644 		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5645 		    e->params[2], e->params[3], e->params[4],
5646 		    e->params[5], e->params[6], e->params[7]);
5647 
5648 		if (++i == nentries)
5649 			i = 0;
5650 	} while (i != first);
5651 
5652 	rc = sbuf_finish(sb);
5653 	sbuf_delete(sb);
5654 done:
5655 	free(buf, M_CXGBE);
5656 	return (rc);
5657 }
5658 
5659 static int
5660 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5661 {
5662 	struct adapter *sc = arg1;
5663 	struct sbuf *sb;
5664 	int rc;
5665 	struct tp_fcoe_stats stats[4];
5666 
5667 	rc = sysctl_wire_old_buffer(req, 0);
5668 	if (rc != 0)
5669 		return (rc);
5670 
5671 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5672 	if (sb == NULL)
5673 		return (ENOMEM);
5674 
5675 	t4_get_fcoe_stats(sc, 0, &stats[0]);
5676 	t4_get_fcoe_stats(sc, 1, &stats[1]);
5677 	t4_get_fcoe_stats(sc, 2, &stats[2]);
5678 	t4_get_fcoe_stats(sc, 3, &stats[3]);
5679 
5680 	sbuf_printf(sb, "                   channel 0        channel 1        "
5681 	    "channel 2        channel 3\n");
5682 	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5683 	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5684 	    stats[3].octetsDDP);
5685 	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5686 	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5687 	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5688 	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5689 	    stats[3].framesDrop);
5690 
5691 	rc = sbuf_finish(sb);
5692 	sbuf_delete(sb);
5693 
5694 	return (rc);
5695 }
5696 
5697 static int
5698 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5699 {
5700 	struct adapter *sc = arg1;
5701 	struct sbuf *sb;
5702 	int rc, i;
5703 	unsigned int map, kbps, ipg, mode;
5704 	unsigned int pace_tab[NTX_SCHED];
5705 
5706 	rc = sysctl_wire_old_buffer(req, 0);
5707 	if (rc != 0)
5708 		return (rc);
5709 
5710 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5711 	if (sb == NULL)
5712 		return (ENOMEM);
5713 
5714 	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5715 	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5716 	t4_read_pace_tbl(sc, pace_tab);
5717 
5718 	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5719 	    "Class IPG (0.1 ns)   Flow IPG (us)");
5720 
5721 	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5722 		t4_get_tx_sched(sc, i, &kbps, &ipg);
5723 		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5724 		    (mode & (1 << i)) ? "flow" : "class", map & 3);
5725 		if (kbps)
5726 			sbuf_printf(sb, "%9u     ", kbps);
5727 		else
5728 			sbuf_printf(sb, " disabled     ");
5729 
5730 		if (ipg)
5731 			sbuf_printf(sb, "%13u        ", ipg);
5732 		else
5733 			sbuf_printf(sb, "     disabled        ");
5734 
5735 		if (pace_tab[i])
5736 			sbuf_printf(sb, "%10u", pace_tab[i]);
5737 		else
5738 			sbuf_printf(sb, "  disabled");
5739 	}
5740 
5741 	rc = sbuf_finish(sb);
5742 	sbuf_delete(sb);
5743 
5744 	return (rc);
5745 }
5746 
5747 static int
5748 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5749 {
5750 	struct adapter *sc = arg1;
5751 	struct sbuf *sb;
5752 	int rc, i, j;
5753 	uint64_t *p0, *p1;
5754 	struct lb_port_stats s[2];
5755 	static const char *stat_name[] = {
5756 		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5757 		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5758 		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
5759 		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5760 		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5761 		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5762 		"BG2FramesTrunc:", "BG3FramesTrunc:"
5763 	};
5764 
5765 	rc = sysctl_wire_old_buffer(req, 0);
5766 	if (rc != 0)
5767 		return (rc);
5768 
5769 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5770 	if (sb == NULL)
5771 		return (ENOMEM);
5772 
5773 	memset(s, 0, sizeof(s));
5774 
5775 	for (i = 0; i < 4; i += 2) {
5776 		t4_get_lb_stats(sc, i, &s[0]);
5777 		t4_get_lb_stats(sc, i + 1, &s[1]);
5778 
5779 		p0 = &s[0].octets;
5780 		p1 = &s[1].octets;
5781 		sbuf_printf(sb, "%s                       Loopback %u"
5782 		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5783 
5784 		for (j = 0; j < nitems(stat_name); j++)
5785 			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5786 				   *p0++, *p1++);
5787 	}
5788 
5789 	rc = sbuf_finish(sb);
5790 	sbuf_delete(sb);
5791 
5792 	return (rc);
5793 }
5794 
5795 static int
5796 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5797 {
5798 	int rc = 0;
5799 	struct port_info *pi = arg1;
5800 	struct sbuf *sb;
5801 	static const char *linkdnreasons[] = {
5802 		"non-specific", "remote fault", "autoneg failed", "reserved3",
5803 		"PHY overheated", "unknown", "rx los", "reserved7"
5804 	};
5805 
5806 	rc = sysctl_wire_old_buffer(req, 0);
5807 	if (rc != 0)
5808 		return(rc);
5809 	sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5810 	if (sb == NULL)
5811 		return (ENOMEM);
5812 
5813 	if (pi->linkdnrc < 0)
5814 		sbuf_printf(sb, "n/a");
5815 	else if (pi->linkdnrc < nitems(linkdnreasons))
5816 		sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5817 	else
5818 		sbuf_printf(sb, "%d", pi->linkdnrc);
5819 
5820 	rc = sbuf_finish(sb);
5821 	sbuf_delete(sb);
5822 
5823 	return (rc);
5824 }
5825 
5826 struct mem_desc {
5827 	unsigned int base;
5828 	unsigned int limit;
5829 	unsigned int idx;
5830 };
5831 
5832 static int
5833 mem_desc_cmp(const void *a, const void *b)
5834 {
5835 	return ((const struct mem_desc *)a)->base -
5836 	       ((const struct mem_desc *)b)->base;
5837 }
5838 
5839 static void
5840 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5841     unsigned int to)
5842 {
5843 	unsigned int size;
5844 
5845 	size = to - from + 1;
5846 	if (size == 0)
5847 		return;
5848 
5849 	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5850 	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5851 }
5852 
5853 static int
5854 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5855 {
5856 	struct adapter *sc = arg1;
5857 	struct sbuf *sb;
5858 	int rc, i, n;
5859 	uint32_t lo, hi, used, alloc;
5860 	static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5861 	static const char *region[] = {
5862 		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5863 		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5864 		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5865 		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5866 		"RQUDP region:", "PBL region:", "TXPBL region:",
5867 		"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5868 		"On-chip queues:"
5869 	};
5870 	struct mem_desc avail[4];
5871 	struct mem_desc mem[nitems(region) + 3];	/* up to 3 holes */
5872 	struct mem_desc *md = mem;
5873 
5874 	rc = sysctl_wire_old_buffer(req, 0);
5875 	if (rc != 0)
5876 		return (rc);
5877 
5878 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5879 	if (sb == NULL)
5880 		return (ENOMEM);
5881 
5882 	for (i = 0; i < nitems(mem); i++) {
5883 		mem[i].limit = 0;
5884 		mem[i].idx = i;
5885 	}
5886 
5887 	/* Find and sort the populated memory ranges */
5888 	i = 0;
5889 	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5890 	if (lo & F_EDRAM0_ENABLE) {
5891 		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5892 		avail[i].base = G_EDRAM0_BASE(hi) << 20;
5893 		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5894 		avail[i].idx = 0;
5895 		i++;
5896 	}
5897 	if (lo & F_EDRAM1_ENABLE) {
5898 		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5899 		avail[i].base = G_EDRAM1_BASE(hi) << 20;
5900 		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5901 		avail[i].idx = 1;
5902 		i++;
5903 	}
5904 	if (lo & F_EXT_MEM_ENABLE) {
5905 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5906 		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5907 		avail[i].limit = avail[i].base +
5908 		    (G_EXT_MEM_SIZE(hi) << 20);
5909 		avail[i].idx = is_t4(sc) ? 2 : 3;	/* Call it MC for T4 */
5910 		i++;
5911 	}
5912 	if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5913 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5914 		avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5915 		avail[i].limit = avail[i].base +
5916 		    (G_EXT_MEM1_SIZE(hi) << 20);
5917 		avail[i].idx = 4;
5918 		i++;
5919 	}
5920 	if (!i)                                    /* no memory available */
5921 		return 0;
5922 	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
5923 
5924 	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
5925 	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
5926 	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
5927 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5928 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
5929 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
5930 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
5931 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
5932 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
5933 
5934 	/* the next few have explicit upper bounds */
5935 	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
5936 	md->limit = md->base - 1 +
5937 		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
5938 		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
5939 	md++;
5940 
5941 	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
5942 	md->limit = md->base - 1 +
5943 		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
5944 		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
5945 	md++;
5946 
5947 	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
5948 		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
5949 		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
5950 		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
5951 	} else {
5952 		md->base = 0;
5953 		md->idx = nitems(region);  /* hide it */
5954 	}
5955 	md++;
5956 
5957 #define ulp_region(reg) \
5958 	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
5959 	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
5960 
5961 	ulp_region(RX_ISCSI);
5962 	ulp_region(RX_TDDP);
5963 	ulp_region(TX_TPT);
5964 	ulp_region(RX_STAG);
5965 	ulp_region(RX_RQ);
5966 	ulp_region(RX_RQUDP);
5967 	ulp_region(RX_PBL);
5968 	ulp_region(TX_PBL);
5969 #undef ulp_region
5970 
5971 	md->base = 0;
5972 	md->idx = nitems(region);
5973 	if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
5974 		md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
5975 		md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
5976 		    A_SGE_DBVFIFO_SIZE))) << 2) - 1;
5977 	}
5978 	md++;
5979 
5980 	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
5981 	md->limit = md->base + sc->tids.ntids - 1;
5982 	md++;
5983 	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
5984 	md->limit = md->base + sc->tids.ntids - 1;
5985 	md++;
5986 
5987 	md->base = sc->vres.ocq.start;
5988 	if (sc->vres.ocq.size)
5989 		md->limit = md->base + sc->vres.ocq.size - 1;
5990 	else
5991 		md->idx = nitems(region);  /* hide it */
5992 	md++;
5993 
5994 	/* add any address-space holes, there can be up to 3 */
5995 	for (n = 0; n < i - 1; n++)
5996 		if (avail[n].limit < avail[n + 1].base)
5997 			(md++)->base = avail[n].limit;
5998 	if (avail[n].limit)
5999 		(md++)->base = avail[n].limit;
6000 
6001 	n = md - mem;
6002 	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6003 
6004 	for (lo = 0; lo < i; lo++)
6005 		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6006 				avail[lo].limit - 1);
6007 
6008 	sbuf_printf(sb, "\n");
6009 	for (i = 0; i < n; i++) {
6010 		if (mem[i].idx >= nitems(region))
6011 			continue;                        /* skip holes */
6012 		if (!mem[i].limit)
6013 			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6014 		mem_region_show(sb, region[mem[i].idx], mem[i].base,
6015 				mem[i].limit);
6016 	}
6017 
6018 	sbuf_printf(sb, "\n");
6019 	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6020 	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6021 	mem_region_show(sb, "uP RAM:", lo, hi);
6022 
6023 	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6024 	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6025 	mem_region_show(sb, "uP Extmem2:", lo, hi);
6026 
6027 	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6028 	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6029 		   G_PMRXMAXPAGE(lo),
6030 		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6031 		   (lo & F_PMRXNUMCHN) ? 2 : 1);
6032 
6033 	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6034 	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6035 	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6036 		   G_PMTXMAXPAGE(lo),
6037 		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6038 		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6039 	sbuf_printf(sb, "%u p-structs\n",
6040 		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6041 
6042 	for (i = 0; i < 4; i++) {
6043 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6044 		if (is_t4(sc)) {
6045 			used = G_USED(lo);
6046 			alloc = G_ALLOC(lo);
6047 		} else {
6048 			used = G_T5_USED(lo);
6049 			alloc = G_T5_ALLOC(lo);
6050 		}
6051 		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6052 			   i, used, alloc);
6053 	}
6054 	for (i = 0; i < 4; i++) {
6055 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6056 		if (is_t4(sc)) {
6057 			used = G_USED(lo);
6058 			alloc = G_ALLOC(lo);
6059 		} else {
6060 			used = G_T5_USED(lo);
6061 			alloc = G_T5_ALLOC(lo);
6062 		}
6063 		sbuf_printf(sb,
6064 			   "\nLoopback %d using %u pages out of %u allocated",
6065 			   i, used, alloc);
6066 	}
6067 
6068 	rc = sbuf_finish(sb);
6069 	sbuf_delete(sb);
6070 
6071 	return (rc);
6072 }
6073 
6074 static inline void
6075 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6076 {
6077 	*mask = x | y;
6078 	y = htobe64(y);
6079 	memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6080 }
6081 
6082 static int
6083 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6084 {
6085 	struct adapter *sc = arg1;
6086 	struct sbuf *sb;
6087 	int rc, i, n;
6088 
6089 	rc = sysctl_wire_old_buffer(req, 0);
6090 	if (rc != 0)
6091 		return (rc);
6092 
6093 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6094 	if (sb == NULL)
6095 		return (ENOMEM);
6096 
6097 	sbuf_printf(sb,
6098 	    "Idx  Ethernet address     Mask     Vld Ports PF"
6099 	    "  VF              Replication             P0 P1 P2 P3  ML");
6100 	n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6101 	    NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6102 	for (i = 0; i < n; i++) {
6103 		uint64_t tcamx, tcamy, mask;
6104 		uint32_t cls_lo, cls_hi;
6105 		uint8_t addr[ETHER_ADDR_LEN];
6106 
6107 		tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6108 		tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6109 		cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6110 		cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6111 
6112 		if (tcamx & tcamy)
6113 			continue;
6114 
6115 		tcamxy2valmask(tcamx, tcamy, addr, &mask);
6116 		sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6117 			   "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6118 			   addr[3], addr[4], addr[5], (uintmax_t)mask,
6119 			   (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6120 			   G_PORTMAP(cls_hi), G_PF(cls_lo),
6121 			   (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6122 
6123 		if (cls_lo & F_REPLICATE) {
6124 			struct fw_ldst_cmd ldst_cmd;
6125 
6126 			memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6127 			ldst_cmd.op_to_addrspace =
6128 			    htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6129 				F_FW_CMD_REQUEST | F_FW_CMD_READ |
6130 				V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6131 			ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6132 			ldst_cmd.u.mps.fid_ctl =
6133 			    htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6134 				V_FW_LDST_CMD_CTL(i));
6135 
6136 			rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6137 			    "t4mps");
6138 			if (rc)
6139 				break;
6140 			rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6141 			    sizeof(ldst_cmd), &ldst_cmd);
6142 			end_synchronized_op(sc, 0);
6143 
6144 			if (rc != 0) {
6145 				sbuf_printf(sb,
6146 				    " ------------ error %3u ------------", rc);
6147 				rc = 0;
6148 			} else {
6149 				sbuf_printf(sb, " %08x %08x %08x %08x",
6150 				    be32toh(ldst_cmd.u.mps.rplc127_96),
6151 				    be32toh(ldst_cmd.u.mps.rplc95_64),
6152 				    be32toh(ldst_cmd.u.mps.rplc63_32),
6153 				    be32toh(ldst_cmd.u.mps.rplc31_0));
6154 			}
6155 		} else
6156 			sbuf_printf(sb, "%36s", "");
6157 
6158 		sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6159 		    G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6160 		    G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6161 	}
6162 
6163 	if (rc)
6164 		(void) sbuf_finish(sb);
6165 	else
6166 		rc = sbuf_finish(sb);
6167 	sbuf_delete(sb);
6168 
6169 	return (rc);
6170 }
6171 
6172 static int
6173 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6174 {
6175 	struct adapter *sc = arg1;
6176 	struct sbuf *sb;
6177 	int rc;
6178 	uint16_t mtus[NMTUS];
6179 
6180 	rc = sysctl_wire_old_buffer(req, 0);
6181 	if (rc != 0)
6182 		return (rc);
6183 
6184 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6185 	if (sb == NULL)
6186 		return (ENOMEM);
6187 
6188 	t4_read_mtu_tbl(sc, mtus, NULL);
6189 
6190 	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6191 	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6192 	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6193 	    mtus[14], mtus[15]);
6194 
6195 	rc = sbuf_finish(sb);
6196 	sbuf_delete(sb);
6197 
6198 	return (rc);
6199 }
6200 
6201 static int
6202 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6203 {
6204 	struct adapter *sc = arg1;
6205 	struct sbuf *sb;
6206 	int rc, i;
6207 	uint32_t cnt[PM_NSTATS];
6208 	uint64_t cyc[PM_NSTATS];
6209 	static const char *rx_stats[] = {
6210 		"Read:", "Write bypass:", "Write mem:", "Flush:"
6211 	};
6212 	static const char *tx_stats[] = {
6213 		"Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6214 	};
6215 
6216 	rc = sysctl_wire_old_buffer(req, 0);
6217 	if (rc != 0)
6218 		return (rc);
6219 
6220 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6221 	if (sb == NULL)
6222 		return (ENOMEM);
6223 
6224 	t4_pmtx_get_stats(sc, cnt, cyc);
6225 	sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6226 	for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6227 		sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6228 		    cyc[i]);
6229 
6230 	t4_pmrx_get_stats(sc, cnt, cyc);
6231 	sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6232 	for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6233 		sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6234 		    cyc[i]);
6235 
6236 	rc = sbuf_finish(sb);
6237 	sbuf_delete(sb);
6238 
6239 	return (rc);
6240 }
6241 
6242 static int
6243 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6244 {
6245 	struct adapter *sc = arg1;
6246 	struct sbuf *sb;
6247 	int rc;
6248 	struct tp_rdma_stats stats;
6249 
6250 	rc = sysctl_wire_old_buffer(req, 0);
6251 	if (rc != 0)
6252 		return (rc);
6253 
6254 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6255 	if (sb == NULL)
6256 		return (ENOMEM);
6257 
6258 	t4_tp_get_rdma_stats(sc, &stats);
6259 	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6260 	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6261 
6262 	rc = sbuf_finish(sb);
6263 	sbuf_delete(sb);
6264 
6265 	return (rc);
6266 }
6267 
6268 static int
6269 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6270 {
6271 	struct adapter *sc = arg1;
6272 	struct sbuf *sb;
6273 	int rc;
6274 	struct tp_tcp_stats v4, v6;
6275 
6276 	rc = sysctl_wire_old_buffer(req, 0);
6277 	if (rc != 0)
6278 		return (rc);
6279 
6280 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6281 	if (sb == NULL)
6282 		return (ENOMEM);
6283 
6284 	t4_tp_get_tcp_stats(sc, &v4, &v6);
6285 	sbuf_printf(sb,
6286 	    "                                IP                 IPv6\n");
6287 	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6288 	    v4.tcpOutRsts, v6.tcpOutRsts);
6289 	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6290 	    v4.tcpInSegs, v6.tcpInSegs);
6291 	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6292 	    v4.tcpOutSegs, v6.tcpOutSegs);
6293 	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6294 	    v4.tcpRetransSegs, v6.tcpRetransSegs);
6295 
6296 	rc = sbuf_finish(sb);
6297 	sbuf_delete(sb);
6298 
6299 	return (rc);
6300 }
6301 
6302 static int
6303 sysctl_tids(SYSCTL_HANDLER_ARGS)
6304 {
6305 	struct adapter *sc = arg1;
6306 	struct sbuf *sb;
6307 	int rc;
6308 	struct tid_info *t = &sc->tids;
6309 
6310 	rc = sysctl_wire_old_buffer(req, 0);
6311 	if (rc != 0)
6312 		return (rc);
6313 
6314 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6315 	if (sb == NULL)
6316 		return (ENOMEM);
6317 
6318 	if (t->natids) {
6319 		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6320 		    t->atids_in_use);
6321 	}
6322 
6323 	if (t->ntids) {
6324 		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6325 			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6326 
6327 			if (b) {
6328 				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6329 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6330 				    t->ntids - 1);
6331 			} else {
6332 				sbuf_printf(sb, "TID range: %u-%u",
6333 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6334 				    t->ntids - 1);
6335 			}
6336 		} else
6337 			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6338 		sbuf_printf(sb, ", in use: %u\n",
6339 		    atomic_load_acq_int(&t->tids_in_use));
6340 	}
6341 
6342 	if (t->nstids) {
6343 		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6344 		    t->stid_base + t->nstids - 1, t->stids_in_use);
6345 	}
6346 
6347 	if (t->nftids) {
6348 		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6349 		    t->ftid_base + t->nftids - 1);
6350 	}
6351 
6352 	if (t->netids) {
6353 		sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6354 		    t->etid_base + t->netids - 1);
6355 	}
6356 
6357 	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6358 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6359 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6360 
6361 	rc = sbuf_finish(sb);
6362 	sbuf_delete(sb);
6363 
6364 	return (rc);
6365 }
6366 
6367 static int
6368 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6369 {
6370 	struct adapter *sc = arg1;
6371 	struct sbuf *sb;
6372 	int rc;
6373 	struct tp_err_stats stats;
6374 
6375 	rc = sysctl_wire_old_buffer(req, 0);
6376 	if (rc != 0)
6377 		return (rc);
6378 
6379 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6380 	if (sb == NULL)
6381 		return (ENOMEM);
6382 
6383 	t4_tp_get_err_stats(sc, &stats);
6384 
6385 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6386 		      "channel 3\n");
6387 	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6388 	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6389 	    stats.macInErrs[3]);
6390 	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6391 	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6392 	    stats.hdrInErrs[3]);
6393 	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6394 	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6395 	    stats.tcpInErrs[3]);
6396 	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6397 	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6398 	    stats.tcp6InErrs[3]);
6399 	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6400 	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6401 	    stats.tnlCongDrops[3]);
6402 	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6403 	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6404 	    stats.tnlTxDrops[3]);
6405 	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6406 	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6407 	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6408 	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6409 	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6410 	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6411 	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6412 	    stats.ofldNoNeigh, stats.ofldCongDefer);
6413 
6414 	rc = sbuf_finish(sb);
6415 	sbuf_delete(sb);
6416 
6417 	return (rc);
6418 }
6419 
6420 struct field_desc {
6421 	const char *name;
6422 	u_int start;
6423 	u_int width;
6424 };
6425 
6426 static void
6427 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6428 {
6429 	char buf[32];
6430 	int line_size = 0;
6431 
6432 	while (f->name) {
6433 		uint64_t mask = (1ULL << f->width) - 1;
6434 		int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6435 		    ((uintmax_t)v >> f->start) & mask);
6436 
6437 		if (line_size + len >= 79) {
6438 			line_size = 8;
6439 			sbuf_printf(sb, "\n        ");
6440 		}
6441 		sbuf_printf(sb, "%s ", buf);
6442 		line_size += len + 1;
6443 		f++;
6444 	}
6445 	sbuf_printf(sb, "\n");
6446 }
6447 
6448 static struct field_desc tp_la0[] = {
6449 	{ "RcfOpCodeOut", 60, 4 },
6450 	{ "State", 56, 4 },
6451 	{ "WcfState", 52, 4 },
6452 	{ "RcfOpcSrcOut", 50, 2 },
6453 	{ "CRxError", 49, 1 },
6454 	{ "ERxError", 48, 1 },
6455 	{ "SanityFailed", 47, 1 },
6456 	{ "SpuriousMsg", 46, 1 },
6457 	{ "FlushInputMsg", 45, 1 },
6458 	{ "FlushInputCpl", 44, 1 },
6459 	{ "RssUpBit", 43, 1 },
6460 	{ "RssFilterHit", 42, 1 },
6461 	{ "Tid", 32, 10 },
6462 	{ "InitTcb", 31, 1 },
6463 	{ "LineNumber", 24, 7 },
6464 	{ "Emsg", 23, 1 },
6465 	{ "EdataOut", 22, 1 },
6466 	{ "Cmsg", 21, 1 },
6467 	{ "CdataOut", 20, 1 },
6468 	{ "EreadPdu", 19, 1 },
6469 	{ "CreadPdu", 18, 1 },
6470 	{ "TunnelPkt", 17, 1 },
6471 	{ "RcfPeerFin", 16, 1 },
6472 	{ "RcfReasonOut", 12, 4 },
6473 	{ "TxCchannel", 10, 2 },
6474 	{ "RcfTxChannel", 8, 2 },
6475 	{ "RxEchannel", 6, 2 },
6476 	{ "RcfRxChannel", 5, 1 },
6477 	{ "RcfDataOutSrdy", 4, 1 },
6478 	{ "RxDvld", 3, 1 },
6479 	{ "RxOoDvld", 2, 1 },
6480 	{ "RxCongestion", 1, 1 },
6481 	{ "TxCongestion", 0, 1 },
6482 	{ NULL }
6483 };
6484 
6485 static struct field_desc tp_la1[] = {
6486 	{ "CplCmdIn", 56, 8 },
6487 	{ "CplCmdOut", 48, 8 },
6488 	{ "ESynOut", 47, 1 },
6489 	{ "EAckOut", 46, 1 },
6490 	{ "EFinOut", 45, 1 },
6491 	{ "ERstOut", 44, 1 },
6492 	{ "SynIn", 43, 1 },
6493 	{ "AckIn", 42, 1 },
6494 	{ "FinIn", 41, 1 },
6495 	{ "RstIn", 40, 1 },
6496 	{ "DataIn", 39, 1 },
6497 	{ "DataInVld", 38, 1 },
6498 	{ "PadIn", 37, 1 },
6499 	{ "RxBufEmpty", 36, 1 },
6500 	{ "RxDdp", 35, 1 },
6501 	{ "RxFbCongestion", 34, 1 },
6502 	{ "TxFbCongestion", 33, 1 },
6503 	{ "TxPktSumSrdy", 32, 1 },
6504 	{ "RcfUlpType", 28, 4 },
6505 	{ "Eread", 27, 1 },
6506 	{ "Ebypass", 26, 1 },
6507 	{ "Esave", 25, 1 },
6508 	{ "Static0", 24, 1 },
6509 	{ "Cread", 23, 1 },
6510 	{ "Cbypass", 22, 1 },
6511 	{ "Csave", 21, 1 },
6512 	{ "CPktOut", 20, 1 },
6513 	{ "RxPagePoolFull", 18, 2 },
6514 	{ "RxLpbkPkt", 17, 1 },
6515 	{ "TxLpbkPkt", 16, 1 },
6516 	{ "RxVfValid", 15, 1 },
6517 	{ "SynLearned", 14, 1 },
6518 	{ "SetDelEntry", 13, 1 },
6519 	{ "SetInvEntry", 12, 1 },
6520 	{ "CpcmdDvld", 11, 1 },
6521 	{ "CpcmdSave", 10, 1 },
6522 	{ "RxPstructsFull", 8, 2 },
6523 	{ "EpcmdDvld", 7, 1 },
6524 	{ "EpcmdFlush", 6, 1 },
6525 	{ "EpcmdTrimPrefix", 5, 1 },
6526 	{ "EpcmdTrimPostfix", 4, 1 },
6527 	{ "ERssIp4Pkt", 3, 1 },
6528 	{ "ERssIp6Pkt", 2, 1 },
6529 	{ "ERssTcpUdpPkt", 1, 1 },
6530 	{ "ERssFceFipPkt", 0, 1 },
6531 	{ NULL }
6532 };
6533 
6534 static struct field_desc tp_la2[] = {
6535 	{ "CplCmdIn", 56, 8 },
6536 	{ "MpsVfVld", 55, 1 },
6537 	{ "MpsPf", 52, 3 },
6538 	{ "MpsVf", 44, 8 },
6539 	{ "SynIn", 43, 1 },
6540 	{ "AckIn", 42, 1 },
6541 	{ "FinIn", 41, 1 },
6542 	{ "RstIn", 40, 1 },
6543 	{ "DataIn", 39, 1 },
6544 	{ "DataInVld", 38, 1 },
6545 	{ "PadIn", 37, 1 },
6546 	{ "RxBufEmpty", 36, 1 },
6547 	{ "RxDdp", 35, 1 },
6548 	{ "RxFbCongestion", 34, 1 },
6549 	{ "TxFbCongestion", 33, 1 },
6550 	{ "TxPktSumSrdy", 32, 1 },
6551 	{ "RcfUlpType", 28, 4 },
6552 	{ "Eread", 27, 1 },
6553 	{ "Ebypass", 26, 1 },
6554 	{ "Esave", 25, 1 },
6555 	{ "Static0", 24, 1 },
6556 	{ "Cread", 23, 1 },
6557 	{ "Cbypass", 22, 1 },
6558 	{ "Csave", 21, 1 },
6559 	{ "CPktOut", 20, 1 },
6560 	{ "RxPagePoolFull", 18, 2 },
6561 	{ "RxLpbkPkt", 17, 1 },
6562 	{ "TxLpbkPkt", 16, 1 },
6563 	{ "RxVfValid", 15, 1 },
6564 	{ "SynLearned", 14, 1 },
6565 	{ "SetDelEntry", 13, 1 },
6566 	{ "SetInvEntry", 12, 1 },
6567 	{ "CpcmdDvld", 11, 1 },
6568 	{ "CpcmdSave", 10, 1 },
6569 	{ "RxPstructsFull", 8, 2 },
6570 	{ "EpcmdDvld", 7, 1 },
6571 	{ "EpcmdFlush", 6, 1 },
6572 	{ "EpcmdTrimPrefix", 5, 1 },
6573 	{ "EpcmdTrimPostfix", 4, 1 },
6574 	{ "ERssIp4Pkt", 3, 1 },
6575 	{ "ERssIp6Pkt", 2, 1 },
6576 	{ "ERssTcpUdpPkt", 1, 1 },
6577 	{ "ERssFceFipPkt", 0, 1 },
6578 	{ NULL }
6579 };
6580 
6581 static void
6582 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6583 {
6584 
6585 	field_desc_show(sb, *p, tp_la0);
6586 }
6587 
6588 static void
6589 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6590 {
6591 
6592 	if (idx)
6593 		sbuf_printf(sb, "\n");
6594 	field_desc_show(sb, p[0], tp_la0);
6595 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6596 		field_desc_show(sb, p[1], tp_la0);
6597 }
6598 
6599 static void
6600 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6601 {
6602 
6603 	if (idx)
6604 		sbuf_printf(sb, "\n");
6605 	field_desc_show(sb, p[0], tp_la0);
6606 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6607 		field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6608 }
6609 
6610 static int
6611 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6612 {
6613 	struct adapter *sc = arg1;
6614 	struct sbuf *sb;
6615 	uint64_t *buf, *p;
6616 	int rc;
6617 	u_int i, inc;
6618 	void (*show_func)(struct sbuf *, uint64_t *, int);
6619 
6620 	rc = sysctl_wire_old_buffer(req, 0);
6621 	if (rc != 0)
6622 		return (rc);
6623 
6624 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6625 	if (sb == NULL)
6626 		return (ENOMEM);
6627 
6628 	buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6629 
6630 	t4_tp_read_la(sc, buf, NULL);
6631 	p = buf;
6632 
6633 	switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6634 	case 2:
6635 		inc = 2;
6636 		show_func = tp_la_show2;
6637 		break;
6638 	case 3:
6639 		inc = 2;
6640 		show_func = tp_la_show3;
6641 		break;
6642 	default:
6643 		inc = 1;
6644 		show_func = tp_la_show;
6645 	}
6646 
6647 	for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6648 		(*show_func)(sb, p, i);
6649 
6650 	rc = sbuf_finish(sb);
6651 	sbuf_delete(sb);
6652 	free(buf, M_CXGBE);
6653 	return (rc);
6654 }
6655 
6656 static int
6657 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6658 {
6659 	struct adapter *sc = arg1;
6660 	struct sbuf *sb;
6661 	int rc;
6662 	u64 nrate[NCHAN], orate[NCHAN];
6663 
6664 	rc = sysctl_wire_old_buffer(req, 0);
6665 	if (rc != 0)
6666 		return (rc);
6667 
6668 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6669 	if (sb == NULL)
6670 		return (ENOMEM);
6671 
6672 	t4_get_chan_txrate(sc, nrate, orate);
6673 	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6674 		 "channel 3\n");
6675 	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6676 	    nrate[0], nrate[1], nrate[2], nrate[3]);
6677 	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6678 	    orate[0], orate[1], orate[2], orate[3]);
6679 
6680 	rc = sbuf_finish(sb);
6681 	sbuf_delete(sb);
6682 
6683 	return (rc);
6684 }
6685 
6686 static int
6687 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6688 {
6689 	struct adapter *sc = arg1;
6690 	struct sbuf *sb;
6691 	uint32_t *buf, *p;
6692 	int rc, i;
6693 
6694 	rc = sysctl_wire_old_buffer(req, 0);
6695 	if (rc != 0)
6696 		return (rc);
6697 
6698 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6699 	if (sb == NULL)
6700 		return (ENOMEM);
6701 
6702 	buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6703 	    M_ZERO | M_WAITOK);
6704 
6705 	t4_ulprx_read_la(sc, buf);
6706 	p = buf;
6707 
6708 	sbuf_printf(sb, "      Pcmd        Type   Message"
6709 	    "                Data");
6710 	for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6711 		sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6712 		    p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6713 	}
6714 
6715 	rc = sbuf_finish(sb);
6716 	sbuf_delete(sb);
6717 	free(buf, M_CXGBE);
6718 	return (rc);
6719 }
6720 
6721 static int
6722 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6723 {
6724 	struct adapter *sc = arg1;
6725 	struct sbuf *sb;
6726 	int rc, v;
6727 
6728 	rc = sysctl_wire_old_buffer(req, 0);
6729 	if (rc != 0)
6730 		return (rc);
6731 
6732 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6733 	if (sb == NULL)
6734 		return (ENOMEM);
6735 
6736 	v = t4_read_reg(sc, A_SGE_STAT_CFG);
6737 	if (G_STATSOURCE_T5(v) == 7) {
6738 		if (G_STATMODE(v) == 0) {
6739 			sbuf_printf(sb, "total %d, incomplete %d",
6740 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
6741 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
6742 		} else if (G_STATMODE(v) == 1) {
6743 			sbuf_printf(sb, "total %d, data overflow %d",
6744 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
6745 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
6746 		}
6747 	}
6748 	rc = sbuf_finish(sb);
6749 	sbuf_delete(sb);
6750 
6751 	return (rc);
6752 }
6753 #endif
6754 
6755 static inline void
6756 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6757 {
6758 	struct buf_ring *br;
6759 	struct mbuf *m;
6760 
6761 	TXQ_LOCK_ASSERT_OWNED(txq);
6762 
6763 	br = txq->br;
6764 	m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6765 	if (m)
6766 		t4_eth_tx(ifp, txq, m);
6767 }
6768 
6769 void
6770 t4_tx_callout(void *arg)
6771 {
6772 	struct sge_eq *eq = arg;
6773 	struct adapter *sc;
6774 
6775 	if (EQ_TRYLOCK(eq) == 0)
6776 		goto reschedule;
6777 
6778 	if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6779 		EQ_UNLOCK(eq);
6780 reschedule:
6781 		if (__predict_true(!(eq->flags && EQ_DOOMED)))
6782 			callout_schedule(&eq->tx_callout, 1);
6783 		return;
6784 	}
6785 
6786 	EQ_LOCK_ASSERT_OWNED(eq);
6787 
6788 	if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6789 
6790 		if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6791 			struct sge_txq *txq = arg;
6792 			struct port_info *pi = txq->ifp->if_softc;
6793 
6794 			sc = pi->adapter;
6795 		} else {
6796 			struct sge_wrq *wrq = arg;
6797 
6798 			sc = wrq->adapter;
6799 		}
6800 
6801 		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6802 	}
6803 
6804 	EQ_UNLOCK(eq);
6805 }
6806 
6807 void
6808 t4_tx_task(void *arg, int count)
6809 {
6810 	struct sge_eq *eq = arg;
6811 
6812 	EQ_LOCK(eq);
6813 	if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6814 		struct sge_txq *txq = arg;
6815 		txq_start(txq->ifp, txq);
6816 	} else {
6817 		struct sge_wrq *wrq = arg;
6818 		t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6819 	}
6820 	EQ_UNLOCK(eq);
6821 }
6822 
6823 static uint32_t
6824 fconf_to_mode(uint32_t fconf)
6825 {
6826 	uint32_t mode;
6827 
6828 	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6829 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6830 
6831 	if (fconf & F_FRAGMENTATION)
6832 		mode |= T4_FILTER_IP_FRAGMENT;
6833 
6834 	if (fconf & F_MPSHITTYPE)
6835 		mode |= T4_FILTER_MPS_HIT_TYPE;
6836 
6837 	if (fconf & F_MACMATCH)
6838 		mode |= T4_FILTER_MAC_IDX;
6839 
6840 	if (fconf & F_ETHERTYPE)
6841 		mode |= T4_FILTER_ETH_TYPE;
6842 
6843 	if (fconf & F_PROTOCOL)
6844 		mode |= T4_FILTER_IP_PROTO;
6845 
6846 	if (fconf & F_TOS)
6847 		mode |= T4_FILTER_IP_TOS;
6848 
6849 	if (fconf & F_VLAN)
6850 		mode |= T4_FILTER_VLAN;
6851 
6852 	if (fconf & F_VNIC_ID)
6853 		mode |= T4_FILTER_VNIC;
6854 
6855 	if (fconf & F_PORT)
6856 		mode |= T4_FILTER_PORT;
6857 
6858 	if (fconf & F_FCOE)
6859 		mode |= T4_FILTER_FCoE;
6860 
6861 	return (mode);
6862 }
6863 
6864 static uint32_t
6865 mode_to_fconf(uint32_t mode)
6866 {
6867 	uint32_t fconf = 0;
6868 
6869 	if (mode & T4_FILTER_IP_FRAGMENT)
6870 		fconf |= F_FRAGMENTATION;
6871 
6872 	if (mode & T4_FILTER_MPS_HIT_TYPE)
6873 		fconf |= F_MPSHITTYPE;
6874 
6875 	if (mode & T4_FILTER_MAC_IDX)
6876 		fconf |= F_MACMATCH;
6877 
6878 	if (mode & T4_FILTER_ETH_TYPE)
6879 		fconf |= F_ETHERTYPE;
6880 
6881 	if (mode & T4_FILTER_IP_PROTO)
6882 		fconf |= F_PROTOCOL;
6883 
6884 	if (mode & T4_FILTER_IP_TOS)
6885 		fconf |= F_TOS;
6886 
6887 	if (mode & T4_FILTER_VLAN)
6888 		fconf |= F_VLAN;
6889 
6890 	if (mode & T4_FILTER_VNIC)
6891 		fconf |= F_VNIC_ID;
6892 
6893 	if (mode & T4_FILTER_PORT)
6894 		fconf |= F_PORT;
6895 
6896 	if (mode & T4_FILTER_FCoE)
6897 		fconf |= F_FCOE;
6898 
6899 	return (fconf);
6900 }
6901 
6902 static uint32_t
6903 fspec_to_fconf(struct t4_filter_specification *fs)
6904 {
6905 	uint32_t fconf = 0;
6906 
6907 	if (fs->val.frag || fs->mask.frag)
6908 		fconf |= F_FRAGMENTATION;
6909 
6910 	if (fs->val.matchtype || fs->mask.matchtype)
6911 		fconf |= F_MPSHITTYPE;
6912 
6913 	if (fs->val.macidx || fs->mask.macidx)
6914 		fconf |= F_MACMATCH;
6915 
6916 	if (fs->val.ethtype || fs->mask.ethtype)
6917 		fconf |= F_ETHERTYPE;
6918 
6919 	if (fs->val.proto || fs->mask.proto)
6920 		fconf |= F_PROTOCOL;
6921 
6922 	if (fs->val.tos || fs->mask.tos)
6923 		fconf |= F_TOS;
6924 
6925 	if (fs->val.vlan_vld || fs->mask.vlan_vld)
6926 		fconf |= F_VLAN;
6927 
6928 	if (fs->val.vnic_vld || fs->mask.vnic_vld)
6929 		fconf |= F_VNIC_ID;
6930 
6931 	if (fs->val.iport || fs->mask.iport)
6932 		fconf |= F_PORT;
6933 
6934 	if (fs->val.fcoe || fs->mask.fcoe)
6935 		fconf |= F_FCOE;
6936 
6937 	return (fconf);
6938 }
6939 
6940 static int
6941 get_filter_mode(struct adapter *sc, uint32_t *mode)
6942 {
6943 	int rc;
6944 	uint32_t fconf;
6945 
6946 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6947 	    "t4getfm");
6948 	if (rc)
6949 		return (rc);
6950 
6951 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
6952 	    A_TP_VLAN_PRI_MAP);
6953 
6954 	if (sc->params.tp.vlan_pri_map != fconf) {
6955 		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
6956 		    device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
6957 		    fconf);
6958 		sc->params.tp.vlan_pri_map = fconf;
6959 	}
6960 
6961 	*mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
6962 
6963 	end_synchronized_op(sc, LOCK_HELD);
6964 	return (0);
6965 }
6966 
6967 static int
6968 set_filter_mode(struct adapter *sc, uint32_t mode)
6969 {
6970 	uint32_t fconf;
6971 	int rc;
6972 
6973 	fconf = mode_to_fconf(mode);
6974 
6975 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6976 	    "t4setfm");
6977 	if (rc)
6978 		return (rc);
6979 
6980 	if (sc->tids.ftids_in_use > 0) {
6981 		rc = EBUSY;
6982 		goto done;
6983 	}
6984 
6985 #ifdef TCP_OFFLOAD
6986 	if (sc->offload_map) {
6987 		rc = EBUSY;
6988 		goto done;
6989 	}
6990 #endif
6991 
6992 #ifdef notyet
6993 	rc = -t4_set_filter_mode(sc, fconf);
6994 	if (rc == 0)
6995 		sc->filter_mode = fconf;
6996 #else
6997 	rc = ENOTSUP;
6998 #endif
6999 
7000 done:
7001 	end_synchronized_op(sc, LOCK_HELD);
7002 	return (rc);
7003 }
7004 
7005 static inline uint64_t
7006 get_filter_hits(struct adapter *sc, uint32_t fid)
7007 {
7008 	uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7009 	uint64_t hits;
7010 
7011 	memwin_info(sc, 0, &mw_base, NULL);
7012 	off = position_memwin(sc, 0,
7013 	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7014 	if (is_t4(sc)) {
7015 		hits = t4_read_reg64(sc, mw_base + off + 16);
7016 		hits = be64toh(hits);
7017 	} else {
7018 		hits = t4_read_reg(sc, mw_base + off + 24);
7019 		hits = be32toh(hits);
7020 	}
7021 
7022 	return (hits);
7023 }
7024 
7025 static int
7026 get_filter(struct adapter *sc, struct t4_filter *t)
7027 {
7028 	int i, rc, nfilters = sc->tids.nftids;
7029 	struct filter_entry *f;
7030 
7031 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7032 	    "t4getf");
7033 	if (rc)
7034 		return (rc);
7035 
7036 	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7037 	    t->idx >= nfilters) {
7038 		t->idx = 0xffffffff;
7039 		goto done;
7040 	}
7041 
7042 	f = &sc->tids.ftid_tab[t->idx];
7043 	for (i = t->idx; i < nfilters; i++, f++) {
7044 		if (f->valid) {
7045 			t->idx = i;
7046 			t->l2tidx = f->l2t ? f->l2t->idx : 0;
7047 			t->smtidx = f->smtidx;
7048 			if (f->fs.hitcnts)
7049 				t->hits = get_filter_hits(sc, t->idx);
7050 			else
7051 				t->hits = UINT64_MAX;
7052 			t->fs = f->fs;
7053 
7054 			goto done;
7055 		}
7056 	}
7057 
7058 	t->idx = 0xffffffff;
7059 done:
7060 	end_synchronized_op(sc, LOCK_HELD);
7061 	return (0);
7062 }
7063 
7064 static int
7065 set_filter(struct adapter *sc, struct t4_filter *t)
7066 {
7067 	unsigned int nfilters, nports;
7068 	struct filter_entry *f;
7069 	int i, rc;
7070 
7071 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7072 	if (rc)
7073 		return (rc);
7074 
7075 	nfilters = sc->tids.nftids;
7076 	nports = sc->params.nports;
7077 
7078 	if (nfilters == 0) {
7079 		rc = ENOTSUP;
7080 		goto done;
7081 	}
7082 
7083 	if (!(sc->flags & FULL_INIT_DONE)) {
7084 		rc = EAGAIN;
7085 		goto done;
7086 	}
7087 
7088 	if (t->idx >= nfilters) {
7089 		rc = EINVAL;
7090 		goto done;
7091 	}
7092 
7093 	/* Validate against the global filter mode */
7094 	if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7095 	    sc->params.tp.vlan_pri_map) {
7096 		rc = E2BIG;
7097 		goto done;
7098 	}
7099 
7100 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7101 		rc = EINVAL;
7102 		goto done;
7103 	}
7104 
7105 	if (t->fs.val.iport >= nports) {
7106 		rc = EINVAL;
7107 		goto done;
7108 	}
7109 
7110 	/* Can't specify an iq if not steering to it */
7111 	if (!t->fs.dirsteer && t->fs.iq) {
7112 		rc = EINVAL;
7113 		goto done;
7114 	}
7115 
7116 	/* IPv6 filter idx must be 4 aligned */
7117 	if (t->fs.type == 1 &&
7118 	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7119 		rc = EINVAL;
7120 		goto done;
7121 	}
7122 
7123 	if (sc->tids.ftid_tab == NULL) {
7124 		KASSERT(sc->tids.ftids_in_use == 0,
7125 		    ("%s: no memory allocated but filters_in_use > 0",
7126 		    __func__));
7127 
7128 		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7129 		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7130 		if (sc->tids.ftid_tab == NULL) {
7131 			rc = ENOMEM;
7132 			goto done;
7133 		}
7134 		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7135 	}
7136 
7137 	for (i = 0; i < 4; i++) {
7138 		f = &sc->tids.ftid_tab[t->idx + i];
7139 
7140 		if (f->pending || f->valid) {
7141 			rc = EBUSY;
7142 			goto done;
7143 		}
7144 		if (f->locked) {
7145 			rc = EPERM;
7146 			goto done;
7147 		}
7148 
7149 		if (t->fs.type == 0)
7150 			break;
7151 	}
7152 
7153 	f = &sc->tids.ftid_tab[t->idx];
7154 	f->fs = t->fs;
7155 
7156 	rc = set_filter_wr(sc, t->idx);
7157 done:
7158 	end_synchronized_op(sc, 0);
7159 
7160 	if (rc == 0) {
7161 		mtx_lock(&sc->tids.ftid_lock);
7162 		for (;;) {
7163 			if (f->pending == 0) {
7164 				rc = f->valid ? 0 : EIO;
7165 				break;
7166 			}
7167 
7168 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7169 			    PCATCH, "t4setfw", 0)) {
7170 				rc = EINPROGRESS;
7171 				break;
7172 			}
7173 		}
7174 		mtx_unlock(&sc->tids.ftid_lock);
7175 	}
7176 	return (rc);
7177 }
7178 
7179 static int
7180 del_filter(struct adapter *sc, struct t4_filter *t)
7181 {
7182 	unsigned int nfilters;
7183 	struct filter_entry *f;
7184 	int rc;
7185 
7186 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7187 	if (rc)
7188 		return (rc);
7189 
7190 	nfilters = sc->tids.nftids;
7191 
7192 	if (nfilters == 0) {
7193 		rc = ENOTSUP;
7194 		goto done;
7195 	}
7196 
7197 	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7198 	    t->idx >= nfilters) {
7199 		rc = EINVAL;
7200 		goto done;
7201 	}
7202 
7203 	if (!(sc->flags & FULL_INIT_DONE)) {
7204 		rc = EAGAIN;
7205 		goto done;
7206 	}
7207 
7208 	f = &sc->tids.ftid_tab[t->idx];
7209 
7210 	if (f->pending) {
7211 		rc = EBUSY;
7212 		goto done;
7213 	}
7214 	if (f->locked) {
7215 		rc = EPERM;
7216 		goto done;
7217 	}
7218 
7219 	if (f->valid) {
7220 		t->fs = f->fs;	/* extra info for the caller */
7221 		rc = del_filter_wr(sc, t->idx);
7222 	}
7223 
7224 done:
7225 	end_synchronized_op(sc, 0);
7226 
7227 	if (rc == 0) {
7228 		mtx_lock(&sc->tids.ftid_lock);
7229 		for (;;) {
7230 			if (f->pending == 0) {
7231 				rc = f->valid ? EIO : 0;
7232 				break;
7233 			}
7234 
7235 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7236 			    PCATCH, "t4delfw", 0)) {
7237 				rc = EINPROGRESS;
7238 				break;
7239 			}
7240 		}
7241 		mtx_unlock(&sc->tids.ftid_lock);
7242 	}
7243 
7244 	return (rc);
7245 }
7246 
7247 static void
7248 clear_filter(struct filter_entry *f)
7249 {
7250 	if (f->l2t)
7251 		t4_l2t_release(f->l2t);
7252 
7253 	bzero(f, sizeof (*f));
7254 }
7255 
7256 static int
7257 set_filter_wr(struct adapter *sc, int fidx)
7258 {
7259 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7260 	struct wrqe *wr;
7261 	struct fw_filter_wr *fwr;
7262 	unsigned int ftid;
7263 
7264 	ASSERT_SYNCHRONIZED_OP(sc);
7265 
7266 	if (f->fs.newdmac || f->fs.newvlan) {
7267 		/* This filter needs an L2T entry; allocate one. */
7268 		f->l2t = t4_l2t_alloc_switching(sc->l2t);
7269 		if (f->l2t == NULL)
7270 			return (EAGAIN);
7271 		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7272 		    f->fs.dmac)) {
7273 			t4_l2t_release(f->l2t);
7274 			f->l2t = NULL;
7275 			return (ENOMEM);
7276 		}
7277 	}
7278 
7279 	ftid = sc->tids.ftid_base + fidx;
7280 
7281 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7282 	if (wr == NULL)
7283 		return (ENOMEM);
7284 
7285 	fwr = wrtod(wr);
7286 	bzero(fwr, sizeof (*fwr));
7287 
7288 	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7289 	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7290 	fwr->tid_to_iq =
7291 	    htobe32(V_FW_FILTER_WR_TID(ftid) |
7292 		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7293 		V_FW_FILTER_WR_NOREPLY(0) |
7294 		V_FW_FILTER_WR_IQ(f->fs.iq));
7295 	fwr->del_filter_to_l2tix =
7296 	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7297 		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7298 		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7299 		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7300 		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7301 		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7302 		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7303 		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7304 		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7305 		    f->fs.newvlan == VLAN_REWRITE) |
7306 		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7307 		    f->fs.newvlan == VLAN_REWRITE) |
7308 		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7309 		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7310 		V_FW_FILTER_WR_PRIO(f->fs.prio) |
7311 		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7312 	fwr->ethtype = htobe16(f->fs.val.ethtype);
7313 	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7314 	fwr->frag_to_ovlan_vldm =
7315 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7316 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7317 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7318 		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7319 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7320 		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7321 	fwr->smac_sel = 0;
7322 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7323 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7324 	fwr->maci_to_matchtypem =
7325 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7326 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7327 		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7328 		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7329 		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7330 		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7331 		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7332 		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7333 	fwr->ptcl = f->fs.val.proto;
7334 	fwr->ptclm = f->fs.mask.proto;
7335 	fwr->ttyp = f->fs.val.tos;
7336 	fwr->ttypm = f->fs.mask.tos;
7337 	fwr->ivlan = htobe16(f->fs.val.vlan);
7338 	fwr->ivlanm = htobe16(f->fs.mask.vlan);
7339 	fwr->ovlan = htobe16(f->fs.val.vnic);
7340 	fwr->ovlanm = htobe16(f->fs.mask.vnic);
7341 	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7342 	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7343 	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7344 	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7345 	fwr->lp = htobe16(f->fs.val.dport);
7346 	fwr->lpm = htobe16(f->fs.mask.dport);
7347 	fwr->fp = htobe16(f->fs.val.sport);
7348 	fwr->fpm = htobe16(f->fs.mask.sport);
7349 	if (f->fs.newsmac)
7350 		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7351 
7352 	f->pending = 1;
7353 	sc->tids.ftids_in_use++;
7354 
7355 	t4_wrq_tx(sc, wr);
7356 	return (0);
7357 }
7358 
7359 static int
7360 del_filter_wr(struct adapter *sc, int fidx)
7361 {
7362 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7363 	struct wrqe *wr;
7364 	struct fw_filter_wr *fwr;
7365 	unsigned int ftid;
7366 
7367 	ftid = sc->tids.ftid_base + fidx;
7368 
7369 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7370 	if (wr == NULL)
7371 		return (ENOMEM);
7372 	fwr = wrtod(wr);
7373 	bzero(fwr, sizeof (*fwr));
7374 
7375 	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7376 
7377 	f->pending = 1;
7378 	t4_wrq_tx(sc, wr);
7379 	return (0);
7380 }
7381 
7382 int
7383 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7384 {
7385 	struct adapter *sc = iq->adapter;
7386 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7387 	unsigned int idx = GET_TID(rpl);
7388 	unsigned int rc;
7389 	struct filter_entry *f;
7390 
7391 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7392 	    rss->opcode));
7393 
7394 	if (is_ftid(sc, idx)) {
7395 
7396 		idx -= sc->tids.ftid_base;
7397 		f = &sc->tids.ftid_tab[idx];
7398 		rc = G_COOKIE(rpl->cookie);
7399 
7400 		mtx_lock(&sc->tids.ftid_lock);
7401 		if (rc == FW_FILTER_WR_FLT_ADDED) {
7402 			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7403 			    __func__, idx));
7404 			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7405 			f->pending = 0;  /* asynchronous setup completed */
7406 			f->valid = 1;
7407 		} else {
7408 			if (rc != FW_FILTER_WR_FLT_DELETED) {
7409 				/* Add or delete failed, display an error */
7410 				log(LOG_ERR,
7411 				    "filter %u setup failed with error %u\n",
7412 				    idx, rc);
7413 			}
7414 
7415 			clear_filter(f);
7416 			sc->tids.ftids_in_use--;
7417 		}
7418 		wakeup(&sc->tids.ftid_tab);
7419 		mtx_unlock(&sc->tids.ftid_lock);
7420 	}
7421 
7422 	return (0);
7423 }
7424 
7425 static int
7426 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7427 {
7428 	int rc;
7429 
7430 	if (cntxt->cid > M_CTXTQID)
7431 		return (EINVAL);
7432 
7433 	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7434 	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7435 		return (EINVAL);
7436 
7437 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7438 	if (rc)
7439 		return (rc);
7440 
7441 	if (sc->flags & FW_OK) {
7442 		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7443 		    &cntxt->data[0]);
7444 		if (rc == 0)
7445 			goto done;
7446 	}
7447 
7448 	/*
7449 	 * Read via firmware failed or wasn't even attempted.  Read directly via
7450 	 * the backdoor.
7451 	 */
7452 	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7453 done:
7454 	end_synchronized_op(sc, 0);
7455 	return (rc);
7456 }
7457 
7458 static int
7459 load_fw(struct adapter *sc, struct t4_data *fw)
7460 {
7461 	int rc;
7462 	uint8_t *fw_data;
7463 
7464 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7465 	if (rc)
7466 		return (rc);
7467 
7468 	if (sc->flags & FULL_INIT_DONE) {
7469 		rc = EBUSY;
7470 		goto done;
7471 	}
7472 
7473 	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7474 	if (fw_data == NULL) {
7475 		rc = ENOMEM;
7476 		goto done;
7477 	}
7478 
7479 	rc = copyin(fw->data, fw_data, fw->len);
7480 	if (rc == 0)
7481 		rc = -t4_load_fw(sc, fw_data, fw->len);
7482 
7483 	free(fw_data, M_CXGBE);
7484 done:
7485 	end_synchronized_op(sc, 0);
7486 	return (rc);
7487 }
7488 
7489 static int
7490 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7491 {
7492 	uint32_t addr, off, remaining, i, n;
7493 	uint32_t *buf, *b;
7494 	uint32_t mw_base, mw_aperture;
7495 	int rc;
7496 	uint8_t *dst;
7497 
7498 	rc = validate_mem_range(sc, mr->addr, mr->len);
7499 	if (rc != 0)
7500 		return (rc);
7501 
7502 	memwin_info(sc, win, &mw_base, &mw_aperture);
7503 	buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7504 	addr = mr->addr;
7505 	remaining = mr->len;
7506 	dst = (void *)mr->data;
7507 
7508 	while (remaining) {
7509 		off = position_memwin(sc, win, addr);
7510 
7511 		/* number of bytes that we'll copy in the inner loop */
7512 		n = min(remaining, mw_aperture - off);
7513 		for (i = 0; i < n; i += 4)
7514 			*b++ = t4_read_reg(sc, mw_base + off + i);
7515 
7516 		rc = copyout(buf, dst, n);
7517 		if (rc != 0)
7518 			break;
7519 
7520 		b = buf;
7521 		dst += n;
7522 		remaining -= n;
7523 		addr += n;
7524 	}
7525 
7526 	free(buf, M_CXGBE);
7527 	return (rc);
7528 }
7529 
7530 static int
7531 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7532 {
7533 	int rc;
7534 
7535 	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7536 		return (EINVAL);
7537 
7538 	if (i2cd->len > 1) {
7539 		/* XXX: need fw support for longer reads in one go */
7540 		return (ENOTSUP);
7541 	}
7542 
7543 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7544 	if (rc)
7545 		return (rc);
7546 	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7547 	    i2cd->offset, &i2cd->data[0]);
7548 	end_synchronized_op(sc, 0);
7549 
7550 	return (rc);
7551 }
7552 
7553 static int
7554 in_range(int val, int lo, int hi)
7555 {
7556 
7557 	return (val < 0 || (val <= hi && val >= lo));
7558 }
7559 
7560 static int
7561 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7562 {
7563 	int fw_subcmd, fw_type, rc;
7564 
7565 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7566 	if (rc)
7567 		return (rc);
7568 
7569 	if (!(sc->flags & FULL_INIT_DONE)) {
7570 		rc = EAGAIN;
7571 		goto done;
7572 	}
7573 
7574 	/*
7575 	 * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7576 	 * sub-command and type are in common locations.)
7577 	 */
7578 	if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7579 		fw_subcmd = FW_SCHED_SC_CONFIG;
7580 	else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7581 		fw_subcmd = FW_SCHED_SC_PARAMS;
7582 	else {
7583 		rc = EINVAL;
7584 		goto done;
7585 	}
7586 	if (p->type == SCHED_CLASS_TYPE_PACKET)
7587 		fw_type = FW_SCHED_TYPE_PKTSCHED;
7588 	else {
7589 		rc = EINVAL;
7590 		goto done;
7591 	}
7592 
7593 	if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7594 		/* Vet our parameters ..*/
7595 		if (p->u.config.minmax < 0) {
7596 			rc = EINVAL;
7597 			goto done;
7598 		}
7599 
7600 		/* And pass the request to the firmware ...*/
7601 		rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
7602 		goto done;
7603 	}
7604 
7605 	if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7606 		int fw_level;
7607 		int fw_mode;
7608 		int fw_rateunit;
7609 		int fw_ratemode;
7610 
7611 		if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7612 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7613 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7614 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7615 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7616 			fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7617 		else {
7618 			rc = EINVAL;
7619 			goto done;
7620 		}
7621 
7622 		if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7623 			fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7624 		else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7625 			fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7626 		else {
7627 			rc = EINVAL;
7628 			goto done;
7629 		}
7630 
7631 		if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7632 			fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7633 		else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7634 			fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7635 		else {
7636 			rc = EINVAL;
7637 			goto done;
7638 		}
7639 
7640 		if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7641 			fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7642 		else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7643 			fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7644 		else {
7645 			rc = EINVAL;
7646 			goto done;
7647 		}
7648 
7649 		/* Vet our parameters ... */
7650 		if (!in_range(p->u.params.channel, 0, 3) ||
7651 		    !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7652 		    !in_range(p->u.params.minrate, 0, 10000000) ||
7653 		    !in_range(p->u.params.maxrate, 0, 10000000) ||
7654 		    !in_range(p->u.params.weight, 0, 100)) {
7655 			rc = ERANGE;
7656 			goto done;
7657 		}
7658 
7659 		/*
7660 		 * Translate any unset parameters into the firmware's
7661 		 * nomenclature and/or fail the call if the parameters
7662 		 * are required ...
7663 		 */
7664 		if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7665 		    p->u.params.channel < 0 || p->u.params.cl < 0) {
7666 			rc = EINVAL;
7667 			goto done;
7668 		}
7669 		if (p->u.params.minrate < 0)
7670 			p->u.params.minrate = 0;
7671 		if (p->u.params.maxrate < 0) {
7672 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7673 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7674 				rc = EINVAL;
7675 				goto done;
7676 			} else
7677 				p->u.params.maxrate = 0;
7678 		}
7679 		if (p->u.params.weight < 0) {
7680 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7681 				rc = EINVAL;
7682 				goto done;
7683 			} else
7684 				p->u.params.weight = 0;
7685 		}
7686 		if (p->u.params.pktsize < 0) {
7687 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7688 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7689 				rc = EINVAL;
7690 				goto done;
7691 			} else
7692 				p->u.params.pktsize = 0;
7693 		}
7694 
7695 		/* See what the firmware thinks of the request ... */
7696 		rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7697 		    fw_rateunit, fw_ratemode, p->u.params.channel,
7698 		    p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7699 		    p->u.params.weight, p->u.params.pktsize);
7700 		goto done;
7701 	}
7702 
7703 	rc = EINVAL;
7704 done:
7705 	end_synchronized_op(sc, 0);
7706 	return (rc);
7707 }
7708 
7709 static int
7710 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7711 {
7712 	struct port_info *pi = NULL;
7713 	struct sge_txq *txq;
7714 	uint32_t fw_mnem, fw_queue, fw_class;
7715 	int i, rc;
7716 
7717 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7718 	if (rc)
7719 		return (rc);
7720 
7721 	if (!(sc->flags & FULL_INIT_DONE)) {
7722 		rc = EAGAIN;
7723 		goto done;
7724 	}
7725 
7726 	if (p->port >= sc->params.nports) {
7727 		rc = EINVAL;
7728 		goto done;
7729 	}
7730 
7731 	pi = sc->port[p->port];
7732 	if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7733 		rc = EINVAL;
7734 		goto done;
7735 	}
7736 
7737 	/*
7738 	 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7739 	 * Scheduling Class in this case).
7740 	 */
7741 	fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7742 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7743 	fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7744 
7745 	/*
7746 	 * If op.queue is non-negative, then we're only changing the scheduling
7747 	 * on a single specified TX queue.
7748 	 */
7749 	if (p->queue >= 0) {
7750 		txq = &sc->sge.txq[pi->first_txq + p->queue];
7751 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7752 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7753 		    &fw_class);
7754 		goto done;
7755 	}
7756 
7757 	/*
7758 	 * Change the scheduling on all the TX queues for the
7759 	 * interface.
7760 	 */
7761 	for_each_txq(pi, i, txq) {
7762 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7763 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7764 		    &fw_class);
7765 		if (rc)
7766 			goto done;
7767 	}
7768 
7769 	rc = 0;
7770 done:
7771 	end_synchronized_op(sc, 0);
7772 	return (rc);
7773 }
7774 
7775 int
7776 t4_os_find_pci_capability(struct adapter *sc, int cap)
7777 {
7778 	int i;
7779 
7780 	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7781 }
7782 
7783 int
7784 t4_os_pci_save_state(struct adapter *sc)
7785 {
7786 	device_t dev;
7787 	struct pci_devinfo *dinfo;
7788 
7789 	dev = sc->dev;
7790 	dinfo = device_get_ivars(dev);
7791 
7792 	pci_cfg_save(dev, dinfo, 0);
7793 	return (0);
7794 }
7795 
7796 int
7797 t4_os_pci_restore_state(struct adapter *sc)
7798 {
7799 	device_t dev;
7800 	struct pci_devinfo *dinfo;
7801 
7802 	dev = sc->dev;
7803 	dinfo = device_get_ivars(dev);
7804 
7805 	pci_cfg_restore(dev, dinfo);
7806 	return (0);
7807 }
7808 
7809 void
7810 t4_os_portmod_changed(const struct adapter *sc, int idx)
7811 {
7812 	struct port_info *pi = sc->port[idx];
7813 	static const char *mod_str[] = {
7814 		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7815 	};
7816 
7817 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7818 		if_printf(pi->ifp, "transceiver unplugged.\n");
7819 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7820 		if_printf(pi->ifp, "unknown transceiver inserted.\n");
7821 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7822 		if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7823 	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7824 		if_printf(pi->ifp, "%s transceiver inserted.\n",
7825 		    mod_str[pi->mod_type]);
7826 	} else {
7827 		if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7828 		    pi->mod_type);
7829 	}
7830 }
7831 
7832 void
7833 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7834 {
7835 	struct port_info *pi = sc->port[idx];
7836 	struct ifnet *ifp = pi->ifp;
7837 
7838 	if (link_stat) {
7839 		pi->linkdnrc = -1;
7840 		ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7841 		if_link_state_change(ifp, LINK_STATE_UP);
7842 	} else {
7843 		if (reason >= 0)
7844 			pi->linkdnrc = reason;
7845 		if_link_state_change(ifp, LINK_STATE_DOWN);
7846 	}
7847 }
7848 
7849 void
7850 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7851 {
7852 	struct adapter *sc;
7853 
7854 	sx_slock(&t4_list_lock);
7855 	SLIST_FOREACH(sc, &t4_list, link) {
7856 		/*
7857 		 * func should not make any assumptions about what state sc is
7858 		 * in - the only guarantee is that sc->sc_lock is a valid lock.
7859 		 */
7860 		func(sc, arg);
7861 	}
7862 	sx_sunlock(&t4_list_lock);
7863 }
7864 
7865 static int
7866 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7867 {
7868        return (0);
7869 }
7870 
7871 static int
7872 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7873 {
7874        return (0);
7875 }
7876 
7877 static int
7878 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7879     struct thread *td)
7880 {
7881 	int rc;
7882 	struct adapter *sc = dev->si_drv1;
7883 
7884 	rc = priv_check(td, PRIV_DRIVER);
7885 	if (rc != 0)
7886 		return (rc);
7887 
7888 	switch (cmd) {
7889 	case CHELSIO_T4_GETREG: {
7890 		struct t4_reg *edata = (struct t4_reg *)data;
7891 
7892 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7893 			return (EFAULT);
7894 
7895 		if (edata->size == 4)
7896 			edata->val = t4_read_reg(sc, edata->addr);
7897 		else if (edata->size == 8)
7898 			edata->val = t4_read_reg64(sc, edata->addr);
7899 		else
7900 			return (EINVAL);
7901 
7902 		break;
7903 	}
7904 	case CHELSIO_T4_SETREG: {
7905 		struct t4_reg *edata = (struct t4_reg *)data;
7906 
7907 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7908 			return (EFAULT);
7909 
7910 		if (edata->size == 4) {
7911 			if (edata->val & 0xffffffff00000000)
7912 				return (EINVAL);
7913 			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7914 		} else if (edata->size == 8)
7915 			t4_write_reg64(sc, edata->addr, edata->val);
7916 		else
7917 			return (EINVAL);
7918 		break;
7919 	}
7920 	case CHELSIO_T4_REGDUMP: {
7921 		struct t4_regdump *regs = (struct t4_regdump *)data;
7922 		int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7923 		uint8_t *buf;
7924 
7925 		if (regs->len < reglen) {
7926 			regs->len = reglen; /* hint to the caller */
7927 			return (ENOBUFS);
7928 		}
7929 
7930 		regs->len = reglen;
7931 		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7932 		t4_get_regs(sc, regs, buf);
7933 		rc = copyout(buf, regs->data, reglen);
7934 		free(buf, M_CXGBE);
7935 		break;
7936 	}
7937 	case CHELSIO_T4_GET_FILTER_MODE:
7938 		rc = get_filter_mode(sc, (uint32_t *)data);
7939 		break;
7940 	case CHELSIO_T4_SET_FILTER_MODE:
7941 		rc = set_filter_mode(sc, *(uint32_t *)data);
7942 		break;
7943 	case CHELSIO_T4_GET_FILTER:
7944 		rc = get_filter(sc, (struct t4_filter *)data);
7945 		break;
7946 	case CHELSIO_T4_SET_FILTER:
7947 		rc = set_filter(sc, (struct t4_filter *)data);
7948 		break;
7949 	case CHELSIO_T4_DEL_FILTER:
7950 		rc = del_filter(sc, (struct t4_filter *)data);
7951 		break;
7952 	case CHELSIO_T4_GET_SGE_CONTEXT:
7953 		rc = get_sge_context(sc, (struct t4_sge_context *)data);
7954 		break;
7955 	case CHELSIO_T4_LOAD_FW:
7956 		rc = load_fw(sc, (struct t4_data *)data);
7957 		break;
7958 	case CHELSIO_T4_GET_MEM:
7959 		rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
7960 		break;
7961 	case CHELSIO_T4_GET_I2C:
7962 		rc = read_i2c(sc, (struct t4_i2c_data *)data);
7963 		break;
7964 	case CHELSIO_T4_CLEAR_STATS: {
7965 		int i;
7966 		u_int port_id = *(uint32_t *)data;
7967 		struct port_info *pi;
7968 
7969 		if (port_id >= sc->params.nports)
7970 			return (EINVAL);
7971 		pi = sc->port[port_id];
7972 
7973 		/* MAC stats */
7974 		t4_clr_port_stats(sc, pi->tx_chan);
7975 
7976 		if (pi->flags & PORT_INIT_DONE) {
7977 			struct sge_rxq *rxq;
7978 			struct sge_txq *txq;
7979 			struct sge_wrq *wrq;
7980 
7981 			for_each_rxq(pi, i, rxq) {
7982 #if defined(INET) || defined(INET6)
7983 				rxq->lro.lro_queued = 0;
7984 				rxq->lro.lro_flushed = 0;
7985 #endif
7986 				rxq->rxcsum = 0;
7987 				rxq->vlan_extraction = 0;
7988 			}
7989 
7990 			for_each_txq(pi, i, txq) {
7991 				txq->txcsum = 0;
7992 				txq->tso_wrs = 0;
7993 				txq->vlan_insertion = 0;
7994 				txq->imm_wrs = 0;
7995 				txq->sgl_wrs = 0;
7996 				txq->txpkt_wrs = 0;
7997 				txq->txpkts_wrs = 0;
7998 				txq->txpkts_pkts = 0;
7999 				txq->br->br_drops = 0;
8000 				txq->no_dmamap = 0;
8001 				txq->no_desc = 0;
8002 			}
8003 
8004 #ifdef TCP_OFFLOAD
8005 			/* nothing to clear for each ofld_rxq */
8006 
8007 			for_each_ofld_txq(pi, i, wrq) {
8008 				wrq->tx_wrs = 0;
8009 				wrq->no_desc = 0;
8010 			}
8011 #endif
8012 			wrq = &sc->sge.ctrlq[pi->port_id];
8013 			wrq->tx_wrs = 0;
8014 			wrq->no_desc = 0;
8015 		}
8016 		break;
8017 	}
8018 	case CHELSIO_T4_SCHED_CLASS:
8019 		rc = set_sched_class(sc, (struct t4_sched_params *)data);
8020 		break;
8021 	case CHELSIO_T4_SCHED_QUEUE:
8022 		rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8023 		break;
8024 	case CHELSIO_T4_GET_TRACER:
8025 		rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8026 		break;
8027 	case CHELSIO_T4_SET_TRACER:
8028 		rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8029 		break;
8030 	default:
8031 		rc = EINVAL;
8032 	}
8033 
8034 	return (rc);
8035 }
8036 
8037 #ifdef TCP_OFFLOAD
8038 static int
8039 toe_capability(struct port_info *pi, int enable)
8040 {
8041 	int rc;
8042 	struct adapter *sc = pi->adapter;
8043 
8044 	ASSERT_SYNCHRONIZED_OP(sc);
8045 
8046 	if (!is_offload(sc))
8047 		return (ENODEV);
8048 
8049 	if (enable) {
8050 		if (!(sc->flags & FULL_INIT_DONE)) {
8051 			rc = cxgbe_init_synchronized(pi);
8052 			if (rc)
8053 				return (rc);
8054 		}
8055 
8056 		if (isset(&sc->offload_map, pi->port_id))
8057 			return (0);
8058 
8059 		if (!(sc->flags & TOM_INIT_DONE)) {
8060 			rc = t4_activate_uld(sc, ULD_TOM);
8061 			if (rc == EAGAIN) {
8062 				log(LOG_WARNING,
8063 				    "You must kldload t4_tom.ko before trying "
8064 				    "to enable TOE on a cxgbe interface.\n");
8065 			}
8066 			if (rc != 0)
8067 				return (rc);
8068 			KASSERT(sc->tom_softc != NULL,
8069 			    ("%s: TOM activated but softc NULL", __func__));
8070 			KASSERT(sc->flags & TOM_INIT_DONE,
8071 			    ("%s: TOM activated but flag not set", __func__));
8072 		}
8073 
8074 		setbit(&sc->offload_map, pi->port_id);
8075 	} else {
8076 		if (!isset(&sc->offload_map, pi->port_id))
8077 			return (0);
8078 
8079 		KASSERT(sc->flags & TOM_INIT_DONE,
8080 		    ("%s: TOM never initialized?", __func__));
8081 		clrbit(&sc->offload_map, pi->port_id);
8082 	}
8083 
8084 	return (0);
8085 }
8086 
8087 /*
8088  * Add an upper layer driver to the global list.
8089  */
8090 int
8091 t4_register_uld(struct uld_info *ui)
8092 {
8093 	int rc = 0;
8094 	struct uld_info *u;
8095 
8096 	sx_xlock(&t4_uld_list_lock);
8097 	SLIST_FOREACH(u, &t4_uld_list, link) {
8098 	    if (u->uld_id == ui->uld_id) {
8099 		    rc = EEXIST;
8100 		    goto done;
8101 	    }
8102 	}
8103 
8104 	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8105 	ui->refcount = 0;
8106 done:
8107 	sx_xunlock(&t4_uld_list_lock);
8108 	return (rc);
8109 }
8110 
8111 int
8112 t4_unregister_uld(struct uld_info *ui)
8113 {
8114 	int rc = EINVAL;
8115 	struct uld_info *u;
8116 
8117 	sx_xlock(&t4_uld_list_lock);
8118 
8119 	SLIST_FOREACH(u, &t4_uld_list, link) {
8120 	    if (u == ui) {
8121 		    if (ui->refcount > 0) {
8122 			    rc = EBUSY;
8123 			    goto done;
8124 		    }
8125 
8126 		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8127 		    rc = 0;
8128 		    goto done;
8129 	    }
8130 	}
8131 done:
8132 	sx_xunlock(&t4_uld_list_lock);
8133 	return (rc);
8134 }
8135 
8136 int
8137 t4_activate_uld(struct adapter *sc, int id)
8138 {
8139 	int rc = EAGAIN;
8140 	struct uld_info *ui;
8141 
8142 	ASSERT_SYNCHRONIZED_OP(sc);
8143 
8144 	sx_slock(&t4_uld_list_lock);
8145 
8146 	SLIST_FOREACH(ui, &t4_uld_list, link) {
8147 		if (ui->uld_id == id) {
8148 			rc = ui->activate(sc);
8149 			if (rc == 0)
8150 				ui->refcount++;
8151 			goto done;
8152 		}
8153 	}
8154 done:
8155 	sx_sunlock(&t4_uld_list_lock);
8156 
8157 	return (rc);
8158 }
8159 
8160 int
8161 t4_deactivate_uld(struct adapter *sc, int id)
8162 {
8163 	int rc = EINVAL;
8164 	struct uld_info *ui;
8165 
8166 	ASSERT_SYNCHRONIZED_OP(sc);
8167 
8168 	sx_slock(&t4_uld_list_lock);
8169 
8170 	SLIST_FOREACH(ui, &t4_uld_list, link) {
8171 		if (ui->uld_id == id) {
8172 			rc = ui->deactivate(sc);
8173 			if (rc == 0)
8174 				ui->refcount--;
8175 			goto done;
8176 		}
8177 	}
8178 done:
8179 	sx_sunlock(&t4_uld_list_lock);
8180 
8181 	return (rc);
8182 }
8183 #endif
8184 
8185 /*
8186  * Come up with reasonable defaults for some of the tunables, provided they're
8187  * not set by the user (in which case we'll use the values as is).
8188  */
8189 static void
8190 tweak_tunables(void)
8191 {
8192 	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
8193 
8194 	if (t4_ntxq10g < 1)
8195 		t4_ntxq10g = min(nc, NTXQ_10G);
8196 
8197 	if (t4_ntxq1g < 1)
8198 		t4_ntxq1g = min(nc, NTXQ_1G);
8199 
8200 	if (t4_nrxq10g < 1)
8201 		t4_nrxq10g = min(nc, NRXQ_10G);
8202 
8203 	if (t4_nrxq1g < 1)
8204 		t4_nrxq1g = min(nc, NRXQ_1G);
8205 
8206 #ifdef TCP_OFFLOAD
8207 	if (t4_nofldtxq10g < 1)
8208 		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8209 
8210 	if (t4_nofldtxq1g < 1)
8211 		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8212 
8213 	if (t4_nofldrxq10g < 1)
8214 		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8215 
8216 	if (t4_nofldrxq1g < 1)
8217 		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8218 
8219 	if (t4_toecaps_allowed == -1)
8220 		t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8221 #else
8222 	if (t4_toecaps_allowed == -1)
8223 		t4_toecaps_allowed = 0;
8224 #endif
8225 
8226 #ifdef DEV_NETMAP
8227 	if (t4_nnmtxq10g < 1)
8228 		t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8229 
8230 	if (t4_nnmtxq1g < 1)
8231 		t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8232 
8233 	if (t4_nnmrxq10g < 1)
8234 		t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8235 
8236 	if (t4_nnmrxq1g < 1)
8237 		t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8238 #endif
8239 
8240 	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8241 		t4_tmr_idx_10g = TMR_IDX_10G;
8242 
8243 	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8244 		t4_pktc_idx_10g = PKTC_IDX_10G;
8245 
8246 	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8247 		t4_tmr_idx_1g = TMR_IDX_1G;
8248 
8249 	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8250 		t4_pktc_idx_1g = PKTC_IDX_1G;
8251 
8252 	if (t4_qsize_txq < 128)
8253 		t4_qsize_txq = 128;
8254 
8255 	if (t4_qsize_rxq < 128)
8256 		t4_qsize_rxq = 128;
8257 	while (t4_qsize_rxq & 7)
8258 		t4_qsize_rxq++;
8259 
8260 	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8261 }
8262 
8263 static int
8264 mod_event(module_t mod, int cmd, void *arg)
8265 {
8266 	int rc = 0;
8267 	static int loaded = 0;
8268 
8269 	switch (cmd) {
8270 	case MOD_LOAD:
8271 		if (atomic_fetchadd_int(&loaded, 1))
8272 			break;
8273 		t4_sge_modload();
8274 		sx_init(&t4_list_lock, "T4/T5 adapters");
8275 		SLIST_INIT(&t4_list);
8276 #ifdef TCP_OFFLOAD
8277 		sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8278 		SLIST_INIT(&t4_uld_list);
8279 #endif
8280 		t4_tracer_modload();
8281 		tweak_tunables();
8282 		break;
8283 
8284 	case MOD_UNLOAD:
8285 		if (atomic_fetchadd_int(&loaded, -1) > 1)
8286 			break;
8287 		t4_tracer_modunload();
8288 #ifdef TCP_OFFLOAD
8289 		sx_slock(&t4_uld_list_lock);
8290 		if (!SLIST_EMPTY(&t4_uld_list)) {
8291 			rc = EBUSY;
8292 			sx_sunlock(&t4_uld_list_lock);
8293 			break;
8294 		}
8295 		sx_sunlock(&t4_uld_list_lock);
8296 		sx_destroy(&t4_uld_list_lock);
8297 #endif
8298 		sx_slock(&t4_list_lock);
8299 		if (!SLIST_EMPTY(&t4_list)) {
8300 			rc = EBUSY;
8301 			sx_sunlock(&t4_list_lock);
8302 			break;
8303 		}
8304 		sx_sunlock(&t4_list_lock);
8305 		sx_destroy(&t4_list_lock);
8306 		break;
8307 	}
8308 
8309 	return (rc);
8310 }
8311 
8312 static devclass_t t4_devclass, t5_devclass;
8313 static devclass_t cxgbe_devclass, cxl_devclass;
8314 
8315 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8316 MODULE_VERSION(t4nex, 1);
8317 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8318 
8319 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8320 MODULE_VERSION(t5nex, 1);
8321 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8322 
8323 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8324 MODULE_VERSION(cxgbe, 1);
8325 
8326 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8327 MODULE_VERSION(cxl, 1);
8328