xref: /freebsd/sys/dev/cxgbe/t4_main.c (revision 13de33a5dc2304b13d595d75d48c51793958474f)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/param.h>
35 #include <sys/conf.h>
36 #include <sys/priv.h>
37 #include <sys/kernel.h>
38 #include <sys/bus.h>
39 #include <sys/module.h>
40 #include <sys/malloc.h>
41 #include <sys/queue.h>
42 #include <sys/taskqueue.h>
43 #include <sys/pciio.h>
44 #include <dev/pci/pcireg.h>
45 #include <dev/pci/pcivar.h>
46 #include <dev/pci/pci_private.h>
47 #include <sys/firmware.h>
48 #include <sys/sbuf.h>
49 #include <sys/smp.h>
50 #include <sys/socket.h>
51 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <net/ethernet.h>
54 #include <net/if.h>
55 #include <net/if_types.h>
56 #include <net/if_dl.h>
57 #include <net/if_vlan_var.h>
58 #if defined(__i386__) || defined(__amd64__)
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #endif
62 
63 #include "common/common.h"
64 #include "common/t4_msg.h"
65 #include "common/t4_regs.h"
66 #include "common/t4_regs_values.h"
67 #include "t4_ioctl.h"
68 #include "t4_l2t.h"
69 
70 /* T4 bus driver interface */
71 static int t4_probe(device_t);
72 static int t4_attach(device_t);
73 static int t4_detach(device_t);
74 static device_method_t t4_methods[] = {
75 	DEVMETHOD(device_probe,		t4_probe),
76 	DEVMETHOD(device_attach,	t4_attach),
77 	DEVMETHOD(device_detach,	t4_detach),
78 
79 	DEVMETHOD_END
80 };
81 static driver_t t4_driver = {
82 	"t4nex",
83 	t4_methods,
84 	sizeof(struct adapter)
85 };
86 
87 
88 /* T4 port (cxgbe) interface */
89 static int cxgbe_probe(device_t);
90 static int cxgbe_attach(device_t);
91 static int cxgbe_detach(device_t);
92 static device_method_t cxgbe_methods[] = {
93 	DEVMETHOD(device_probe,		cxgbe_probe),
94 	DEVMETHOD(device_attach,	cxgbe_attach),
95 	DEVMETHOD(device_detach,	cxgbe_detach),
96 	{ 0, 0 }
97 };
98 static driver_t cxgbe_driver = {
99 	"cxgbe",
100 	cxgbe_methods,
101 	sizeof(struct port_info)
102 };
103 
104 static d_ioctl_t t4_ioctl;
105 static d_open_t t4_open;
106 static d_close_t t4_close;
107 
108 static struct cdevsw t4_cdevsw = {
109        .d_version = D_VERSION,
110        .d_flags = 0,
111        .d_open = t4_open,
112        .d_close = t4_close,
113        .d_ioctl = t4_ioctl,
114        .d_name = "t4nex",
115 };
116 
117 /* T5 bus driver interface */
118 static int t5_probe(device_t);
119 static device_method_t t5_methods[] = {
120 	DEVMETHOD(device_probe,		t5_probe),
121 	DEVMETHOD(device_attach,	t4_attach),
122 	DEVMETHOD(device_detach,	t4_detach),
123 
124 	DEVMETHOD_END
125 };
126 static driver_t t5_driver = {
127 	"t5nex",
128 	t5_methods,
129 	sizeof(struct adapter)
130 };
131 
132 
133 /* T5 port (cxl) interface */
134 static driver_t cxl_driver = {
135 	"cxl",
136 	cxgbe_methods,
137 	sizeof(struct port_info)
138 };
139 
140 static struct cdevsw t5_cdevsw = {
141        .d_version = D_VERSION,
142        .d_flags = 0,
143        .d_open = t4_open,
144        .d_close = t4_close,
145        .d_ioctl = t4_ioctl,
146        .d_name = "t5nex",
147 };
148 
149 /* ifnet + media interface */
150 static void cxgbe_init(void *);
151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
153 static void cxgbe_qflush(struct ifnet *);
154 static int cxgbe_media_change(struct ifnet *);
155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
156 
157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
158 
159 /*
160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
161  * then ADAPTER_LOCK, then t4_uld_list_lock.
162  */
163 static struct sx t4_list_lock;
164 SLIST_HEAD(, adapter) t4_list;
165 #ifdef TCP_OFFLOAD
166 static struct sx t4_uld_list_lock;
167 SLIST_HEAD(, uld_info) t4_uld_list;
168 #endif
169 
170 /*
171  * Tunables.  See tweak_tunables() too.
172  *
173  * Each tunable is set to a default value here if it's known at compile-time.
174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
175  * provide a reasonable default when the driver is loaded.
176  *
177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
178  * T5 are under hw.cxl.
179  */
180 
181 /*
182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
183  */
184 #define NTXQ_10G 16
185 static int t4_ntxq10g = -1;
186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
187 
188 #define NRXQ_10G 8
189 static int t4_nrxq10g = -1;
190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
191 
192 #define NTXQ_1G 4
193 static int t4_ntxq1g = -1;
194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
195 
196 #define NRXQ_1G 2
197 static int t4_nrxq1g = -1;
198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
199 
200 #ifdef TCP_OFFLOAD
201 #define NOFLDTXQ_10G 8
202 static int t4_nofldtxq10g = -1;
203 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
204 
205 #define NOFLDRXQ_10G 2
206 static int t4_nofldrxq10g = -1;
207 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
208 
209 #define NOFLDTXQ_1G 2
210 static int t4_nofldtxq1g = -1;
211 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
212 
213 #define NOFLDRXQ_1G 1
214 static int t4_nofldrxq1g = -1;
215 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
216 #endif
217 
218 /*
219  * Holdoff parameters for 10G and 1G ports.
220  */
221 #define TMR_IDX_10G 1
222 static int t4_tmr_idx_10g = TMR_IDX_10G;
223 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
224 
225 #define PKTC_IDX_10G (-1)
226 static int t4_pktc_idx_10g = PKTC_IDX_10G;
227 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
228 
229 #define TMR_IDX_1G 1
230 static int t4_tmr_idx_1g = TMR_IDX_1G;
231 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
232 
233 #define PKTC_IDX_1G (-1)
234 static int t4_pktc_idx_1g = PKTC_IDX_1G;
235 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
236 
237 /*
238  * Size (# of entries) of each tx and rx queue.
239  */
240 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
241 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
242 
243 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
244 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
245 
246 /*
247  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
248  */
249 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
250 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
251 
252 /*
253  * Configuration file.
254  */
255 #define DEFAULT_CF	"default"
256 #define FLASH_CF	"flash"
257 #define UWIRE_CF	"uwire"
258 #define FPGA_CF		"fpga"
259 static char t4_cfg_file[32] = DEFAULT_CF;
260 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
261 
262 /*
263  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
264  * encouraged respectively).
265  */
266 static unsigned int t4_fw_install = 1;
267 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
268 
269 /*
270  * ASIC features that will be used.  Disable the ones you don't want so that the
271  * chip resources aren't wasted on features that will not be used.
272  */
273 static int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
274 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
275 
276 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
277 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
278 
279 static int t4_toecaps_allowed = -1;
280 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
281 
282 static int t4_rdmacaps_allowed = 0;
283 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
284 
285 static int t4_iscsicaps_allowed = 0;
286 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
287 
288 static int t4_fcoecaps_allowed = 0;
289 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
290 
291 static int t5_write_combine = 0;
292 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
293 
294 struct intrs_and_queues {
295 	int intr_type;		/* INTx, MSI, or MSI-X */
296 	int nirq;		/* Number of vectors */
297 	int intr_flags;
298 	int ntxq10g;		/* # of NIC txq's for each 10G port */
299 	int nrxq10g;		/* # of NIC rxq's for each 10G port */
300 	int ntxq1g;		/* # of NIC txq's for each 1G port */
301 	int nrxq1g;		/* # of NIC rxq's for each 1G port */
302 #ifdef TCP_OFFLOAD
303 	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
304 	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
305 	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
306 	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
307 #endif
308 };
309 
310 struct filter_entry {
311         uint32_t valid:1;	/* filter allocated and valid */
312         uint32_t locked:1;	/* filter is administratively locked */
313         uint32_t pending:1;	/* filter action is pending firmware reply */
314 	uint32_t smtidx:8;	/* Source MAC Table index for smac */
315 	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
316 
317         struct t4_filter_specification fs;
318 };
319 
320 enum {
321 	XGMAC_MTU	= (1 << 0),
322 	XGMAC_PROMISC	= (1 << 1),
323 	XGMAC_ALLMULTI	= (1 << 2),
324 	XGMAC_VLANEX	= (1 << 3),
325 	XGMAC_UCADDR	= (1 << 4),
326 	XGMAC_MCADDRS	= (1 << 5),
327 
328 	XGMAC_ALL	= 0xffff
329 };
330 
331 static int map_bars_0_and_4(struct adapter *);
332 static int map_bar_2(struct adapter *);
333 static void setup_memwin(struct adapter *);
334 static int validate_mem_range(struct adapter *, uint32_t, int);
335 static int fwmtype_to_hwmtype(int);
336 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
337     uint32_t *);
338 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
339 static uint32_t position_memwin(struct adapter *, int, uint32_t);
340 static int cfg_itype_and_nqueues(struct adapter *, int, int,
341     struct intrs_and_queues *);
342 static int prep_firmware(struct adapter *);
343 static int partition_resources(struct adapter *, const struct firmware *,
344     const char *);
345 static int get_params__pre_init(struct adapter *);
346 static int get_params__post_init(struct adapter *);
347 static int set_params__post_init(struct adapter *);
348 static void t4_set_desc(struct adapter *);
349 static void build_medialist(struct port_info *);
350 static int update_mac_settings(struct port_info *, int);
351 static int cxgbe_init_synchronized(struct port_info *);
352 static int cxgbe_uninit_synchronized(struct port_info *);
353 static int setup_intr_handlers(struct adapter *);
354 static int adapter_full_init(struct adapter *);
355 static int adapter_full_uninit(struct adapter *);
356 static int port_full_init(struct port_info *);
357 static int port_full_uninit(struct port_info *);
358 static void quiesce_eq(struct adapter *, struct sge_eq *);
359 static void quiesce_iq(struct adapter *, struct sge_iq *);
360 static void quiesce_fl(struct adapter *, struct sge_fl *);
361 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
362     driver_intr_t *, void *, char *);
363 static int t4_free_irq(struct adapter *, struct irq *);
364 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
365     unsigned int);
366 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
367 static void cxgbe_tick(void *);
368 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
369 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
370     struct mbuf *);
371 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
372 static int fw_msg_not_handled(struct adapter *, const __be64 *);
373 static int t4_sysctls(struct adapter *);
374 static int cxgbe_sysctls(struct port_info *);
375 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
376 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
377 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
378 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
379 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
380 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
381 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
382 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
383 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
384 #ifdef SBUF_DRAIN
385 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
386 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
387 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
388 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
389 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
390 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
391 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
392 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
393 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
394 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
395 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
396 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
397 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
398 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
399 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
400 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
401 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
402 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
403 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
404 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
405 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
406 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
407 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
408 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
409 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
410 #endif
411 static inline void txq_start(struct ifnet *, struct sge_txq *);
412 static uint32_t fconf_to_mode(uint32_t);
413 static uint32_t mode_to_fconf(uint32_t);
414 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
415 static int get_filter_mode(struct adapter *, uint32_t *);
416 static int set_filter_mode(struct adapter *, uint32_t);
417 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
418 static int get_filter(struct adapter *, struct t4_filter *);
419 static int set_filter(struct adapter *, struct t4_filter *);
420 static int del_filter(struct adapter *, struct t4_filter *);
421 static void clear_filter(struct filter_entry *);
422 static int set_filter_wr(struct adapter *, int);
423 static int del_filter_wr(struct adapter *, int);
424 static int get_sge_context(struct adapter *, struct t4_sge_context *);
425 static int load_fw(struct adapter *, struct t4_data *);
426 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
427 static int read_i2c(struct adapter *, struct t4_i2c_data *);
428 static int set_sched_class(struct adapter *, struct t4_sched_params *);
429 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
430 #ifdef TCP_OFFLOAD
431 static int toe_capability(struct port_info *, int);
432 #endif
433 static int mod_event(module_t, int, void *);
434 
435 struct {
436 	uint16_t device;
437 	char *desc;
438 } t4_pciids[] = {
439 	{0xa000, "Chelsio Terminator 4 FPGA"},
440 	{0x4400, "Chelsio T440-dbg"},
441 	{0x4401, "Chelsio T420-CR"},
442 	{0x4402, "Chelsio T422-CR"},
443 	{0x4403, "Chelsio T440-CR"},
444 	{0x4404, "Chelsio T420-BCH"},
445 	{0x4405, "Chelsio T440-BCH"},
446 	{0x4406, "Chelsio T440-CH"},
447 	{0x4407, "Chelsio T420-SO"},
448 	{0x4408, "Chelsio T420-CX"},
449 	{0x4409, "Chelsio T420-BT"},
450 	{0x440a, "Chelsio T404-BT"},
451 	{0x440e, "Chelsio T440-LP-CR"},
452 }, t5_pciids[] = {
453 	{0xb000, "Chelsio Terminator 5 FPGA"},
454 	{0x5400, "Chelsio T580-dbg"},
455 	{0x5401,  "Chelsio T520-CR"},		/* 2 x 10G */
456 	{0x5402,  "Chelsio T522-CR"},		/* 2 x 10G, 2 X 1G */
457 	{0x5403,  "Chelsio T540-CR"},		/* 4 x 10G */
458 	{0x5407,  "Chelsio T520-SO"},		/* 2 x 10G, nomem */
459 	{0x5409,  "Chelsio T520-BT"},		/* 2 x 10GBaseT */
460 	{0x540a,  "Chelsio T504-BT"},		/* 4 x 1G */
461 	{0x540d,  "Chelsio T580-CR"},		/* 2 x 40G */
462 	{0x540e,  "Chelsio T540-LP-CR"},	/* 4 x 10G */
463 	{0x5410,  "Chelsio T580-LP-CR"},	/* 2 x 40G */
464 	{0x5411,  "Chelsio T520-LL-CR"},	/* 2 x 10G */
465 	{0x5412,  "Chelsio T560-CR"},		/* 1 x 40G, 2 x 10G */
466 	{0x5414,  "Chelsio T580-LP-SO-CR"},	/* 2 x 40G, nomem */
467 #ifdef notyet
468 	{0x5404,  "Chelsio T520-BCH"},
469 	{0x5405,  "Chelsio T540-BCH"},
470 	{0x5406,  "Chelsio T540-CH"},
471 	{0x5408,  "Chelsio T520-CX"},
472 	{0x540b,  "Chelsio B520-SR"},
473 	{0x540c,  "Chelsio B504-BT"},
474 	{0x540f,  "Chelsio Amsterdam"},
475 	{0x5413,  "Chelsio T580-CHR"},
476 #endif
477 };
478 
479 #ifdef TCP_OFFLOAD
480 /*
481  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
482  * exactly the same for both rxq and ofld_rxq.
483  */
484 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
485 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
486 #endif
487 
488 /* No easy way to include t4_msg.h before adapter.h so we check this way */
489 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
490 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
491 
492 static int
493 t4_probe(device_t dev)
494 {
495 	int i;
496 	uint16_t v = pci_get_vendor(dev);
497 	uint16_t d = pci_get_device(dev);
498 	uint8_t f = pci_get_function(dev);
499 
500 	if (v != PCI_VENDOR_ID_CHELSIO)
501 		return (ENXIO);
502 
503 	/* Attach only to PF0 of the FPGA */
504 	if (d == 0xa000 && f != 0)
505 		return (ENXIO);
506 
507 	for (i = 0; i < nitems(t4_pciids); i++) {
508 		if (d == t4_pciids[i].device) {
509 			device_set_desc(dev, t4_pciids[i].desc);
510 			return (BUS_PROBE_DEFAULT);
511 		}
512 	}
513 
514 	return (ENXIO);
515 }
516 
517 static int
518 t5_probe(device_t dev)
519 {
520 	int i;
521 	uint16_t v = pci_get_vendor(dev);
522 	uint16_t d = pci_get_device(dev);
523 	uint8_t f = pci_get_function(dev);
524 
525 	if (v != PCI_VENDOR_ID_CHELSIO)
526 		return (ENXIO);
527 
528 	/* Attach only to PF0 of the FPGA */
529 	if (d == 0xb000 && f != 0)
530 		return (ENXIO);
531 
532 	for (i = 0; i < nitems(t5_pciids); i++) {
533 		if (d == t5_pciids[i].device) {
534 			device_set_desc(dev, t5_pciids[i].desc);
535 			return (BUS_PROBE_DEFAULT);
536 		}
537 	}
538 
539 	return (ENXIO);
540 }
541 
542 static int
543 t4_attach(device_t dev)
544 {
545 	struct adapter *sc;
546 	int rc = 0, i, n10g, n1g, rqidx, tqidx;
547 	struct intrs_and_queues iaq;
548 	struct sge *s;
549 #ifdef TCP_OFFLOAD
550 	int ofld_rqidx, ofld_tqidx;
551 #endif
552 	const char *pcie_ts;
553 
554 	sc = device_get_softc(dev);
555 	sc->dev = dev;
556 
557 	pci_enable_busmaster(dev);
558 	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
559 		uint32_t v;
560 
561 		pci_set_max_read_req(dev, 4096);
562 		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
563 		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
564 		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
565 	}
566 
567 	sc->traceq = -1;
568 	mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
569 	snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
570 	    device_get_nameunit(dev));
571 
572 	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
573 	    device_get_nameunit(dev));
574 	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
575 	sx_xlock(&t4_list_lock);
576 	SLIST_INSERT_HEAD(&t4_list, sc, link);
577 	sx_xunlock(&t4_list_lock);
578 
579 	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
580 	TAILQ_INIT(&sc->sfl);
581 	callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
582 
583 	rc = map_bars_0_and_4(sc);
584 	if (rc != 0)
585 		goto done; /* error message displayed already */
586 
587 	/*
588 	 * This is the real PF# to which we're attaching.  Works from within PCI
589 	 * passthrough environments too, where pci_get_function() could return a
590 	 * different PF# depending on the passthrough configuration.  We need to
591 	 * use the real PF# in all our communication with the firmware.
592 	 */
593 	sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
594 	sc->mbox = sc->pf;
595 
596 	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
597 	sc->an_handler = an_not_handled;
598 	for (i = 0; i < nitems(sc->cpl_handler); i++)
599 		sc->cpl_handler[i] = cpl_not_handled;
600 	for (i = 0; i < nitems(sc->fw_msg_handler); i++)
601 		sc->fw_msg_handler[i] = fw_msg_not_handled;
602 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
603 	t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
604 	t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
605 	t4_init_sge_cpl_handlers(sc);
606 
607 	/* Prepare the adapter for operation */
608 	rc = -t4_prep_adapter(sc);
609 	if (rc != 0) {
610 		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
611 		goto done;
612 	}
613 
614 	/*
615 	 * Do this really early, with the memory windows set up even before the
616 	 * character device.  The userland tool's register i/o and mem read
617 	 * will work even in "recovery mode".
618 	 */
619 	setup_memwin(sc);
620 	sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
621 	    device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
622 	    device_get_nameunit(dev));
623 	if (sc->cdev == NULL)
624 		device_printf(dev, "failed to create nexus char device.\n");
625 	else
626 		sc->cdev->si_drv1 = sc;
627 
628 	/* Go no further if recovery mode has been requested. */
629 	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
630 		device_printf(dev, "recovery mode.\n");
631 		goto done;
632 	}
633 
634 	/* Prepare the firmware for operation */
635 	rc = prep_firmware(sc);
636 	if (rc != 0)
637 		goto done; /* error message displayed already */
638 
639 	rc = get_params__post_init(sc);
640 	if (rc != 0)
641 		goto done; /* error message displayed already */
642 
643 	rc = set_params__post_init(sc);
644 	if (rc != 0)
645 		goto done; /* error message displayed already */
646 
647 	rc = map_bar_2(sc);
648 	if (rc != 0)
649 		goto done; /* error message displayed already */
650 
651 	rc = t4_create_dma_tag(sc);
652 	if (rc != 0)
653 		goto done; /* error message displayed already */
654 
655 	/*
656 	 * First pass over all the ports - allocate VIs and initialize some
657 	 * basic parameters like mac address, port type, etc.  We also figure
658 	 * out whether a port is 10G or 1G and use that information when
659 	 * calculating how many interrupts to attempt to allocate.
660 	 */
661 	n10g = n1g = 0;
662 	for_each_port(sc, i) {
663 		struct port_info *pi;
664 
665 		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
666 		sc->port[i] = pi;
667 
668 		/* These must be set before t4_port_init */
669 		pi->adapter = sc;
670 		pi->port_id = i;
671 
672 		/* Allocate the vi and initialize parameters like mac addr */
673 		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
674 		if (rc != 0) {
675 			device_printf(dev, "unable to initialize port %d: %d\n",
676 			    i, rc);
677 			free(pi, M_CXGBE);
678 			sc->port[i] = NULL;
679 			goto done;
680 		}
681 
682 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
683 		    device_get_nameunit(dev), i);
684 		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
685 		sc->chan_map[pi->tx_chan] = i;
686 
687 		if (is_10G_port(pi) || is_40G_port(pi)) {
688 			n10g++;
689 			pi->tmr_idx = t4_tmr_idx_10g;
690 			pi->pktc_idx = t4_pktc_idx_10g;
691 		} else {
692 			n1g++;
693 			pi->tmr_idx = t4_tmr_idx_1g;
694 			pi->pktc_idx = t4_pktc_idx_1g;
695 		}
696 
697 		pi->xact_addr_filt = -1;
698 		pi->linkdnrc = -1;
699 
700 		pi->qsize_rxq = t4_qsize_rxq;
701 		pi->qsize_txq = t4_qsize_txq;
702 
703 		pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
704 		if (pi->dev == NULL) {
705 			device_printf(dev,
706 			    "failed to add device for port %d.\n", i);
707 			rc = ENXIO;
708 			goto done;
709 		}
710 		device_set_softc(pi->dev, pi);
711 	}
712 
713 	/*
714 	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
715 	 */
716 	rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
717 	if (rc != 0)
718 		goto done; /* error message displayed already */
719 
720 	sc->intr_type = iaq.intr_type;
721 	sc->intr_count = iaq.nirq;
722 	sc->flags |= iaq.intr_flags;
723 
724 	s = &sc->sge;
725 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
726 	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
727 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
728 	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
729 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
730 
731 #ifdef TCP_OFFLOAD
732 	if (is_offload(sc)) {
733 
734 		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
735 		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
736 		s->neq += s->nofldtxq + s->nofldrxq;
737 		s->niq += s->nofldrxq;
738 
739 		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
740 		    M_CXGBE, M_ZERO | M_WAITOK);
741 		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
742 		    M_CXGBE, M_ZERO | M_WAITOK);
743 	}
744 #endif
745 
746 	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
747 	    M_ZERO | M_WAITOK);
748 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
749 	    M_ZERO | M_WAITOK);
750 	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
751 	    M_ZERO | M_WAITOK);
752 	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
753 	    M_ZERO | M_WAITOK);
754 	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
755 	    M_ZERO | M_WAITOK);
756 
757 	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
758 	    M_ZERO | M_WAITOK);
759 
760 	t4_init_l2t(sc, M_WAITOK);
761 
762 	/*
763 	 * Second pass over the ports.  This time we know the number of rx and
764 	 * tx queues that each port should get.
765 	 */
766 	rqidx = tqidx = 0;
767 #ifdef TCP_OFFLOAD
768 	ofld_rqidx = ofld_tqidx = 0;
769 #endif
770 	for_each_port(sc, i) {
771 		struct port_info *pi = sc->port[i];
772 
773 		if (pi == NULL)
774 			continue;
775 
776 		pi->first_rxq = rqidx;
777 		pi->first_txq = tqidx;
778 		if (is_10G_port(pi) || is_40G_port(pi)) {
779 			pi->nrxq = iaq.nrxq10g;
780 			pi->ntxq = iaq.ntxq10g;
781 		} else {
782 			pi->nrxq = iaq.nrxq1g;
783 			pi->ntxq = iaq.ntxq1g;
784 		}
785 
786 		rqidx += pi->nrxq;
787 		tqidx += pi->ntxq;
788 
789 #ifdef TCP_OFFLOAD
790 		if (is_offload(sc)) {
791 			pi->first_ofld_rxq = ofld_rqidx;
792 			pi->first_ofld_txq = ofld_tqidx;
793 			if (is_10G_port(pi) || is_40G_port(pi)) {
794 				pi->nofldrxq = iaq.nofldrxq10g;
795 				pi->nofldtxq = iaq.nofldtxq10g;
796 			} else {
797 				pi->nofldrxq = iaq.nofldrxq1g;
798 				pi->nofldtxq = iaq.nofldtxq1g;
799 			}
800 			ofld_rqidx += pi->nofldrxq;
801 			ofld_tqidx += pi->nofldtxq;
802 		}
803 #endif
804 	}
805 
806 	rc = setup_intr_handlers(sc);
807 	if (rc != 0) {
808 		device_printf(dev,
809 		    "failed to setup interrupt handlers: %d\n", rc);
810 		goto done;
811 	}
812 
813 	rc = bus_generic_attach(dev);
814 	if (rc != 0) {
815 		device_printf(dev,
816 		    "failed to attach all child ports: %d\n", rc);
817 		goto done;
818 	}
819 
820 	switch (sc->params.pci.speed) {
821 		case 0x1:
822 			pcie_ts = "2.5";
823 			break;
824 		case 0x2:
825 			pcie_ts = "5.0";
826 			break;
827 		case 0x3:
828 			pcie_ts = "8.0";
829 			break;
830 		default:
831 			pcie_ts = "??";
832 			break;
833 	}
834 	device_printf(dev,
835 	    "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
836 	    sc->params.pci.width, pcie_ts, sc->params.pci.speed,
837 	    sc->params.nports, sc->intr_count,
838 	    sc->intr_type == INTR_MSIX ? "MSI-X" :
839 	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
840 	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
841 
842 	t4_set_desc(sc);
843 
844 done:
845 	if (rc != 0 && sc->cdev) {
846 		/* cdev was created and so cxgbetool works; recover that way. */
847 		device_printf(dev,
848 		    "error during attach, adapter is now in recovery mode.\n");
849 		rc = 0;
850 	}
851 
852 	if (rc != 0)
853 		t4_detach(dev);
854 	else
855 		t4_sysctls(sc);
856 
857 	return (rc);
858 }
859 
860 /*
861  * Idempotent
862  */
863 static int
864 t4_detach(device_t dev)
865 {
866 	struct adapter *sc;
867 	struct port_info *pi;
868 	int i, rc;
869 
870 	sc = device_get_softc(dev);
871 
872 	if (sc->flags & FULL_INIT_DONE)
873 		t4_intr_disable(sc);
874 
875 	if (sc->cdev) {
876 		destroy_dev(sc->cdev);
877 		sc->cdev = NULL;
878 	}
879 
880 	rc = bus_generic_detach(dev);
881 	if (rc) {
882 		device_printf(dev,
883 		    "failed to detach child devices: %d\n", rc);
884 		return (rc);
885 	}
886 
887 	for (i = 0; i < sc->intr_count; i++)
888 		t4_free_irq(sc, &sc->irq[i]);
889 
890 	for (i = 0; i < MAX_NPORTS; i++) {
891 		pi = sc->port[i];
892 		if (pi) {
893 			t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
894 			if (pi->dev)
895 				device_delete_child(dev, pi->dev);
896 
897 			mtx_destroy(&pi->pi_lock);
898 			free(pi, M_CXGBE);
899 		}
900 	}
901 
902 	if (sc->flags & FULL_INIT_DONE)
903 		adapter_full_uninit(sc);
904 
905 	if (sc->flags & FW_OK)
906 		t4_fw_bye(sc, sc->mbox);
907 
908 	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
909 		pci_release_msi(dev);
910 
911 	if (sc->regs_res)
912 		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
913 		    sc->regs_res);
914 
915 	if (sc->udbs_res)
916 		bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
917 		    sc->udbs_res);
918 
919 	if (sc->msix_res)
920 		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
921 		    sc->msix_res);
922 
923 	if (sc->l2t)
924 		t4_free_l2t(sc->l2t);
925 
926 #ifdef TCP_OFFLOAD
927 	free(sc->sge.ofld_rxq, M_CXGBE);
928 	free(sc->sge.ofld_txq, M_CXGBE);
929 #endif
930 	free(sc->irq, M_CXGBE);
931 	free(sc->sge.rxq, M_CXGBE);
932 	free(sc->sge.txq, M_CXGBE);
933 	free(sc->sge.ctrlq, M_CXGBE);
934 	free(sc->sge.iqmap, M_CXGBE);
935 	free(sc->sge.eqmap, M_CXGBE);
936 	free(sc->tids.ftid_tab, M_CXGBE);
937 	t4_destroy_dma_tag(sc);
938 	if (mtx_initialized(&sc->sc_lock)) {
939 		sx_xlock(&t4_list_lock);
940 		SLIST_REMOVE(&t4_list, sc, adapter, link);
941 		sx_xunlock(&t4_list_lock);
942 		mtx_destroy(&sc->sc_lock);
943 	}
944 
945 	if (mtx_initialized(&sc->tids.ftid_lock))
946 		mtx_destroy(&sc->tids.ftid_lock);
947 	if (mtx_initialized(&sc->sfl_lock))
948 		mtx_destroy(&sc->sfl_lock);
949 	if (mtx_initialized(&sc->ifp_lock))
950 		mtx_destroy(&sc->ifp_lock);
951 
952 	bzero(sc, sizeof(*sc));
953 
954 	return (0);
955 }
956 
957 
958 static int
959 cxgbe_probe(device_t dev)
960 {
961 	char buf[128];
962 	struct port_info *pi = device_get_softc(dev);
963 
964 	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
965 	device_set_desc_copy(dev, buf);
966 
967 	return (BUS_PROBE_DEFAULT);
968 }
969 
970 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
971     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
972     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
973 #define T4_CAP_ENABLE (T4_CAP)
974 
975 static int
976 cxgbe_attach(device_t dev)
977 {
978 	struct port_info *pi = device_get_softc(dev);
979 	struct ifnet *ifp;
980 
981 	/* Allocate an ifnet and set it up */
982 	ifp = if_alloc(IFT_ETHER);
983 	if (ifp == NULL) {
984 		device_printf(dev, "Cannot allocate ifnet\n");
985 		return (ENOMEM);
986 	}
987 	pi->ifp = ifp;
988 	ifp->if_softc = pi;
989 
990 	callout_init(&pi->tick, CALLOUT_MPSAFE);
991 
992 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
993 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
994 
995 	ifp->if_init = cxgbe_init;
996 	ifp->if_ioctl = cxgbe_ioctl;
997 	ifp->if_transmit = cxgbe_transmit;
998 	ifp->if_qflush = cxgbe_qflush;
999 
1000 	ifp->if_capabilities = T4_CAP;
1001 #ifdef TCP_OFFLOAD
1002 	if (is_offload(pi->adapter))
1003 		ifp->if_capabilities |= IFCAP_TOE;
1004 #endif
1005 	ifp->if_capenable = T4_CAP_ENABLE;
1006 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1007 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1008 
1009 	/* Initialize ifmedia for this port */
1010 	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1011 	    cxgbe_media_status);
1012 	build_medialist(pi);
1013 
1014 	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1015 	    EVENTHANDLER_PRI_ANY);
1016 
1017 	ether_ifattach(ifp, pi->hw_addr);
1018 
1019 #ifdef TCP_OFFLOAD
1020 	if (is_offload(pi->adapter)) {
1021 		device_printf(dev,
1022 		    "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
1023 		    pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
1024 	} else
1025 #endif
1026 		device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
1027 
1028 	cxgbe_sysctls(pi);
1029 
1030 	return (0);
1031 }
1032 
1033 static int
1034 cxgbe_detach(device_t dev)
1035 {
1036 	struct port_info *pi = device_get_softc(dev);
1037 	struct adapter *sc = pi->adapter;
1038 	struct ifnet *ifp = pi->ifp;
1039 
1040 	/* Tell if_ioctl and if_init that the port is going away */
1041 	ADAPTER_LOCK(sc);
1042 	SET_DOOMED(pi);
1043 	wakeup(&sc->flags);
1044 	while (IS_BUSY(sc))
1045 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1046 	SET_BUSY(sc);
1047 #ifdef INVARIANTS
1048 	sc->last_op = "t4detach";
1049 	sc->last_op_thr = curthread;
1050 #endif
1051 	ADAPTER_UNLOCK(sc);
1052 
1053 	if (pi->flags & HAS_TRACEQ) {
1054 		sc->traceq = -1;	/* cloner should not create ifnet */
1055 		t4_tracer_port_detach(sc);
1056 	}
1057 
1058 	if (pi->vlan_c)
1059 		EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1060 
1061 	PORT_LOCK(pi);
1062 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1063 	callout_stop(&pi->tick);
1064 	PORT_UNLOCK(pi);
1065 	callout_drain(&pi->tick);
1066 
1067 	/* Let detach proceed even if these fail. */
1068 	cxgbe_uninit_synchronized(pi);
1069 	port_full_uninit(pi);
1070 
1071 	ifmedia_removeall(&pi->media);
1072 	ether_ifdetach(pi->ifp);
1073 	if_free(pi->ifp);
1074 
1075 	ADAPTER_LOCK(sc);
1076 	CLR_BUSY(sc);
1077 	wakeup(&sc->flags);
1078 	ADAPTER_UNLOCK(sc);
1079 
1080 	return (0);
1081 }
1082 
1083 static void
1084 cxgbe_init(void *arg)
1085 {
1086 	struct port_info *pi = arg;
1087 	struct adapter *sc = pi->adapter;
1088 
1089 	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1090 		return;
1091 	cxgbe_init_synchronized(pi);
1092 	end_synchronized_op(sc, 0);
1093 }
1094 
1095 static int
1096 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1097 {
1098 	int rc = 0, mtu, flags;
1099 	struct port_info *pi = ifp->if_softc;
1100 	struct adapter *sc = pi->adapter;
1101 	struct ifreq *ifr = (struct ifreq *)data;
1102 	uint32_t mask;
1103 
1104 	switch (cmd) {
1105 	case SIOCSIFMTU:
1106 		mtu = ifr->ifr_mtu;
1107 		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1108 			return (EINVAL);
1109 
1110 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1111 		if (rc)
1112 			return (rc);
1113 		ifp->if_mtu = mtu;
1114 		if (pi->flags & PORT_INIT_DONE) {
1115 			t4_update_fl_bufsize(ifp);
1116 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1117 				rc = update_mac_settings(pi, XGMAC_MTU);
1118 		}
1119 		end_synchronized_op(sc, 0);
1120 		break;
1121 
1122 	case SIOCSIFFLAGS:
1123 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1124 		if (rc)
1125 			return (rc);
1126 
1127 		if (ifp->if_flags & IFF_UP) {
1128 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1129 				flags = pi->if_flags;
1130 				if ((ifp->if_flags ^ flags) &
1131 				    (IFF_PROMISC | IFF_ALLMULTI)) {
1132 					rc = update_mac_settings(pi,
1133 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
1134 				}
1135 			} else
1136 				rc = cxgbe_init_synchronized(pi);
1137 			pi->if_flags = ifp->if_flags;
1138 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1139 			rc = cxgbe_uninit_synchronized(pi);
1140 		end_synchronized_op(sc, 0);
1141 		break;
1142 
1143 	case SIOCADDMULTI:
1144 	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1145 		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1146 		if (rc)
1147 			return (rc);
1148 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1149 			rc = update_mac_settings(pi, XGMAC_MCADDRS);
1150 		end_synchronized_op(sc, LOCK_HELD);
1151 		break;
1152 
1153 	case SIOCSIFCAP:
1154 		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1155 		if (rc)
1156 			return (rc);
1157 
1158 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1159 		if (mask & IFCAP_TXCSUM) {
1160 			ifp->if_capenable ^= IFCAP_TXCSUM;
1161 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1162 
1163 			if (IFCAP_TSO4 & ifp->if_capenable &&
1164 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1165 				ifp->if_capenable &= ~IFCAP_TSO4;
1166 				if_printf(ifp,
1167 				    "tso4 disabled due to -txcsum.\n");
1168 			}
1169 		}
1170 		if (mask & IFCAP_TXCSUM_IPV6) {
1171 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1172 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1173 
1174 			if (IFCAP_TSO6 & ifp->if_capenable &&
1175 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1176 				ifp->if_capenable &= ~IFCAP_TSO6;
1177 				if_printf(ifp,
1178 				    "tso6 disabled due to -txcsum6.\n");
1179 			}
1180 		}
1181 		if (mask & IFCAP_RXCSUM)
1182 			ifp->if_capenable ^= IFCAP_RXCSUM;
1183 		if (mask & IFCAP_RXCSUM_IPV6)
1184 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1185 
1186 		/*
1187 		 * Note that we leave CSUM_TSO alone (it is always set).  The
1188 		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1189 		 * sending a TSO request our way, so it's sufficient to toggle
1190 		 * IFCAP_TSOx only.
1191 		 */
1192 		if (mask & IFCAP_TSO4) {
1193 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1194 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1195 				if_printf(ifp, "enable txcsum first.\n");
1196 				rc = EAGAIN;
1197 				goto fail;
1198 			}
1199 			ifp->if_capenable ^= IFCAP_TSO4;
1200 		}
1201 		if (mask & IFCAP_TSO6) {
1202 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1203 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1204 				if_printf(ifp, "enable txcsum6 first.\n");
1205 				rc = EAGAIN;
1206 				goto fail;
1207 			}
1208 			ifp->if_capenable ^= IFCAP_TSO6;
1209 		}
1210 		if (mask & IFCAP_LRO) {
1211 #if defined(INET) || defined(INET6)
1212 			int i;
1213 			struct sge_rxq *rxq;
1214 
1215 			ifp->if_capenable ^= IFCAP_LRO;
1216 			for_each_rxq(pi, i, rxq) {
1217 				if (ifp->if_capenable & IFCAP_LRO)
1218 					rxq->iq.flags |= IQ_LRO_ENABLED;
1219 				else
1220 					rxq->iq.flags &= ~IQ_LRO_ENABLED;
1221 			}
1222 #endif
1223 		}
1224 #ifdef TCP_OFFLOAD
1225 		if (mask & IFCAP_TOE) {
1226 			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1227 
1228 			rc = toe_capability(pi, enable);
1229 			if (rc != 0)
1230 				goto fail;
1231 
1232 			ifp->if_capenable ^= mask;
1233 		}
1234 #endif
1235 		if (mask & IFCAP_VLAN_HWTAGGING) {
1236 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1237 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1238 				rc = update_mac_settings(pi, XGMAC_VLANEX);
1239 		}
1240 		if (mask & IFCAP_VLAN_MTU) {
1241 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
1242 
1243 			/* Need to find out how to disable auto-mtu-inflation */
1244 		}
1245 		if (mask & IFCAP_VLAN_HWTSO)
1246 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1247 		if (mask & IFCAP_VLAN_HWCSUM)
1248 			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1249 
1250 #ifdef VLAN_CAPABILITIES
1251 		VLAN_CAPABILITIES(ifp);
1252 #endif
1253 fail:
1254 		end_synchronized_op(sc, 0);
1255 		break;
1256 
1257 	case SIOCSIFMEDIA:
1258 	case SIOCGIFMEDIA:
1259 		ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1260 		break;
1261 
1262 	default:
1263 		rc = ether_ioctl(ifp, cmd, data);
1264 	}
1265 
1266 	return (rc);
1267 }
1268 
1269 static int
1270 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1271 {
1272 	struct port_info *pi = ifp->if_softc;
1273 	struct adapter *sc = pi->adapter;
1274 	struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1275 	struct buf_ring *br;
1276 	int rc;
1277 
1278 	M_ASSERTPKTHDR(m);
1279 
1280 	if (__predict_false(pi->link_cfg.link_ok == 0)) {
1281 		m_freem(m);
1282 		return (ENETDOWN);
1283 	}
1284 
1285 	if (m->m_flags & M_FLOWID)
1286 		txq += (m->m_pkthdr.flowid % pi->ntxq);
1287 	br = txq->br;
1288 
1289 	if (TXQ_TRYLOCK(txq) == 0) {
1290 		struct sge_eq *eq = &txq->eq;
1291 
1292 		/*
1293 		 * It is possible that t4_eth_tx finishes up and releases the
1294 		 * lock between the TRYLOCK above and the drbr_enqueue here.  We
1295 		 * need to make sure that this mbuf doesn't just sit there in
1296 		 * the drbr.
1297 		 */
1298 
1299 		rc = drbr_enqueue(ifp, br, m);
1300 		if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1301 		    !(eq->flags & EQ_DOOMED))
1302 			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1303 		return (rc);
1304 	}
1305 
1306 	/*
1307 	 * txq->m is the mbuf that is held up due to a temporary shortage of
1308 	 * resources and it should be put on the wire first.  Then what's in
1309 	 * drbr and finally the mbuf that was just passed in to us.
1310 	 *
1311 	 * Return code should indicate the fate of the mbuf that was passed in
1312 	 * this time.
1313 	 */
1314 
1315 	TXQ_LOCK_ASSERT_OWNED(txq);
1316 	if (drbr_needs_enqueue(ifp, br) || txq->m) {
1317 
1318 		/* Queued for transmission. */
1319 
1320 		rc = drbr_enqueue(ifp, br, m);
1321 		m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1322 		(void) t4_eth_tx(ifp, txq, m);
1323 		TXQ_UNLOCK(txq);
1324 		return (rc);
1325 	}
1326 
1327 	/* Direct transmission. */
1328 	rc = t4_eth_tx(ifp, txq, m);
1329 	if (rc != 0 && txq->m)
1330 		rc = 0;	/* held, will be transmitted soon (hopefully) */
1331 
1332 	TXQ_UNLOCK(txq);
1333 	return (rc);
1334 }
1335 
1336 static void
1337 cxgbe_qflush(struct ifnet *ifp)
1338 {
1339 	struct port_info *pi = ifp->if_softc;
1340 	struct sge_txq *txq;
1341 	int i;
1342 	struct mbuf *m;
1343 
1344 	/* queues do not exist if !PORT_INIT_DONE. */
1345 	if (pi->flags & PORT_INIT_DONE) {
1346 		for_each_txq(pi, i, txq) {
1347 			TXQ_LOCK(txq);
1348 			m_freem(txq->m);
1349 			txq->m = NULL;
1350 			while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1351 				m_freem(m);
1352 			TXQ_UNLOCK(txq);
1353 		}
1354 	}
1355 	if_qflush(ifp);
1356 }
1357 
1358 static int
1359 cxgbe_media_change(struct ifnet *ifp)
1360 {
1361 	struct port_info *pi = ifp->if_softc;
1362 
1363 	device_printf(pi->dev, "%s unimplemented.\n", __func__);
1364 
1365 	return (EOPNOTSUPP);
1366 }
1367 
1368 static void
1369 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1370 {
1371 	struct port_info *pi = ifp->if_softc;
1372 	struct ifmedia_entry *cur = pi->media.ifm_cur;
1373 	int speed = pi->link_cfg.speed;
1374 	int data = (pi->port_type << 8) | pi->mod_type;
1375 
1376 	if (cur->ifm_data != data) {
1377 		build_medialist(pi);
1378 		cur = pi->media.ifm_cur;
1379 	}
1380 
1381 	ifmr->ifm_status = IFM_AVALID;
1382 	if (!pi->link_cfg.link_ok)
1383 		return;
1384 
1385 	ifmr->ifm_status |= IFM_ACTIVE;
1386 
1387 	/* active and current will differ iff current media is autoselect. */
1388 	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1389 		return;
1390 
1391 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1392 	if (speed == SPEED_10000)
1393 		ifmr->ifm_active |= IFM_10G_T;
1394 	else if (speed == SPEED_1000)
1395 		ifmr->ifm_active |= IFM_1000_T;
1396 	else if (speed == SPEED_100)
1397 		ifmr->ifm_active |= IFM_100_TX;
1398 	else if (speed == SPEED_10)
1399 		ifmr->ifm_active |= IFM_10_T;
1400 	else
1401 		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1402 			    speed));
1403 }
1404 
1405 void
1406 t4_fatal_err(struct adapter *sc)
1407 {
1408 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1409 	t4_intr_disable(sc);
1410 	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1411 	    device_get_nameunit(sc->dev));
1412 }
1413 
1414 static int
1415 map_bars_0_and_4(struct adapter *sc)
1416 {
1417 	sc->regs_rid = PCIR_BAR(0);
1418 	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1419 	    &sc->regs_rid, RF_ACTIVE);
1420 	if (sc->regs_res == NULL) {
1421 		device_printf(sc->dev, "cannot map registers.\n");
1422 		return (ENXIO);
1423 	}
1424 	sc->bt = rman_get_bustag(sc->regs_res);
1425 	sc->bh = rman_get_bushandle(sc->regs_res);
1426 	sc->mmio_len = rman_get_size(sc->regs_res);
1427 	setbit(&sc->doorbells, DOORBELL_KDB);
1428 
1429 	sc->msix_rid = PCIR_BAR(4);
1430 	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1431 	    &sc->msix_rid, RF_ACTIVE);
1432 	if (sc->msix_res == NULL) {
1433 		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1434 		return (ENXIO);
1435 	}
1436 
1437 	return (0);
1438 }
1439 
1440 static int
1441 map_bar_2(struct adapter *sc)
1442 {
1443 
1444 	/*
1445 	 * T4: only iWARP driver uses the userspace doorbells.  There is no need
1446 	 * to map it if RDMA is disabled.
1447 	 */
1448 	if (is_t4(sc) && sc->rdmacaps == 0)
1449 		return (0);
1450 
1451 	sc->udbs_rid = PCIR_BAR(2);
1452 	sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1453 	    &sc->udbs_rid, RF_ACTIVE);
1454 	if (sc->udbs_res == NULL) {
1455 		device_printf(sc->dev, "cannot map doorbell BAR.\n");
1456 		return (ENXIO);
1457 	}
1458 	sc->udbs_base = rman_get_virtual(sc->udbs_res);
1459 
1460 	if (is_t5(sc)) {
1461 		setbit(&sc->doorbells, DOORBELL_UDB);
1462 #if defined(__i386__) || defined(__amd64__)
1463 		if (t5_write_combine) {
1464 			int rc;
1465 
1466 			/*
1467 			 * Enable write combining on BAR2.  This is the
1468 			 * userspace doorbell BAR and is split into 128B
1469 			 * (UDBS_SEG_SIZE) doorbell regions, each associated
1470 			 * with an egress queue.  The first 64B has the doorbell
1471 			 * and the second 64B can be used to submit a tx work
1472 			 * request with an implicit doorbell.
1473 			 */
1474 
1475 			rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1476 			    rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1477 			if (rc == 0) {
1478 				clrbit(&sc->doorbells, DOORBELL_UDB);
1479 				setbit(&sc->doorbells, DOORBELL_WCWR);
1480 				setbit(&sc->doorbells, DOORBELL_UDBWC);
1481 			} else {
1482 				device_printf(sc->dev,
1483 				    "couldn't enable write combining: %d\n",
1484 				    rc);
1485 			}
1486 
1487 			t4_write_reg(sc, A_SGE_STAT_CFG,
1488 			    V_STATSOURCE_T5(7) | V_STATMODE(0));
1489 		}
1490 #endif
1491 	}
1492 
1493 	return (0);
1494 }
1495 
1496 static const struct memwin t4_memwin[] = {
1497 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1498 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1499 	{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1500 };
1501 
1502 static const struct memwin t5_memwin[] = {
1503 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1504 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1505 	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1506 };
1507 
1508 static void
1509 setup_memwin(struct adapter *sc)
1510 {
1511 	const struct memwin *mw;
1512 	int i, n;
1513 	uint32_t bar0;
1514 
1515 	if (is_t4(sc)) {
1516 		/*
1517 		 * Read low 32b of bar0 indirectly via the hardware backdoor
1518 		 * mechanism.  Works from within PCI passthrough environments
1519 		 * too, where rman_get_start() can return a different value.  We
1520 		 * need to program the T4 memory window decoders with the actual
1521 		 * addresses that will be coming across the PCIe link.
1522 		 */
1523 		bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1524 		bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1525 
1526 		mw = &t4_memwin[0];
1527 		n = nitems(t4_memwin);
1528 	} else {
1529 		/* T5 uses the relative offset inside the PCIe BAR */
1530 		bar0 = 0;
1531 
1532 		mw = &t5_memwin[0];
1533 		n = nitems(t5_memwin);
1534 	}
1535 
1536 	for (i = 0; i < n; i++, mw++) {
1537 		t4_write_reg(sc,
1538 		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1539 		    (mw->base + bar0) | V_BIR(0) |
1540 		    V_WINDOW(ilog2(mw->aperture) - 10));
1541 	}
1542 
1543 	/* flush */
1544 	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1545 }
1546 
1547 /*
1548  * Verify that the memory range specified by the addr/len pair is valid and lies
1549  * entirely within a single region (EDCx or MCx).
1550  */
1551 static int
1552 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1553 {
1554 	uint32_t em, addr_len, maddr, mlen;
1555 
1556 	/* Memory can only be accessed in naturally aligned 4 byte units */
1557 	if (addr & 3 || len & 3 || len == 0)
1558 		return (EINVAL);
1559 
1560 	/* Enabled memories */
1561 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1562 	if (em & F_EDRAM0_ENABLE) {
1563 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1564 		maddr = G_EDRAM0_BASE(addr_len) << 20;
1565 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
1566 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1567 		    addr + len <= maddr + mlen)
1568 			return (0);
1569 	}
1570 	if (em & F_EDRAM1_ENABLE) {
1571 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1572 		maddr = G_EDRAM1_BASE(addr_len) << 20;
1573 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
1574 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1575 		    addr + len <= maddr + mlen)
1576 			return (0);
1577 	}
1578 	if (em & F_EXT_MEM_ENABLE) {
1579 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1580 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
1581 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1582 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1583 		    addr + len <= maddr + mlen)
1584 			return (0);
1585 	}
1586 	if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1587 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1588 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1589 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1590 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1591 		    addr + len <= maddr + mlen)
1592 			return (0);
1593 	}
1594 
1595 	return (EFAULT);
1596 }
1597 
1598 static int
1599 fwmtype_to_hwmtype(int mtype)
1600 {
1601 
1602 	switch (mtype) {
1603 	case FW_MEMTYPE_EDC0:
1604 		return (MEM_EDC0);
1605 	case FW_MEMTYPE_EDC1:
1606 		return (MEM_EDC1);
1607 	case FW_MEMTYPE_EXTMEM:
1608 		return (MEM_MC0);
1609 	case FW_MEMTYPE_EXTMEM1:
1610 		return (MEM_MC1);
1611 	default:
1612 		panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1613 	}
1614 }
1615 
1616 /*
1617  * Verify that the memory range specified by the memtype/offset/len pair is
1618  * valid and lies entirely within the memtype specified.  The global address of
1619  * the start of the range is returned in addr.
1620  */
1621 static int
1622 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1623     uint32_t *addr)
1624 {
1625 	uint32_t em, addr_len, maddr, mlen;
1626 
1627 	/* Memory can only be accessed in naturally aligned 4 byte units */
1628 	if (off & 3 || len & 3 || len == 0)
1629 		return (EINVAL);
1630 
1631 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1632 	switch (fwmtype_to_hwmtype(mtype)) {
1633 	case MEM_EDC0:
1634 		if (!(em & F_EDRAM0_ENABLE))
1635 			return (EINVAL);
1636 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1637 		maddr = G_EDRAM0_BASE(addr_len) << 20;
1638 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
1639 		break;
1640 	case MEM_EDC1:
1641 		if (!(em & F_EDRAM1_ENABLE))
1642 			return (EINVAL);
1643 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1644 		maddr = G_EDRAM1_BASE(addr_len) << 20;
1645 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
1646 		break;
1647 	case MEM_MC:
1648 		if (!(em & F_EXT_MEM_ENABLE))
1649 			return (EINVAL);
1650 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1651 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
1652 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1653 		break;
1654 	case MEM_MC1:
1655 		if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1656 			return (EINVAL);
1657 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1658 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1659 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1660 		break;
1661 	default:
1662 		return (EINVAL);
1663 	}
1664 
1665 	if (mlen > 0 && off < mlen && off + len <= mlen) {
1666 		*addr = maddr + off;	/* global address */
1667 		return (0);
1668 	}
1669 
1670 	return (EFAULT);
1671 }
1672 
1673 static void
1674 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1675 {
1676 	const struct memwin *mw;
1677 
1678 	if (is_t4(sc)) {
1679 		KASSERT(win >= 0 && win < nitems(t4_memwin),
1680 		    ("%s: incorrect memwin# (%d)", __func__, win));
1681 		mw = &t4_memwin[win];
1682 	} else {
1683 		KASSERT(win >= 0 && win < nitems(t5_memwin),
1684 		    ("%s: incorrect memwin# (%d)", __func__, win));
1685 		mw = &t5_memwin[win];
1686 	}
1687 
1688 	if (base != NULL)
1689 		*base = mw->base;
1690 	if (aperture != NULL)
1691 		*aperture = mw->aperture;
1692 }
1693 
1694 /*
1695  * Positions the memory window such that it can be used to access the specified
1696  * address in the chip's address space.  The return value is the offset of addr
1697  * from the start of the window.
1698  */
1699 static uint32_t
1700 position_memwin(struct adapter *sc, int n, uint32_t addr)
1701 {
1702 	uint32_t start, pf;
1703 	uint32_t reg;
1704 
1705 	KASSERT(n >= 0 && n <= 3,
1706 	    ("%s: invalid window %d.", __func__, n));
1707 	KASSERT((addr & 3) == 0,
1708 	    ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1709 
1710 	if (is_t4(sc)) {
1711 		pf = 0;
1712 		start = addr & ~0xf;	/* start must be 16B aligned */
1713 	} else {
1714 		pf = V_PFNUM(sc->pf);
1715 		start = addr & ~0x7f;	/* start must be 128B aligned */
1716 	}
1717 	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1718 
1719 	t4_write_reg(sc, reg, start | pf);
1720 	t4_read_reg(sc, reg);
1721 
1722 	return (addr - start);
1723 }
1724 
1725 static int
1726 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1727     struct intrs_and_queues *iaq)
1728 {
1729 	int rc, itype, navail, nrxq10g, nrxq1g, n;
1730 	int nofldrxq10g = 0, nofldrxq1g = 0;
1731 
1732 	bzero(iaq, sizeof(*iaq));
1733 
1734 	iaq->ntxq10g = t4_ntxq10g;
1735 	iaq->ntxq1g = t4_ntxq1g;
1736 	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1737 	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1738 #ifdef TCP_OFFLOAD
1739 	if (is_offload(sc)) {
1740 		iaq->nofldtxq10g = t4_nofldtxq10g;
1741 		iaq->nofldtxq1g = t4_nofldtxq1g;
1742 		iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1743 		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1744 	}
1745 #endif
1746 
1747 	for (itype = INTR_MSIX; itype; itype >>= 1) {
1748 
1749 		if ((itype & t4_intr_types) == 0)
1750 			continue;	/* not allowed */
1751 
1752 		if (itype == INTR_MSIX)
1753 			navail = pci_msix_count(sc->dev);
1754 		else if (itype == INTR_MSI)
1755 			navail = pci_msi_count(sc->dev);
1756 		else
1757 			navail = 1;
1758 restart:
1759 		if (navail == 0)
1760 			continue;
1761 
1762 		iaq->intr_type = itype;
1763 		iaq->intr_flags = 0;
1764 
1765 		/*
1766 		 * Best option: an interrupt vector for errors, one for the
1767 		 * firmware event queue, and one each for each rxq (NIC as well
1768 		 * as offload).
1769 		 */
1770 		iaq->nirq = T4_EXTRA_INTR;
1771 		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1772 		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1773 		if (iaq->nirq <= navail &&
1774 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
1775 			iaq->intr_flags |= INTR_DIRECT;
1776 			goto allocate;
1777 		}
1778 
1779 		/*
1780 		 * Second best option: an interrupt vector for errors, one for
1781 		 * the firmware event queue, and one each for either NIC or
1782 		 * offload rxq's.
1783 		 */
1784 		iaq->nirq = T4_EXTRA_INTR;
1785 		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1786 		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1787 		if (iaq->nirq <= navail &&
1788 		    (itype != INTR_MSI || powerof2(iaq->nirq)))
1789 			goto allocate;
1790 
1791 		/*
1792 		 * Next best option: an interrupt vector for errors, one for the
1793 		 * firmware event queue, and at least one per port.  At this
1794 		 * point we know we'll have to downsize nrxq or nofldrxq to fit
1795 		 * what's available to us.
1796 		 */
1797 		iaq->nirq = T4_EXTRA_INTR;
1798 		iaq->nirq += n10g + n1g;
1799 		if (iaq->nirq <= navail) {
1800 			int leftover = navail - iaq->nirq;
1801 
1802 			if (n10g > 0) {
1803 				int target = max(nrxq10g, nofldrxq10g);
1804 
1805 				n = 1;
1806 				while (n < target && leftover >= n10g) {
1807 					leftover -= n10g;
1808 					iaq->nirq += n10g;
1809 					n++;
1810 				}
1811 				iaq->nrxq10g = min(n, nrxq10g);
1812 #ifdef TCP_OFFLOAD
1813 				if (is_offload(sc))
1814 					iaq->nofldrxq10g = min(n, nofldrxq10g);
1815 #endif
1816 			}
1817 
1818 			if (n1g > 0) {
1819 				int target = max(nrxq1g, nofldrxq1g);
1820 
1821 				n = 1;
1822 				while (n < target && leftover >= n1g) {
1823 					leftover -= n1g;
1824 					iaq->nirq += n1g;
1825 					n++;
1826 				}
1827 				iaq->nrxq1g = min(n, nrxq1g);
1828 #ifdef TCP_OFFLOAD
1829 				if (is_offload(sc))
1830 					iaq->nofldrxq1g = min(n, nofldrxq1g);
1831 #endif
1832 			}
1833 
1834 			if (itype != INTR_MSI || powerof2(iaq->nirq))
1835 				goto allocate;
1836 		}
1837 
1838 		/*
1839 		 * Least desirable option: one interrupt vector for everything.
1840 		 */
1841 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1842 #ifdef TCP_OFFLOAD
1843 		if (is_offload(sc))
1844 			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1845 #endif
1846 
1847 allocate:
1848 		navail = iaq->nirq;
1849 		rc = 0;
1850 		if (itype == INTR_MSIX)
1851 			rc = pci_alloc_msix(sc->dev, &navail);
1852 		else if (itype == INTR_MSI)
1853 			rc = pci_alloc_msi(sc->dev, &navail);
1854 
1855 		if (rc == 0) {
1856 			if (navail == iaq->nirq)
1857 				return (0);
1858 
1859 			/*
1860 			 * Didn't get the number requested.  Use whatever number
1861 			 * the kernel is willing to allocate (it's in navail).
1862 			 */
1863 			device_printf(sc->dev, "fewer vectors than requested, "
1864 			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1865 			    itype, iaq->nirq, navail);
1866 			pci_release_msi(sc->dev);
1867 			goto restart;
1868 		}
1869 
1870 		device_printf(sc->dev,
1871 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1872 		    itype, rc, iaq->nirq, navail);
1873 	}
1874 
1875 	device_printf(sc->dev,
1876 	    "failed to find a usable interrupt type.  "
1877 	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1878 	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1879 
1880 	return (ENXIO);
1881 }
1882 
1883 #define FW_VERSION(chip) ( \
1884     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
1885     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
1886     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
1887     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
1888 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
1889 
1890 struct fw_info {
1891 	uint8_t chip;
1892 	char *kld_name;
1893 	char *fw_mod_name;
1894 	struct fw_hdr fw_hdr;	/* XXX: waste of space, need a sparse struct */
1895 } fw_info[] = {
1896 	{
1897 		.chip = CHELSIO_T4,
1898 		.kld_name = "t4fw_cfg",
1899 		.fw_mod_name = "t4fw",
1900 		.fw_hdr = {
1901 			.chip = FW_HDR_CHIP_T4,
1902 			.fw_ver = htobe32_const(FW_VERSION(T4)),
1903 			.intfver_nic = FW_INTFVER(T4, NIC),
1904 			.intfver_vnic = FW_INTFVER(T4, VNIC),
1905 			.intfver_ofld = FW_INTFVER(T4, OFLD),
1906 			.intfver_ri = FW_INTFVER(T4, RI),
1907 			.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
1908 			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
1909 			.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
1910 			.intfver_fcoe = FW_INTFVER(T4, FCOE),
1911 		},
1912 	}, {
1913 		.chip = CHELSIO_T5,
1914 		.kld_name = "t5fw_cfg",
1915 		.fw_mod_name = "t5fw",
1916 		.fw_hdr = {
1917 			.chip = FW_HDR_CHIP_T5,
1918 			.fw_ver = htobe32_const(FW_VERSION(T5)),
1919 			.intfver_nic = FW_INTFVER(T5, NIC),
1920 			.intfver_vnic = FW_INTFVER(T5, VNIC),
1921 			.intfver_ofld = FW_INTFVER(T5, OFLD),
1922 			.intfver_ri = FW_INTFVER(T5, RI),
1923 			.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
1924 			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
1925 			.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
1926 			.intfver_fcoe = FW_INTFVER(T5, FCOE),
1927 		},
1928 	}
1929 };
1930 
1931 static struct fw_info *
1932 find_fw_info(int chip)
1933 {
1934 	int i;
1935 
1936 	for (i = 0; i < nitems(fw_info); i++) {
1937 		if (fw_info[i].chip == chip)
1938 			return (&fw_info[i]);
1939 	}
1940 	return (NULL);
1941 }
1942 
1943 /*
1944  * Is the given firmware API compatible with the one the driver was compiled
1945  * with?
1946  */
1947 static int
1948 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
1949 {
1950 
1951 	/* short circuit if it's the exact same firmware version */
1952 	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
1953 		return (1);
1954 
1955 	/*
1956 	 * XXX: Is this too conservative?  Perhaps I should limit this to the
1957 	 * features that are supported in the driver.
1958 	 */
1959 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
1960 	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
1961 	    SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
1962 	    SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
1963 		return (1);
1964 #undef SAME_INTF
1965 
1966 	return (0);
1967 }
1968 
1969 /*
1970  * The firmware in the KLD is usable, but should it be installed?  This routine
1971  * explains itself in detail if it indicates the KLD firmware should be
1972  * installed.
1973  */
1974 static int
1975 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
1976 {
1977 	const char *reason;
1978 
1979 	if (!card_fw_usable) {
1980 		reason = "incompatible or unusable";
1981 		goto install;
1982 	}
1983 
1984 	if (k > c) {
1985 		reason = "older than the version bundled with this driver";
1986 		goto install;
1987 	}
1988 
1989 	if (t4_fw_install == 2 && k != c) {
1990 		reason = "different than the version bundled with this driver";
1991 		goto install;
1992 	}
1993 
1994 	return (0);
1995 
1996 install:
1997 	if (t4_fw_install == 0) {
1998 		device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
1999 		    "but the driver is prohibited from installing a different "
2000 		    "firmware on the card.\n",
2001 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2002 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2003 
2004 		return (0);
2005 	}
2006 
2007 	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2008 	    "installing firmware %u.%u.%u.%u on card.\n",
2009 	    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2010 	    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2011 	    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2012 	    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2013 
2014 	return (1);
2015 }
2016 /*
2017  * Establish contact with the firmware and determine if we are the master driver
2018  * or not, and whether we are responsible for chip initialization.
2019  */
2020 static int
2021 prep_firmware(struct adapter *sc)
2022 {
2023 	const struct firmware *fw = NULL, *default_cfg;
2024 	int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2025 	enum dev_state state;
2026 	struct fw_info *fw_info;
2027 	struct fw_hdr *card_fw;		/* fw on the card */
2028 	const struct fw_hdr *kld_fw;	/* fw in the KLD */
2029 	const struct fw_hdr *drv_fw;	/* fw header the driver was compiled
2030 					   against */
2031 
2032 	/* Contact firmware. */
2033 	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2034 	if (rc < 0 || state == DEV_STATE_ERR) {
2035 		rc = -rc;
2036 		device_printf(sc->dev,
2037 		    "failed to connect to the firmware: %d, %d.\n", rc, state);
2038 		return (rc);
2039 	}
2040 	pf = rc;
2041 	if (pf == sc->mbox)
2042 		sc->flags |= MASTER_PF;
2043 	else if (state == DEV_STATE_UNINIT) {
2044 		/*
2045 		 * We didn't get to be the master so we definitely won't be
2046 		 * configuring the chip.  It's a bug if someone else hasn't
2047 		 * configured it already.
2048 		 */
2049 		device_printf(sc->dev, "couldn't be master(%d), "
2050 		    "device not already initialized either(%d).\n", rc, state);
2051 		return (EDOOFUS);
2052 	}
2053 
2054 	/* This is the firmware whose headers the driver was compiled against */
2055 	fw_info = find_fw_info(chip_id(sc));
2056 	if (fw_info == NULL) {
2057 		device_printf(sc->dev,
2058 		    "unable to look up firmware information for chip %d.\n",
2059 		    chip_id(sc));
2060 		return (EINVAL);
2061 	}
2062 	drv_fw = &fw_info->fw_hdr;
2063 
2064 	/*
2065 	 * The firmware KLD contains many modules.  The KLD name is also the
2066 	 * name of the module that contains the default config file.
2067 	 */
2068 	default_cfg = firmware_get(fw_info->kld_name);
2069 
2070 	/* Read the header of the firmware on the card */
2071 	card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2072 	rc = -t4_read_flash(sc, FLASH_FW_START,
2073 	    sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2074 	if (rc == 0)
2075 		card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2076 	else {
2077 		device_printf(sc->dev,
2078 		    "Unable to read card's firmware header: %d\n", rc);
2079 		card_fw_usable = 0;
2080 	}
2081 
2082 	/* This is the firmware in the KLD */
2083 	fw = firmware_get(fw_info->fw_mod_name);
2084 	if (fw != NULL) {
2085 		kld_fw = (const void *)fw->data;
2086 		kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2087 	} else {
2088 		kld_fw = NULL;
2089 		kld_fw_usable = 0;
2090 	}
2091 
2092 	if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2093 	    (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2094 		/*
2095 		 * Common case: the firmware on the card is an exact match and
2096 		 * the KLD is an exact match too, or the KLD is
2097 		 * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2098 		 * here -- use cxgbetool loadfw if you want to reinstall the
2099 		 * same firmware as the one on the card.
2100 		 */
2101 	} else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2102 	    should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2103 	    be32toh(card_fw->fw_ver))) {
2104 
2105 		rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2106 		if (rc != 0) {
2107 			device_printf(sc->dev,
2108 			    "failed to install firmware: %d\n", rc);
2109 			goto done;
2110 		}
2111 
2112 		/* Installed successfully, update the cached header too. */
2113 		memcpy(card_fw, kld_fw, sizeof(*card_fw));
2114 		card_fw_usable = 1;
2115 		need_fw_reset = 0;	/* already reset as part of load_fw */
2116 	}
2117 
2118 	if (!card_fw_usable) {
2119 		uint32_t d, c, k;
2120 
2121 		d = ntohl(drv_fw->fw_ver);
2122 		c = ntohl(card_fw->fw_ver);
2123 		k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2124 
2125 		device_printf(sc->dev, "Cannot find a usable firmware: "
2126 		    "fw_install %d, chip state %d, "
2127 		    "driver compiled with %d.%d.%d.%d, "
2128 		    "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2129 		    t4_fw_install, state,
2130 		    G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2131 		    G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2132 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2133 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2134 		    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2135 		    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2136 		rc = EINVAL;
2137 		goto done;
2138 	}
2139 
2140 	/* We're using whatever's on the card and it's known to be good. */
2141 	sc->params.fw_vers = ntohl(card_fw->fw_ver);
2142 	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2143 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2144 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2145 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2146 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2147 	t4_get_tp_version(sc, &sc->params.tp_vers);
2148 
2149 	/* Reset device */
2150 	if (need_fw_reset &&
2151 	    (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2152 		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2153 		if (rc != ETIMEDOUT && rc != EIO)
2154 			t4_fw_bye(sc, sc->mbox);
2155 		goto done;
2156 	}
2157 	sc->flags |= FW_OK;
2158 
2159 	rc = get_params__pre_init(sc);
2160 	if (rc != 0)
2161 		goto done; /* error message displayed already */
2162 
2163 	/* Partition adapter resources as specified in the config file. */
2164 	if (state == DEV_STATE_UNINIT) {
2165 
2166 		KASSERT(sc->flags & MASTER_PF,
2167 		    ("%s: trying to change chip settings when not master.",
2168 		    __func__));
2169 
2170 		rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2171 		if (rc != 0)
2172 			goto done;	/* error message displayed already */
2173 
2174 		t4_tweak_chip_settings(sc);
2175 
2176 		/* get basic stuff going */
2177 		rc = -t4_fw_initialize(sc, sc->mbox);
2178 		if (rc != 0) {
2179 			device_printf(sc->dev, "fw init failed: %d.\n", rc);
2180 			goto done;
2181 		}
2182 	} else {
2183 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2184 		sc->cfcsum = 0;
2185 	}
2186 
2187 done:
2188 	free(card_fw, M_CXGBE);
2189 	if (fw != NULL)
2190 		firmware_put(fw, FIRMWARE_UNLOAD);
2191 	if (default_cfg != NULL)
2192 		firmware_put(default_cfg, FIRMWARE_UNLOAD);
2193 
2194 	return (rc);
2195 }
2196 
2197 #define FW_PARAM_DEV(param) \
2198 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2199 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2200 #define FW_PARAM_PFVF(param) \
2201 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2202 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2203 
2204 /*
2205  * Partition chip resources for use between various PFs, VFs, etc.
2206  */
2207 static int
2208 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2209     const char *name_prefix)
2210 {
2211 	const struct firmware *cfg = NULL;
2212 	int rc = 0;
2213 	struct fw_caps_config_cmd caps;
2214 	uint32_t mtype, moff, finicsum, cfcsum;
2215 
2216 	/*
2217 	 * Figure out what configuration file to use.  Pick the default config
2218 	 * file for the card if the user hasn't specified one explicitly.
2219 	 */
2220 	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2221 	if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2222 		/* Card specific overrides go here. */
2223 		if (pci_get_device(sc->dev) == 0x440a)
2224 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2225 		if (is_fpga(sc))
2226 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2227 	}
2228 
2229 	/*
2230 	 * We need to load another module if the profile is anything except
2231 	 * "default" or "flash".
2232 	 */
2233 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2234 	    strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2235 		char s[32];
2236 
2237 		snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2238 		cfg = firmware_get(s);
2239 		if (cfg == NULL) {
2240 			if (default_cfg != NULL) {
2241 				device_printf(sc->dev,
2242 				    "unable to load module \"%s\" for "
2243 				    "configuration profile \"%s\", will use "
2244 				    "the default config file instead.\n",
2245 				    s, sc->cfg_file);
2246 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2247 				    "%s", DEFAULT_CF);
2248 			} else {
2249 				device_printf(sc->dev,
2250 				    "unable to load module \"%s\" for "
2251 				    "configuration profile \"%s\", will use "
2252 				    "the config file on the card's flash "
2253 				    "instead.\n", s, sc->cfg_file);
2254 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2255 				    "%s", FLASH_CF);
2256 			}
2257 		}
2258 	}
2259 
2260 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2261 	    default_cfg == NULL) {
2262 		device_printf(sc->dev,
2263 		    "default config file not available, will use the config "
2264 		    "file on the card's flash instead.\n");
2265 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2266 	}
2267 
2268 	if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2269 		u_int cflen, i, n;
2270 		const uint32_t *cfdata;
2271 		uint32_t param, val, addr, off, mw_base, mw_aperture;
2272 
2273 		KASSERT(cfg != NULL || default_cfg != NULL,
2274 		    ("%s: no config to upload", __func__));
2275 
2276 		/*
2277 		 * Ask the firmware where it wants us to upload the config file.
2278 		 */
2279 		param = FW_PARAM_DEV(CF);
2280 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2281 		if (rc != 0) {
2282 			/* No support for config file?  Shouldn't happen. */
2283 			device_printf(sc->dev,
2284 			    "failed to query config file location: %d.\n", rc);
2285 			goto done;
2286 		}
2287 		mtype = G_FW_PARAMS_PARAM_Y(val);
2288 		moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2289 
2290 		/*
2291 		 * XXX: sheer laziness.  We deliberately added 4 bytes of
2292 		 * useless stuffing/comments at the end of the config file so
2293 		 * it's ok to simply throw away the last remaining bytes when
2294 		 * the config file is not an exact multiple of 4.  This also
2295 		 * helps with the validate_mt_off_len check.
2296 		 */
2297 		if (cfg != NULL) {
2298 			cflen = cfg->datasize & ~3;
2299 			cfdata = cfg->data;
2300 		} else {
2301 			cflen = default_cfg->datasize & ~3;
2302 			cfdata = default_cfg->data;
2303 		}
2304 
2305 		if (cflen > FLASH_CFG_MAX_SIZE) {
2306 			device_printf(sc->dev,
2307 			    "config file too long (%d, max allowed is %d).  "
2308 			    "Will try to use the config on the card, if any.\n",
2309 			    cflen, FLASH_CFG_MAX_SIZE);
2310 			goto use_config_on_flash;
2311 		}
2312 
2313 		rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2314 		if (rc != 0) {
2315 			device_printf(sc->dev,
2316 			    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2317 			    "Will try to use the config on the card, if any.\n",
2318 			    __func__, mtype, moff, cflen, rc);
2319 			goto use_config_on_flash;
2320 		}
2321 
2322 		memwin_info(sc, 2, &mw_base, &mw_aperture);
2323 		while (cflen) {
2324 			off = position_memwin(sc, 2, addr);
2325 			n = min(cflen, mw_aperture - off);
2326 			for (i = 0; i < n; i += 4)
2327 				t4_write_reg(sc, mw_base + off + i, *cfdata++);
2328 			cflen -= n;
2329 			addr += n;
2330 		}
2331 	} else {
2332 use_config_on_flash:
2333 		mtype = FW_MEMTYPE_FLASH;
2334 		moff = t4_flash_cfg_addr(sc);
2335 	}
2336 
2337 	bzero(&caps, sizeof(caps));
2338 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2339 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2340 	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2341 	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2342 	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2343 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2344 	if (rc != 0) {
2345 		device_printf(sc->dev,
2346 		    "failed to pre-process config file: %d "
2347 		    "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2348 		goto done;
2349 	}
2350 
2351 	finicsum = be32toh(caps.finicsum);
2352 	cfcsum = be32toh(caps.cfcsum);
2353 	if (finicsum != cfcsum) {
2354 		device_printf(sc->dev,
2355 		    "WARNING: config file checksum mismatch: %08x %08x\n",
2356 		    finicsum, cfcsum);
2357 	}
2358 	sc->cfcsum = cfcsum;
2359 
2360 #define LIMIT_CAPS(x) do { \
2361 	caps.x &= htobe16(t4_##x##_allowed); \
2362 	sc->x = htobe16(caps.x); \
2363 } while (0)
2364 
2365 	/*
2366 	 * Let the firmware know what features will (not) be used so it can tune
2367 	 * things accordingly.
2368 	 */
2369 	LIMIT_CAPS(linkcaps);
2370 	LIMIT_CAPS(niccaps);
2371 	LIMIT_CAPS(toecaps);
2372 	LIMIT_CAPS(rdmacaps);
2373 	LIMIT_CAPS(iscsicaps);
2374 	LIMIT_CAPS(fcoecaps);
2375 #undef LIMIT_CAPS
2376 
2377 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2378 	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2379 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2380 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2381 	if (rc != 0) {
2382 		device_printf(sc->dev,
2383 		    "failed to process config file: %d.\n", rc);
2384 	}
2385 done:
2386 	if (cfg != NULL)
2387 		firmware_put(cfg, FIRMWARE_UNLOAD);
2388 	return (rc);
2389 }
2390 
2391 /*
2392  * Retrieve parameters that are needed (or nice to have) very early.
2393  */
2394 static int
2395 get_params__pre_init(struct adapter *sc)
2396 {
2397 	int rc;
2398 	uint32_t param[2], val[2];
2399 	struct fw_devlog_cmd cmd;
2400 	struct devlog_params *dlog = &sc->params.devlog;
2401 
2402 	param[0] = FW_PARAM_DEV(PORTVEC);
2403 	param[1] = FW_PARAM_DEV(CCLK);
2404 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2405 	if (rc != 0) {
2406 		device_printf(sc->dev,
2407 		    "failed to query parameters (pre_init): %d.\n", rc);
2408 		return (rc);
2409 	}
2410 
2411 	sc->params.portvec = val[0];
2412 	sc->params.nports = bitcount32(val[0]);
2413 	sc->params.vpd.cclk = val[1];
2414 
2415 	/* Read device log parameters. */
2416 	bzero(&cmd, sizeof(cmd));
2417 	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2418 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2419 	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2420 	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2421 	if (rc != 0) {
2422 		device_printf(sc->dev,
2423 		    "failed to get devlog parameters: %d.\n", rc);
2424 		bzero(dlog, sizeof (*dlog));
2425 		rc = 0;	/* devlog isn't critical for device operation */
2426 	} else {
2427 		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2428 		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2429 		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2430 		dlog->size = be32toh(cmd.memsize_devlog);
2431 	}
2432 
2433 	return (rc);
2434 }
2435 
2436 /*
2437  * Retrieve various parameters that are of interest to the driver.  The device
2438  * has been initialized by the firmware at this point.
2439  */
2440 static int
2441 get_params__post_init(struct adapter *sc)
2442 {
2443 	int rc;
2444 	uint32_t param[7], val[7];
2445 	struct fw_caps_config_cmd caps;
2446 
2447 	param[0] = FW_PARAM_PFVF(IQFLINT_START);
2448 	param[1] = FW_PARAM_PFVF(EQ_START);
2449 	param[2] = FW_PARAM_PFVF(FILTER_START);
2450 	param[3] = FW_PARAM_PFVF(FILTER_END);
2451 	param[4] = FW_PARAM_PFVF(L2T_START);
2452 	param[5] = FW_PARAM_PFVF(L2T_END);
2453 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2454 	if (rc != 0) {
2455 		device_printf(sc->dev,
2456 		    "failed to query parameters (post_init): %d.\n", rc);
2457 		return (rc);
2458 	}
2459 
2460 	sc->sge.iq_start = val[0];
2461 	sc->sge.eq_start = val[1];
2462 	sc->tids.ftid_base = val[2];
2463 	sc->tids.nftids = val[3] - val[2] + 1;
2464 	sc->vres.l2t.start = val[4];
2465 	sc->vres.l2t.size = val[5] - val[4] + 1;
2466 	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2467 	    ("%s: L2 table size (%u) larger than expected (%u)",
2468 	    __func__, sc->vres.l2t.size, L2T_SIZE));
2469 
2470 	/* get capabilites */
2471 	bzero(&caps, sizeof(caps));
2472 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2473 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2474 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2475 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2476 	if (rc != 0) {
2477 		device_printf(sc->dev,
2478 		    "failed to get card capabilities: %d.\n", rc);
2479 		return (rc);
2480 	}
2481 
2482 	if (caps.toecaps) {
2483 		/* query offload-related parameters */
2484 		param[0] = FW_PARAM_DEV(NTID);
2485 		param[1] = FW_PARAM_PFVF(SERVER_START);
2486 		param[2] = FW_PARAM_PFVF(SERVER_END);
2487 		param[3] = FW_PARAM_PFVF(TDDP_START);
2488 		param[4] = FW_PARAM_PFVF(TDDP_END);
2489 		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2490 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2491 		if (rc != 0) {
2492 			device_printf(sc->dev,
2493 			    "failed to query TOE parameters: %d.\n", rc);
2494 			return (rc);
2495 		}
2496 		sc->tids.ntids = val[0];
2497 		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2498 		sc->tids.stid_base = val[1];
2499 		sc->tids.nstids = val[2] - val[1] + 1;
2500 		sc->vres.ddp.start = val[3];
2501 		sc->vres.ddp.size = val[4] - val[3] + 1;
2502 		sc->params.ofldq_wr_cred = val[5];
2503 		sc->params.offload = 1;
2504 	}
2505 	if (caps.rdmacaps) {
2506 		param[0] = FW_PARAM_PFVF(STAG_START);
2507 		param[1] = FW_PARAM_PFVF(STAG_END);
2508 		param[2] = FW_PARAM_PFVF(RQ_START);
2509 		param[3] = FW_PARAM_PFVF(RQ_END);
2510 		param[4] = FW_PARAM_PFVF(PBL_START);
2511 		param[5] = FW_PARAM_PFVF(PBL_END);
2512 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2513 		if (rc != 0) {
2514 			device_printf(sc->dev,
2515 			    "failed to query RDMA parameters(1): %d.\n", rc);
2516 			return (rc);
2517 		}
2518 		sc->vres.stag.start = val[0];
2519 		sc->vres.stag.size = val[1] - val[0] + 1;
2520 		sc->vres.rq.start = val[2];
2521 		sc->vres.rq.size = val[3] - val[2] + 1;
2522 		sc->vres.pbl.start = val[4];
2523 		sc->vres.pbl.size = val[5] - val[4] + 1;
2524 
2525 		param[0] = FW_PARAM_PFVF(SQRQ_START);
2526 		param[1] = FW_PARAM_PFVF(SQRQ_END);
2527 		param[2] = FW_PARAM_PFVF(CQ_START);
2528 		param[3] = FW_PARAM_PFVF(CQ_END);
2529 		param[4] = FW_PARAM_PFVF(OCQ_START);
2530 		param[5] = FW_PARAM_PFVF(OCQ_END);
2531 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2532 		if (rc != 0) {
2533 			device_printf(sc->dev,
2534 			    "failed to query RDMA parameters(2): %d.\n", rc);
2535 			return (rc);
2536 		}
2537 		sc->vres.qp.start = val[0];
2538 		sc->vres.qp.size = val[1] - val[0] + 1;
2539 		sc->vres.cq.start = val[2];
2540 		sc->vres.cq.size = val[3] - val[2] + 1;
2541 		sc->vres.ocq.start = val[4];
2542 		sc->vres.ocq.size = val[5] - val[4] + 1;
2543 	}
2544 	if (caps.iscsicaps) {
2545 		param[0] = FW_PARAM_PFVF(ISCSI_START);
2546 		param[1] = FW_PARAM_PFVF(ISCSI_END);
2547 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2548 		if (rc != 0) {
2549 			device_printf(sc->dev,
2550 			    "failed to query iSCSI parameters: %d.\n", rc);
2551 			return (rc);
2552 		}
2553 		sc->vres.iscsi.start = val[0];
2554 		sc->vres.iscsi.size = val[1] - val[0] + 1;
2555 	}
2556 
2557 	/*
2558 	 * We've got the params we wanted to query via the firmware.  Now grab
2559 	 * some others directly from the chip.
2560 	 */
2561 	rc = t4_read_chip_settings(sc);
2562 
2563 	return (rc);
2564 }
2565 
2566 static int
2567 set_params__post_init(struct adapter *sc)
2568 {
2569 	uint32_t param, val;
2570 
2571 	/* ask for encapsulated CPLs */
2572 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2573 	val = 1;
2574 	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2575 
2576 	return (0);
2577 }
2578 
2579 #undef FW_PARAM_PFVF
2580 #undef FW_PARAM_DEV
2581 
2582 static void
2583 t4_set_desc(struct adapter *sc)
2584 {
2585 	char buf[128];
2586 	struct adapter_params *p = &sc->params;
2587 
2588 	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2589 	    "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2590 	    chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2591 
2592 	device_set_desc_copy(sc->dev, buf);
2593 }
2594 
2595 static void
2596 build_medialist(struct port_info *pi)
2597 {
2598 	struct ifmedia *media = &pi->media;
2599 	int data, m;
2600 
2601 	PORT_LOCK(pi);
2602 
2603 	ifmedia_removeall(media);
2604 
2605 	m = IFM_ETHER | IFM_FDX;
2606 	data = (pi->port_type << 8) | pi->mod_type;
2607 
2608 	switch(pi->port_type) {
2609 	case FW_PORT_TYPE_BT_XFI:
2610 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2611 		break;
2612 
2613 	case FW_PORT_TYPE_BT_XAUI:
2614 		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2615 		/* fall through */
2616 
2617 	case FW_PORT_TYPE_BT_SGMII:
2618 		ifmedia_add(media, m | IFM_1000_T, data, NULL);
2619 		ifmedia_add(media, m | IFM_100_TX, data, NULL);
2620 		ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2621 		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2622 		break;
2623 
2624 	case FW_PORT_TYPE_CX4:
2625 		ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2626 		ifmedia_set(media, m | IFM_10G_CX4);
2627 		break;
2628 
2629 	case FW_PORT_TYPE_SFP:
2630 	case FW_PORT_TYPE_FIBER_XFI:
2631 	case FW_PORT_TYPE_FIBER_XAUI:
2632 		switch (pi->mod_type) {
2633 
2634 		case FW_PORT_MOD_TYPE_LR:
2635 			ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2636 			ifmedia_set(media, m | IFM_10G_LR);
2637 			break;
2638 
2639 		case FW_PORT_MOD_TYPE_SR:
2640 			ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2641 			ifmedia_set(media, m | IFM_10G_SR);
2642 			break;
2643 
2644 		case FW_PORT_MOD_TYPE_LRM:
2645 			ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2646 			ifmedia_set(media, m | IFM_10G_LRM);
2647 			break;
2648 
2649 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2650 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2651 			ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2652 			ifmedia_set(media, m | IFM_10G_TWINAX);
2653 			break;
2654 
2655 		case FW_PORT_MOD_TYPE_NONE:
2656 			m &= ~IFM_FDX;
2657 			ifmedia_add(media, m | IFM_NONE, data, NULL);
2658 			ifmedia_set(media, m | IFM_NONE);
2659 			break;
2660 
2661 		case FW_PORT_MOD_TYPE_NA:
2662 		case FW_PORT_MOD_TYPE_ER:
2663 		default:
2664 			device_printf(pi->dev,
2665 			    "unknown port_type (%d), mod_type (%d)\n",
2666 			    pi->port_type, pi->mod_type);
2667 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2668 			ifmedia_set(media, m | IFM_UNKNOWN);
2669 			break;
2670 		}
2671 		break;
2672 
2673 	case FW_PORT_TYPE_QSFP:
2674 		switch (pi->mod_type) {
2675 
2676 		case FW_PORT_MOD_TYPE_LR:
2677 			ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2678 			ifmedia_set(media, m | IFM_40G_LR4);
2679 			break;
2680 
2681 		case FW_PORT_MOD_TYPE_SR:
2682 			ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2683 			ifmedia_set(media, m | IFM_40G_SR4);
2684 			break;
2685 
2686 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2687 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2688 			ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2689 			ifmedia_set(media, m | IFM_40G_CR4);
2690 			break;
2691 
2692 		case FW_PORT_MOD_TYPE_NONE:
2693 			m &= ~IFM_FDX;
2694 			ifmedia_add(media, m | IFM_NONE, data, NULL);
2695 			ifmedia_set(media, m | IFM_NONE);
2696 			break;
2697 
2698 		default:
2699 			device_printf(pi->dev,
2700 			    "unknown port_type (%d), mod_type (%d)\n",
2701 			    pi->port_type, pi->mod_type);
2702 			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2703 			ifmedia_set(media, m | IFM_UNKNOWN);
2704 			break;
2705 		}
2706 		break;
2707 
2708 	default:
2709 		device_printf(pi->dev,
2710 		    "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2711 		    pi->mod_type);
2712 		ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2713 		ifmedia_set(media, m | IFM_UNKNOWN);
2714 		break;
2715 	}
2716 
2717 	PORT_UNLOCK(pi);
2718 }
2719 
2720 #define FW_MAC_EXACT_CHUNK	7
2721 
2722 /*
2723  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2724  * indicates which parameters should be programmed (the rest are left alone).
2725  */
2726 static int
2727 update_mac_settings(struct port_info *pi, int flags)
2728 {
2729 	int rc;
2730 	struct ifnet *ifp = pi->ifp;
2731 	struct adapter *sc = pi->adapter;
2732 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2733 
2734 	ASSERT_SYNCHRONIZED_OP(sc);
2735 	KASSERT(flags, ("%s: not told what to update.", __func__));
2736 
2737 	if (flags & XGMAC_MTU)
2738 		mtu = ifp->if_mtu;
2739 
2740 	if (flags & XGMAC_PROMISC)
2741 		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2742 
2743 	if (flags & XGMAC_ALLMULTI)
2744 		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2745 
2746 	if (flags & XGMAC_VLANEX)
2747 		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2748 
2749 	rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2750 	    vlanex, false);
2751 	if (rc) {
2752 		if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2753 		return (rc);
2754 	}
2755 
2756 	if (flags & XGMAC_UCADDR) {
2757 		uint8_t ucaddr[ETHER_ADDR_LEN];
2758 
2759 		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2760 		rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2761 		    ucaddr, true, true);
2762 		if (rc < 0) {
2763 			rc = -rc;
2764 			if_printf(ifp, "change_mac failed: %d\n", rc);
2765 			return (rc);
2766 		} else {
2767 			pi->xact_addr_filt = rc;
2768 			rc = 0;
2769 		}
2770 	}
2771 
2772 	if (flags & XGMAC_MCADDRS) {
2773 		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2774 		int del = 1;
2775 		uint64_t hash = 0;
2776 		struct ifmultiaddr *ifma;
2777 		int i = 0, j;
2778 
2779 		if_maddr_rlock(ifp);
2780 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2781 			if (ifma->ifma_addr->sa_family != AF_LINK)
2782 				continue;
2783 			mcaddr[i++] =
2784 			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2785 
2786 			if (i == FW_MAC_EXACT_CHUNK) {
2787 				rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2788 				    del, i, mcaddr, NULL, &hash, 0);
2789 				if (rc < 0) {
2790 					rc = -rc;
2791 					for (j = 0; j < i; j++) {
2792 						if_printf(ifp,
2793 						    "failed to add mc address"
2794 						    " %02x:%02x:%02x:"
2795 						    "%02x:%02x:%02x rc=%d\n",
2796 						    mcaddr[j][0], mcaddr[j][1],
2797 						    mcaddr[j][2], mcaddr[j][3],
2798 						    mcaddr[j][4], mcaddr[j][5],
2799 						    rc);
2800 					}
2801 					goto mcfail;
2802 				}
2803 				del = 0;
2804 				i = 0;
2805 			}
2806 		}
2807 		if (i > 0) {
2808 			rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2809 			    del, i, mcaddr, NULL, &hash, 0);
2810 			if (rc < 0) {
2811 				rc = -rc;
2812 				for (j = 0; j < i; j++) {
2813 					if_printf(ifp,
2814 					    "failed to add mc address"
2815 					    " %02x:%02x:%02x:"
2816 					    "%02x:%02x:%02x rc=%d\n",
2817 					    mcaddr[j][0], mcaddr[j][1],
2818 					    mcaddr[j][2], mcaddr[j][3],
2819 					    mcaddr[j][4], mcaddr[j][5],
2820 					    rc);
2821 				}
2822 				goto mcfail;
2823 			}
2824 		}
2825 
2826 		rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2827 		if (rc != 0)
2828 			if_printf(ifp, "failed to set mc address hash: %d", rc);
2829 mcfail:
2830 		if_maddr_runlock(ifp);
2831 	}
2832 
2833 	return (rc);
2834 }
2835 
2836 int
2837 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2838     char *wmesg)
2839 {
2840 	int rc, pri;
2841 
2842 #ifdef WITNESS
2843 	/* the caller thinks it's ok to sleep, but is it really? */
2844 	if (flags & SLEEP_OK)
2845 		pause("t4slptst", 1);
2846 #endif
2847 
2848 	if (INTR_OK)
2849 		pri = PCATCH;
2850 	else
2851 		pri = 0;
2852 
2853 	ADAPTER_LOCK(sc);
2854 	for (;;) {
2855 
2856 		if (pi && IS_DOOMED(pi)) {
2857 			rc = ENXIO;
2858 			goto done;
2859 		}
2860 
2861 		if (!IS_BUSY(sc)) {
2862 			rc = 0;
2863 			break;
2864 		}
2865 
2866 		if (!(flags & SLEEP_OK)) {
2867 			rc = EBUSY;
2868 			goto done;
2869 		}
2870 
2871 		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2872 			rc = EINTR;
2873 			goto done;
2874 		}
2875 	}
2876 
2877 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2878 	SET_BUSY(sc);
2879 #ifdef INVARIANTS
2880 	sc->last_op = wmesg;
2881 	sc->last_op_thr = curthread;
2882 #endif
2883 
2884 done:
2885 	if (!(flags & HOLD_LOCK) || rc)
2886 		ADAPTER_UNLOCK(sc);
2887 
2888 	return (rc);
2889 }
2890 
2891 void
2892 end_synchronized_op(struct adapter *sc, int flags)
2893 {
2894 
2895 	if (flags & LOCK_HELD)
2896 		ADAPTER_LOCK_ASSERT_OWNED(sc);
2897 	else
2898 		ADAPTER_LOCK(sc);
2899 
2900 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2901 	CLR_BUSY(sc);
2902 	wakeup(&sc->flags);
2903 	ADAPTER_UNLOCK(sc);
2904 }
2905 
2906 static int
2907 cxgbe_init_synchronized(struct port_info *pi)
2908 {
2909 	struct adapter *sc = pi->adapter;
2910 	struct ifnet *ifp = pi->ifp;
2911 	int rc = 0;
2912 
2913 	ASSERT_SYNCHRONIZED_OP(sc);
2914 
2915 	if (isset(&sc->open_device_map, pi->port_id)) {
2916 		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2917 		    ("mismatch between open_device_map and if_drv_flags"));
2918 		return (0);	/* already running */
2919 	}
2920 
2921 	if (!(sc->flags & FULL_INIT_DONE) &&
2922 	    ((rc = adapter_full_init(sc)) != 0))
2923 		return (rc);	/* error message displayed already */
2924 
2925 	if (!(pi->flags & PORT_INIT_DONE) &&
2926 	    ((rc = port_full_init(pi)) != 0))
2927 		return (rc); /* error message displayed already */
2928 
2929 	rc = update_mac_settings(pi, XGMAC_ALL);
2930 	if (rc)
2931 		goto done;	/* error message displayed already */
2932 
2933 	rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2934 	if (rc != 0) {
2935 		if_printf(ifp, "start_link failed: %d\n", rc);
2936 		goto done;
2937 	}
2938 
2939 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2940 	if (rc != 0) {
2941 		if_printf(ifp, "enable_vi failed: %d\n", rc);
2942 		goto done;
2943 	}
2944 
2945 	/*
2946 	 * The first iq of the first port to come up is used for tracing.
2947 	 */
2948 	if (sc->traceq < 0) {
2949 		sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
2950 		t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
2951 		    A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
2952 		    V_QUEUENUMBER(sc->traceq));
2953 		pi->flags |= HAS_TRACEQ;
2954 	}
2955 
2956 	/* all ok */
2957 	setbit(&sc->open_device_map, pi->port_id);
2958 	PORT_LOCK(pi);
2959 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
2960 	PORT_UNLOCK(pi);
2961 
2962 	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2963 done:
2964 	if (rc != 0)
2965 		cxgbe_uninit_synchronized(pi);
2966 
2967 	return (rc);
2968 }
2969 
2970 /*
2971  * Idempotent.
2972  */
2973 static int
2974 cxgbe_uninit_synchronized(struct port_info *pi)
2975 {
2976 	struct adapter *sc = pi->adapter;
2977 	struct ifnet *ifp = pi->ifp;
2978 	int rc;
2979 
2980 	ASSERT_SYNCHRONIZED_OP(sc);
2981 
2982 	/*
2983 	 * Disable the VI so that all its data in either direction is discarded
2984 	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2985 	 * tick) intact as the TP can deliver negative advice or data that it's
2986 	 * holding in its RAM (for an offloaded connection) even after the VI is
2987 	 * disabled.
2988 	 */
2989 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2990 	if (rc) {
2991 		if_printf(ifp, "disable_vi failed: %d\n", rc);
2992 		return (rc);
2993 	}
2994 
2995 	clrbit(&sc->open_device_map, pi->port_id);
2996 	PORT_LOCK(pi);
2997 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2998 	PORT_UNLOCK(pi);
2999 
3000 	pi->link_cfg.link_ok = 0;
3001 	pi->link_cfg.speed = 0;
3002 	pi->linkdnrc = -1;
3003 	t4_os_link_changed(sc, pi->port_id, 0, -1);
3004 
3005 	return (0);
3006 }
3007 
3008 /*
3009  * It is ok for this function to fail midway and return right away.  t4_detach
3010  * will walk the entire sc->irq list and clean up whatever is valid.
3011  */
3012 static int
3013 setup_intr_handlers(struct adapter *sc)
3014 {
3015 	int rc, rid, p, q;
3016 	char s[8];
3017 	struct irq *irq;
3018 	struct port_info *pi;
3019 	struct sge_rxq *rxq;
3020 #ifdef TCP_OFFLOAD
3021 	struct sge_ofld_rxq *ofld_rxq;
3022 #endif
3023 
3024 	/*
3025 	 * Setup interrupts.
3026 	 */
3027 	irq = &sc->irq[0];
3028 	rid = sc->intr_type == INTR_INTX ? 0 : 1;
3029 	if (sc->intr_count == 1) {
3030 		KASSERT(!(sc->flags & INTR_DIRECT),
3031 		    ("%s: single interrupt && INTR_DIRECT?", __func__));
3032 
3033 		rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
3034 		if (rc != 0)
3035 			return (rc);
3036 	} else {
3037 		/* Multiple interrupts. */
3038 		KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3039 		    ("%s: too few intr.", __func__));
3040 
3041 		/* The first one is always error intr */
3042 		rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3043 		if (rc != 0)
3044 			return (rc);
3045 		irq++;
3046 		rid++;
3047 
3048 		/* The second one is always the firmware event queue */
3049 		rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
3050 		    "evt");
3051 		if (rc != 0)
3052 			return (rc);
3053 		irq++;
3054 		rid++;
3055 
3056 		/*
3057 		 * Note that if INTR_DIRECT is not set then either the NIC rx
3058 		 * queues or (exclusive or) the TOE rx queueus will be taking
3059 		 * direct interrupts.
3060 		 *
3061 		 * There is no need to check for is_offload(sc) as nofldrxq
3062 		 * will be 0 if offload is disabled.
3063 		 */
3064 		for_each_port(sc, p) {
3065 			pi = sc->port[p];
3066 
3067 #ifdef TCP_OFFLOAD
3068 			/*
3069 			 * Skip over the NIC queues if they aren't taking direct
3070 			 * interrupts.
3071 			 */
3072 			if (!(sc->flags & INTR_DIRECT) &&
3073 			    pi->nofldrxq > pi->nrxq)
3074 				goto ofld_queues;
3075 #endif
3076 			rxq = &sc->sge.rxq[pi->first_rxq];
3077 			for (q = 0; q < pi->nrxq; q++, rxq++) {
3078 				snprintf(s, sizeof(s), "%d.%d", p, q);
3079 				rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3080 				    s);
3081 				if (rc != 0)
3082 					return (rc);
3083 				irq++;
3084 				rid++;
3085 			}
3086 
3087 #ifdef TCP_OFFLOAD
3088 			/*
3089 			 * Skip over the offload queues if they aren't taking
3090 			 * direct interrupts.
3091 			 */
3092 			if (!(sc->flags & INTR_DIRECT))
3093 				continue;
3094 ofld_queues:
3095 			ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
3096 			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
3097 				snprintf(s, sizeof(s), "%d,%d", p, q);
3098 				rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3099 				    ofld_rxq, s);
3100 				if (rc != 0)
3101 					return (rc);
3102 				irq++;
3103 				rid++;
3104 			}
3105 #endif
3106 		}
3107 	}
3108 
3109 	return (0);
3110 }
3111 
3112 static int
3113 adapter_full_init(struct adapter *sc)
3114 {
3115 	int rc, i;
3116 
3117 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3118 	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3119 	    ("%s: FULL_INIT_DONE already", __func__));
3120 
3121 	/*
3122 	 * queues that belong to the adapter (not any particular port).
3123 	 */
3124 	rc = t4_setup_adapter_queues(sc);
3125 	if (rc != 0)
3126 		goto done;
3127 
3128 	for (i = 0; i < nitems(sc->tq); i++) {
3129 		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3130 		    taskqueue_thread_enqueue, &sc->tq[i]);
3131 		if (sc->tq[i] == NULL) {
3132 			device_printf(sc->dev,
3133 			    "failed to allocate task queue %d\n", i);
3134 			rc = ENOMEM;
3135 			goto done;
3136 		}
3137 		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3138 		    device_get_nameunit(sc->dev), i);
3139 	}
3140 
3141 	t4_intr_enable(sc);
3142 	sc->flags |= FULL_INIT_DONE;
3143 done:
3144 	if (rc != 0)
3145 		adapter_full_uninit(sc);
3146 
3147 	return (rc);
3148 }
3149 
3150 static int
3151 adapter_full_uninit(struct adapter *sc)
3152 {
3153 	int i;
3154 
3155 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3156 
3157 	t4_teardown_adapter_queues(sc);
3158 
3159 	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3160 		taskqueue_free(sc->tq[i]);
3161 		sc->tq[i] = NULL;
3162 	}
3163 
3164 	sc->flags &= ~FULL_INIT_DONE;
3165 
3166 	return (0);
3167 }
3168 
3169 static int
3170 port_full_init(struct port_info *pi)
3171 {
3172 	struct adapter *sc = pi->adapter;
3173 	struct ifnet *ifp = pi->ifp;
3174 	uint16_t *rss;
3175 	struct sge_rxq *rxq;
3176 	int rc, i, j;
3177 
3178 	ASSERT_SYNCHRONIZED_OP(sc);
3179 	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3180 	    ("%s: PORT_INIT_DONE already", __func__));
3181 
3182 	sysctl_ctx_init(&pi->ctx);
3183 	pi->flags |= PORT_SYSCTL_CTX;
3184 
3185 	/*
3186 	 * Allocate tx/rx/fl queues for this port.
3187 	 */
3188 	rc = t4_setup_port_queues(pi);
3189 	if (rc != 0)
3190 		goto done;	/* error message displayed already */
3191 
3192 	/*
3193 	 * Setup RSS for this port.  Save a copy of the RSS table for later use.
3194 	 */
3195 	rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3196 	for (i = 0; i < pi->rss_size;) {
3197 		for_each_rxq(pi, j, rxq) {
3198 			rss[i++] = rxq->iq.abs_id;
3199 			if (i == pi->rss_size)
3200 				break;
3201 		}
3202 	}
3203 
3204 	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3205 	    pi->rss_size);
3206 	if (rc != 0) {
3207 		if_printf(ifp, "rss_config failed: %d\n", rc);
3208 		goto done;
3209 	}
3210 
3211 	pi->rss = rss;
3212 	pi->flags |= PORT_INIT_DONE;
3213 done:
3214 	if (rc != 0)
3215 		port_full_uninit(pi);
3216 
3217 	return (rc);
3218 }
3219 
3220 /*
3221  * Idempotent.
3222  */
3223 static int
3224 port_full_uninit(struct port_info *pi)
3225 {
3226 	struct adapter *sc = pi->adapter;
3227 	int i;
3228 	struct sge_rxq *rxq;
3229 	struct sge_txq *txq;
3230 #ifdef TCP_OFFLOAD
3231 	struct sge_ofld_rxq *ofld_rxq;
3232 	struct sge_wrq *ofld_txq;
3233 #endif
3234 
3235 	if (pi->flags & PORT_INIT_DONE) {
3236 
3237 		/* Need to quiesce queues.  XXX: ctrl queues? */
3238 
3239 		for_each_txq(pi, i, txq) {
3240 			quiesce_eq(sc, &txq->eq);
3241 		}
3242 
3243 #ifdef TCP_OFFLOAD
3244 		for_each_ofld_txq(pi, i, ofld_txq) {
3245 			quiesce_eq(sc, &ofld_txq->eq);
3246 		}
3247 #endif
3248 
3249 		for_each_rxq(pi, i, rxq) {
3250 			quiesce_iq(sc, &rxq->iq);
3251 			quiesce_fl(sc, &rxq->fl);
3252 		}
3253 
3254 #ifdef TCP_OFFLOAD
3255 		for_each_ofld_rxq(pi, i, ofld_rxq) {
3256 			quiesce_iq(sc, &ofld_rxq->iq);
3257 			quiesce_fl(sc, &ofld_rxq->fl);
3258 		}
3259 #endif
3260 		free(pi->rss, M_CXGBE);
3261 	}
3262 
3263 	t4_teardown_port_queues(pi);
3264 	pi->flags &= ~PORT_INIT_DONE;
3265 
3266 	return (0);
3267 }
3268 
3269 static void
3270 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3271 {
3272 	EQ_LOCK(eq);
3273 	eq->flags |= EQ_DOOMED;
3274 
3275 	/*
3276 	 * Wait for the response to a credit flush if one's
3277 	 * pending.
3278 	 */
3279 	while (eq->flags & EQ_CRFLUSHED)
3280 		mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3281 	EQ_UNLOCK(eq);
3282 
3283 	callout_drain(&eq->tx_callout);	/* XXX: iffy */
3284 	pause("callout", 10);		/* Still iffy */
3285 
3286 	taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3287 }
3288 
3289 static void
3290 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3291 {
3292 	(void) sc;	/* unused */
3293 
3294 	/* Synchronize with the interrupt handler */
3295 	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3296 		pause("iqfree", 1);
3297 }
3298 
3299 static void
3300 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3301 {
3302 	mtx_lock(&sc->sfl_lock);
3303 	FL_LOCK(fl);
3304 	fl->flags |= FL_DOOMED;
3305 	FL_UNLOCK(fl);
3306 	mtx_unlock(&sc->sfl_lock);
3307 
3308 	callout_drain(&sc->sfl_callout);
3309 	KASSERT((fl->flags & FL_STARVING) == 0,
3310 	    ("%s: still starving", __func__));
3311 }
3312 
3313 static int
3314 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3315     driver_intr_t *handler, void *arg, char *name)
3316 {
3317 	int rc;
3318 
3319 	irq->rid = rid;
3320 	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3321 	    RF_SHAREABLE | RF_ACTIVE);
3322 	if (irq->res == NULL) {
3323 		device_printf(sc->dev,
3324 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3325 		return (ENOMEM);
3326 	}
3327 
3328 	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3329 	    NULL, handler, arg, &irq->tag);
3330 	if (rc != 0) {
3331 		device_printf(sc->dev,
3332 		    "failed to setup interrupt for rid %d, name %s: %d\n",
3333 		    rid, name, rc);
3334 	} else if (name)
3335 		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3336 
3337 	return (rc);
3338 }
3339 
3340 static int
3341 t4_free_irq(struct adapter *sc, struct irq *irq)
3342 {
3343 	if (irq->tag)
3344 		bus_teardown_intr(sc->dev, irq->res, irq->tag);
3345 	if (irq->res)
3346 		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3347 
3348 	bzero(irq, sizeof(*irq));
3349 
3350 	return (0);
3351 }
3352 
3353 static void
3354 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3355     unsigned int end)
3356 {
3357 	uint32_t *p = (uint32_t *)(buf + start);
3358 
3359 	for ( ; start <= end; start += sizeof(uint32_t))
3360 		*p++ = t4_read_reg(sc, start);
3361 }
3362 
3363 static void
3364 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3365 {
3366 	int i, n;
3367 	const unsigned int *reg_ranges;
3368 	static const unsigned int t4_reg_ranges[] = {
3369 		0x1008, 0x1108,
3370 		0x1180, 0x11b4,
3371 		0x11fc, 0x123c,
3372 		0x1300, 0x173c,
3373 		0x1800, 0x18fc,
3374 		0x3000, 0x30d8,
3375 		0x30e0, 0x5924,
3376 		0x5960, 0x59d4,
3377 		0x5a00, 0x5af8,
3378 		0x6000, 0x6098,
3379 		0x6100, 0x6150,
3380 		0x6200, 0x6208,
3381 		0x6240, 0x6248,
3382 		0x6280, 0x6338,
3383 		0x6370, 0x638c,
3384 		0x6400, 0x643c,
3385 		0x6500, 0x6524,
3386 		0x6a00, 0x6a38,
3387 		0x6a60, 0x6a78,
3388 		0x6b00, 0x6b84,
3389 		0x6bf0, 0x6c84,
3390 		0x6cf0, 0x6d84,
3391 		0x6df0, 0x6e84,
3392 		0x6ef0, 0x6f84,
3393 		0x6ff0, 0x7084,
3394 		0x70f0, 0x7184,
3395 		0x71f0, 0x7284,
3396 		0x72f0, 0x7384,
3397 		0x73f0, 0x7450,
3398 		0x7500, 0x7530,
3399 		0x7600, 0x761c,
3400 		0x7680, 0x76cc,
3401 		0x7700, 0x7798,
3402 		0x77c0, 0x77fc,
3403 		0x7900, 0x79fc,
3404 		0x7b00, 0x7c38,
3405 		0x7d00, 0x7efc,
3406 		0x8dc0, 0x8e1c,
3407 		0x8e30, 0x8e78,
3408 		0x8ea0, 0x8f6c,
3409 		0x8fc0, 0x9074,
3410 		0x90fc, 0x90fc,
3411 		0x9400, 0x9458,
3412 		0x9600, 0x96bc,
3413 		0x9800, 0x9808,
3414 		0x9820, 0x983c,
3415 		0x9850, 0x9864,
3416 		0x9c00, 0x9c6c,
3417 		0x9c80, 0x9cec,
3418 		0x9d00, 0x9d6c,
3419 		0x9d80, 0x9dec,
3420 		0x9e00, 0x9e6c,
3421 		0x9e80, 0x9eec,
3422 		0x9f00, 0x9f6c,
3423 		0x9f80, 0x9fec,
3424 		0xd004, 0xd03c,
3425 		0xdfc0, 0xdfe0,
3426 		0xe000, 0xea7c,
3427 		0xf000, 0x11110,
3428 		0x11118, 0x11190,
3429 		0x19040, 0x1906c,
3430 		0x19078, 0x19080,
3431 		0x1908c, 0x19124,
3432 		0x19150, 0x191b0,
3433 		0x191d0, 0x191e8,
3434 		0x19238, 0x1924c,
3435 		0x193f8, 0x19474,
3436 		0x19490, 0x194f8,
3437 		0x19800, 0x19f30,
3438 		0x1a000, 0x1a06c,
3439 		0x1a0b0, 0x1a120,
3440 		0x1a128, 0x1a138,
3441 		0x1a190, 0x1a1c4,
3442 		0x1a1fc, 0x1a1fc,
3443 		0x1e040, 0x1e04c,
3444 		0x1e284, 0x1e28c,
3445 		0x1e2c0, 0x1e2c0,
3446 		0x1e2e0, 0x1e2e0,
3447 		0x1e300, 0x1e384,
3448 		0x1e3c0, 0x1e3c8,
3449 		0x1e440, 0x1e44c,
3450 		0x1e684, 0x1e68c,
3451 		0x1e6c0, 0x1e6c0,
3452 		0x1e6e0, 0x1e6e0,
3453 		0x1e700, 0x1e784,
3454 		0x1e7c0, 0x1e7c8,
3455 		0x1e840, 0x1e84c,
3456 		0x1ea84, 0x1ea8c,
3457 		0x1eac0, 0x1eac0,
3458 		0x1eae0, 0x1eae0,
3459 		0x1eb00, 0x1eb84,
3460 		0x1ebc0, 0x1ebc8,
3461 		0x1ec40, 0x1ec4c,
3462 		0x1ee84, 0x1ee8c,
3463 		0x1eec0, 0x1eec0,
3464 		0x1eee0, 0x1eee0,
3465 		0x1ef00, 0x1ef84,
3466 		0x1efc0, 0x1efc8,
3467 		0x1f040, 0x1f04c,
3468 		0x1f284, 0x1f28c,
3469 		0x1f2c0, 0x1f2c0,
3470 		0x1f2e0, 0x1f2e0,
3471 		0x1f300, 0x1f384,
3472 		0x1f3c0, 0x1f3c8,
3473 		0x1f440, 0x1f44c,
3474 		0x1f684, 0x1f68c,
3475 		0x1f6c0, 0x1f6c0,
3476 		0x1f6e0, 0x1f6e0,
3477 		0x1f700, 0x1f784,
3478 		0x1f7c0, 0x1f7c8,
3479 		0x1f840, 0x1f84c,
3480 		0x1fa84, 0x1fa8c,
3481 		0x1fac0, 0x1fac0,
3482 		0x1fae0, 0x1fae0,
3483 		0x1fb00, 0x1fb84,
3484 		0x1fbc0, 0x1fbc8,
3485 		0x1fc40, 0x1fc4c,
3486 		0x1fe84, 0x1fe8c,
3487 		0x1fec0, 0x1fec0,
3488 		0x1fee0, 0x1fee0,
3489 		0x1ff00, 0x1ff84,
3490 		0x1ffc0, 0x1ffc8,
3491 		0x20000, 0x2002c,
3492 		0x20100, 0x2013c,
3493 		0x20190, 0x201c8,
3494 		0x20200, 0x20318,
3495 		0x20400, 0x20528,
3496 		0x20540, 0x20614,
3497 		0x21000, 0x21040,
3498 		0x2104c, 0x21060,
3499 		0x210c0, 0x210ec,
3500 		0x21200, 0x21268,
3501 		0x21270, 0x21284,
3502 		0x212fc, 0x21388,
3503 		0x21400, 0x21404,
3504 		0x21500, 0x21518,
3505 		0x2152c, 0x2153c,
3506 		0x21550, 0x21554,
3507 		0x21600, 0x21600,
3508 		0x21608, 0x21628,
3509 		0x21630, 0x2163c,
3510 		0x21700, 0x2171c,
3511 		0x21780, 0x2178c,
3512 		0x21800, 0x21c38,
3513 		0x21c80, 0x21d7c,
3514 		0x21e00, 0x21e04,
3515 		0x22000, 0x2202c,
3516 		0x22100, 0x2213c,
3517 		0x22190, 0x221c8,
3518 		0x22200, 0x22318,
3519 		0x22400, 0x22528,
3520 		0x22540, 0x22614,
3521 		0x23000, 0x23040,
3522 		0x2304c, 0x23060,
3523 		0x230c0, 0x230ec,
3524 		0x23200, 0x23268,
3525 		0x23270, 0x23284,
3526 		0x232fc, 0x23388,
3527 		0x23400, 0x23404,
3528 		0x23500, 0x23518,
3529 		0x2352c, 0x2353c,
3530 		0x23550, 0x23554,
3531 		0x23600, 0x23600,
3532 		0x23608, 0x23628,
3533 		0x23630, 0x2363c,
3534 		0x23700, 0x2371c,
3535 		0x23780, 0x2378c,
3536 		0x23800, 0x23c38,
3537 		0x23c80, 0x23d7c,
3538 		0x23e00, 0x23e04,
3539 		0x24000, 0x2402c,
3540 		0x24100, 0x2413c,
3541 		0x24190, 0x241c8,
3542 		0x24200, 0x24318,
3543 		0x24400, 0x24528,
3544 		0x24540, 0x24614,
3545 		0x25000, 0x25040,
3546 		0x2504c, 0x25060,
3547 		0x250c0, 0x250ec,
3548 		0x25200, 0x25268,
3549 		0x25270, 0x25284,
3550 		0x252fc, 0x25388,
3551 		0x25400, 0x25404,
3552 		0x25500, 0x25518,
3553 		0x2552c, 0x2553c,
3554 		0x25550, 0x25554,
3555 		0x25600, 0x25600,
3556 		0x25608, 0x25628,
3557 		0x25630, 0x2563c,
3558 		0x25700, 0x2571c,
3559 		0x25780, 0x2578c,
3560 		0x25800, 0x25c38,
3561 		0x25c80, 0x25d7c,
3562 		0x25e00, 0x25e04,
3563 		0x26000, 0x2602c,
3564 		0x26100, 0x2613c,
3565 		0x26190, 0x261c8,
3566 		0x26200, 0x26318,
3567 		0x26400, 0x26528,
3568 		0x26540, 0x26614,
3569 		0x27000, 0x27040,
3570 		0x2704c, 0x27060,
3571 		0x270c0, 0x270ec,
3572 		0x27200, 0x27268,
3573 		0x27270, 0x27284,
3574 		0x272fc, 0x27388,
3575 		0x27400, 0x27404,
3576 		0x27500, 0x27518,
3577 		0x2752c, 0x2753c,
3578 		0x27550, 0x27554,
3579 		0x27600, 0x27600,
3580 		0x27608, 0x27628,
3581 		0x27630, 0x2763c,
3582 		0x27700, 0x2771c,
3583 		0x27780, 0x2778c,
3584 		0x27800, 0x27c38,
3585 		0x27c80, 0x27d7c,
3586 		0x27e00, 0x27e04
3587 	};
3588 	static const unsigned int t5_reg_ranges[] = {
3589 		0x1008, 0x1148,
3590 		0x1180, 0x11b4,
3591 		0x11fc, 0x123c,
3592 		0x1280, 0x173c,
3593 		0x1800, 0x18fc,
3594 		0x3000, 0x3028,
3595 		0x3060, 0x30d8,
3596 		0x30e0, 0x30fc,
3597 		0x3140, 0x357c,
3598 		0x35a8, 0x35cc,
3599 		0x35ec, 0x35ec,
3600 		0x3600, 0x5624,
3601 		0x56cc, 0x575c,
3602 		0x580c, 0x5814,
3603 		0x5890, 0x58bc,
3604 		0x5940, 0x59dc,
3605 		0x59fc, 0x5a18,
3606 		0x5a60, 0x5a9c,
3607 		0x5b94, 0x5bfc,
3608 		0x6000, 0x6040,
3609 		0x6058, 0x614c,
3610 		0x7700, 0x7798,
3611 		0x77c0, 0x78fc,
3612 		0x7b00, 0x7c54,
3613 		0x7d00, 0x7efc,
3614 		0x8dc0, 0x8de0,
3615 		0x8df8, 0x8e84,
3616 		0x8ea0, 0x8f84,
3617 		0x8fc0, 0x90f8,
3618 		0x9400, 0x9470,
3619 		0x9600, 0x96f4,
3620 		0x9800, 0x9808,
3621 		0x9820, 0x983c,
3622 		0x9850, 0x9864,
3623 		0x9c00, 0x9c6c,
3624 		0x9c80, 0x9cec,
3625 		0x9d00, 0x9d6c,
3626 		0x9d80, 0x9dec,
3627 		0x9e00, 0x9e6c,
3628 		0x9e80, 0x9eec,
3629 		0x9f00, 0x9f6c,
3630 		0x9f80, 0xa020,
3631 		0xd004, 0xd03c,
3632 		0xdfc0, 0xdfe0,
3633 		0xe000, 0x11088,
3634 		0x1109c, 0x11110,
3635 		0x11118, 0x1117c,
3636 		0x11190, 0x11204,
3637 		0x19040, 0x1906c,
3638 		0x19078, 0x19080,
3639 		0x1908c, 0x19124,
3640 		0x19150, 0x191b0,
3641 		0x191d0, 0x191e8,
3642 		0x19238, 0x19290,
3643 		0x193f8, 0x19474,
3644 		0x19490, 0x194cc,
3645 		0x194f0, 0x194f8,
3646 		0x19c00, 0x19c60,
3647 		0x19c94, 0x19e10,
3648 		0x19e50, 0x19f34,
3649 		0x19f40, 0x19f50,
3650 		0x19f90, 0x19fe4,
3651 		0x1a000, 0x1a06c,
3652 		0x1a0b0, 0x1a120,
3653 		0x1a128, 0x1a138,
3654 		0x1a190, 0x1a1c4,
3655 		0x1a1fc, 0x1a1fc,
3656 		0x1e008, 0x1e00c,
3657 		0x1e040, 0x1e04c,
3658 		0x1e284, 0x1e290,
3659 		0x1e2c0, 0x1e2c0,
3660 		0x1e2e0, 0x1e2e0,
3661 		0x1e300, 0x1e384,
3662 		0x1e3c0, 0x1e3c8,
3663 		0x1e408, 0x1e40c,
3664 		0x1e440, 0x1e44c,
3665 		0x1e684, 0x1e690,
3666 		0x1e6c0, 0x1e6c0,
3667 		0x1e6e0, 0x1e6e0,
3668 		0x1e700, 0x1e784,
3669 		0x1e7c0, 0x1e7c8,
3670 		0x1e808, 0x1e80c,
3671 		0x1e840, 0x1e84c,
3672 		0x1ea84, 0x1ea90,
3673 		0x1eac0, 0x1eac0,
3674 		0x1eae0, 0x1eae0,
3675 		0x1eb00, 0x1eb84,
3676 		0x1ebc0, 0x1ebc8,
3677 		0x1ec08, 0x1ec0c,
3678 		0x1ec40, 0x1ec4c,
3679 		0x1ee84, 0x1ee90,
3680 		0x1eec0, 0x1eec0,
3681 		0x1eee0, 0x1eee0,
3682 		0x1ef00, 0x1ef84,
3683 		0x1efc0, 0x1efc8,
3684 		0x1f008, 0x1f00c,
3685 		0x1f040, 0x1f04c,
3686 		0x1f284, 0x1f290,
3687 		0x1f2c0, 0x1f2c0,
3688 		0x1f2e0, 0x1f2e0,
3689 		0x1f300, 0x1f384,
3690 		0x1f3c0, 0x1f3c8,
3691 		0x1f408, 0x1f40c,
3692 		0x1f440, 0x1f44c,
3693 		0x1f684, 0x1f690,
3694 		0x1f6c0, 0x1f6c0,
3695 		0x1f6e0, 0x1f6e0,
3696 		0x1f700, 0x1f784,
3697 		0x1f7c0, 0x1f7c8,
3698 		0x1f808, 0x1f80c,
3699 		0x1f840, 0x1f84c,
3700 		0x1fa84, 0x1fa90,
3701 		0x1fac0, 0x1fac0,
3702 		0x1fae0, 0x1fae0,
3703 		0x1fb00, 0x1fb84,
3704 		0x1fbc0, 0x1fbc8,
3705 		0x1fc08, 0x1fc0c,
3706 		0x1fc40, 0x1fc4c,
3707 		0x1fe84, 0x1fe90,
3708 		0x1fec0, 0x1fec0,
3709 		0x1fee0, 0x1fee0,
3710 		0x1ff00, 0x1ff84,
3711 		0x1ffc0, 0x1ffc8,
3712 		0x30000, 0x30030,
3713 		0x30100, 0x30144,
3714 		0x30190, 0x301d0,
3715 		0x30200, 0x30318,
3716 		0x30400, 0x3052c,
3717 		0x30540, 0x3061c,
3718 		0x30800, 0x30834,
3719 		0x308c0, 0x30908,
3720 		0x30910, 0x309ac,
3721 		0x30a00, 0x30a2c,
3722 		0x30a44, 0x30a50,
3723 		0x30a74, 0x30c24,
3724 		0x30d00, 0x30d00,
3725 		0x30d08, 0x30d14,
3726 		0x30d1c, 0x30d20,
3727 		0x30d3c, 0x30d50,
3728 		0x31200, 0x3120c,
3729 		0x31220, 0x31220,
3730 		0x31240, 0x31240,
3731 		0x31600, 0x3160c,
3732 		0x31a00, 0x31a1c,
3733 		0x31e00, 0x31e20,
3734 		0x31e38, 0x31e3c,
3735 		0x31e80, 0x31e80,
3736 		0x31e88, 0x31ea8,
3737 		0x31eb0, 0x31eb4,
3738 		0x31ec8, 0x31ed4,
3739 		0x31fb8, 0x32004,
3740 		0x32200, 0x32200,
3741 		0x32208, 0x32240,
3742 		0x32248, 0x32280,
3743 		0x32288, 0x322c0,
3744 		0x322c8, 0x322fc,
3745 		0x32600, 0x32630,
3746 		0x32a00, 0x32abc,
3747 		0x32b00, 0x32b70,
3748 		0x33000, 0x33048,
3749 		0x33060, 0x3309c,
3750 		0x330f0, 0x33148,
3751 		0x33160, 0x3319c,
3752 		0x331f0, 0x332e4,
3753 		0x332f8, 0x333e4,
3754 		0x333f8, 0x33448,
3755 		0x33460, 0x3349c,
3756 		0x334f0, 0x33548,
3757 		0x33560, 0x3359c,
3758 		0x335f0, 0x336e4,
3759 		0x336f8, 0x337e4,
3760 		0x337f8, 0x337fc,
3761 		0x33814, 0x33814,
3762 		0x3382c, 0x3382c,
3763 		0x33880, 0x3388c,
3764 		0x338e8, 0x338ec,
3765 		0x33900, 0x33948,
3766 		0x33960, 0x3399c,
3767 		0x339f0, 0x33ae4,
3768 		0x33af8, 0x33b10,
3769 		0x33b28, 0x33b28,
3770 		0x33b3c, 0x33b50,
3771 		0x33bf0, 0x33c10,
3772 		0x33c28, 0x33c28,
3773 		0x33c3c, 0x33c50,
3774 		0x33cf0, 0x33cfc,
3775 		0x34000, 0x34030,
3776 		0x34100, 0x34144,
3777 		0x34190, 0x341d0,
3778 		0x34200, 0x34318,
3779 		0x34400, 0x3452c,
3780 		0x34540, 0x3461c,
3781 		0x34800, 0x34834,
3782 		0x348c0, 0x34908,
3783 		0x34910, 0x349ac,
3784 		0x34a00, 0x34a2c,
3785 		0x34a44, 0x34a50,
3786 		0x34a74, 0x34c24,
3787 		0x34d00, 0x34d00,
3788 		0x34d08, 0x34d14,
3789 		0x34d1c, 0x34d20,
3790 		0x34d3c, 0x34d50,
3791 		0x35200, 0x3520c,
3792 		0x35220, 0x35220,
3793 		0x35240, 0x35240,
3794 		0x35600, 0x3560c,
3795 		0x35a00, 0x35a1c,
3796 		0x35e00, 0x35e20,
3797 		0x35e38, 0x35e3c,
3798 		0x35e80, 0x35e80,
3799 		0x35e88, 0x35ea8,
3800 		0x35eb0, 0x35eb4,
3801 		0x35ec8, 0x35ed4,
3802 		0x35fb8, 0x36004,
3803 		0x36200, 0x36200,
3804 		0x36208, 0x36240,
3805 		0x36248, 0x36280,
3806 		0x36288, 0x362c0,
3807 		0x362c8, 0x362fc,
3808 		0x36600, 0x36630,
3809 		0x36a00, 0x36abc,
3810 		0x36b00, 0x36b70,
3811 		0x37000, 0x37048,
3812 		0x37060, 0x3709c,
3813 		0x370f0, 0x37148,
3814 		0x37160, 0x3719c,
3815 		0x371f0, 0x372e4,
3816 		0x372f8, 0x373e4,
3817 		0x373f8, 0x37448,
3818 		0x37460, 0x3749c,
3819 		0x374f0, 0x37548,
3820 		0x37560, 0x3759c,
3821 		0x375f0, 0x376e4,
3822 		0x376f8, 0x377e4,
3823 		0x377f8, 0x377fc,
3824 		0x37814, 0x37814,
3825 		0x3782c, 0x3782c,
3826 		0x37880, 0x3788c,
3827 		0x378e8, 0x378ec,
3828 		0x37900, 0x37948,
3829 		0x37960, 0x3799c,
3830 		0x379f0, 0x37ae4,
3831 		0x37af8, 0x37b10,
3832 		0x37b28, 0x37b28,
3833 		0x37b3c, 0x37b50,
3834 		0x37bf0, 0x37c10,
3835 		0x37c28, 0x37c28,
3836 		0x37c3c, 0x37c50,
3837 		0x37cf0, 0x37cfc,
3838 		0x38000, 0x38030,
3839 		0x38100, 0x38144,
3840 		0x38190, 0x381d0,
3841 		0x38200, 0x38318,
3842 		0x38400, 0x3852c,
3843 		0x38540, 0x3861c,
3844 		0x38800, 0x38834,
3845 		0x388c0, 0x38908,
3846 		0x38910, 0x389ac,
3847 		0x38a00, 0x38a2c,
3848 		0x38a44, 0x38a50,
3849 		0x38a74, 0x38c24,
3850 		0x38d00, 0x38d00,
3851 		0x38d08, 0x38d14,
3852 		0x38d1c, 0x38d20,
3853 		0x38d3c, 0x38d50,
3854 		0x39200, 0x3920c,
3855 		0x39220, 0x39220,
3856 		0x39240, 0x39240,
3857 		0x39600, 0x3960c,
3858 		0x39a00, 0x39a1c,
3859 		0x39e00, 0x39e20,
3860 		0x39e38, 0x39e3c,
3861 		0x39e80, 0x39e80,
3862 		0x39e88, 0x39ea8,
3863 		0x39eb0, 0x39eb4,
3864 		0x39ec8, 0x39ed4,
3865 		0x39fb8, 0x3a004,
3866 		0x3a200, 0x3a200,
3867 		0x3a208, 0x3a240,
3868 		0x3a248, 0x3a280,
3869 		0x3a288, 0x3a2c0,
3870 		0x3a2c8, 0x3a2fc,
3871 		0x3a600, 0x3a630,
3872 		0x3aa00, 0x3aabc,
3873 		0x3ab00, 0x3ab70,
3874 		0x3b000, 0x3b048,
3875 		0x3b060, 0x3b09c,
3876 		0x3b0f0, 0x3b148,
3877 		0x3b160, 0x3b19c,
3878 		0x3b1f0, 0x3b2e4,
3879 		0x3b2f8, 0x3b3e4,
3880 		0x3b3f8, 0x3b448,
3881 		0x3b460, 0x3b49c,
3882 		0x3b4f0, 0x3b548,
3883 		0x3b560, 0x3b59c,
3884 		0x3b5f0, 0x3b6e4,
3885 		0x3b6f8, 0x3b7e4,
3886 		0x3b7f8, 0x3b7fc,
3887 		0x3b814, 0x3b814,
3888 		0x3b82c, 0x3b82c,
3889 		0x3b880, 0x3b88c,
3890 		0x3b8e8, 0x3b8ec,
3891 		0x3b900, 0x3b948,
3892 		0x3b960, 0x3b99c,
3893 		0x3b9f0, 0x3bae4,
3894 		0x3baf8, 0x3bb10,
3895 		0x3bb28, 0x3bb28,
3896 		0x3bb3c, 0x3bb50,
3897 		0x3bbf0, 0x3bc10,
3898 		0x3bc28, 0x3bc28,
3899 		0x3bc3c, 0x3bc50,
3900 		0x3bcf0, 0x3bcfc,
3901 		0x3c000, 0x3c030,
3902 		0x3c100, 0x3c144,
3903 		0x3c190, 0x3c1d0,
3904 		0x3c200, 0x3c318,
3905 		0x3c400, 0x3c52c,
3906 		0x3c540, 0x3c61c,
3907 		0x3c800, 0x3c834,
3908 		0x3c8c0, 0x3c908,
3909 		0x3c910, 0x3c9ac,
3910 		0x3ca00, 0x3ca2c,
3911 		0x3ca44, 0x3ca50,
3912 		0x3ca74, 0x3cc24,
3913 		0x3cd00, 0x3cd00,
3914 		0x3cd08, 0x3cd14,
3915 		0x3cd1c, 0x3cd20,
3916 		0x3cd3c, 0x3cd50,
3917 		0x3d200, 0x3d20c,
3918 		0x3d220, 0x3d220,
3919 		0x3d240, 0x3d240,
3920 		0x3d600, 0x3d60c,
3921 		0x3da00, 0x3da1c,
3922 		0x3de00, 0x3de20,
3923 		0x3de38, 0x3de3c,
3924 		0x3de80, 0x3de80,
3925 		0x3de88, 0x3dea8,
3926 		0x3deb0, 0x3deb4,
3927 		0x3dec8, 0x3ded4,
3928 		0x3dfb8, 0x3e004,
3929 		0x3e200, 0x3e200,
3930 		0x3e208, 0x3e240,
3931 		0x3e248, 0x3e280,
3932 		0x3e288, 0x3e2c0,
3933 		0x3e2c8, 0x3e2fc,
3934 		0x3e600, 0x3e630,
3935 		0x3ea00, 0x3eabc,
3936 		0x3eb00, 0x3eb70,
3937 		0x3f000, 0x3f048,
3938 		0x3f060, 0x3f09c,
3939 		0x3f0f0, 0x3f148,
3940 		0x3f160, 0x3f19c,
3941 		0x3f1f0, 0x3f2e4,
3942 		0x3f2f8, 0x3f3e4,
3943 		0x3f3f8, 0x3f448,
3944 		0x3f460, 0x3f49c,
3945 		0x3f4f0, 0x3f548,
3946 		0x3f560, 0x3f59c,
3947 		0x3f5f0, 0x3f6e4,
3948 		0x3f6f8, 0x3f7e4,
3949 		0x3f7f8, 0x3f7fc,
3950 		0x3f814, 0x3f814,
3951 		0x3f82c, 0x3f82c,
3952 		0x3f880, 0x3f88c,
3953 		0x3f8e8, 0x3f8ec,
3954 		0x3f900, 0x3f948,
3955 		0x3f960, 0x3f99c,
3956 		0x3f9f0, 0x3fae4,
3957 		0x3faf8, 0x3fb10,
3958 		0x3fb28, 0x3fb28,
3959 		0x3fb3c, 0x3fb50,
3960 		0x3fbf0, 0x3fc10,
3961 		0x3fc28, 0x3fc28,
3962 		0x3fc3c, 0x3fc50,
3963 		0x3fcf0, 0x3fcfc,
3964 		0x40000, 0x4000c,
3965 		0x40040, 0x40068,
3966 		0x4007c, 0x40144,
3967 		0x40180, 0x4018c,
3968 		0x40200, 0x40298,
3969 		0x402ac, 0x4033c,
3970 		0x403f8, 0x403fc,
3971 		0x41304, 0x413c4,
3972 		0x41400, 0x4141c,
3973 		0x41480, 0x414d0,
3974 		0x44000, 0x44078,
3975 		0x440c0, 0x44278,
3976 		0x442c0, 0x44478,
3977 		0x444c0, 0x44678,
3978 		0x446c0, 0x44878,
3979 		0x448c0, 0x449fc,
3980 		0x45000, 0x45068,
3981 		0x45080, 0x45084,
3982 		0x450a0, 0x450b0,
3983 		0x45200, 0x45268,
3984 		0x45280, 0x45284,
3985 		0x452a0, 0x452b0,
3986 		0x460c0, 0x460e4,
3987 		0x47000, 0x4708c,
3988 		0x47200, 0x47250,
3989 		0x47400, 0x47420,
3990 		0x47600, 0x47618,
3991 		0x47800, 0x47814,
3992 		0x48000, 0x4800c,
3993 		0x48040, 0x48068,
3994 		0x4807c, 0x48144,
3995 		0x48180, 0x4818c,
3996 		0x48200, 0x48298,
3997 		0x482ac, 0x4833c,
3998 		0x483f8, 0x483fc,
3999 		0x49304, 0x493c4,
4000 		0x49400, 0x4941c,
4001 		0x49480, 0x494d0,
4002 		0x4c000, 0x4c078,
4003 		0x4c0c0, 0x4c278,
4004 		0x4c2c0, 0x4c478,
4005 		0x4c4c0, 0x4c678,
4006 		0x4c6c0, 0x4c878,
4007 		0x4c8c0, 0x4c9fc,
4008 		0x4d000, 0x4d068,
4009 		0x4d080, 0x4d084,
4010 		0x4d0a0, 0x4d0b0,
4011 		0x4d200, 0x4d268,
4012 		0x4d280, 0x4d284,
4013 		0x4d2a0, 0x4d2b0,
4014 		0x4e0c0, 0x4e0e4,
4015 		0x4f000, 0x4f08c,
4016 		0x4f200, 0x4f250,
4017 		0x4f400, 0x4f420,
4018 		0x4f600, 0x4f618,
4019 		0x4f800, 0x4f814,
4020 		0x50000, 0x500cc,
4021 		0x50400, 0x50400,
4022 		0x50800, 0x508cc,
4023 		0x50c00, 0x50c00,
4024 		0x51000, 0x5101c,
4025 		0x51300, 0x51308,
4026 	};
4027 
4028 	if (is_t4(sc)) {
4029 		reg_ranges = &t4_reg_ranges[0];
4030 		n = nitems(t4_reg_ranges);
4031 	} else {
4032 		reg_ranges = &t5_reg_ranges[0];
4033 		n = nitems(t5_reg_ranges);
4034 	}
4035 
4036 	regs->version = chip_id(sc) | chip_rev(sc) << 10;
4037 	for (i = 0; i < n; i += 2)
4038 		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4039 }
4040 
4041 static void
4042 cxgbe_tick(void *arg)
4043 {
4044 	struct port_info *pi = arg;
4045 	struct ifnet *ifp = pi->ifp;
4046 	struct sge_txq *txq;
4047 	int i, drops;
4048 	struct port_stats *s = &pi->stats;
4049 
4050 	PORT_LOCK(pi);
4051 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4052 		PORT_UNLOCK(pi);
4053 		return;	/* without scheduling another callout */
4054 	}
4055 
4056 	t4_get_port_stats(pi->adapter, pi->tx_chan, s);
4057 
4058 	ifp->if_opackets = s->tx_frames - s->tx_pause;
4059 	ifp->if_ipackets = s->rx_frames - s->rx_pause;
4060 	ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4061 	ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4062 	ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4063 	ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4064 	ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4065 	    s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4066 	    s->rx_trunc3;
4067 
4068 	drops = s->tx_drop;
4069 	for_each_txq(pi, i, txq)
4070 		drops += txq->br->br_drops;
4071 	ifp->if_snd.ifq_drops = drops;
4072 
4073 	ifp->if_oerrors = s->tx_error_frames;
4074 	ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4075 	    s->rx_fcs_err + s->rx_len_err;
4076 
4077 	callout_schedule(&pi->tick, hz);
4078 	PORT_UNLOCK(pi);
4079 }
4080 
4081 static void
4082 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4083 {
4084 	struct ifnet *vlan;
4085 
4086 	if (arg != ifp || ifp->if_type != IFT_ETHER)
4087 		return;
4088 
4089 	vlan = VLAN_DEVAT(ifp, vid);
4090 	VLAN_SETCOOKIE(vlan, ifp);
4091 }
4092 
4093 static int
4094 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4095 {
4096 
4097 #ifdef INVARIANTS
4098 	panic("%s: opcode 0x%02x on iq %p with payload %p",
4099 	    __func__, rss->opcode, iq, m);
4100 #else
4101 	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4102 	    __func__, rss->opcode, iq, m);
4103 	m_freem(m);
4104 #endif
4105 	return (EDOOFUS);
4106 }
4107 
4108 int
4109 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4110 {
4111 	uintptr_t *loc, new;
4112 
4113 	if (opcode >= nitems(sc->cpl_handler))
4114 		return (EINVAL);
4115 
4116 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4117 	loc = (uintptr_t *) &sc->cpl_handler[opcode];
4118 	atomic_store_rel_ptr(loc, new);
4119 
4120 	return (0);
4121 }
4122 
4123 static int
4124 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4125 {
4126 
4127 #ifdef INVARIANTS
4128 	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4129 #else
4130 	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4131 	    __func__, iq, ctrl);
4132 #endif
4133 	return (EDOOFUS);
4134 }
4135 
4136 int
4137 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4138 {
4139 	uintptr_t *loc, new;
4140 
4141 	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4142 	loc = (uintptr_t *) &sc->an_handler;
4143 	atomic_store_rel_ptr(loc, new);
4144 
4145 	return (0);
4146 }
4147 
4148 static int
4149 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4150 {
4151 	const struct cpl_fw6_msg *cpl =
4152 	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
4153 
4154 #ifdef INVARIANTS
4155 	panic("%s: fw_msg type %d", __func__, cpl->type);
4156 #else
4157 	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4158 #endif
4159 	return (EDOOFUS);
4160 }
4161 
4162 int
4163 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4164 {
4165 	uintptr_t *loc, new;
4166 
4167 	if (type >= nitems(sc->fw_msg_handler))
4168 		return (EINVAL);
4169 
4170 	/*
4171 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4172 	 * handler dispatch table.  Reject any attempt to install a handler for
4173 	 * this subtype.
4174 	 */
4175 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4176 		return (EINVAL);
4177 
4178 	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4179 	loc = (uintptr_t *) &sc->fw_msg_handler[type];
4180 	atomic_store_rel_ptr(loc, new);
4181 
4182 	return (0);
4183 }
4184 
4185 static int
4186 t4_sysctls(struct adapter *sc)
4187 {
4188 	struct sysctl_ctx_list *ctx;
4189 	struct sysctl_oid *oid;
4190 	struct sysctl_oid_list *children, *c0;
4191 	static char *caps[] = {
4192 		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
4193 		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"	/* caps[1] niccaps */
4194 		    "\6HASHFILTER\7ETHOFLD",
4195 		"\20\1TOE",				/* caps[2] toecaps */
4196 		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
4197 		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
4198 		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4199 		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4200 		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
4201 		    "\4PO_INITIAOR\5PO_TARGET"
4202 	};
4203 	static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4204 
4205 	ctx = device_get_sysctl_ctx(sc->dev);
4206 
4207 	/*
4208 	 * dev.t4nex.X.
4209 	 */
4210 	oid = device_get_sysctl_tree(sc->dev);
4211 	c0 = children = SYSCTL_CHILDREN(oid);
4212 
4213 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4214 	    sc->params.nports, "# of ports");
4215 
4216 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4217 	    NULL, chip_rev(sc), "chip hardware revision");
4218 
4219 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4220 	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4221 
4222 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4223 	    CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4224 
4225 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4226 	    sc->cfcsum, "config file checksum");
4227 
4228 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4229 	    CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4230 	    sysctl_bitfield, "A", "available doorbells");
4231 
4232 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4233 	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4234 	    sysctl_bitfield, "A", "available link capabilities");
4235 
4236 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4237 	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4238 	    sysctl_bitfield, "A", "available NIC capabilities");
4239 
4240 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4241 	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4242 	    sysctl_bitfield, "A", "available TCP offload capabilities");
4243 
4244 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4245 	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4246 	    sysctl_bitfield, "A", "available RDMA capabilities");
4247 
4248 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4249 	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4250 	    sysctl_bitfield, "A", "available iSCSI capabilities");
4251 
4252 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4253 	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4254 	    sysctl_bitfield, "A", "available FCoE capabilities");
4255 
4256 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4257 	    sc->params.vpd.cclk, "core clock frequency (in KHz)");
4258 
4259 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4260 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4261 	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4262 	    "interrupt holdoff timer values (us)");
4263 
4264 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4265 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4266 	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4267 	    "interrupt holdoff packet counter values");
4268 
4269 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4270 	    NULL, sc->tids.nftids, "number of filters");
4271 
4272 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4273 	    CTLFLAG_RD, sc, 0, sysctl_temperature, "A",
4274 	    "chip temperature (in Celsius)");
4275 
4276 	t4_sge_sysctls(sc, ctx, children);
4277 
4278 	sc->lro_timeout = 100;
4279 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4280 	    &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4281 
4282 #ifdef SBUF_DRAIN
4283 	/*
4284 	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4285 	 */
4286 	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4287 	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4288 	    "logs and miscellaneous information");
4289 	children = SYSCTL_CHILDREN(oid);
4290 
4291 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4292 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4293 	    sysctl_cctrl, "A", "congestion control");
4294 
4295 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4296 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4297 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4298 
4299 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4300 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4301 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4302 
4303 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4304 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4305 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4306 
4307 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4308 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4309 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4310 
4311 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4312 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4313 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4314 
4315 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4316 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4317 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4318 
4319 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4320 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4321 	    sysctl_cim_la, "A", "CIM logic analyzer");
4322 
4323 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4324 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4325 	    sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4326 
4327 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4328 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4329 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4330 
4331 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4332 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4333 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4334 
4335 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4336 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4337 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4338 
4339 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4340 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4341 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4342 
4343 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4344 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4345 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4346 
4347 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4348 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4349 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4350 
4351 	if (is_t5(sc)) {
4352 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4353 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4354 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4355 
4356 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4357 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4358 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4359 	}
4360 
4361 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4362 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4363 	    sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4364 
4365 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4366 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4367 	    sysctl_cim_qcfg, "A", "CIM queue configuration");
4368 
4369 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4370 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4371 	    sysctl_cpl_stats, "A", "CPL statistics");
4372 
4373 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4374 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4375 	    sysctl_ddp_stats, "A", "DDP statistics");
4376 
4377 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4378 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4379 	    sysctl_devlog, "A", "firmware's device log");
4380 
4381 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4382 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4383 	    sysctl_fcoe_stats, "A", "FCoE statistics");
4384 
4385 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4386 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4387 	    sysctl_hw_sched, "A", "hardware scheduler ");
4388 
4389 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4390 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4391 	    sysctl_l2t, "A", "hardware L2 table");
4392 
4393 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4394 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4395 	    sysctl_lb_stats, "A", "loopback statistics");
4396 
4397 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4398 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4399 	    sysctl_meminfo, "A", "memory regions");
4400 
4401 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4402 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4403 	    sysctl_mps_tcam, "A", "MPS TCAM entries");
4404 
4405 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4406 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4407 	    sysctl_path_mtus, "A", "path MTUs");
4408 
4409 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4410 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4411 	    sysctl_pm_stats, "A", "PM statistics");
4412 
4413 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4414 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4415 	    sysctl_rdma_stats, "A", "RDMA statistics");
4416 
4417 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4418 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4419 	    sysctl_tcp_stats, "A", "TCP statistics");
4420 
4421 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4422 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4423 	    sysctl_tids, "A", "TID information");
4424 
4425 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4426 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4427 	    sysctl_tp_err_stats, "A", "TP error statistics");
4428 
4429 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4430 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4431 	    sysctl_tp_la, "A", "TP logic analyzer");
4432 
4433 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4434 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4435 	    sysctl_tx_rate, "A", "Tx rate");
4436 
4437 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4438 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4439 	    sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4440 
4441 	if (is_t5(sc)) {
4442 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4443 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4444 		    sysctl_wcwr_stats, "A", "write combined work requests");
4445 	}
4446 #endif
4447 
4448 #ifdef TCP_OFFLOAD
4449 	if (is_offload(sc)) {
4450 		/*
4451 		 * dev.t4nex.X.toe.
4452 		 */
4453 		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4454 		    NULL, "TOE parameters");
4455 		children = SYSCTL_CHILDREN(oid);
4456 
4457 		sc->tt.sndbuf = 256 * 1024;
4458 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4459 		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
4460 
4461 		sc->tt.ddp = 0;
4462 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4463 		    &sc->tt.ddp, 0, "DDP allowed");
4464 
4465 		sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4466 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4467 		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
4468 
4469 		sc->tt.ddp_thres =
4470 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4471 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4472 		    &sc->tt.ddp_thres, 0, "DDP threshold");
4473 
4474 		sc->tt.rx_coalesce = 1;
4475 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4476 		    CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4477 	}
4478 #endif
4479 
4480 
4481 	return (0);
4482 }
4483 
4484 static int
4485 cxgbe_sysctls(struct port_info *pi)
4486 {
4487 	struct sysctl_ctx_list *ctx;
4488 	struct sysctl_oid *oid;
4489 	struct sysctl_oid_list *children;
4490 
4491 	ctx = device_get_sysctl_ctx(pi->dev);
4492 
4493 	/*
4494 	 * dev.cxgbe.X.
4495 	 */
4496 	oid = device_get_sysctl_tree(pi->dev);
4497 	children = SYSCTL_CHILDREN(oid);
4498 
4499 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4500 	   CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4501 	if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4502 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4503 		    CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4504 		    "PHY temperature (in Celsius)");
4505 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4506 		    CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4507 		    "PHY firmware version");
4508 	}
4509 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4510 	    &pi->nrxq, 0, "# of rx queues");
4511 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4512 	    &pi->ntxq, 0, "# of tx queues");
4513 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4514 	    &pi->first_rxq, 0, "index of first rx queue");
4515 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4516 	    &pi->first_txq, 0, "index of first tx queue");
4517 
4518 #ifdef TCP_OFFLOAD
4519 	if (is_offload(pi->adapter)) {
4520 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4521 		    &pi->nofldrxq, 0,
4522 		    "# of rx queues for offloaded TCP connections");
4523 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4524 		    &pi->nofldtxq, 0,
4525 		    "# of tx queues for offloaded TCP connections");
4526 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4527 		    CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4528 		    "index of first TOE rx queue");
4529 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4530 		    CTLFLAG_RD, &pi->first_ofld_txq, 0,
4531 		    "index of first TOE tx queue");
4532 	}
4533 #endif
4534 
4535 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4536 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4537 	    "holdoff timer index");
4538 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4539 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4540 	    "holdoff packet counter index");
4541 
4542 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4543 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4544 	    "rx queue size");
4545 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4546 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4547 	    "tx queue size");
4548 
4549 	/*
4550 	 * dev.cxgbe.X.stats.
4551 	 */
4552 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4553 	    NULL, "port statistics");
4554 	children = SYSCTL_CHILDREN(oid);
4555 
4556 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4557 	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4558 	    CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
4559 	    sysctl_handle_t4_reg64, "QU", desc)
4560 
4561 	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4562 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4563 	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4564 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4565 	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4566 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4567 	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4568 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4569 	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4570 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4571 	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4572 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4573 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4574 	    "# of tx frames in this range",
4575 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4576 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4577 	    "# of tx frames in this range",
4578 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4579 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4580 	    "# of tx frames in this range",
4581 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4582 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4583 	    "# of tx frames in this range",
4584 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4585 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4586 	    "# of tx frames in this range",
4587 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4588 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4589 	    "# of tx frames in this range",
4590 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4591 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4592 	    "# of tx frames in this range",
4593 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4594 	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4595 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4596 	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4597 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4598 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4599 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4600 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4601 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4602 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4603 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4604 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4605 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4606 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4607 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4608 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4609 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4610 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4611 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4612 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4613 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4614 
4615 	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4616 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4617 	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4618 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4619 	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4620 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4621 	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4622 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4623 	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4624 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4625 	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4626 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4627 	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4628 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4629 	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4630 	    "# of frames received with bad FCS",
4631 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4632 	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4633 	    "# of frames received with length error",
4634 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4635 	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4636 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4637 	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4638 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4639 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4640 	    "# of rx frames in this range",
4641 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4642 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4643 	    "# of rx frames in this range",
4644 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4645 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4646 	    "# of rx frames in this range",
4647 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4648 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4649 	    "# of rx frames in this range",
4650 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4651 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4652 	    "# of rx frames in this range",
4653 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4654 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4655 	    "# of rx frames in this range",
4656 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4657 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4658 	    "# of rx frames in this range",
4659 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4660 	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4661 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4662 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4663 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4664 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4665 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4666 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4667 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4668 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4669 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4670 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4671 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4672 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4673 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4674 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4675 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4676 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4677 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4678 
4679 #undef SYSCTL_ADD_T4_REG64
4680 
4681 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4682 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4683 	    &pi->stats.name, desc)
4684 
4685 	/* We get these from port_stats and they may be stale by upto 1s */
4686 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4687 	    "# drops due to buffer-group 0 overflows");
4688 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4689 	    "# drops due to buffer-group 1 overflows");
4690 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4691 	    "# drops due to buffer-group 2 overflows");
4692 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4693 	    "# drops due to buffer-group 3 overflows");
4694 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4695 	    "# of buffer-group 0 truncated packets");
4696 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4697 	    "# of buffer-group 1 truncated packets");
4698 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4699 	    "# of buffer-group 2 truncated packets");
4700 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4701 	    "# of buffer-group 3 truncated packets");
4702 
4703 #undef SYSCTL_ADD_T4_PORTSTAT
4704 
4705 	return (0);
4706 }
4707 
4708 static int
4709 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4710 {
4711 	int rc, *i;
4712 	struct sbuf sb;
4713 
4714 	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4715 	for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4716 		sbuf_printf(&sb, "%d ", *i);
4717 	sbuf_trim(&sb);
4718 	sbuf_finish(&sb);
4719 	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4720 	sbuf_delete(&sb);
4721 	return (rc);
4722 }
4723 
4724 static int
4725 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4726 {
4727 	int rc;
4728 	struct sbuf *sb;
4729 
4730 	rc = sysctl_wire_old_buffer(req, 0);
4731 	if (rc != 0)
4732 		return(rc);
4733 
4734 	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4735 	if (sb == NULL)
4736 		return (ENOMEM);
4737 
4738 	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4739 	rc = sbuf_finish(sb);
4740 	sbuf_delete(sb);
4741 
4742 	return (rc);
4743 }
4744 
4745 static int
4746 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4747 {
4748 	struct port_info *pi = arg1;
4749 	int op = arg2;
4750 	struct adapter *sc = pi->adapter;
4751 	u_int v;
4752 	int rc;
4753 
4754 	rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4755 	if (rc)
4756 		return (rc);
4757 	/* XXX: magic numbers */
4758 	rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4759 	    &v);
4760 	end_synchronized_op(sc, 0);
4761 	if (rc)
4762 		return (rc);
4763 	if (op == 0)
4764 		v /= 256;
4765 
4766 	rc = sysctl_handle_int(oidp, &v, 0, req);
4767 	return (rc);
4768 }
4769 
4770 static int
4771 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
4772 {
4773 	struct port_info *pi = arg1;
4774 	struct adapter *sc = pi->adapter;
4775 	int idx, rc, i;
4776 	struct sge_rxq *rxq;
4777 #ifdef TCP_OFFLOAD
4778 	struct sge_ofld_rxq *ofld_rxq;
4779 #endif
4780 	uint8_t v;
4781 
4782 	idx = pi->tmr_idx;
4783 
4784 	rc = sysctl_handle_int(oidp, &idx, 0, req);
4785 	if (rc != 0 || req->newptr == NULL)
4786 		return (rc);
4787 
4788 	if (idx < 0 || idx >= SGE_NTIMERS)
4789 		return (EINVAL);
4790 
4791 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4792 	    "t4tmr");
4793 	if (rc)
4794 		return (rc);
4795 
4796 	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
4797 	for_each_rxq(pi, i, rxq) {
4798 #ifdef atomic_store_rel_8
4799 		atomic_store_rel_8(&rxq->iq.intr_params, v);
4800 #else
4801 		rxq->iq.intr_params = v;
4802 #endif
4803 	}
4804 #ifdef TCP_OFFLOAD
4805 	for_each_ofld_rxq(pi, i, ofld_rxq) {
4806 #ifdef atomic_store_rel_8
4807 		atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
4808 #else
4809 		ofld_rxq->iq.intr_params = v;
4810 #endif
4811 	}
4812 #endif
4813 	pi->tmr_idx = idx;
4814 
4815 	end_synchronized_op(sc, LOCK_HELD);
4816 	return (0);
4817 }
4818 
4819 static int
4820 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
4821 {
4822 	struct port_info *pi = arg1;
4823 	struct adapter *sc = pi->adapter;
4824 	int idx, rc;
4825 
4826 	idx = pi->pktc_idx;
4827 
4828 	rc = sysctl_handle_int(oidp, &idx, 0, req);
4829 	if (rc != 0 || req->newptr == NULL)
4830 		return (rc);
4831 
4832 	if (idx < -1 || idx >= SGE_NCOUNTERS)
4833 		return (EINVAL);
4834 
4835 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4836 	    "t4pktc");
4837 	if (rc)
4838 		return (rc);
4839 
4840 	if (pi->flags & PORT_INIT_DONE)
4841 		rc = EBUSY; /* cannot be changed once the queues are created */
4842 	else
4843 		pi->pktc_idx = idx;
4844 
4845 	end_synchronized_op(sc, LOCK_HELD);
4846 	return (rc);
4847 }
4848 
4849 static int
4850 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
4851 {
4852 	struct port_info *pi = arg1;
4853 	struct adapter *sc = pi->adapter;
4854 	int qsize, rc;
4855 
4856 	qsize = pi->qsize_rxq;
4857 
4858 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
4859 	if (rc != 0 || req->newptr == NULL)
4860 		return (rc);
4861 
4862 	if (qsize < 128 || (qsize & 7))
4863 		return (EINVAL);
4864 
4865 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4866 	    "t4rxqs");
4867 	if (rc)
4868 		return (rc);
4869 
4870 	if (pi->flags & PORT_INIT_DONE)
4871 		rc = EBUSY; /* cannot be changed once the queues are created */
4872 	else
4873 		pi->qsize_rxq = qsize;
4874 
4875 	end_synchronized_op(sc, LOCK_HELD);
4876 	return (rc);
4877 }
4878 
4879 static int
4880 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
4881 {
4882 	struct port_info *pi = arg1;
4883 	struct adapter *sc = pi->adapter;
4884 	int qsize, rc;
4885 
4886 	qsize = pi->qsize_txq;
4887 
4888 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
4889 	if (rc != 0 || req->newptr == NULL)
4890 		return (rc);
4891 
4892 	/* bufring size must be powerof2 */
4893 	if (qsize < 128 || !powerof2(qsize))
4894 		return (EINVAL);
4895 
4896 	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4897 	    "t4txqs");
4898 	if (rc)
4899 		return (rc);
4900 
4901 	if (pi->flags & PORT_INIT_DONE)
4902 		rc = EBUSY; /* cannot be changed once the queues are created */
4903 	else
4904 		pi->qsize_txq = qsize;
4905 
4906 	end_synchronized_op(sc, LOCK_HELD);
4907 	return (rc);
4908 }
4909 
4910 static int
4911 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
4912 {
4913 	struct adapter *sc = arg1;
4914 	int reg = arg2;
4915 	uint64_t val;
4916 
4917 	val = t4_read_reg64(sc, reg);
4918 
4919 	return (sysctl_handle_64(oidp, &val, 0, req));
4920 }
4921 
4922 static int
4923 sysctl_temperature(SYSCTL_HANDLER_ARGS)
4924 {
4925 	struct adapter *sc = arg1;
4926 	int rc, t;
4927 	uint32_t param, val;
4928 
4929 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
4930 	if (rc)
4931 		return (rc);
4932 	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4933 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
4934 	    V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
4935 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
4936 	end_synchronized_op(sc, 0);
4937 	if (rc)
4938 		return (rc);
4939 
4940 	/* unknown is returned as 0 but we display -1 in that case */
4941 	t = val == 0 ? -1 : val;
4942 
4943 	rc = sysctl_handle_int(oidp, &t, 0, req);
4944 	return (rc);
4945 }
4946 
4947 #ifdef SBUF_DRAIN
4948 static int
4949 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
4950 {
4951 	struct adapter *sc = arg1;
4952 	struct sbuf *sb;
4953 	int rc, i;
4954 	uint16_t incr[NMTUS][NCCTRL_WIN];
4955 	static const char *dec_fac[] = {
4956 		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
4957 		"0.9375"
4958 	};
4959 
4960 	rc = sysctl_wire_old_buffer(req, 0);
4961 	if (rc != 0)
4962 		return (rc);
4963 
4964 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4965 	if (sb == NULL)
4966 		return (ENOMEM);
4967 
4968 	t4_read_cong_tbl(sc, incr);
4969 
4970 	for (i = 0; i < NCCTRL_WIN; ++i) {
4971 		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
4972 		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
4973 		    incr[5][i], incr[6][i], incr[7][i]);
4974 		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
4975 		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
4976 		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
4977 		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
4978 	}
4979 
4980 	rc = sbuf_finish(sb);
4981 	sbuf_delete(sb);
4982 
4983 	return (rc);
4984 }
4985 
4986 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
4987 	"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",	/* ibq's */
4988 	"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI",	/* obq's */
4989 	"SGE0-RX", "SGE1-RX"	/* additional obq's (T5 onwards) */
4990 };
4991 
4992 static int
4993 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
4994 {
4995 	struct adapter *sc = arg1;
4996 	struct sbuf *sb;
4997 	int rc, i, n, qid = arg2;
4998 	uint32_t *buf, *p;
4999 	char *qtype;
5000 	u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5001 
5002 	KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5003 	    ("%s: bad qid %d\n", __func__, qid));
5004 
5005 	if (qid < CIM_NUM_IBQ) {
5006 		/* inbound queue */
5007 		qtype = "IBQ";
5008 		n = 4 * CIM_IBQ_SIZE;
5009 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5010 		rc = t4_read_cim_ibq(sc, qid, buf, n);
5011 	} else {
5012 		/* outbound queue */
5013 		qtype = "OBQ";
5014 		qid -= CIM_NUM_IBQ;
5015 		n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5016 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5017 		rc = t4_read_cim_obq(sc, qid, buf, n);
5018 	}
5019 
5020 	if (rc < 0) {
5021 		rc = -rc;
5022 		goto done;
5023 	}
5024 	n = rc * sizeof(uint32_t);	/* rc has # of words actually read */
5025 
5026 	rc = sysctl_wire_old_buffer(req, 0);
5027 	if (rc != 0)
5028 		goto done;
5029 
5030 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5031 	if (sb == NULL) {
5032 		rc = ENOMEM;
5033 		goto done;
5034 	}
5035 
5036 	sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5037 	for (i = 0, p = buf; i < n; i += 16, p += 4)
5038 		sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5039 		    p[2], p[3]);
5040 
5041 	rc = sbuf_finish(sb);
5042 	sbuf_delete(sb);
5043 done:
5044 	free(buf, M_CXGBE);
5045 	return (rc);
5046 }
5047 
5048 static int
5049 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5050 {
5051 	struct adapter *sc = arg1;
5052 	u_int cfg;
5053 	struct sbuf *sb;
5054 	uint32_t *buf, *p;
5055 	int rc;
5056 
5057 	rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5058 	if (rc != 0)
5059 		return (rc);
5060 
5061 	rc = sysctl_wire_old_buffer(req, 0);
5062 	if (rc != 0)
5063 		return (rc);
5064 
5065 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5066 	if (sb == NULL)
5067 		return (ENOMEM);
5068 
5069 	buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5070 	    M_ZERO | M_WAITOK);
5071 
5072 	rc = -t4_cim_read_la(sc, buf, NULL);
5073 	if (rc != 0)
5074 		goto done;
5075 
5076 	sbuf_printf(sb, "Status   Data      PC%s",
5077 	    cfg & F_UPDBGLACAPTPCONLY ? "" :
5078 	    "     LS0Stat  LS0Addr             LS0Data");
5079 
5080 	KASSERT((sc->params.cim_la_size & 7) == 0,
5081 	    ("%s: p will walk off the end of buf", __func__));
5082 
5083 	for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5084 		if (cfg & F_UPDBGLACAPTPCONLY) {
5085 			sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5086 			    p[6], p[7]);
5087 			sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5088 			    (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5089 			    p[4] & 0xff, p[5] >> 8);
5090 			sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5091 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5092 			    p[1] & 0xf, p[2] >> 4);
5093 		} else {
5094 			sbuf_printf(sb,
5095 			    "\n  %02x   %x%07x %x%07x %08x %08x "
5096 			    "%08x%08x%08x%08x",
5097 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5098 			    p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5099 			    p[6], p[7]);
5100 		}
5101 	}
5102 
5103 	rc = sbuf_finish(sb);
5104 	sbuf_delete(sb);
5105 done:
5106 	free(buf, M_CXGBE);
5107 	return (rc);
5108 }
5109 
5110 static int
5111 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5112 {
5113 	struct adapter *sc = arg1;
5114 	u_int i;
5115 	struct sbuf *sb;
5116 	uint32_t *buf, *p;
5117 	int rc;
5118 
5119 	rc = sysctl_wire_old_buffer(req, 0);
5120 	if (rc != 0)
5121 		return (rc);
5122 
5123 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5124 	if (sb == NULL)
5125 		return (ENOMEM);
5126 
5127 	buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5128 	    M_ZERO | M_WAITOK);
5129 
5130 	t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5131 	p = buf;
5132 
5133 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5134 		sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5135 		    p[1], p[0]);
5136 	}
5137 
5138 	sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5139 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5140 		sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5141 		    (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5142 		    (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5143 		    (p[1] >> 2) | ((p[2] & 3) << 30),
5144 		    (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5145 		    p[0] & 1);
5146 	}
5147 
5148 	rc = sbuf_finish(sb);
5149 	sbuf_delete(sb);
5150 	free(buf, M_CXGBE);
5151 	return (rc);
5152 }
5153 
5154 static int
5155 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5156 {
5157 	struct adapter *sc = arg1;
5158 	u_int i;
5159 	struct sbuf *sb;
5160 	uint32_t *buf, *p;
5161 	int rc;
5162 
5163 	rc = sysctl_wire_old_buffer(req, 0);
5164 	if (rc != 0)
5165 		return (rc);
5166 
5167 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5168 	if (sb == NULL)
5169 		return (ENOMEM);
5170 
5171 	buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5172 	    M_ZERO | M_WAITOK);
5173 
5174 	t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5175 	p = buf;
5176 
5177 	sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5178 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5179 		sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5180 		    (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5181 		    p[4], p[3], p[2], p[1], p[0]);
5182 	}
5183 
5184 	sbuf_printf(sb, "\n\nCntl ID               Data");
5185 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5186 		sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5187 		    (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5188 	}
5189 
5190 	rc = sbuf_finish(sb);
5191 	sbuf_delete(sb);
5192 	free(buf, M_CXGBE);
5193 	return (rc);
5194 }
5195 
5196 static int
5197 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5198 {
5199 	struct adapter *sc = arg1;
5200 	struct sbuf *sb;
5201 	int rc, i;
5202 	uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5203 	uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5204 	uint16_t thres[CIM_NUM_IBQ];
5205 	uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5206 	uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5207 	u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5208 
5209 	if (is_t4(sc)) {
5210 		cim_num_obq = CIM_NUM_OBQ;
5211 		ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5212 		obq_rdaddr = A_UP_OBQ_0_REALADDR;
5213 	} else {
5214 		cim_num_obq = CIM_NUM_OBQ_T5;
5215 		ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5216 		obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5217 	}
5218 	nq = CIM_NUM_IBQ + cim_num_obq;
5219 
5220 	rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5221 	if (rc == 0)
5222 		rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5223 	if (rc != 0)
5224 		return (rc);
5225 
5226 	t4_read_cimq_cfg(sc, base, size, thres);
5227 
5228 	rc = sysctl_wire_old_buffer(req, 0);
5229 	if (rc != 0)
5230 		return (rc);
5231 
5232 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5233 	if (sb == NULL)
5234 		return (ENOMEM);
5235 
5236 	sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5237 
5238 	for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5239 		sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5240 		    qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5241 		    G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5242 		    G_QUEREMFLITS(p[2]) * 16);
5243 	for ( ; i < nq; i++, p += 4, wr += 2)
5244 		sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5245 		    base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5246 		    wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5247 		    G_QUEREMFLITS(p[2]) * 16);
5248 
5249 	rc = sbuf_finish(sb);
5250 	sbuf_delete(sb);
5251 
5252 	return (rc);
5253 }
5254 
5255 static int
5256 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5257 {
5258 	struct adapter *sc = arg1;
5259 	struct sbuf *sb;
5260 	int rc;
5261 	struct tp_cpl_stats stats;
5262 
5263 	rc = sysctl_wire_old_buffer(req, 0);
5264 	if (rc != 0)
5265 		return (rc);
5266 
5267 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5268 	if (sb == NULL)
5269 		return (ENOMEM);
5270 
5271 	t4_tp_get_cpl_stats(sc, &stats);
5272 
5273 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5274 	    "channel 3\n");
5275 	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5276 		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5277 	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5278 		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5279 
5280 	rc = sbuf_finish(sb);
5281 	sbuf_delete(sb);
5282 
5283 	return (rc);
5284 }
5285 
5286 static int
5287 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5288 {
5289 	struct adapter *sc = arg1;
5290 	struct sbuf *sb;
5291 	int rc;
5292 	struct tp_usm_stats stats;
5293 
5294 	rc = sysctl_wire_old_buffer(req, 0);
5295 	if (rc != 0)
5296 		return(rc);
5297 
5298 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5299 	if (sb == NULL)
5300 		return (ENOMEM);
5301 
5302 	t4_get_usm_stats(sc, &stats);
5303 
5304 	sbuf_printf(sb, "Frames: %u\n", stats.frames);
5305 	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5306 	sbuf_printf(sb, "Drops:  %u", stats.drops);
5307 
5308 	rc = sbuf_finish(sb);
5309 	sbuf_delete(sb);
5310 
5311 	return (rc);
5312 }
5313 
5314 const char *devlog_level_strings[] = {
5315 	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
5316 	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
5317 	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
5318 	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
5319 	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
5320 	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
5321 };
5322 
5323 const char *devlog_facility_strings[] = {
5324 	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
5325 	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
5326 	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
5327 	[FW_DEVLOG_FACILITY_RES]	= "RES",
5328 	[FW_DEVLOG_FACILITY_HW]		= "HW",
5329 	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
5330 	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
5331 	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
5332 	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
5333 	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
5334 	[FW_DEVLOG_FACILITY_VI]		= "VI",
5335 	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
5336 	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
5337 	[FW_DEVLOG_FACILITY_TM]		= "TM",
5338 	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
5339 	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
5340 	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
5341 	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
5342 	[FW_DEVLOG_FACILITY_RI]		= "RI",
5343 	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
5344 	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
5345 	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
5346 	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
5347 };
5348 
5349 static int
5350 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5351 {
5352 	struct adapter *sc = arg1;
5353 	struct devlog_params *dparams = &sc->params.devlog;
5354 	struct fw_devlog_e *buf, *e;
5355 	int i, j, rc, nentries, first = 0, m;
5356 	struct sbuf *sb;
5357 	uint64_t ftstamp = UINT64_MAX;
5358 
5359 	if (dparams->start == 0) {
5360 		dparams->memtype = FW_MEMTYPE_EDC0;
5361 		dparams->start = 0x84000;
5362 		dparams->size = 32768;
5363 	}
5364 
5365 	nentries = dparams->size / sizeof(struct fw_devlog_e);
5366 
5367 	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5368 	if (buf == NULL)
5369 		return (ENOMEM);
5370 
5371 	m = fwmtype_to_hwmtype(dparams->memtype);
5372 	rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5373 	if (rc != 0)
5374 		goto done;
5375 
5376 	for (i = 0; i < nentries; i++) {
5377 		e = &buf[i];
5378 
5379 		if (e->timestamp == 0)
5380 			break;	/* end */
5381 
5382 		e->timestamp = be64toh(e->timestamp);
5383 		e->seqno = be32toh(e->seqno);
5384 		for (j = 0; j < 8; j++)
5385 			e->params[j] = be32toh(e->params[j]);
5386 
5387 		if (e->timestamp < ftstamp) {
5388 			ftstamp = e->timestamp;
5389 			first = i;
5390 		}
5391 	}
5392 
5393 	if (buf[first].timestamp == 0)
5394 		goto done;	/* nothing in the log */
5395 
5396 	rc = sysctl_wire_old_buffer(req, 0);
5397 	if (rc != 0)
5398 		goto done;
5399 
5400 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5401 	if (sb == NULL) {
5402 		rc = ENOMEM;
5403 		goto done;
5404 	}
5405 	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5406 	    "Seq#", "Tstamp", "Level", "Facility", "Message");
5407 
5408 	i = first;
5409 	do {
5410 		e = &buf[i];
5411 		if (e->timestamp == 0)
5412 			break;	/* end */
5413 
5414 		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5415 		    e->seqno, e->timestamp,
5416 		    (e->level < nitems(devlog_level_strings) ?
5417 			devlog_level_strings[e->level] : "UNKNOWN"),
5418 		    (e->facility < nitems(devlog_facility_strings) ?
5419 			devlog_facility_strings[e->facility] : "UNKNOWN"));
5420 		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5421 		    e->params[2], e->params[3], e->params[4],
5422 		    e->params[5], e->params[6], e->params[7]);
5423 
5424 		if (++i == nentries)
5425 			i = 0;
5426 	} while (i != first);
5427 
5428 	rc = sbuf_finish(sb);
5429 	sbuf_delete(sb);
5430 done:
5431 	free(buf, M_CXGBE);
5432 	return (rc);
5433 }
5434 
5435 static int
5436 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5437 {
5438 	struct adapter *sc = arg1;
5439 	struct sbuf *sb;
5440 	int rc;
5441 	struct tp_fcoe_stats stats[4];
5442 
5443 	rc = sysctl_wire_old_buffer(req, 0);
5444 	if (rc != 0)
5445 		return (rc);
5446 
5447 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5448 	if (sb == NULL)
5449 		return (ENOMEM);
5450 
5451 	t4_get_fcoe_stats(sc, 0, &stats[0]);
5452 	t4_get_fcoe_stats(sc, 1, &stats[1]);
5453 	t4_get_fcoe_stats(sc, 2, &stats[2]);
5454 	t4_get_fcoe_stats(sc, 3, &stats[3]);
5455 
5456 	sbuf_printf(sb, "                   channel 0        channel 1        "
5457 	    "channel 2        channel 3\n");
5458 	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5459 	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5460 	    stats[3].octetsDDP);
5461 	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5462 	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5463 	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5464 	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5465 	    stats[3].framesDrop);
5466 
5467 	rc = sbuf_finish(sb);
5468 	sbuf_delete(sb);
5469 
5470 	return (rc);
5471 }
5472 
5473 static int
5474 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5475 {
5476 	struct adapter *sc = arg1;
5477 	struct sbuf *sb;
5478 	int rc, i;
5479 	unsigned int map, kbps, ipg, mode;
5480 	unsigned int pace_tab[NTX_SCHED];
5481 
5482 	rc = sysctl_wire_old_buffer(req, 0);
5483 	if (rc != 0)
5484 		return (rc);
5485 
5486 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5487 	if (sb == NULL)
5488 		return (ENOMEM);
5489 
5490 	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5491 	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5492 	t4_read_pace_tbl(sc, pace_tab);
5493 
5494 	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5495 	    "Class IPG (0.1 ns)   Flow IPG (us)");
5496 
5497 	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5498 		t4_get_tx_sched(sc, i, &kbps, &ipg);
5499 		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5500 		    (mode & (1 << i)) ? "flow" : "class", map & 3);
5501 		if (kbps)
5502 			sbuf_printf(sb, "%9u     ", kbps);
5503 		else
5504 			sbuf_printf(sb, " disabled     ");
5505 
5506 		if (ipg)
5507 			sbuf_printf(sb, "%13u        ", ipg);
5508 		else
5509 			sbuf_printf(sb, "     disabled        ");
5510 
5511 		if (pace_tab[i])
5512 			sbuf_printf(sb, "%10u", pace_tab[i]);
5513 		else
5514 			sbuf_printf(sb, "  disabled");
5515 	}
5516 
5517 	rc = sbuf_finish(sb);
5518 	sbuf_delete(sb);
5519 
5520 	return (rc);
5521 }
5522 
5523 static int
5524 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5525 {
5526 	struct adapter *sc = arg1;
5527 	struct sbuf *sb;
5528 	int rc, i, j;
5529 	uint64_t *p0, *p1;
5530 	struct lb_port_stats s[2];
5531 	static const char *stat_name[] = {
5532 		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5533 		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5534 		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
5535 		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5536 		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5537 		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5538 		"BG2FramesTrunc:", "BG3FramesTrunc:"
5539 	};
5540 
5541 	rc = sysctl_wire_old_buffer(req, 0);
5542 	if (rc != 0)
5543 		return (rc);
5544 
5545 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5546 	if (sb == NULL)
5547 		return (ENOMEM);
5548 
5549 	memset(s, 0, sizeof(s));
5550 
5551 	for (i = 0; i < 4; i += 2) {
5552 		t4_get_lb_stats(sc, i, &s[0]);
5553 		t4_get_lb_stats(sc, i + 1, &s[1]);
5554 
5555 		p0 = &s[0].octets;
5556 		p1 = &s[1].octets;
5557 		sbuf_printf(sb, "%s                       Loopback %u"
5558 		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5559 
5560 		for (j = 0; j < nitems(stat_name); j++)
5561 			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5562 				   *p0++, *p1++);
5563 	}
5564 
5565 	rc = sbuf_finish(sb);
5566 	sbuf_delete(sb);
5567 
5568 	return (rc);
5569 }
5570 
5571 static int
5572 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5573 {
5574 	int rc = 0;
5575 	struct port_info *pi = arg1;
5576 	struct sbuf *sb;
5577 	static const char *linkdnreasons[] = {
5578 		"non-specific", "remote fault", "autoneg failed", "reserved3",
5579 		"PHY overheated", "unknown", "rx los", "reserved7"
5580 	};
5581 
5582 	rc = sysctl_wire_old_buffer(req, 0);
5583 	if (rc != 0)
5584 		return(rc);
5585 	sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5586 	if (sb == NULL)
5587 		return (ENOMEM);
5588 
5589 	if (pi->linkdnrc < 0)
5590 		sbuf_printf(sb, "n/a");
5591 	else if (pi->linkdnrc < nitems(linkdnreasons))
5592 		sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5593 	else
5594 		sbuf_printf(sb, "%d", pi->linkdnrc);
5595 
5596 	rc = sbuf_finish(sb);
5597 	sbuf_delete(sb);
5598 
5599 	return (rc);
5600 }
5601 
5602 struct mem_desc {
5603 	unsigned int base;
5604 	unsigned int limit;
5605 	unsigned int idx;
5606 };
5607 
5608 static int
5609 mem_desc_cmp(const void *a, const void *b)
5610 {
5611 	return ((const struct mem_desc *)a)->base -
5612 	       ((const struct mem_desc *)b)->base;
5613 }
5614 
5615 static void
5616 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5617     unsigned int to)
5618 {
5619 	unsigned int size;
5620 
5621 	size = to - from + 1;
5622 	if (size == 0)
5623 		return;
5624 
5625 	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5626 	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5627 }
5628 
5629 static int
5630 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5631 {
5632 	struct adapter *sc = arg1;
5633 	struct sbuf *sb;
5634 	int rc, i, n;
5635 	uint32_t lo, hi, used, alloc;
5636 	static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5637 	static const char *region[] = {
5638 		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5639 		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5640 		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5641 		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5642 		"RQUDP region:", "PBL region:", "TXPBL region:",
5643 		"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5644 		"On-chip queues:"
5645 	};
5646 	struct mem_desc avail[4];
5647 	struct mem_desc mem[nitems(region) + 3];	/* up to 3 holes */
5648 	struct mem_desc *md = mem;
5649 
5650 	rc = sysctl_wire_old_buffer(req, 0);
5651 	if (rc != 0)
5652 		return (rc);
5653 
5654 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5655 	if (sb == NULL)
5656 		return (ENOMEM);
5657 
5658 	for (i = 0; i < nitems(mem); i++) {
5659 		mem[i].limit = 0;
5660 		mem[i].idx = i;
5661 	}
5662 
5663 	/* Find and sort the populated memory ranges */
5664 	i = 0;
5665 	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5666 	if (lo & F_EDRAM0_ENABLE) {
5667 		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5668 		avail[i].base = G_EDRAM0_BASE(hi) << 20;
5669 		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5670 		avail[i].idx = 0;
5671 		i++;
5672 	}
5673 	if (lo & F_EDRAM1_ENABLE) {
5674 		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5675 		avail[i].base = G_EDRAM1_BASE(hi) << 20;
5676 		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5677 		avail[i].idx = 1;
5678 		i++;
5679 	}
5680 	if (lo & F_EXT_MEM_ENABLE) {
5681 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5682 		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5683 		avail[i].limit = avail[i].base +
5684 		    (G_EXT_MEM_SIZE(hi) << 20);
5685 		avail[i].idx = is_t4(sc) ? 2 : 3;	/* Call it MC for T4 */
5686 		i++;
5687 	}
5688 	if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5689 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5690 		avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5691 		avail[i].limit = avail[i].base +
5692 		    (G_EXT_MEM1_SIZE(hi) << 20);
5693 		avail[i].idx = 4;
5694 		i++;
5695 	}
5696 	if (!i)                                    /* no memory available */
5697 		return 0;
5698 	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
5699 
5700 	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
5701 	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
5702 	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
5703 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5704 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
5705 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
5706 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
5707 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
5708 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
5709 
5710 	/* the next few have explicit upper bounds */
5711 	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
5712 	md->limit = md->base - 1 +
5713 		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
5714 		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
5715 	md++;
5716 
5717 	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
5718 	md->limit = md->base - 1 +
5719 		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
5720 		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
5721 	md++;
5722 
5723 	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
5724 		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
5725 		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
5726 		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
5727 	} else {
5728 		md->base = 0;
5729 		md->idx = nitems(region);  /* hide it */
5730 	}
5731 	md++;
5732 
5733 #define ulp_region(reg) \
5734 	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
5735 	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
5736 
5737 	ulp_region(RX_ISCSI);
5738 	ulp_region(RX_TDDP);
5739 	ulp_region(TX_TPT);
5740 	ulp_region(RX_STAG);
5741 	ulp_region(RX_RQ);
5742 	ulp_region(RX_RQUDP);
5743 	ulp_region(RX_PBL);
5744 	ulp_region(TX_PBL);
5745 #undef ulp_region
5746 
5747 	md->base = 0;
5748 	md->idx = nitems(region);
5749 	if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
5750 		md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
5751 		md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
5752 		    A_SGE_DBVFIFO_SIZE))) << 2) - 1;
5753 	}
5754 	md++;
5755 
5756 	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
5757 	md->limit = md->base + sc->tids.ntids - 1;
5758 	md++;
5759 	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
5760 	md->limit = md->base + sc->tids.ntids - 1;
5761 	md++;
5762 
5763 	md->base = sc->vres.ocq.start;
5764 	if (sc->vres.ocq.size)
5765 		md->limit = md->base + sc->vres.ocq.size - 1;
5766 	else
5767 		md->idx = nitems(region);  /* hide it */
5768 	md++;
5769 
5770 	/* add any address-space holes, there can be up to 3 */
5771 	for (n = 0; n < i - 1; n++)
5772 		if (avail[n].limit < avail[n + 1].base)
5773 			(md++)->base = avail[n].limit;
5774 	if (avail[n].limit)
5775 		(md++)->base = avail[n].limit;
5776 
5777 	n = md - mem;
5778 	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
5779 
5780 	for (lo = 0; lo < i; lo++)
5781 		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
5782 				avail[lo].limit - 1);
5783 
5784 	sbuf_printf(sb, "\n");
5785 	for (i = 0; i < n; i++) {
5786 		if (mem[i].idx >= nitems(region))
5787 			continue;                        /* skip holes */
5788 		if (!mem[i].limit)
5789 			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
5790 		mem_region_show(sb, region[mem[i].idx], mem[i].base,
5791 				mem[i].limit);
5792 	}
5793 
5794 	sbuf_printf(sb, "\n");
5795 	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
5796 	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
5797 	mem_region_show(sb, "uP RAM:", lo, hi);
5798 
5799 	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
5800 	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
5801 	mem_region_show(sb, "uP Extmem2:", lo, hi);
5802 
5803 	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
5804 	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
5805 		   G_PMRXMAXPAGE(lo),
5806 		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
5807 		   (lo & F_PMRXNUMCHN) ? 2 : 1);
5808 
5809 	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
5810 	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
5811 	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
5812 		   G_PMTXMAXPAGE(lo),
5813 		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
5814 		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
5815 	sbuf_printf(sb, "%u p-structs\n",
5816 		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
5817 
5818 	for (i = 0; i < 4; i++) {
5819 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
5820 		if (is_t4(sc)) {
5821 			used = G_USED(lo);
5822 			alloc = G_ALLOC(lo);
5823 		} else {
5824 			used = G_T5_USED(lo);
5825 			alloc = G_T5_ALLOC(lo);
5826 		}
5827 		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
5828 			   i, used, alloc);
5829 	}
5830 	for (i = 0; i < 4; i++) {
5831 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
5832 		if (is_t4(sc)) {
5833 			used = G_USED(lo);
5834 			alloc = G_ALLOC(lo);
5835 		} else {
5836 			used = G_T5_USED(lo);
5837 			alloc = G_T5_ALLOC(lo);
5838 		}
5839 		sbuf_printf(sb,
5840 			   "\nLoopback %d using %u pages out of %u allocated",
5841 			   i, used, alloc);
5842 	}
5843 
5844 	rc = sbuf_finish(sb);
5845 	sbuf_delete(sb);
5846 
5847 	return (rc);
5848 }
5849 
5850 static inline void
5851 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
5852 {
5853 	*mask = x | y;
5854 	y = htobe64(y);
5855 	memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
5856 }
5857 
5858 static int
5859 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
5860 {
5861 	struct adapter *sc = arg1;
5862 	struct sbuf *sb;
5863 	int rc, i, n;
5864 
5865 	rc = sysctl_wire_old_buffer(req, 0);
5866 	if (rc != 0)
5867 		return (rc);
5868 
5869 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5870 	if (sb == NULL)
5871 		return (ENOMEM);
5872 
5873 	sbuf_printf(sb,
5874 	    "Idx  Ethernet address     Mask     Vld Ports PF"
5875 	    "  VF              Replication             P0 P1 P2 P3  ML");
5876 	n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
5877 	    NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
5878 	for (i = 0; i < n; i++) {
5879 		uint64_t tcamx, tcamy, mask;
5880 		uint32_t cls_lo, cls_hi;
5881 		uint8_t addr[ETHER_ADDR_LEN];
5882 
5883 		tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
5884 		tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
5885 		cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
5886 		cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
5887 
5888 		if (tcamx & tcamy)
5889 			continue;
5890 
5891 		tcamxy2valmask(tcamx, tcamy, addr, &mask);
5892 		sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
5893 			   "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
5894 			   addr[3], addr[4], addr[5], (uintmax_t)mask,
5895 			   (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
5896 			   G_PORTMAP(cls_hi), G_PF(cls_lo),
5897 			   (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
5898 
5899 		if (cls_lo & F_REPLICATE) {
5900 			struct fw_ldst_cmd ldst_cmd;
5901 
5902 			memset(&ldst_cmd, 0, sizeof(ldst_cmd));
5903 			ldst_cmd.op_to_addrspace =
5904 			    htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
5905 				F_FW_CMD_REQUEST | F_FW_CMD_READ |
5906 				V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
5907 			ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
5908 			ldst_cmd.u.mps.fid_ctl =
5909 			    htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
5910 				V_FW_LDST_CMD_CTL(i));
5911 
5912 			rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
5913 			    "t4mps");
5914 			if (rc)
5915 				break;
5916 			rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
5917 			    sizeof(ldst_cmd), &ldst_cmd);
5918 			end_synchronized_op(sc, 0);
5919 
5920 			if (rc != 0) {
5921 				sbuf_printf(sb,
5922 				    " ------------ error %3u ------------", rc);
5923 				rc = 0;
5924 			} else {
5925 				sbuf_printf(sb, " %08x %08x %08x %08x",
5926 				    be32toh(ldst_cmd.u.mps.rplc127_96),
5927 				    be32toh(ldst_cmd.u.mps.rplc95_64),
5928 				    be32toh(ldst_cmd.u.mps.rplc63_32),
5929 				    be32toh(ldst_cmd.u.mps.rplc31_0));
5930 			}
5931 		} else
5932 			sbuf_printf(sb, "%36s", "");
5933 
5934 		sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
5935 		    G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
5936 		    G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
5937 	}
5938 
5939 	if (rc)
5940 		(void) sbuf_finish(sb);
5941 	else
5942 		rc = sbuf_finish(sb);
5943 	sbuf_delete(sb);
5944 
5945 	return (rc);
5946 }
5947 
5948 static int
5949 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
5950 {
5951 	struct adapter *sc = arg1;
5952 	struct sbuf *sb;
5953 	int rc;
5954 	uint16_t mtus[NMTUS];
5955 
5956 	rc = sysctl_wire_old_buffer(req, 0);
5957 	if (rc != 0)
5958 		return (rc);
5959 
5960 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5961 	if (sb == NULL)
5962 		return (ENOMEM);
5963 
5964 	t4_read_mtu_tbl(sc, mtus, NULL);
5965 
5966 	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
5967 	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
5968 	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
5969 	    mtus[14], mtus[15]);
5970 
5971 	rc = sbuf_finish(sb);
5972 	sbuf_delete(sb);
5973 
5974 	return (rc);
5975 }
5976 
5977 static int
5978 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
5979 {
5980 	struct adapter *sc = arg1;
5981 	struct sbuf *sb;
5982 	int rc, i;
5983 	uint32_t cnt[PM_NSTATS];
5984 	uint64_t cyc[PM_NSTATS];
5985 	static const char *rx_stats[] = {
5986 		"Read:", "Write bypass:", "Write mem:", "Flush:"
5987 	};
5988 	static const char *tx_stats[] = {
5989 		"Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
5990 	};
5991 
5992 	rc = sysctl_wire_old_buffer(req, 0);
5993 	if (rc != 0)
5994 		return (rc);
5995 
5996 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5997 	if (sb == NULL)
5998 		return (ENOMEM);
5999 
6000 	t4_pmtx_get_stats(sc, cnt, cyc);
6001 	sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6002 	for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6003 		sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6004 		    cyc[i]);
6005 
6006 	t4_pmrx_get_stats(sc, cnt, cyc);
6007 	sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6008 	for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6009 		sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6010 		    cyc[i]);
6011 
6012 	rc = sbuf_finish(sb);
6013 	sbuf_delete(sb);
6014 
6015 	return (rc);
6016 }
6017 
6018 static int
6019 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6020 {
6021 	struct adapter *sc = arg1;
6022 	struct sbuf *sb;
6023 	int rc;
6024 	struct tp_rdma_stats stats;
6025 
6026 	rc = sysctl_wire_old_buffer(req, 0);
6027 	if (rc != 0)
6028 		return (rc);
6029 
6030 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6031 	if (sb == NULL)
6032 		return (ENOMEM);
6033 
6034 	t4_tp_get_rdma_stats(sc, &stats);
6035 	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6036 	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6037 
6038 	rc = sbuf_finish(sb);
6039 	sbuf_delete(sb);
6040 
6041 	return (rc);
6042 }
6043 
6044 static int
6045 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6046 {
6047 	struct adapter *sc = arg1;
6048 	struct sbuf *sb;
6049 	int rc;
6050 	struct tp_tcp_stats v4, v6;
6051 
6052 	rc = sysctl_wire_old_buffer(req, 0);
6053 	if (rc != 0)
6054 		return (rc);
6055 
6056 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6057 	if (sb == NULL)
6058 		return (ENOMEM);
6059 
6060 	t4_tp_get_tcp_stats(sc, &v4, &v6);
6061 	sbuf_printf(sb,
6062 	    "                                IP                 IPv6\n");
6063 	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6064 	    v4.tcpOutRsts, v6.tcpOutRsts);
6065 	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6066 	    v4.tcpInSegs, v6.tcpInSegs);
6067 	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6068 	    v4.tcpOutSegs, v6.tcpOutSegs);
6069 	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6070 	    v4.tcpRetransSegs, v6.tcpRetransSegs);
6071 
6072 	rc = sbuf_finish(sb);
6073 	sbuf_delete(sb);
6074 
6075 	return (rc);
6076 }
6077 
6078 static int
6079 sysctl_tids(SYSCTL_HANDLER_ARGS)
6080 {
6081 	struct adapter *sc = arg1;
6082 	struct sbuf *sb;
6083 	int rc;
6084 	struct tid_info *t = &sc->tids;
6085 
6086 	rc = sysctl_wire_old_buffer(req, 0);
6087 	if (rc != 0)
6088 		return (rc);
6089 
6090 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6091 	if (sb == NULL)
6092 		return (ENOMEM);
6093 
6094 	if (t->natids) {
6095 		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6096 		    t->atids_in_use);
6097 	}
6098 
6099 	if (t->ntids) {
6100 		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6101 			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6102 
6103 			if (b) {
6104 				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6105 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6106 				    t->ntids - 1);
6107 			} else {
6108 				sbuf_printf(sb, "TID range: %u-%u",
6109 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6110 				    t->ntids - 1);
6111 			}
6112 		} else
6113 			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6114 		sbuf_printf(sb, ", in use: %u\n",
6115 		    atomic_load_acq_int(&t->tids_in_use));
6116 	}
6117 
6118 	if (t->nstids) {
6119 		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6120 		    t->stid_base + t->nstids - 1, t->stids_in_use);
6121 	}
6122 
6123 	if (t->nftids) {
6124 		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6125 		    t->ftid_base + t->nftids - 1);
6126 	}
6127 
6128 	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6129 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6130 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6131 
6132 	rc = sbuf_finish(sb);
6133 	sbuf_delete(sb);
6134 
6135 	return (rc);
6136 }
6137 
6138 static int
6139 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6140 {
6141 	struct adapter *sc = arg1;
6142 	struct sbuf *sb;
6143 	int rc;
6144 	struct tp_err_stats stats;
6145 
6146 	rc = sysctl_wire_old_buffer(req, 0);
6147 	if (rc != 0)
6148 		return (rc);
6149 
6150 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6151 	if (sb == NULL)
6152 		return (ENOMEM);
6153 
6154 	t4_tp_get_err_stats(sc, &stats);
6155 
6156 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6157 		      "channel 3\n");
6158 	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6159 	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6160 	    stats.macInErrs[3]);
6161 	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6162 	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6163 	    stats.hdrInErrs[3]);
6164 	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6165 	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6166 	    stats.tcpInErrs[3]);
6167 	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6168 	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6169 	    stats.tcp6InErrs[3]);
6170 	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6171 	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6172 	    stats.tnlCongDrops[3]);
6173 	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6174 	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6175 	    stats.tnlTxDrops[3]);
6176 	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6177 	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6178 	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6179 	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6180 	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6181 	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6182 	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6183 	    stats.ofldNoNeigh, stats.ofldCongDefer);
6184 
6185 	rc = sbuf_finish(sb);
6186 	sbuf_delete(sb);
6187 
6188 	return (rc);
6189 }
6190 
6191 struct field_desc {
6192 	const char *name;
6193 	u_int start;
6194 	u_int width;
6195 };
6196 
6197 static void
6198 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6199 {
6200 	char buf[32];
6201 	int line_size = 0;
6202 
6203 	while (f->name) {
6204 		uint64_t mask = (1ULL << f->width) - 1;
6205 		int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6206 		    ((uintmax_t)v >> f->start) & mask);
6207 
6208 		if (line_size + len >= 79) {
6209 			line_size = 8;
6210 			sbuf_printf(sb, "\n        ");
6211 		}
6212 		sbuf_printf(sb, "%s ", buf);
6213 		line_size += len + 1;
6214 		f++;
6215 	}
6216 	sbuf_printf(sb, "\n");
6217 }
6218 
6219 static struct field_desc tp_la0[] = {
6220 	{ "RcfOpCodeOut", 60, 4 },
6221 	{ "State", 56, 4 },
6222 	{ "WcfState", 52, 4 },
6223 	{ "RcfOpcSrcOut", 50, 2 },
6224 	{ "CRxError", 49, 1 },
6225 	{ "ERxError", 48, 1 },
6226 	{ "SanityFailed", 47, 1 },
6227 	{ "SpuriousMsg", 46, 1 },
6228 	{ "FlushInputMsg", 45, 1 },
6229 	{ "FlushInputCpl", 44, 1 },
6230 	{ "RssUpBit", 43, 1 },
6231 	{ "RssFilterHit", 42, 1 },
6232 	{ "Tid", 32, 10 },
6233 	{ "InitTcb", 31, 1 },
6234 	{ "LineNumber", 24, 7 },
6235 	{ "Emsg", 23, 1 },
6236 	{ "EdataOut", 22, 1 },
6237 	{ "Cmsg", 21, 1 },
6238 	{ "CdataOut", 20, 1 },
6239 	{ "EreadPdu", 19, 1 },
6240 	{ "CreadPdu", 18, 1 },
6241 	{ "TunnelPkt", 17, 1 },
6242 	{ "RcfPeerFin", 16, 1 },
6243 	{ "RcfReasonOut", 12, 4 },
6244 	{ "TxCchannel", 10, 2 },
6245 	{ "RcfTxChannel", 8, 2 },
6246 	{ "RxEchannel", 6, 2 },
6247 	{ "RcfRxChannel", 5, 1 },
6248 	{ "RcfDataOutSrdy", 4, 1 },
6249 	{ "RxDvld", 3, 1 },
6250 	{ "RxOoDvld", 2, 1 },
6251 	{ "RxCongestion", 1, 1 },
6252 	{ "TxCongestion", 0, 1 },
6253 	{ NULL }
6254 };
6255 
6256 static struct field_desc tp_la1[] = {
6257 	{ "CplCmdIn", 56, 8 },
6258 	{ "CplCmdOut", 48, 8 },
6259 	{ "ESynOut", 47, 1 },
6260 	{ "EAckOut", 46, 1 },
6261 	{ "EFinOut", 45, 1 },
6262 	{ "ERstOut", 44, 1 },
6263 	{ "SynIn", 43, 1 },
6264 	{ "AckIn", 42, 1 },
6265 	{ "FinIn", 41, 1 },
6266 	{ "RstIn", 40, 1 },
6267 	{ "DataIn", 39, 1 },
6268 	{ "DataInVld", 38, 1 },
6269 	{ "PadIn", 37, 1 },
6270 	{ "RxBufEmpty", 36, 1 },
6271 	{ "RxDdp", 35, 1 },
6272 	{ "RxFbCongestion", 34, 1 },
6273 	{ "TxFbCongestion", 33, 1 },
6274 	{ "TxPktSumSrdy", 32, 1 },
6275 	{ "RcfUlpType", 28, 4 },
6276 	{ "Eread", 27, 1 },
6277 	{ "Ebypass", 26, 1 },
6278 	{ "Esave", 25, 1 },
6279 	{ "Static0", 24, 1 },
6280 	{ "Cread", 23, 1 },
6281 	{ "Cbypass", 22, 1 },
6282 	{ "Csave", 21, 1 },
6283 	{ "CPktOut", 20, 1 },
6284 	{ "RxPagePoolFull", 18, 2 },
6285 	{ "RxLpbkPkt", 17, 1 },
6286 	{ "TxLpbkPkt", 16, 1 },
6287 	{ "RxVfValid", 15, 1 },
6288 	{ "SynLearned", 14, 1 },
6289 	{ "SetDelEntry", 13, 1 },
6290 	{ "SetInvEntry", 12, 1 },
6291 	{ "CpcmdDvld", 11, 1 },
6292 	{ "CpcmdSave", 10, 1 },
6293 	{ "RxPstructsFull", 8, 2 },
6294 	{ "EpcmdDvld", 7, 1 },
6295 	{ "EpcmdFlush", 6, 1 },
6296 	{ "EpcmdTrimPrefix", 5, 1 },
6297 	{ "EpcmdTrimPostfix", 4, 1 },
6298 	{ "ERssIp4Pkt", 3, 1 },
6299 	{ "ERssIp6Pkt", 2, 1 },
6300 	{ "ERssTcpUdpPkt", 1, 1 },
6301 	{ "ERssFceFipPkt", 0, 1 },
6302 	{ NULL }
6303 };
6304 
6305 static struct field_desc tp_la2[] = {
6306 	{ "CplCmdIn", 56, 8 },
6307 	{ "MpsVfVld", 55, 1 },
6308 	{ "MpsPf", 52, 3 },
6309 	{ "MpsVf", 44, 8 },
6310 	{ "SynIn", 43, 1 },
6311 	{ "AckIn", 42, 1 },
6312 	{ "FinIn", 41, 1 },
6313 	{ "RstIn", 40, 1 },
6314 	{ "DataIn", 39, 1 },
6315 	{ "DataInVld", 38, 1 },
6316 	{ "PadIn", 37, 1 },
6317 	{ "RxBufEmpty", 36, 1 },
6318 	{ "RxDdp", 35, 1 },
6319 	{ "RxFbCongestion", 34, 1 },
6320 	{ "TxFbCongestion", 33, 1 },
6321 	{ "TxPktSumSrdy", 32, 1 },
6322 	{ "RcfUlpType", 28, 4 },
6323 	{ "Eread", 27, 1 },
6324 	{ "Ebypass", 26, 1 },
6325 	{ "Esave", 25, 1 },
6326 	{ "Static0", 24, 1 },
6327 	{ "Cread", 23, 1 },
6328 	{ "Cbypass", 22, 1 },
6329 	{ "Csave", 21, 1 },
6330 	{ "CPktOut", 20, 1 },
6331 	{ "RxPagePoolFull", 18, 2 },
6332 	{ "RxLpbkPkt", 17, 1 },
6333 	{ "TxLpbkPkt", 16, 1 },
6334 	{ "RxVfValid", 15, 1 },
6335 	{ "SynLearned", 14, 1 },
6336 	{ "SetDelEntry", 13, 1 },
6337 	{ "SetInvEntry", 12, 1 },
6338 	{ "CpcmdDvld", 11, 1 },
6339 	{ "CpcmdSave", 10, 1 },
6340 	{ "RxPstructsFull", 8, 2 },
6341 	{ "EpcmdDvld", 7, 1 },
6342 	{ "EpcmdFlush", 6, 1 },
6343 	{ "EpcmdTrimPrefix", 5, 1 },
6344 	{ "EpcmdTrimPostfix", 4, 1 },
6345 	{ "ERssIp4Pkt", 3, 1 },
6346 	{ "ERssIp6Pkt", 2, 1 },
6347 	{ "ERssTcpUdpPkt", 1, 1 },
6348 	{ "ERssFceFipPkt", 0, 1 },
6349 	{ NULL }
6350 };
6351 
6352 static void
6353 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6354 {
6355 
6356 	field_desc_show(sb, *p, tp_la0);
6357 }
6358 
6359 static void
6360 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6361 {
6362 
6363 	if (idx)
6364 		sbuf_printf(sb, "\n");
6365 	field_desc_show(sb, p[0], tp_la0);
6366 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6367 		field_desc_show(sb, p[1], tp_la0);
6368 }
6369 
6370 static void
6371 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6372 {
6373 
6374 	if (idx)
6375 		sbuf_printf(sb, "\n");
6376 	field_desc_show(sb, p[0], tp_la0);
6377 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6378 		field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6379 }
6380 
6381 static int
6382 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6383 {
6384 	struct adapter *sc = arg1;
6385 	struct sbuf *sb;
6386 	uint64_t *buf, *p;
6387 	int rc;
6388 	u_int i, inc;
6389 	void (*show_func)(struct sbuf *, uint64_t *, int);
6390 
6391 	rc = sysctl_wire_old_buffer(req, 0);
6392 	if (rc != 0)
6393 		return (rc);
6394 
6395 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6396 	if (sb == NULL)
6397 		return (ENOMEM);
6398 
6399 	buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6400 
6401 	t4_tp_read_la(sc, buf, NULL);
6402 	p = buf;
6403 
6404 	switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6405 	case 2:
6406 		inc = 2;
6407 		show_func = tp_la_show2;
6408 		break;
6409 	case 3:
6410 		inc = 2;
6411 		show_func = tp_la_show3;
6412 		break;
6413 	default:
6414 		inc = 1;
6415 		show_func = tp_la_show;
6416 	}
6417 
6418 	for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6419 		(*show_func)(sb, p, i);
6420 
6421 	rc = sbuf_finish(sb);
6422 	sbuf_delete(sb);
6423 	free(buf, M_CXGBE);
6424 	return (rc);
6425 }
6426 
6427 static int
6428 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6429 {
6430 	struct adapter *sc = arg1;
6431 	struct sbuf *sb;
6432 	int rc;
6433 	u64 nrate[NCHAN], orate[NCHAN];
6434 
6435 	rc = sysctl_wire_old_buffer(req, 0);
6436 	if (rc != 0)
6437 		return (rc);
6438 
6439 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6440 	if (sb == NULL)
6441 		return (ENOMEM);
6442 
6443 	t4_get_chan_txrate(sc, nrate, orate);
6444 	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6445 		 "channel 3\n");
6446 	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6447 	    nrate[0], nrate[1], nrate[2], nrate[3]);
6448 	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6449 	    orate[0], orate[1], orate[2], orate[3]);
6450 
6451 	rc = sbuf_finish(sb);
6452 	sbuf_delete(sb);
6453 
6454 	return (rc);
6455 }
6456 
6457 static int
6458 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6459 {
6460 	struct adapter *sc = arg1;
6461 	struct sbuf *sb;
6462 	uint32_t *buf, *p;
6463 	int rc, i;
6464 
6465 	rc = sysctl_wire_old_buffer(req, 0);
6466 	if (rc != 0)
6467 		return (rc);
6468 
6469 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6470 	if (sb == NULL)
6471 		return (ENOMEM);
6472 
6473 	buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6474 	    M_ZERO | M_WAITOK);
6475 
6476 	t4_ulprx_read_la(sc, buf);
6477 	p = buf;
6478 
6479 	sbuf_printf(sb, "      Pcmd        Type   Message"
6480 	    "                Data");
6481 	for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6482 		sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6483 		    p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6484 	}
6485 
6486 	rc = sbuf_finish(sb);
6487 	sbuf_delete(sb);
6488 	free(buf, M_CXGBE);
6489 	return (rc);
6490 }
6491 
6492 static int
6493 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6494 {
6495 	struct adapter *sc = arg1;
6496 	struct sbuf *sb;
6497 	int rc, v;
6498 
6499 	rc = sysctl_wire_old_buffer(req, 0);
6500 	if (rc != 0)
6501 		return (rc);
6502 
6503 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6504 	if (sb == NULL)
6505 		return (ENOMEM);
6506 
6507 	v = t4_read_reg(sc, A_SGE_STAT_CFG);
6508 	if (G_STATSOURCE_T5(v) == 7) {
6509 		if (G_STATMODE(v) == 0) {
6510 			sbuf_printf(sb, "total %d, incomplete %d",
6511 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
6512 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
6513 		} else if (G_STATMODE(v) == 1) {
6514 			sbuf_printf(sb, "total %d, data overflow %d",
6515 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
6516 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
6517 		}
6518 	}
6519 	rc = sbuf_finish(sb);
6520 	sbuf_delete(sb);
6521 
6522 	return (rc);
6523 }
6524 #endif
6525 
6526 static inline void
6527 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6528 {
6529 	struct buf_ring *br;
6530 	struct mbuf *m;
6531 
6532 	TXQ_LOCK_ASSERT_OWNED(txq);
6533 
6534 	br = txq->br;
6535 	m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6536 	if (m)
6537 		t4_eth_tx(ifp, txq, m);
6538 }
6539 
6540 void
6541 t4_tx_callout(void *arg)
6542 {
6543 	struct sge_eq *eq = arg;
6544 	struct adapter *sc;
6545 
6546 	if (EQ_TRYLOCK(eq) == 0)
6547 		goto reschedule;
6548 
6549 	if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6550 		EQ_UNLOCK(eq);
6551 reschedule:
6552 		if (__predict_true(!(eq->flags && EQ_DOOMED)))
6553 			callout_schedule(&eq->tx_callout, 1);
6554 		return;
6555 	}
6556 
6557 	EQ_LOCK_ASSERT_OWNED(eq);
6558 
6559 	if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6560 
6561 		if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6562 			struct sge_txq *txq = arg;
6563 			struct port_info *pi = txq->ifp->if_softc;
6564 
6565 			sc = pi->adapter;
6566 		} else {
6567 			struct sge_wrq *wrq = arg;
6568 
6569 			sc = wrq->adapter;
6570 		}
6571 
6572 		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6573 	}
6574 
6575 	EQ_UNLOCK(eq);
6576 }
6577 
6578 void
6579 t4_tx_task(void *arg, int count)
6580 {
6581 	struct sge_eq *eq = arg;
6582 
6583 	EQ_LOCK(eq);
6584 	if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6585 		struct sge_txq *txq = arg;
6586 		txq_start(txq->ifp, txq);
6587 	} else {
6588 		struct sge_wrq *wrq = arg;
6589 		t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6590 	}
6591 	EQ_UNLOCK(eq);
6592 }
6593 
6594 static uint32_t
6595 fconf_to_mode(uint32_t fconf)
6596 {
6597 	uint32_t mode;
6598 
6599 	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6600 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6601 
6602 	if (fconf & F_FRAGMENTATION)
6603 		mode |= T4_FILTER_IP_FRAGMENT;
6604 
6605 	if (fconf & F_MPSHITTYPE)
6606 		mode |= T4_FILTER_MPS_HIT_TYPE;
6607 
6608 	if (fconf & F_MACMATCH)
6609 		mode |= T4_FILTER_MAC_IDX;
6610 
6611 	if (fconf & F_ETHERTYPE)
6612 		mode |= T4_FILTER_ETH_TYPE;
6613 
6614 	if (fconf & F_PROTOCOL)
6615 		mode |= T4_FILTER_IP_PROTO;
6616 
6617 	if (fconf & F_TOS)
6618 		mode |= T4_FILTER_IP_TOS;
6619 
6620 	if (fconf & F_VLAN)
6621 		mode |= T4_FILTER_VLAN;
6622 
6623 	if (fconf & F_VNIC_ID)
6624 		mode |= T4_FILTER_VNIC;
6625 
6626 	if (fconf & F_PORT)
6627 		mode |= T4_FILTER_PORT;
6628 
6629 	if (fconf & F_FCOE)
6630 		mode |= T4_FILTER_FCoE;
6631 
6632 	return (mode);
6633 }
6634 
6635 static uint32_t
6636 mode_to_fconf(uint32_t mode)
6637 {
6638 	uint32_t fconf = 0;
6639 
6640 	if (mode & T4_FILTER_IP_FRAGMENT)
6641 		fconf |= F_FRAGMENTATION;
6642 
6643 	if (mode & T4_FILTER_MPS_HIT_TYPE)
6644 		fconf |= F_MPSHITTYPE;
6645 
6646 	if (mode & T4_FILTER_MAC_IDX)
6647 		fconf |= F_MACMATCH;
6648 
6649 	if (mode & T4_FILTER_ETH_TYPE)
6650 		fconf |= F_ETHERTYPE;
6651 
6652 	if (mode & T4_FILTER_IP_PROTO)
6653 		fconf |= F_PROTOCOL;
6654 
6655 	if (mode & T4_FILTER_IP_TOS)
6656 		fconf |= F_TOS;
6657 
6658 	if (mode & T4_FILTER_VLAN)
6659 		fconf |= F_VLAN;
6660 
6661 	if (mode & T4_FILTER_VNIC)
6662 		fconf |= F_VNIC_ID;
6663 
6664 	if (mode & T4_FILTER_PORT)
6665 		fconf |= F_PORT;
6666 
6667 	if (mode & T4_FILTER_FCoE)
6668 		fconf |= F_FCOE;
6669 
6670 	return (fconf);
6671 }
6672 
6673 static uint32_t
6674 fspec_to_fconf(struct t4_filter_specification *fs)
6675 {
6676 	uint32_t fconf = 0;
6677 
6678 	if (fs->val.frag || fs->mask.frag)
6679 		fconf |= F_FRAGMENTATION;
6680 
6681 	if (fs->val.matchtype || fs->mask.matchtype)
6682 		fconf |= F_MPSHITTYPE;
6683 
6684 	if (fs->val.macidx || fs->mask.macidx)
6685 		fconf |= F_MACMATCH;
6686 
6687 	if (fs->val.ethtype || fs->mask.ethtype)
6688 		fconf |= F_ETHERTYPE;
6689 
6690 	if (fs->val.proto || fs->mask.proto)
6691 		fconf |= F_PROTOCOL;
6692 
6693 	if (fs->val.tos || fs->mask.tos)
6694 		fconf |= F_TOS;
6695 
6696 	if (fs->val.vlan_vld || fs->mask.vlan_vld)
6697 		fconf |= F_VLAN;
6698 
6699 	if (fs->val.vnic_vld || fs->mask.vnic_vld)
6700 		fconf |= F_VNIC_ID;
6701 
6702 	if (fs->val.iport || fs->mask.iport)
6703 		fconf |= F_PORT;
6704 
6705 	if (fs->val.fcoe || fs->mask.fcoe)
6706 		fconf |= F_FCOE;
6707 
6708 	return (fconf);
6709 }
6710 
6711 static int
6712 get_filter_mode(struct adapter *sc, uint32_t *mode)
6713 {
6714 	int rc;
6715 	uint32_t fconf;
6716 
6717 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6718 	    "t4getfm");
6719 	if (rc)
6720 		return (rc);
6721 
6722 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
6723 	    A_TP_VLAN_PRI_MAP);
6724 
6725 	if (sc->params.tp.vlan_pri_map != fconf) {
6726 		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
6727 		    device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
6728 		    fconf);
6729 		sc->params.tp.vlan_pri_map = fconf;
6730 	}
6731 
6732 	*mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
6733 
6734 	end_synchronized_op(sc, LOCK_HELD);
6735 	return (0);
6736 }
6737 
6738 static int
6739 set_filter_mode(struct adapter *sc, uint32_t mode)
6740 {
6741 	uint32_t fconf;
6742 	int rc;
6743 
6744 	fconf = mode_to_fconf(mode);
6745 
6746 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6747 	    "t4setfm");
6748 	if (rc)
6749 		return (rc);
6750 
6751 	if (sc->tids.ftids_in_use > 0) {
6752 		rc = EBUSY;
6753 		goto done;
6754 	}
6755 
6756 #ifdef TCP_OFFLOAD
6757 	if (sc->offload_map) {
6758 		rc = EBUSY;
6759 		goto done;
6760 	}
6761 #endif
6762 
6763 #ifdef notyet
6764 	rc = -t4_set_filter_mode(sc, fconf);
6765 	if (rc == 0)
6766 		sc->filter_mode = fconf;
6767 #else
6768 	rc = ENOTSUP;
6769 #endif
6770 
6771 done:
6772 	end_synchronized_op(sc, LOCK_HELD);
6773 	return (rc);
6774 }
6775 
6776 static inline uint64_t
6777 get_filter_hits(struct adapter *sc, uint32_t fid)
6778 {
6779 	uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6780 	uint64_t hits;
6781 
6782 	memwin_info(sc, 0, &mw_base, NULL);
6783 	off = position_memwin(sc, 0,
6784 	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
6785 	if (is_t4(sc)) {
6786 		hits = t4_read_reg64(sc, mw_base + off + 16);
6787 		hits = be64toh(hits);
6788 	} else {
6789 		hits = t4_read_reg(sc, mw_base + off + 24);
6790 		hits = be32toh(hits);
6791 	}
6792 
6793 	return (hits);
6794 }
6795 
6796 static int
6797 get_filter(struct adapter *sc, struct t4_filter *t)
6798 {
6799 	int i, rc, nfilters = sc->tids.nftids;
6800 	struct filter_entry *f;
6801 
6802 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6803 	    "t4getf");
6804 	if (rc)
6805 		return (rc);
6806 
6807 	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
6808 	    t->idx >= nfilters) {
6809 		t->idx = 0xffffffff;
6810 		goto done;
6811 	}
6812 
6813 	f = &sc->tids.ftid_tab[t->idx];
6814 	for (i = t->idx; i < nfilters; i++, f++) {
6815 		if (f->valid) {
6816 			t->idx = i;
6817 			t->l2tidx = f->l2t ? f->l2t->idx : 0;
6818 			t->smtidx = f->smtidx;
6819 			if (f->fs.hitcnts)
6820 				t->hits = get_filter_hits(sc, t->idx);
6821 			else
6822 				t->hits = UINT64_MAX;
6823 			t->fs = f->fs;
6824 
6825 			goto done;
6826 		}
6827 	}
6828 
6829 	t->idx = 0xffffffff;
6830 done:
6831 	end_synchronized_op(sc, LOCK_HELD);
6832 	return (0);
6833 }
6834 
6835 static int
6836 set_filter(struct adapter *sc, struct t4_filter *t)
6837 {
6838 	unsigned int nfilters, nports;
6839 	struct filter_entry *f;
6840 	int i, rc;
6841 
6842 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
6843 	if (rc)
6844 		return (rc);
6845 
6846 	nfilters = sc->tids.nftids;
6847 	nports = sc->params.nports;
6848 
6849 	if (nfilters == 0) {
6850 		rc = ENOTSUP;
6851 		goto done;
6852 	}
6853 
6854 	if (!(sc->flags & FULL_INIT_DONE)) {
6855 		rc = EAGAIN;
6856 		goto done;
6857 	}
6858 
6859 	if (t->idx >= nfilters) {
6860 		rc = EINVAL;
6861 		goto done;
6862 	}
6863 
6864 	/* Validate against the global filter mode */
6865 	if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
6866 	    sc->params.tp.vlan_pri_map) {
6867 		rc = E2BIG;
6868 		goto done;
6869 	}
6870 
6871 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
6872 		rc = EINVAL;
6873 		goto done;
6874 	}
6875 
6876 	if (t->fs.val.iport >= nports) {
6877 		rc = EINVAL;
6878 		goto done;
6879 	}
6880 
6881 	/* Can't specify an iq if not steering to it */
6882 	if (!t->fs.dirsteer && t->fs.iq) {
6883 		rc = EINVAL;
6884 		goto done;
6885 	}
6886 
6887 	/* IPv6 filter idx must be 4 aligned */
6888 	if (t->fs.type == 1 &&
6889 	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
6890 		rc = EINVAL;
6891 		goto done;
6892 	}
6893 
6894 	if (sc->tids.ftid_tab == NULL) {
6895 		KASSERT(sc->tids.ftids_in_use == 0,
6896 		    ("%s: no memory allocated but filters_in_use > 0",
6897 		    __func__));
6898 
6899 		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
6900 		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
6901 		if (sc->tids.ftid_tab == NULL) {
6902 			rc = ENOMEM;
6903 			goto done;
6904 		}
6905 		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
6906 	}
6907 
6908 	for (i = 0; i < 4; i++) {
6909 		f = &sc->tids.ftid_tab[t->idx + i];
6910 
6911 		if (f->pending || f->valid) {
6912 			rc = EBUSY;
6913 			goto done;
6914 		}
6915 		if (f->locked) {
6916 			rc = EPERM;
6917 			goto done;
6918 		}
6919 
6920 		if (t->fs.type == 0)
6921 			break;
6922 	}
6923 
6924 	f = &sc->tids.ftid_tab[t->idx];
6925 	f->fs = t->fs;
6926 
6927 	rc = set_filter_wr(sc, t->idx);
6928 done:
6929 	end_synchronized_op(sc, 0);
6930 
6931 	if (rc == 0) {
6932 		mtx_lock(&sc->tids.ftid_lock);
6933 		for (;;) {
6934 			if (f->pending == 0) {
6935 				rc = f->valid ? 0 : EIO;
6936 				break;
6937 			}
6938 
6939 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
6940 			    PCATCH, "t4setfw", 0)) {
6941 				rc = EINPROGRESS;
6942 				break;
6943 			}
6944 		}
6945 		mtx_unlock(&sc->tids.ftid_lock);
6946 	}
6947 	return (rc);
6948 }
6949 
6950 static int
6951 del_filter(struct adapter *sc, struct t4_filter *t)
6952 {
6953 	unsigned int nfilters;
6954 	struct filter_entry *f;
6955 	int rc;
6956 
6957 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
6958 	if (rc)
6959 		return (rc);
6960 
6961 	nfilters = sc->tids.nftids;
6962 
6963 	if (nfilters == 0) {
6964 		rc = ENOTSUP;
6965 		goto done;
6966 	}
6967 
6968 	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
6969 	    t->idx >= nfilters) {
6970 		rc = EINVAL;
6971 		goto done;
6972 	}
6973 
6974 	if (!(sc->flags & FULL_INIT_DONE)) {
6975 		rc = EAGAIN;
6976 		goto done;
6977 	}
6978 
6979 	f = &sc->tids.ftid_tab[t->idx];
6980 
6981 	if (f->pending) {
6982 		rc = EBUSY;
6983 		goto done;
6984 	}
6985 	if (f->locked) {
6986 		rc = EPERM;
6987 		goto done;
6988 	}
6989 
6990 	if (f->valid) {
6991 		t->fs = f->fs;	/* extra info for the caller */
6992 		rc = del_filter_wr(sc, t->idx);
6993 	}
6994 
6995 done:
6996 	end_synchronized_op(sc, 0);
6997 
6998 	if (rc == 0) {
6999 		mtx_lock(&sc->tids.ftid_lock);
7000 		for (;;) {
7001 			if (f->pending == 0) {
7002 				rc = f->valid ? EIO : 0;
7003 				break;
7004 			}
7005 
7006 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7007 			    PCATCH, "t4delfw", 0)) {
7008 				rc = EINPROGRESS;
7009 				break;
7010 			}
7011 		}
7012 		mtx_unlock(&sc->tids.ftid_lock);
7013 	}
7014 
7015 	return (rc);
7016 }
7017 
7018 static void
7019 clear_filter(struct filter_entry *f)
7020 {
7021 	if (f->l2t)
7022 		t4_l2t_release(f->l2t);
7023 
7024 	bzero(f, sizeof (*f));
7025 }
7026 
7027 static int
7028 set_filter_wr(struct adapter *sc, int fidx)
7029 {
7030 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7031 	struct wrqe *wr;
7032 	struct fw_filter_wr *fwr;
7033 	unsigned int ftid;
7034 
7035 	ASSERT_SYNCHRONIZED_OP(sc);
7036 
7037 	if (f->fs.newdmac || f->fs.newvlan) {
7038 		/* This filter needs an L2T entry; allocate one. */
7039 		f->l2t = t4_l2t_alloc_switching(sc->l2t);
7040 		if (f->l2t == NULL)
7041 			return (EAGAIN);
7042 		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7043 		    f->fs.dmac)) {
7044 			t4_l2t_release(f->l2t);
7045 			f->l2t = NULL;
7046 			return (ENOMEM);
7047 		}
7048 	}
7049 
7050 	ftid = sc->tids.ftid_base + fidx;
7051 
7052 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7053 	if (wr == NULL)
7054 		return (ENOMEM);
7055 
7056 	fwr = wrtod(wr);
7057 	bzero(fwr, sizeof (*fwr));
7058 
7059 	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7060 	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7061 	fwr->tid_to_iq =
7062 	    htobe32(V_FW_FILTER_WR_TID(ftid) |
7063 		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7064 		V_FW_FILTER_WR_NOREPLY(0) |
7065 		V_FW_FILTER_WR_IQ(f->fs.iq));
7066 	fwr->del_filter_to_l2tix =
7067 	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7068 		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7069 		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7070 		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7071 		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7072 		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7073 		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7074 		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7075 		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7076 		    f->fs.newvlan == VLAN_REWRITE) |
7077 		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7078 		    f->fs.newvlan == VLAN_REWRITE) |
7079 		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7080 		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7081 		V_FW_FILTER_WR_PRIO(f->fs.prio) |
7082 		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7083 	fwr->ethtype = htobe16(f->fs.val.ethtype);
7084 	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7085 	fwr->frag_to_ovlan_vldm =
7086 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7087 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7088 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7089 		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7090 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7091 		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7092 	fwr->smac_sel = 0;
7093 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7094 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7095 	fwr->maci_to_matchtypem =
7096 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7097 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7098 		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7099 		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7100 		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7101 		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7102 		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7103 		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7104 	fwr->ptcl = f->fs.val.proto;
7105 	fwr->ptclm = f->fs.mask.proto;
7106 	fwr->ttyp = f->fs.val.tos;
7107 	fwr->ttypm = f->fs.mask.tos;
7108 	fwr->ivlan = htobe16(f->fs.val.vlan);
7109 	fwr->ivlanm = htobe16(f->fs.mask.vlan);
7110 	fwr->ovlan = htobe16(f->fs.val.vnic);
7111 	fwr->ovlanm = htobe16(f->fs.mask.vnic);
7112 	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7113 	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7114 	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7115 	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7116 	fwr->lp = htobe16(f->fs.val.dport);
7117 	fwr->lpm = htobe16(f->fs.mask.dport);
7118 	fwr->fp = htobe16(f->fs.val.sport);
7119 	fwr->fpm = htobe16(f->fs.mask.sport);
7120 	if (f->fs.newsmac)
7121 		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7122 
7123 	f->pending = 1;
7124 	sc->tids.ftids_in_use++;
7125 
7126 	t4_wrq_tx(sc, wr);
7127 	return (0);
7128 }
7129 
7130 static int
7131 del_filter_wr(struct adapter *sc, int fidx)
7132 {
7133 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7134 	struct wrqe *wr;
7135 	struct fw_filter_wr *fwr;
7136 	unsigned int ftid;
7137 
7138 	ftid = sc->tids.ftid_base + fidx;
7139 
7140 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7141 	if (wr == NULL)
7142 		return (ENOMEM);
7143 	fwr = wrtod(wr);
7144 	bzero(fwr, sizeof (*fwr));
7145 
7146 	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7147 
7148 	f->pending = 1;
7149 	t4_wrq_tx(sc, wr);
7150 	return (0);
7151 }
7152 
7153 int
7154 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7155 {
7156 	struct adapter *sc = iq->adapter;
7157 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7158 	unsigned int idx = GET_TID(rpl);
7159 
7160 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7161 	    rss->opcode));
7162 
7163 	if (idx >= sc->tids.ftid_base &&
7164 	    (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
7165 		unsigned int rc = G_COOKIE(rpl->cookie);
7166 		struct filter_entry *f = &sc->tids.ftid_tab[idx];
7167 
7168 		mtx_lock(&sc->tids.ftid_lock);
7169 		if (rc == FW_FILTER_WR_FLT_ADDED) {
7170 			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7171 			    __func__, idx));
7172 			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7173 			f->pending = 0;  /* asynchronous setup completed */
7174 			f->valid = 1;
7175 		} else {
7176 			if (rc != FW_FILTER_WR_FLT_DELETED) {
7177 				/* Add or delete failed, display an error */
7178 				log(LOG_ERR,
7179 				    "filter %u setup failed with error %u\n",
7180 				    idx, rc);
7181 			}
7182 
7183 			clear_filter(f);
7184 			sc->tids.ftids_in_use--;
7185 		}
7186 		wakeup(&sc->tids.ftid_tab);
7187 		mtx_unlock(&sc->tids.ftid_lock);
7188 	}
7189 
7190 	return (0);
7191 }
7192 
7193 static int
7194 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7195 {
7196 	int rc;
7197 
7198 	if (cntxt->cid > M_CTXTQID)
7199 		return (EINVAL);
7200 
7201 	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7202 	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7203 		return (EINVAL);
7204 
7205 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7206 	if (rc)
7207 		return (rc);
7208 
7209 	if (sc->flags & FW_OK) {
7210 		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7211 		    &cntxt->data[0]);
7212 		if (rc == 0)
7213 			goto done;
7214 	}
7215 
7216 	/*
7217 	 * Read via firmware failed or wasn't even attempted.  Read directly via
7218 	 * the backdoor.
7219 	 */
7220 	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7221 done:
7222 	end_synchronized_op(sc, 0);
7223 	return (rc);
7224 }
7225 
7226 static int
7227 load_fw(struct adapter *sc, struct t4_data *fw)
7228 {
7229 	int rc;
7230 	uint8_t *fw_data;
7231 
7232 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7233 	if (rc)
7234 		return (rc);
7235 
7236 	if (sc->flags & FULL_INIT_DONE) {
7237 		rc = EBUSY;
7238 		goto done;
7239 	}
7240 
7241 	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7242 	if (fw_data == NULL) {
7243 		rc = ENOMEM;
7244 		goto done;
7245 	}
7246 
7247 	rc = copyin(fw->data, fw_data, fw->len);
7248 	if (rc == 0)
7249 		rc = -t4_load_fw(sc, fw_data, fw->len);
7250 
7251 	free(fw_data, M_CXGBE);
7252 done:
7253 	end_synchronized_op(sc, 0);
7254 	return (rc);
7255 }
7256 
7257 static int
7258 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7259 {
7260 	uint32_t addr, off, remaining, i, n;
7261 	uint32_t *buf, *b;
7262 	uint32_t mw_base, mw_aperture;
7263 	int rc;
7264 	uint8_t *dst;
7265 
7266 	rc = validate_mem_range(sc, mr->addr, mr->len);
7267 	if (rc != 0)
7268 		return (rc);
7269 
7270 	memwin_info(sc, win, &mw_base, &mw_aperture);
7271 	buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7272 	addr = mr->addr;
7273 	remaining = mr->len;
7274 	dst = (void *)mr->data;
7275 
7276 	while (remaining) {
7277 		off = position_memwin(sc, win, addr);
7278 
7279 		/* number of bytes that we'll copy in the inner loop */
7280 		n = min(remaining, mw_aperture - off);
7281 		for (i = 0; i < n; i += 4)
7282 			*b++ = t4_read_reg(sc, mw_base + off + i);
7283 
7284 		rc = copyout(buf, dst, n);
7285 		if (rc != 0)
7286 			break;
7287 
7288 		b = buf;
7289 		dst += n;
7290 		remaining -= n;
7291 		addr += n;
7292 	}
7293 
7294 	free(buf, M_CXGBE);
7295 	return (rc);
7296 }
7297 
7298 static int
7299 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7300 {
7301 	int rc;
7302 
7303 	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7304 		return (EINVAL);
7305 
7306 	if (i2cd->len > 1) {
7307 		/* XXX: need fw support for longer reads in one go */
7308 		return (ENOTSUP);
7309 	}
7310 
7311 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7312 	if (rc)
7313 		return (rc);
7314 	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7315 	    i2cd->offset, &i2cd->data[0]);
7316 	end_synchronized_op(sc, 0);
7317 
7318 	return (rc);
7319 }
7320 
7321 static int
7322 in_range(int val, int lo, int hi)
7323 {
7324 
7325 	return (val < 0 || (val <= hi && val >= lo));
7326 }
7327 
7328 static int
7329 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7330 {
7331 	int fw_subcmd, fw_type, rc;
7332 
7333 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7334 	if (rc)
7335 		return (rc);
7336 
7337 	if (!(sc->flags & FULL_INIT_DONE)) {
7338 		rc = EAGAIN;
7339 		goto done;
7340 	}
7341 
7342 	/*
7343 	 * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7344 	 * sub-command and type are in common locations.)
7345 	 */
7346 	if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7347 		fw_subcmd = FW_SCHED_SC_CONFIG;
7348 	else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7349 		fw_subcmd = FW_SCHED_SC_PARAMS;
7350 	else {
7351 		rc = EINVAL;
7352 		goto done;
7353 	}
7354 	if (p->type == SCHED_CLASS_TYPE_PACKET)
7355 		fw_type = FW_SCHED_TYPE_PKTSCHED;
7356 	else {
7357 		rc = EINVAL;
7358 		goto done;
7359 	}
7360 
7361 	if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7362 		/* Vet our parameters ..*/
7363 		if (p->u.config.minmax < 0) {
7364 			rc = EINVAL;
7365 			goto done;
7366 		}
7367 
7368 		/* And pass the request to the firmware ...*/
7369 		rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
7370 		goto done;
7371 	}
7372 
7373 	if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7374 		int fw_level;
7375 		int fw_mode;
7376 		int fw_rateunit;
7377 		int fw_ratemode;
7378 
7379 		if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7380 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7381 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7382 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7383 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7384 			fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7385 		else {
7386 			rc = EINVAL;
7387 			goto done;
7388 		}
7389 
7390 		if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7391 			fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7392 		else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7393 			fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7394 		else {
7395 			rc = EINVAL;
7396 			goto done;
7397 		}
7398 
7399 		if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7400 			fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7401 		else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7402 			fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7403 		else {
7404 			rc = EINVAL;
7405 			goto done;
7406 		}
7407 
7408 		if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7409 			fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7410 		else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7411 			fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7412 		else {
7413 			rc = EINVAL;
7414 			goto done;
7415 		}
7416 
7417 		/* Vet our parameters ... */
7418 		if (!in_range(p->u.params.channel, 0, 3) ||
7419 		    !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7420 		    !in_range(p->u.params.minrate, 0, 10000000) ||
7421 		    !in_range(p->u.params.maxrate, 0, 10000000) ||
7422 		    !in_range(p->u.params.weight, 0, 100)) {
7423 			rc = ERANGE;
7424 			goto done;
7425 		}
7426 
7427 		/*
7428 		 * Translate any unset parameters into the firmware's
7429 		 * nomenclature and/or fail the call if the parameters
7430 		 * are required ...
7431 		 */
7432 		if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7433 		    p->u.params.channel < 0 || p->u.params.cl < 0) {
7434 			rc = EINVAL;
7435 			goto done;
7436 		}
7437 		if (p->u.params.minrate < 0)
7438 			p->u.params.minrate = 0;
7439 		if (p->u.params.maxrate < 0) {
7440 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7441 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7442 				rc = EINVAL;
7443 				goto done;
7444 			} else
7445 				p->u.params.maxrate = 0;
7446 		}
7447 		if (p->u.params.weight < 0) {
7448 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7449 				rc = EINVAL;
7450 				goto done;
7451 			} else
7452 				p->u.params.weight = 0;
7453 		}
7454 		if (p->u.params.pktsize < 0) {
7455 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7456 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7457 				rc = EINVAL;
7458 				goto done;
7459 			} else
7460 				p->u.params.pktsize = 0;
7461 		}
7462 
7463 		/* See what the firmware thinks of the request ... */
7464 		rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7465 		    fw_rateunit, fw_ratemode, p->u.params.channel,
7466 		    p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7467 		    p->u.params.weight, p->u.params.pktsize);
7468 		goto done;
7469 	}
7470 
7471 	rc = EINVAL;
7472 done:
7473 	end_synchronized_op(sc, 0);
7474 	return (rc);
7475 }
7476 
7477 static int
7478 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7479 {
7480 	struct port_info *pi = NULL;
7481 	struct sge_txq *txq;
7482 	uint32_t fw_mnem, fw_queue, fw_class;
7483 	int i, rc;
7484 
7485 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7486 	if (rc)
7487 		return (rc);
7488 
7489 	if (!(sc->flags & FULL_INIT_DONE)) {
7490 		rc = EAGAIN;
7491 		goto done;
7492 	}
7493 
7494 	if (p->port >= sc->params.nports) {
7495 		rc = EINVAL;
7496 		goto done;
7497 	}
7498 
7499 	pi = sc->port[p->port];
7500 	if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7501 		rc = EINVAL;
7502 		goto done;
7503 	}
7504 
7505 	/*
7506 	 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7507 	 * Scheduling Class in this case).
7508 	 */
7509 	fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7510 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7511 	fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7512 
7513 	/*
7514 	 * If op.queue is non-negative, then we're only changing the scheduling
7515 	 * on a single specified TX queue.
7516 	 */
7517 	if (p->queue >= 0) {
7518 		txq = &sc->sge.txq[pi->first_txq + p->queue];
7519 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7520 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7521 		    &fw_class);
7522 		goto done;
7523 	}
7524 
7525 	/*
7526 	 * Change the scheduling on all the TX queues for the
7527 	 * interface.
7528 	 */
7529 	for_each_txq(pi, i, txq) {
7530 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7531 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7532 		    &fw_class);
7533 		if (rc)
7534 			goto done;
7535 	}
7536 
7537 	rc = 0;
7538 done:
7539 	end_synchronized_op(sc, 0);
7540 	return (rc);
7541 }
7542 
7543 int
7544 t4_os_find_pci_capability(struct adapter *sc, int cap)
7545 {
7546 	int i;
7547 
7548 	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7549 }
7550 
7551 int
7552 t4_os_pci_save_state(struct adapter *sc)
7553 {
7554 	device_t dev;
7555 	struct pci_devinfo *dinfo;
7556 
7557 	dev = sc->dev;
7558 	dinfo = device_get_ivars(dev);
7559 
7560 	pci_cfg_save(dev, dinfo, 0);
7561 	return (0);
7562 }
7563 
7564 int
7565 t4_os_pci_restore_state(struct adapter *sc)
7566 {
7567 	device_t dev;
7568 	struct pci_devinfo *dinfo;
7569 
7570 	dev = sc->dev;
7571 	dinfo = device_get_ivars(dev);
7572 
7573 	pci_cfg_restore(dev, dinfo);
7574 	return (0);
7575 }
7576 
7577 void
7578 t4_os_portmod_changed(const struct adapter *sc, int idx)
7579 {
7580 	struct port_info *pi = sc->port[idx];
7581 	static const char *mod_str[] = {
7582 		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7583 	};
7584 
7585 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7586 		if_printf(pi->ifp, "transceiver unplugged.\n");
7587 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7588 		if_printf(pi->ifp, "unknown transceiver inserted.\n");
7589 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7590 		if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7591 	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7592 		if_printf(pi->ifp, "%s transceiver inserted.\n",
7593 		    mod_str[pi->mod_type]);
7594 	} else {
7595 		if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7596 		    pi->mod_type);
7597 	}
7598 }
7599 
7600 void
7601 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7602 {
7603 	struct port_info *pi = sc->port[idx];
7604 	struct ifnet *ifp = pi->ifp;
7605 
7606 	if (link_stat) {
7607 		pi->linkdnrc = -1;
7608 		ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7609 		if_link_state_change(ifp, LINK_STATE_UP);
7610 	} else {
7611 		if (reason >= 0)
7612 			pi->linkdnrc = reason;
7613 		if_link_state_change(ifp, LINK_STATE_DOWN);
7614 	}
7615 }
7616 
7617 void
7618 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7619 {
7620 	struct adapter *sc;
7621 
7622 	sx_slock(&t4_list_lock);
7623 	SLIST_FOREACH(sc, &t4_list, link) {
7624 		/*
7625 		 * func should not make any assumptions about what state sc is
7626 		 * in - the only guarantee is that sc->sc_lock is a valid lock.
7627 		 */
7628 		func(sc, arg);
7629 	}
7630 	sx_sunlock(&t4_list_lock);
7631 }
7632 
7633 static int
7634 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7635 {
7636        return (0);
7637 }
7638 
7639 static int
7640 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7641 {
7642        return (0);
7643 }
7644 
7645 static int
7646 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7647     struct thread *td)
7648 {
7649 	int rc;
7650 	struct adapter *sc = dev->si_drv1;
7651 
7652 	rc = priv_check(td, PRIV_DRIVER);
7653 	if (rc != 0)
7654 		return (rc);
7655 
7656 	switch (cmd) {
7657 	case CHELSIO_T4_GETREG: {
7658 		struct t4_reg *edata = (struct t4_reg *)data;
7659 
7660 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7661 			return (EFAULT);
7662 
7663 		if (edata->size == 4)
7664 			edata->val = t4_read_reg(sc, edata->addr);
7665 		else if (edata->size == 8)
7666 			edata->val = t4_read_reg64(sc, edata->addr);
7667 		else
7668 			return (EINVAL);
7669 
7670 		break;
7671 	}
7672 	case CHELSIO_T4_SETREG: {
7673 		struct t4_reg *edata = (struct t4_reg *)data;
7674 
7675 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7676 			return (EFAULT);
7677 
7678 		if (edata->size == 4) {
7679 			if (edata->val & 0xffffffff00000000)
7680 				return (EINVAL);
7681 			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7682 		} else if (edata->size == 8)
7683 			t4_write_reg64(sc, edata->addr, edata->val);
7684 		else
7685 			return (EINVAL);
7686 		break;
7687 	}
7688 	case CHELSIO_T4_REGDUMP: {
7689 		struct t4_regdump *regs = (struct t4_regdump *)data;
7690 		int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7691 		uint8_t *buf;
7692 
7693 		if (regs->len < reglen) {
7694 			regs->len = reglen; /* hint to the caller */
7695 			return (ENOBUFS);
7696 		}
7697 
7698 		regs->len = reglen;
7699 		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7700 		t4_get_regs(sc, regs, buf);
7701 		rc = copyout(buf, regs->data, reglen);
7702 		free(buf, M_CXGBE);
7703 		break;
7704 	}
7705 	case CHELSIO_T4_GET_FILTER_MODE:
7706 		rc = get_filter_mode(sc, (uint32_t *)data);
7707 		break;
7708 	case CHELSIO_T4_SET_FILTER_MODE:
7709 		rc = set_filter_mode(sc, *(uint32_t *)data);
7710 		break;
7711 	case CHELSIO_T4_GET_FILTER:
7712 		rc = get_filter(sc, (struct t4_filter *)data);
7713 		break;
7714 	case CHELSIO_T4_SET_FILTER:
7715 		rc = set_filter(sc, (struct t4_filter *)data);
7716 		break;
7717 	case CHELSIO_T4_DEL_FILTER:
7718 		rc = del_filter(sc, (struct t4_filter *)data);
7719 		break;
7720 	case CHELSIO_T4_GET_SGE_CONTEXT:
7721 		rc = get_sge_context(sc, (struct t4_sge_context *)data);
7722 		break;
7723 	case CHELSIO_T4_LOAD_FW:
7724 		rc = load_fw(sc, (struct t4_data *)data);
7725 		break;
7726 	case CHELSIO_T4_GET_MEM:
7727 		rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
7728 		break;
7729 	case CHELSIO_T4_GET_I2C:
7730 		rc = read_i2c(sc, (struct t4_i2c_data *)data);
7731 		break;
7732 	case CHELSIO_T4_CLEAR_STATS: {
7733 		int i;
7734 		u_int port_id = *(uint32_t *)data;
7735 		struct port_info *pi;
7736 
7737 		if (port_id >= sc->params.nports)
7738 			return (EINVAL);
7739 
7740 		/* MAC stats */
7741 		t4_clr_port_stats(sc, port_id);
7742 
7743 		pi = sc->port[port_id];
7744 		if (pi->flags & PORT_INIT_DONE) {
7745 			struct sge_rxq *rxq;
7746 			struct sge_txq *txq;
7747 			struct sge_wrq *wrq;
7748 
7749 			for_each_rxq(pi, i, rxq) {
7750 #if defined(INET) || defined(INET6)
7751 				rxq->lro.lro_queued = 0;
7752 				rxq->lro.lro_flushed = 0;
7753 #endif
7754 				rxq->rxcsum = 0;
7755 				rxq->vlan_extraction = 0;
7756 			}
7757 
7758 			for_each_txq(pi, i, txq) {
7759 				txq->txcsum = 0;
7760 				txq->tso_wrs = 0;
7761 				txq->vlan_insertion = 0;
7762 				txq->imm_wrs = 0;
7763 				txq->sgl_wrs = 0;
7764 				txq->txpkt_wrs = 0;
7765 				txq->txpkts_wrs = 0;
7766 				txq->txpkts_pkts = 0;
7767 				txq->br->br_drops = 0;
7768 				txq->no_dmamap = 0;
7769 				txq->no_desc = 0;
7770 			}
7771 
7772 #ifdef TCP_OFFLOAD
7773 			/* nothing to clear for each ofld_rxq */
7774 
7775 			for_each_ofld_txq(pi, i, wrq) {
7776 				wrq->tx_wrs = 0;
7777 				wrq->no_desc = 0;
7778 			}
7779 #endif
7780 			wrq = &sc->sge.ctrlq[pi->port_id];
7781 			wrq->tx_wrs = 0;
7782 			wrq->no_desc = 0;
7783 		}
7784 		break;
7785 	}
7786 	case CHELSIO_T4_SCHED_CLASS:
7787 		rc = set_sched_class(sc, (struct t4_sched_params *)data);
7788 		break;
7789 	case CHELSIO_T4_SCHED_QUEUE:
7790 		rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
7791 		break;
7792 	case CHELSIO_T4_GET_TRACER:
7793 		rc = t4_get_tracer(sc, (struct t4_tracer *)data);
7794 		break;
7795 	case CHELSIO_T4_SET_TRACER:
7796 		rc = t4_set_tracer(sc, (struct t4_tracer *)data);
7797 		break;
7798 	default:
7799 		rc = EINVAL;
7800 	}
7801 
7802 	return (rc);
7803 }
7804 
7805 #ifdef TCP_OFFLOAD
7806 static int
7807 toe_capability(struct port_info *pi, int enable)
7808 {
7809 	int rc;
7810 	struct adapter *sc = pi->adapter;
7811 
7812 	ASSERT_SYNCHRONIZED_OP(sc);
7813 
7814 	if (!is_offload(sc))
7815 		return (ENODEV);
7816 
7817 	if (enable) {
7818 		if (!(sc->flags & FULL_INIT_DONE)) {
7819 			rc = cxgbe_init_synchronized(pi);
7820 			if (rc)
7821 				return (rc);
7822 		}
7823 
7824 		if (isset(&sc->offload_map, pi->port_id))
7825 			return (0);
7826 
7827 		if (!(sc->flags & TOM_INIT_DONE)) {
7828 			rc = t4_activate_uld(sc, ULD_TOM);
7829 			if (rc == EAGAIN) {
7830 				log(LOG_WARNING,
7831 				    "You must kldload t4_tom.ko before trying "
7832 				    "to enable TOE on a cxgbe interface.\n");
7833 			}
7834 			if (rc != 0)
7835 				return (rc);
7836 			KASSERT(sc->tom_softc != NULL,
7837 			    ("%s: TOM activated but softc NULL", __func__));
7838 			KASSERT(sc->flags & TOM_INIT_DONE,
7839 			    ("%s: TOM activated but flag not set", __func__));
7840 		}
7841 
7842 		setbit(&sc->offload_map, pi->port_id);
7843 	} else {
7844 		if (!isset(&sc->offload_map, pi->port_id))
7845 			return (0);
7846 
7847 		KASSERT(sc->flags & TOM_INIT_DONE,
7848 		    ("%s: TOM never initialized?", __func__));
7849 		clrbit(&sc->offload_map, pi->port_id);
7850 	}
7851 
7852 	return (0);
7853 }
7854 
7855 /*
7856  * Add an upper layer driver to the global list.
7857  */
7858 int
7859 t4_register_uld(struct uld_info *ui)
7860 {
7861 	int rc = 0;
7862 	struct uld_info *u;
7863 
7864 	sx_xlock(&t4_uld_list_lock);
7865 	SLIST_FOREACH(u, &t4_uld_list, link) {
7866 	    if (u->uld_id == ui->uld_id) {
7867 		    rc = EEXIST;
7868 		    goto done;
7869 	    }
7870 	}
7871 
7872 	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
7873 	ui->refcount = 0;
7874 done:
7875 	sx_xunlock(&t4_uld_list_lock);
7876 	return (rc);
7877 }
7878 
7879 int
7880 t4_unregister_uld(struct uld_info *ui)
7881 {
7882 	int rc = EINVAL;
7883 	struct uld_info *u;
7884 
7885 	sx_xlock(&t4_uld_list_lock);
7886 
7887 	SLIST_FOREACH(u, &t4_uld_list, link) {
7888 	    if (u == ui) {
7889 		    if (ui->refcount > 0) {
7890 			    rc = EBUSY;
7891 			    goto done;
7892 		    }
7893 
7894 		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
7895 		    rc = 0;
7896 		    goto done;
7897 	    }
7898 	}
7899 done:
7900 	sx_xunlock(&t4_uld_list_lock);
7901 	return (rc);
7902 }
7903 
7904 int
7905 t4_activate_uld(struct adapter *sc, int id)
7906 {
7907 	int rc = EAGAIN;
7908 	struct uld_info *ui;
7909 
7910 	ASSERT_SYNCHRONIZED_OP(sc);
7911 
7912 	sx_slock(&t4_uld_list_lock);
7913 
7914 	SLIST_FOREACH(ui, &t4_uld_list, link) {
7915 		if (ui->uld_id == id) {
7916 			rc = ui->activate(sc);
7917 			if (rc == 0)
7918 				ui->refcount++;
7919 			goto done;
7920 		}
7921 	}
7922 done:
7923 	sx_sunlock(&t4_uld_list_lock);
7924 
7925 	return (rc);
7926 }
7927 
7928 int
7929 t4_deactivate_uld(struct adapter *sc, int id)
7930 {
7931 	int rc = EINVAL;
7932 	struct uld_info *ui;
7933 
7934 	ASSERT_SYNCHRONIZED_OP(sc);
7935 
7936 	sx_slock(&t4_uld_list_lock);
7937 
7938 	SLIST_FOREACH(ui, &t4_uld_list, link) {
7939 		if (ui->uld_id == id) {
7940 			rc = ui->deactivate(sc);
7941 			if (rc == 0)
7942 				ui->refcount--;
7943 			goto done;
7944 		}
7945 	}
7946 done:
7947 	sx_sunlock(&t4_uld_list_lock);
7948 
7949 	return (rc);
7950 }
7951 #endif
7952 
7953 /*
7954  * Come up with reasonable defaults for some of the tunables, provided they're
7955  * not set by the user (in which case we'll use the values as is).
7956  */
7957 static void
7958 tweak_tunables(void)
7959 {
7960 	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
7961 
7962 	if (t4_ntxq10g < 1)
7963 		t4_ntxq10g = min(nc, NTXQ_10G);
7964 
7965 	if (t4_ntxq1g < 1)
7966 		t4_ntxq1g = min(nc, NTXQ_1G);
7967 
7968 	if (t4_nrxq10g < 1)
7969 		t4_nrxq10g = min(nc, NRXQ_10G);
7970 
7971 	if (t4_nrxq1g < 1)
7972 		t4_nrxq1g = min(nc, NRXQ_1G);
7973 
7974 #ifdef TCP_OFFLOAD
7975 	if (t4_nofldtxq10g < 1)
7976 		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
7977 
7978 	if (t4_nofldtxq1g < 1)
7979 		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
7980 
7981 	if (t4_nofldrxq10g < 1)
7982 		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
7983 
7984 	if (t4_nofldrxq1g < 1)
7985 		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
7986 
7987 	if (t4_toecaps_allowed == -1)
7988 		t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
7989 #else
7990 	if (t4_toecaps_allowed == -1)
7991 		t4_toecaps_allowed = 0;
7992 #endif
7993 
7994 	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
7995 		t4_tmr_idx_10g = TMR_IDX_10G;
7996 
7997 	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
7998 		t4_pktc_idx_10g = PKTC_IDX_10G;
7999 
8000 	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8001 		t4_tmr_idx_1g = TMR_IDX_1G;
8002 
8003 	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8004 		t4_pktc_idx_1g = PKTC_IDX_1G;
8005 
8006 	if (t4_qsize_txq < 128)
8007 		t4_qsize_txq = 128;
8008 
8009 	if (t4_qsize_rxq < 128)
8010 		t4_qsize_rxq = 128;
8011 	while (t4_qsize_rxq & 7)
8012 		t4_qsize_rxq++;
8013 
8014 	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8015 }
8016 
8017 static int
8018 mod_event(module_t mod, int cmd, void *arg)
8019 {
8020 	int rc = 0;
8021 	static int loaded = 0;
8022 
8023 	switch (cmd) {
8024 	case MOD_LOAD:
8025 		if (atomic_fetchadd_int(&loaded, 1))
8026 			break;
8027 		t4_sge_modload();
8028 		sx_init(&t4_list_lock, "T4/T5 adapters");
8029 		SLIST_INIT(&t4_list);
8030 #ifdef TCP_OFFLOAD
8031 		sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8032 		SLIST_INIT(&t4_uld_list);
8033 #endif
8034 		t4_tracer_modload();
8035 		tweak_tunables();
8036 		break;
8037 
8038 	case MOD_UNLOAD:
8039 		if (atomic_fetchadd_int(&loaded, -1) > 1)
8040 			break;
8041 		t4_tracer_modunload();
8042 #ifdef TCP_OFFLOAD
8043 		sx_slock(&t4_uld_list_lock);
8044 		if (!SLIST_EMPTY(&t4_uld_list)) {
8045 			rc = EBUSY;
8046 			sx_sunlock(&t4_uld_list_lock);
8047 			break;
8048 		}
8049 		sx_sunlock(&t4_uld_list_lock);
8050 		sx_destroy(&t4_uld_list_lock);
8051 #endif
8052 		sx_slock(&t4_list_lock);
8053 		if (!SLIST_EMPTY(&t4_list)) {
8054 			rc = EBUSY;
8055 			sx_sunlock(&t4_list_lock);
8056 			break;
8057 		}
8058 		sx_sunlock(&t4_list_lock);
8059 		sx_destroy(&t4_list_lock);
8060 		break;
8061 	}
8062 
8063 	return (rc);
8064 }
8065 
8066 static devclass_t t4_devclass, t5_devclass;
8067 static devclass_t cxgbe_devclass, cxl_devclass;
8068 
8069 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8070 MODULE_VERSION(t4nex, 1);
8071 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8072 
8073 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8074 MODULE_VERSION(t5nex, 1);
8075 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8076 
8077 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8078 MODULE_VERSION(cxgbe, 1);
8079 
8080 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8081 MODULE_VERSION(cxl, 1);
8082