xref: /freebsd/sys/dev/cxgbe/t4_main.c (revision 839529caa9c35f92b638dbe074655598e7a6bb6f)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 #include "opt_rss.h"
34 
35 #include <sys/param.h>
36 #include <sys/conf.h>
37 #include <sys/priv.h>
38 #include <sys/kernel.h>
39 #include <sys/bus.h>
40 #include <sys/module.h>
41 #include <sys/malloc.h>
42 #include <sys/queue.h>
43 #include <sys/taskqueue.h>
44 #include <sys/pciio.h>
45 #include <dev/pci/pcireg.h>
46 #include <dev/pci/pcivar.h>
47 #include <dev/pci/pci_private.h>
48 #include <sys/firmware.h>
49 #include <sys/sbuf.h>
50 #include <sys/smp.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
53 #include <sys/sysctl.h>
54 #include <net/ethernet.h>
55 #include <net/if.h>
56 #include <net/if_types.h>
57 #include <net/if_dl.h>
58 #include <net/if_vlan_var.h>
59 #ifdef RSS
60 #include <net/rss_config.h>
61 #endif
62 #if defined(__i386__) || defined(__amd64__)
63 #include <vm/vm.h>
64 #include <vm/pmap.h>
65 #endif
66 
67 #include "common/common.h"
68 #include "common/t4_msg.h"
69 #include "common/t4_regs.h"
70 #include "common/t4_regs_values.h"
71 #include "t4_ioctl.h"
72 #include "t4_l2t.h"
73 #include "t4_mp_ring.h"
74 
75 /* T4 bus driver interface */
76 static int t4_probe(device_t);
77 static int t4_attach(device_t);
78 static int t4_detach(device_t);
79 static device_method_t t4_methods[] = {
80 	DEVMETHOD(device_probe,		t4_probe),
81 	DEVMETHOD(device_attach,	t4_attach),
82 	DEVMETHOD(device_detach,	t4_detach),
83 
84 	DEVMETHOD_END
85 };
86 static driver_t t4_driver = {
87 	"t4nex",
88 	t4_methods,
89 	sizeof(struct adapter)
90 };
91 
92 
93 /* T4 port (cxgbe) interface */
94 static int cxgbe_probe(device_t);
95 static int cxgbe_attach(device_t);
96 static int cxgbe_detach(device_t);
97 static device_method_t cxgbe_methods[] = {
98 	DEVMETHOD(device_probe,		cxgbe_probe),
99 	DEVMETHOD(device_attach,	cxgbe_attach),
100 	DEVMETHOD(device_detach,	cxgbe_detach),
101 	{ 0, 0 }
102 };
103 static driver_t cxgbe_driver = {
104 	"cxgbe",
105 	cxgbe_methods,
106 	sizeof(struct port_info)
107 };
108 
109 /* T4 VI (vcxgbe) interface */
110 static int vcxgbe_probe(device_t);
111 static int vcxgbe_attach(device_t);
112 static int vcxgbe_detach(device_t);
113 static device_method_t vcxgbe_methods[] = {
114 	DEVMETHOD(device_probe,		vcxgbe_probe),
115 	DEVMETHOD(device_attach,	vcxgbe_attach),
116 	DEVMETHOD(device_detach,	vcxgbe_detach),
117 	{ 0, 0 }
118 };
119 static driver_t vcxgbe_driver = {
120 	"vcxgbe",
121 	vcxgbe_methods,
122 	sizeof(struct vi_info)
123 };
124 
125 static d_ioctl_t t4_ioctl;
126 static d_open_t t4_open;
127 static d_close_t t4_close;
128 
129 static struct cdevsw t4_cdevsw = {
130        .d_version = D_VERSION,
131        .d_flags = 0,
132        .d_open = t4_open,
133        .d_close = t4_close,
134        .d_ioctl = t4_ioctl,
135        .d_name = "t4nex",
136 };
137 
138 /* T5 bus driver interface */
139 static int t5_probe(device_t);
140 static device_method_t t5_methods[] = {
141 	DEVMETHOD(device_probe,		t5_probe),
142 	DEVMETHOD(device_attach,	t4_attach),
143 	DEVMETHOD(device_detach,	t4_detach),
144 
145 	DEVMETHOD_END
146 };
147 static driver_t t5_driver = {
148 	"t5nex",
149 	t5_methods,
150 	sizeof(struct adapter)
151 };
152 
153 
154 /* T5 port (cxl) interface */
155 static driver_t cxl_driver = {
156 	"cxl",
157 	cxgbe_methods,
158 	sizeof(struct port_info)
159 };
160 
161 /* T5 VI (vcxl) interface */
162 static driver_t vcxl_driver = {
163 	"vcxl",
164 	vcxgbe_methods,
165 	sizeof(struct vi_info)
166 };
167 
168 static struct cdevsw t5_cdevsw = {
169        .d_version = D_VERSION,
170        .d_flags = 0,
171        .d_open = t4_open,
172        .d_close = t4_close,
173        .d_ioctl = t4_ioctl,
174        .d_name = "t5nex",
175 };
176 
177 /* ifnet + media interface */
178 static void cxgbe_init(void *);
179 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
180 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
181 static void cxgbe_qflush(struct ifnet *);
182 static int cxgbe_media_change(struct ifnet *);
183 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
184 
185 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
186 
187 /*
188  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
189  * then ADAPTER_LOCK, then t4_uld_list_lock.
190  */
191 static struct sx t4_list_lock;
192 SLIST_HEAD(, adapter) t4_list;
193 #ifdef TCP_OFFLOAD
194 static struct sx t4_uld_list_lock;
195 SLIST_HEAD(, uld_info) t4_uld_list;
196 #endif
197 
198 /*
199  * Tunables.  See tweak_tunables() too.
200  *
201  * Each tunable is set to a default value here if it's known at compile-time.
202  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
203  * provide a reasonable default when the driver is loaded.
204  *
205  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
206  * T5 are under hw.cxl.
207  */
208 
209 /*
210  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
211  */
212 #define NTXQ_10G 16
213 static int t4_ntxq10g = -1;
214 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
215 
216 #define NRXQ_10G 8
217 static int t4_nrxq10g = -1;
218 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
219 
220 #define NTXQ_1G 4
221 static int t4_ntxq1g = -1;
222 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
223 
224 #define NRXQ_1G 2
225 static int t4_nrxq1g = -1;
226 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
227 
228 static int t4_rsrv_noflowq = 0;
229 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
230 
231 #ifdef TCP_OFFLOAD
232 #define NOFLDTXQ_10G 8
233 static int t4_nofldtxq10g = -1;
234 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
235 
236 #define NOFLDRXQ_10G 2
237 static int t4_nofldrxq10g = -1;
238 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
239 
240 #define NOFLDTXQ_1G 2
241 static int t4_nofldtxq1g = -1;
242 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
243 
244 #define NOFLDRXQ_1G 1
245 static int t4_nofldrxq1g = -1;
246 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
247 #endif
248 
249 #ifdef DEV_NETMAP
250 #define NNMTXQ_10G 2
251 static int t4_nnmtxq10g = -1;
252 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
253 
254 #define NNMRXQ_10G 2
255 static int t4_nnmrxq10g = -1;
256 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
257 
258 #define NNMTXQ_1G 1
259 static int t4_nnmtxq1g = -1;
260 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
261 
262 #define NNMRXQ_1G 1
263 static int t4_nnmrxq1g = -1;
264 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
265 #endif
266 
267 /*
268  * Holdoff parameters for 10G and 1G ports.
269  */
270 #define TMR_IDX_10G 1
271 static int t4_tmr_idx_10g = TMR_IDX_10G;
272 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
273 
274 #define PKTC_IDX_10G (-1)
275 static int t4_pktc_idx_10g = PKTC_IDX_10G;
276 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
277 
278 #define TMR_IDX_1G 1
279 static int t4_tmr_idx_1g = TMR_IDX_1G;
280 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
281 
282 #define PKTC_IDX_1G (-1)
283 static int t4_pktc_idx_1g = PKTC_IDX_1G;
284 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
285 
286 /*
287  * Size (# of entries) of each tx and rx queue.
288  */
289 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
290 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
291 
292 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
293 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
294 
295 /*
296  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
297  */
298 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
299 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
300 
301 /*
302  * Configuration file.
303  */
304 #define DEFAULT_CF	"default"
305 #define FLASH_CF	"flash"
306 #define UWIRE_CF	"uwire"
307 #define FPGA_CF		"fpga"
308 static char t4_cfg_file[32] = DEFAULT_CF;
309 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
310 
311 /*
312  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
313  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
314  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
315  *            mark or when signalled to do so, 0 to never emit PAUSE.
316  */
317 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
318 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
319 
320 /*
321  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
322  * encouraged respectively).
323  */
324 static unsigned int t4_fw_install = 1;
325 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
326 
327 /*
328  * ASIC features that will be used.  Disable the ones you don't want so that the
329  * chip resources aren't wasted on features that will not be used.
330  */
331 static int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
332 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
333 
334 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
335 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
336 
337 static int t4_toecaps_allowed = -1;
338 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
339 
340 static int t4_rdmacaps_allowed = 0;
341 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
342 
343 static int t4_iscsicaps_allowed = 0;
344 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
345 
346 static int t4_fcoecaps_allowed = 0;
347 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
348 
349 static int t5_write_combine = 0;
350 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
351 
352 static int t4_num_vis = 1;
353 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
354 
355 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */
356 static int vi_mac_funcs[] = {
357 	FW_VI_FUNC_OFLD,
358 	FW_VI_FUNC_IWARP,
359 	FW_VI_FUNC_OPENISCSI,
360 	FW_VI_FUNC_OPENFCOE,
361 	FW_VI_FUNC_FOISCSI,
362 	FW_VI_FUNC_FOFCOE,
363 };
364 
365 struct intrs_and_queues {
366 	uint16_t intr_type;	/* INTx, MSI, or MSI-X */
367 	uint16_t nirq;		/* Total # of vectors */
368 	uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
369 	uint16_t intr_flags_1g;	/* Interrupt flags for each 1G port */
370 	uint16_t ntxq10g;	/* # of NIC txq's for each 10G port */
371 	uint16_t nrxq10g;	/* # of NIC rxq's for each 10G port */
372 	uint16_t ntxq1g;	/* # of NIC txq's for each 1G port */
373 	uint16_t nrxq1g;	/* # of NIC rxq's for each 1G port */
374 	uint16_t rsrv_noflowq;	/* Flag whether to reserve queue 0 */
375 #ifdef TCP_OFFLOAD
376 	uint16_t nofldtxq10g;	/* # of TOE txq's for each 10G port */
377 	uint16_t nofldrxq10g;	/* # of TOE rxq's for each 10G port */
378 	uint16_t nofldtxq1g;	/* # of TOE txq's for each 1G port */
379 	uint16_t nofldrxq1g;	/* # of TOE rxq's for each 1G port */
380 #endif
381 #ifdef DEV_NETMAP
382 	uint16_t nnmtxq10g;	/* # of netmap txq's for each 10G port */
383 	uint16_t nnmrxq10g;	/* # of netmap rxq's for each 10G port */
384 	uint16_t nnmtxq1g;	/* # of netmap txq's for each 1G port */
385 	uint16_t nnmrxq1g;	/* # of netmap rxq's for each 1G port */
386 #endif
387 };
388 
389 struct filter_entry {
390         uint32_t valid:1;	/* filter allocated and valid */
391         uint32_t locked:1;	/* filter is administratively locked */
392         uint32_t pending:1;	/* filter action is pending firmware reply */
393 	uint32_t smtidx:8;	/* Source MAC Table index for smac */
394 	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
395 
396         struct t4_filter_specification fs;
397 };
398 
399 static int map_bars_0_and_4(struct adapter *);
400 static int map_bar_2(struct adapter *);
401 static void setup_memwin(struct adapter *);
402 static int validate_mem_range(struct adapter *, uint32_t, int);
403 static int fwmtype_to_hwmtype(int);
404 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
405     uint32_t *);
406 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
407 static uint32_t position_memwin(struct adapter *, int, uint32_t);
408 static int cfg_itype_and_nqueues(struct adapter *, int, int, int,
409     struct intrs_and_queues *);
410 static int prep_firmware(struct adapter *);
411 static int partition_resources(struct adapter *, const struct firmware *,
412     const char *);
413 static int get_params__pre_init(struct adapter *);
414 static int get_params__post_init(struct adapter *);
415 static int set_params__post_init(struct adapter *);
416 static void t4_set_desc(struct adapter *);
417 static void build_medialist(struct port_info *, struct ifmedia *);
418 static int cxgbe_init_synchronized(struct vi_info *);
419 static int cxgbe_uninit_synchronized(struct vi_info *);
420 static int setup_intr_handlers(struct adapter *);
421 static void quiesce_txq(struct adapter *, struct sge_txq *);
422 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
423 static void quiesce_iq(struct adapter *, struct sge_iq *);
424 static void quiesce_fl(struct adapter *, struct sge_fl *);
425 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
426     driver_intr_t *, void *, char *);
427 static int t4_free_irq(struct adapter *, struct irq *);
428 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
429     unsigned int);
430 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
431 static void vi_refresh_stats(struct adapter *, struct vi_info *);
432 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
433 static void cxgbe_tick(void *);
434 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
435 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
436     struct mbuf *);
437 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
438 static int fw_msg_not_handled(struct adapter *, const __be64 *);
439 static void t4_sysctls(struct adapter *);
440 static void cxgbe_sysctls(struct port_info *);
441 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
442 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
443 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
444 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
445 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
446 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
447 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
448 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
449 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
450 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
451 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
452 #ifdef SBUF_DRAIN
453 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
454 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
455 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
456 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
457 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
458 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
459 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
460 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
461 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
462 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
463 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
464 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
465 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
466 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
467 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
468 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
469 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
470 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
471 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
472 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
473 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
474 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
475 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
476 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
477 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
478 #endif
479 static uint32_t fconf_to_mode(uint32_t);
480 static uint32_t mode_to_fconf(uint32_t);
481 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
482 static int get_filter_mode(struct adapter *, uint32_t *);
483 static int set_filter_mode(struct adapter *, uint32_t);
484 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
485 static int get_filter(struct adapter *, struct t4_filter *);
486 static int set_filter(struct adapter *, struct t4_filter *);
487 static int del_filter(struct adapter *, struct t4_filter *);
488 static void clear_filter(struct filter_entry *);
489 static int set_filter_wr(struct adapter *, int);
490 static int del_filter_wr(struct adapter *, int);
491 static int get_sge_context(struct adapter *, struct t4_sge_context *);
492 static int load_fw(struct adapter *, struct t4_data *);
493 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
494 static int read_i2c(struct adapter *, struct t4_i2c_data *);
495 static int set_sched_class(struct adapter *, struct t4_sched_params *);
496 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
497 #ifdef TCP_OFFLOAD
498 static int toe_capability(struct vi_info *, int);
499 #endif
500 static int mod_event(module_t, int, void *);
501 
502 struct {
503 	uint16_t device;
504 	char *desc;
505 } t4_pciids[] = {
506 	{0xa000, "Chelsio Terminator 4 FPGA"},
507 	{0x4400, "Chelsio T440-dbg"},
508 	{0x4401, "Chelsio T420-CR"},
509 	{0x4402, "Chelsio T422-CR"},
510 	{0x4403, "Chelsio T440-CR"},
511 	{0x4404, "Chelsio T420-BCH"},
512 	{0x4405, "Chelsio T440-BCH"},
513 	{0x4406, "Chelsio T440-CH"},
514 	{0x4407, "Chelsio T420-SO"},
515 	{0x4408, "Chelsio T420-CX"},
516 	{0x4409, "Chelsio T420-BT"},
517 	{0x440a, "Chelsio T404-BT"},
518 	{0x440e, "Chelsio T440-LP-CR"},
519 }, t5_pciids[] = {
520 	{0xb000, "Chelsio Terminator 5 FPGA"},
521 	{0x5400, "Chelsio T580-dbg"},
522 	{0x5401,  "Chelsio T520-CR"},		/* 2 x 10G */
523 	{0x5402,  "Chelsio T522-CR"},		/* 2 x 10G, 2 X 1G */
524 	{0x5403,  "Chelsio T540-CR"},		/* 4 x 10G */
525 	{0x5407,  "Chelsio T520-SO"},		/* 2 x 10G, nomem */
526 	{0x5409,  "Chelsio T520-BT"},		/* 2 x 10GBaseT */
527 	{0x540a,  "Chelsio T504-BT"},		/* 4 x 1G */
528 	{0x540d,  "Chelsio T580-CR"},		/* 2 x 40G */
529 	{0x540e,  "Chelsio T540-LP-CR"},	/* 4 x 10G */
530 	{0x5410,  "Chelsio T580-LP-CR"},	/* 2 x 40G */
531 	{0x5411,  "Chelsio T520-LL-CR"},	/* 2 x 10G */
532 	{0x5412,  "Chelsio T560-CR"},		/* 1 x 40G, 2 x 10G */
533 	{0x5414,  "Chelsio T580-LP-SO-CR"},	/* 2 x 40G, nomem */
534 	{0x5415,  "Chelsio T502-BT"},		/* 2 x 1G */
535 #ifdef notyet
536 	{0x5404,  "Chelsio T520-BCH"},
537 	{0x5405,  "Chelsio T540-BCH"},
538 	{0x5406,  "Chelsio T540-CH"},
539 	{0x5408,  "Chelsio T520-CX"},
540 	{0x540b,  "Chelsio B520-SR"},
541 	{0x540c,  "Chelsio B504-BT"},
542 	{0x540f,  "Chelsio Amsterdam"},
543 	{0x5413,  "Chelsio T580-CHR"},
544 #endif
545 };
546 
547 #ifdef TCP_OFFLOAD
548 /*
549  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
550  * exactly the same for both rxq and ofld_rxq.
551  */
552 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
553 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
554 #endif
555 
556 /* No easy way to include t4_msg.h before adapter.h so we check this way */
557 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
558 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
559 
560 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
561 
562 static int
563 t4_probe(device_t dev)
564 {
565 	int i;
566 	uint16_t v = pci_get_vendor(dev);
567 	uint16_t d = pci_get_device(dev);
568 	uint8_t f = pci_get_function(dev);
569 
570 	if (v != PCI_VENDOR_ID_CHELSIO)
571 		return (ENXIO);
572 
573 	/* Attach only to PF0 of the FPGA */
574 	if (d == 0xa000 && f != 0)
575 		return (ENXIO);
576 
577 	for (i = 0; i < nitems(t4_pciids); i++) {
578 		if (d == t4_pciids[i].device) {
579 			device_set_desc(dev, t4_pciids[i].desc);
580 			return (BUS_PROBE_DEFAULT);
581 		}
582 	}
583 
584 	return (ENXIO);
585 }
586 
587 static int
588 t5_probe(device_t dev)
589 {
590 	int i;
591 	uint16_t v = pci_get_vendor(dev);
592 	uint16_t d = pci_get_device(dev);
593 	uint8_t f = pci_get_function(dev);
594 
595 	if (v != PCI_VENDOR_ID_CHELSIO)
596 		return (ENXIO);
597 
598 	/* Attach only to PF0 of the FPGA */
599 	if (d == 0xb000 && f != 0)
600 		return (ENXIO);
601 
602 	for (i = 0; i < nitems(t5_pciids); i++) {
603 		if (d == t5_pciids[i].device) {
604 			device_set_desc(dev, t5_pciids[i].desc);
605 			return (BUS_PROBE_DEFAULT);
606 		}
607 	}
608 
609 	return (ENXIO);
610 }
611 
612 static void
613 t5_attribute_workaround(device_t dev)
614 {
615 	device_t root_port;
616 	uint32_t v;
617 
618 	/*
619 	 * The T5 chips do not properly echo the No Snoop and Relaxed
620 	 * Ordering attributes when replying to a TLP from a Root
621 	 * Port.  As a workaround, find the parent Root Port and
622 	 * disable No Snoop and Relaxed Ordering.  Note that this
623 	 * affects all devices under this root port.
624 	 */
625 	root_port = pci_find_pcie_root_port(dev);
626 	if (root_port == NULL) {
627 		device_printf(dev, "Unable to find parent root port\n");
628 		return;
629 	}
630 
631 	v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
632 	    PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
633 	if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
634 	    0)
635 		device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
636 		    device_get_nameunit(root_port));
637 }
638 
639 static int
640 t4_attach(device_t dev)
641 {
642 	struct adapter *sc;
643 	int rc = 0, i, j, n10g, n1g, rqidx, tqidx;
644 	struct intrs_and_queues iaq;
645 	struct sge *s;
646 #ifdef TCP_OFFLOAD
647 	int ofld_rqidx, ofld_tqidx;
648 #endif
649 #ifdef DEV_NETMAP
650 	int nm_rqidx, nm_tqidx;
651 #endif
652 	int num_vis;
653 
654 	sc = device_get_softc(dev);
655 	sc->dev = dev;
656 	TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
657 
658 	if ((pci_get_device(dev) & 0xff00) == 0x5400)
659 		t5_attribute_workaround(dev);
660 	pci_enable_busmaster(dev);
661 	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
662 		uint32_t v;
663 
664 		pci_set_max_read_req(dev, 4096);
665 		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
666 		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
667 		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
668 
669 		sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
670 	}
671 
672 	sc->traceq = -1;
673 	mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
674 	snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
675 	    device_get_nameunit(dev));
676 
677 	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
678 	    device_get_nameunit(dev));
679 	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
680 	sx_xlock(&t4_list_lock);
681 	SLIST_INSERT_HEAD(&t4_list, sc, link);
682 	sx_xunlock(&t4_list_lock);
683 
684 	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
685 	TAILQ_INIT(&sc->sfl);
686 	callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
687 
688 	mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
689 
690 	rc = map_bars_0_and_4(sc);
691 	if (rc != 0)
692 		goto done; /* error message displayed already */
693 
694 	/*
695 	 * This is the real PF# to which we're attaching.  Works from within PCI
696 	 * passthrough environments too, where pci_get_function() could return a
697 	 * different PF# depending on the passthrough configuration.  We need to
698 	 * use the real PF# in all our communication with the firmware.
699 	 */
700 	sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
701 	sc->mbox = sc->pf;
702 
703 	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
704 	sc->an_handler = an_not_handled;
705 	for (i = 0; i < nitems(sc->cpl_handler); i++)
706 		sc->cpl_handler[i] = cpl_not_handled;
707 	for (i = 0; i < nitems(sc->fw_msg_handler); i++)
708 		sc->fw_msg_handler[i] = fw_msg_not_handled;
709 	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
710 	t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
711 	t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt);
712 	t4_init_sge_cpl_handlers(sc);
713 
714 	/* Prepare the adapter for operation */
715 	rc = -t4_prep_adapter(sc);
716 	if (rc != 0) {
717 		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
718 		goto done;
719 	}
720 
721 	/*
722 	 * Do this really early, with the memory windows set up even before the
723 	 * character device.  The userland tool's register i/o and mem read
724 	 * will work even in "recovery mode".
725 	 */
726 	setup_memwin(sc);
727 	sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
728 	    device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
729 	    device_get_nameunit(dev));
730 	if (sc->cdev == NULL)
731 		device_printf(dev, "failed to create nexus char device.\n");
732 	else
733 		sc->cdev->si_drv1 = sc;
734 
735 	/* Go no further if recovery mode has been requested. */
736 	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
737 		device_printf(dev, "recovery mode.\n");
738 		goto done;
739 	}
740 
741 #if defined(__i386__)
742 	if ((cpu_feature & CPUID_CX8) == 0) {
743 		device_printf(dev, "64 bit atomics not available.\n");
744 		rc = ENOTSUP;
745 		goto done;
746 	}
747 #endif
748 
749 	/* Prepare the firmware for operation */
750 	rc = prep_firmware(sc);
751 	if (rc != 0)
752 		goto done; /* error message displayed already */
753 
754 	rc = get_params__post_init(sc);
755 	if (rc != 0)
756 		goto done; /* error message displayed already */
757 
758 	rc = set_params__post_init(sc);
759 	if (rc != 0)
760 		goto done; /* error message displayed already */
761 
762 	rc = map_bar_2(sc);
763 	if (rc != 0)
764 		goto done; /* error message displayed already */
765 
766 	rc = t4_create_dma_tag(sc);
767 	if (rc != 0)
768 		goto done; /* error message displayed already */
769 
770 	/*
771 	 * Number of VIs to create per-port.  The first VI is the
772 	 * "main" regular VI for the port.  The second VI is used for
773 	 * netmap if present, and any remaining VIs are used for
774 	 * additional virtual interfaces.
775 	 *
776 	 * Limit the number of VIs per port to the number of available
777 	 * MAC addresses per port.
778 	 */
779 	if (t4_num_vis >= 1)
780 		num_vis = t4_num_vis;
781 	else
782 		num_vis = 1;
783 #ifdef DEV_NETMAP
784 	num_vis++;
785 #endif
786 	if (num_vis > nitems(vi_mac_funcs)) {
787 		num_vis = nitems(vi_mac_funcs);
788 		device_printf(dev, "Number of VIs limited to %d\n", num_vis);
789 	}
790 
791 	/*
792 	 * First pass over all the ports - allocate VIs and initialize some
793 	 * basic parameters like mac address, port type, etc.  We also figure
794 	 * out whether a port is 10G or 1G and use that information when
795 	 * calculating how many interrupts to attempt to allocate.
796 	 */
797 	n10g = n1g = 0;
798 	for_each_port(sc, i) {
799 		struct port_info *pi;
800 		struct vi_info *vi;
801 
802 		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
803 		sc->port[i] = pi;
804 
805 		/* These must be set before t4_port_init */
806 		pi->adapter = sc;
807 		pi->port_id = i;
808 		pi->nvi = num_vis;
809 		pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE,
810 		    M_ZERO | M_WAITOK);
811 
812 		/*
813 		 * Allocate the "main" VI and initialize parameters
814 		 * like mac addr.
815 		 */
816 		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
817 		if (rc != 0) {
818 			device_printf(dev, "unable to initialize port %d: %d\n",
819 			    i, rc);
820 			free(pi->vi, M_CXGBE);
821 			free(pi, M_CXGBE);
822 			sc->port[i] = NULL;
823 			goto done;
824 		}
825 
826 		pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
827 		pi->link_cfg.requested_fc |= t4_pause_settings;
828 		pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
829 		pi->link_cfg.fc |= t4_pause_settings;
830 
831 		rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
832 		if (rc != 0) {
833 			device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
834 			free(pi->vi, M_CXGBE);
835 			free(pi, M_CXGBE);
836 			sc->port[i] = NULL;
837 			goto done;
838 		}
839 
840 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
841 		    device_get_nameunit(dev), i);
842 		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
843 		sc->chan_map[pi->tx_chan] = i;
844 
845 		if (is_10G_port(pi) || is_40G_port(pi)) {
846 			n10g++;
847 			for_each_vi(pi, j, vi) {
848 				vi->tmr_idx = t4_tmr_idx_10g;
849 				vi->pktc_idx = t4_pktc_idx_10g;
850 			}
851 		} else {
852 			n1g++;
853 			for_each_vi(pi, j, vi) {
854 				vi->tmr_idx = t4_tmr_idx_1g;
855 				vi->pktc_idx = t4_pktc_idx_1g;
856 			}
857 		}
858 
859 		pi->linkdnrc = -1;
860 
861 		for_each_vi(pi, j, vi) {
862 			vi->qsize_rxq = t4_qsize_rxq;
863 			vi->qsize_txq = t4_qsize_txq;
864 			vi->pi = pi;
865 		}
866 
867 		pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
868 		if (pi->dev == NULL) {
869 			device_printf(dev,
870 			    "failed to add device for port %d.\n", i);
871 			rc = ENXIO;
872 			goto done;
873 		}
874 		pi->vi[0].dev = pi->dev;
875 		device_set_softc(pi->dev, pi);
876 	}
877 
878 	/*
879 	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
880 	 */
881 #ifdef DEV_NETMAP
882 	num_vis--;
883 #endif
884 	rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq);
885 	if (rc != 0)
886 		goto done; /* error message displayed already */
887 
888 	sc->intr_type = iaq.intr_type;
889 	sc->intr_count = iaq.nirq;
890 
891 	s = &sc->sge;
892 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
893 	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
894 	if (num_vis > 1) {
895 		s->nrxq += (n10g + n1g) * (num_vis - 1);
896 		s->ntxq += (n10g + n1g) * (num_vis - 1);
897 	}
898 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
899 	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
900 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
901 #ifdef TCP_OFFLOAD
902 	if (is_offload(sc)) {
903 		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
904 		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
905 		if (num_vis > 1) {
906 			s->nofldrxq += (n10g + n1g) * (num_vis - 1);
907 			s->nofldtxq += (n10g + n1g) * (num_vis - 1);
908 		}
909 		s->neq += s->nofldtxq + s->nofldrxq;
910 		s->niq += s->nofldrxq;
911 
912 		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
913 		    M_CXGBE, M_ZERO | M_WAITOK);
914 		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
915 		    M_CXGBE, M_ZERO | M_WAITOK);
916 	}
917 #endif
918 #ifdef DEV_NETMAP
919 	s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
920 	s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
921 	s->neq += s->nnmtxq + s->nnmrxq;
922 	s->niq += s->nnmrxq;
923 
924 	s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
925 	    M_CXGBE, M_ZERO | M_WAITOK);
926 	s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
927 	    M_CXGBE, M_ZERO | M_WAITOK);
928 #endif
929 
930 	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
931 	    M_ZERO | M_WAITOK);
932 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
933 	    M_ZERO | M_WAITOK);
934 	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
935 	    M_ZERO | M_WAITOK);
936 	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
937 	    M_ZERO | M_WAITOK);
938 	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
939 	    M_ZERO | M_WAITOK);
940 
941 	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
942 	    M_ZERO | M_WAITOK);
943 
944 	t4_init_l2t(sc, M_WAITOK);
945 
946 	/*
947 	 * Second pass over the ports.  This time we know the number of rx and
948 	 * tx queues that each port should get.
949 	 */
950 	rqidx = tqidx = 0;
951 #ifdef TCP_OFFLOAD
952 	ofld_rqidx = ofld_tqidx = 0;
953 #endif
954 #ifdef DEV_NETMAP
955 	nm_rqidx = nm_tqidx = 0;
956 #endif
957 	for_each_port(sc, i) {
958 		struct port_info *pi = sc->port[i];
959 		struct vi_info *vi;
960 
961 		if (pi == NULL)
962 			continue;
963 
964 		for_each_vi(pi, j, vi) {
965 #ifdef DEV_NETMAP
966 			if (j == 1) {
967 				vi->flags |= VI_NETMAP | INTR_RXQ;
968 				vi->first_rxq = nm_rqidx;
969 				vi->first_txq = nm_tqidx;
970 				if (is_10G_port(pi) || is_40G_port(pi)) {
971 					vi->nrxq = iaq.nnmrxq10g;
972 					vi->ntxq = iaq.nnmtxq10g;
973 				} else {
974 					vi->nrxq = iaq.nnmrxq1g;
975 					vi->ntxq = iaq.nnmtxq1g;
976 				}
977 				nm_rqidx += vi->nrxq;
978 				nm_tqidx += vi->ntxq;
979 				continue;
980 			}
981 #endif
982 
983 			vi->first_rxq = rqidx;
984 			vi->first_txq = tqidx;
985 			if (is_10G_port(pi) || is_40G_port(pi)) {
986 				vi->flags |= iaq.intr_flags_10g & INTR_RXQ;
987 				vi->nrxq = j == 0 ? iaq.nrxq10g : 1;
988 				vi->ntxq = j == 0 ? iaq.ntxq10g : 1;
989 			} else {
990 				vi->flags |= iaq.intr_flags_1g & INTR_RXQ;
991 				vi->nrxq = j == 0 ? iaq.nrxq1g : 1;
992 				vi->ntxq = j == 0 ? iaq.ntxq1g : 1;
993 			}
994 
995 			if (vi->ntxq > 1)
996 				vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
997 			else
998 				vi->rsrv_noflowq = 0;
999 
1000 			rqidx += vi->nrxq;
1001 			tqidx += vi->ntxq;
1002 
1003 #ifdef TCP_OFFLOAD
1004 			if (!is_offload(sc))
1005 				continue;
1006 			vi->first_ofld_rxq = ofld_rqidx;
1007 			vi->first_ofld_txq = ofld_tqidx;
1008 			if (is_10G_port(pi) || is_40G_port(pi)) {
1009 				vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ;
1010 				vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1;
1011 				vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1;
1012 			} else {
1013 				vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ;
1014 				vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1;
1015 				vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1;
1016 			}
1017 			ofld_rqidx += vi->nofldrxq;
1018 			ofld_tqidx += vi->nofldtxq;
1019 #endif
1020 		}
1021 	}
1022 
1023 	rc = setup_intr_handlers(sc);
1024 	if (rc != 0) {
1025 		device_printf(dev,
1026 		    "failed to setup interrupt handlers: %d\n", rc);
1027 		goto done;
1028 	}
1029 
1030 	rc = bus_generic_attach(dev);
1031 	if (rc != 0) {
1032 		device_printf(dev,
1033 		    "failed to attach all child ports: %d\n", rc);
1034 		goto done;
1035 	}
1036 
1037 	device_printf(dev,
1038 	    "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1039 	    sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1040 	    sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1041 	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1042 	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1043 
1044 	t4_set_desc(sc);
1045 
1046 done:
1047 	if (rc != 0 && sc->cdev) {
1048 		/* cdev was created and so cxgbetool works; recover that way. */
1049 		device_printf(dev,
1050 		    "error during attach, adapter is now in recovery mode.\n");
1051 		rc = 0;
1052 	}
1053 
1054 	if (rc != 0)
1055 		t4_detach(dev);
1056 	else
1057 		t4_sysctls(sc);
1058 
1059 	return (rc);
1060 }
1061 
1062 /*
1063  * Idempotent
1064  */
1065 static int
1066 t4_detach(device_t dev)
1067 {
1068 	struct adapter *sc;
1069 	struct port_info *pi;
1070 	int i, rc;
1071 
1072 	sc = device_get_softc(dev);
1073 
1074 	if (sc->flags & FULL_INIT_DONE)
1075 		t4_intr_disable(sc);
1076 
1077 	if (sc->cdev) {
1078 		destroy_dev(sc->cdev);
1079 		sc->cdev = NULL;
1080 	}
1081 
1082 	rc = bus_generic_detach(dev);
1083 	if (rc) {
1084 		device_printf(dev,
1085 		    "failed to detach child devices: %d\n", rc);
1086 		return (rc);
1087 	}
1088 
1089 	for (i = 0; i < sc->intr_count; i++)
1090 		t4_free_irq(sc, &sc->irq[i]);
1091 
1092 	for (i = 0; i < MAX_NPORTS; i++) {
1093 		pi = sc->port[i];
1094 		if (pi) {
1095 			t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1096 			if (pi->dev)
1097 				device_delete_child(dev, pi->dev);
1098 
1099 			mtx_destroy(&pi->pi_lock);
1100 			free(pi->vi, M_CXGBE);
1101 			free(pi, M_CXGBE);
1102 		}
1103 	}
1104 
1105 	if (sc->flags & FULL_INIT_DONE)
1106 		adapter_full_uninit(sc);
1107 
1108 	if (sc->flags & FW_OK)
1109 		t4_fw_bye(sc, sc->mbox);
1110 
1111 	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1112 		pci_release_msi(dev);
1113 
1114 	if (sc->regs_res)
1115 		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1116 		    sc->regs_res);
1117 
1118 	if (sc->udbs_res)
1119 		bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1120 		    sc->udbs_res);
1121 
1122 	if (sc->msix_res)
1123 		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1124 		    sc->msix_res);
1125 
1126 	if (sc->l2t)
1127 		t4_free_l2t(sc->l2t);
1128 
1129 #ifdef TCP_OFFLOAD
1130 	free(sc->sge.ofld_rxq, M_CXGBE);
1131 	free(sc->sge.ofld_txq, M_CXGBE);
1132 #endif
1133 #ifdef DEV_NETMAP
1134 	free(sc->sge.nm_rxq, M_CXGBE);
1135 	free(sc->sge.nm_txq, M_CXGBE);
1136 #endif
1137 	free(sc->irq, M_CXGBE);
1138 	free(sc->sge.rxq, M_CXGBE);
1139 	free(sc->sge.txq, M_CXGBE);
1140 	free(sc->sge.ctrlq, M_CXGBE);
1141 	free(sc->sge.iqmap, M_CXGBE);
1142 	free(sc->sge.eqmap, M_CXGBE);
1143 	free(sc->tids.ftid_tab, M_CXGBE);
1144 	t4_destroy_dma_tag(sc);
1145 	if (mtx_initialized(&sc->sc_lock)) {
1146 		sx_xlock(&t4_list_lock);
1147 		SLIST_REMOVE(&t4_list, sc, adapter, link);
1148 		sx_xunlock(&t4_list_lock);
1149 		mtx_destroy(&sc->sc_lock);
1150 	}
1151 
1152 	callout_drain(&sc->sfl_callout);
1153 	if (mtx_initialized(&sc->tids.ftid_lock))
1154 		mtx_destroy(&sc->tids.ftid_lock);
1155 	if (mtx_initialized(&sc->sfl_lock))
1156 		mtx_destroy(&sc->sfl_lock);
1157 	if (mtx_initialized(&sc->ifp_lock))
1158 		mtx_destroy(&sc->ifp_lock);
1159 	if (mtx_initialized(&sc->regwin_lock))
1160 		mtx_destroy(&sc->regwin_lock);
1161 
1162 	bzero(sc, sizeof(*sc));
1163 
1164 	return (0);
1165 }
1166 
1167 static int
1168 cxgbe_probe(device_t dev)
1169 {
1170 	char buf[128];
1171 	struct port_info *pi = device_get_softc(dev);
1172 
1173 	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1174 	device_set_desc_copy(dev, buf);
1175 
1176 	return (BUS_PROBE_DEFAULT);
1177 }
1178 
1179 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1180     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1181     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1182 #define T4_CAP_ENABLE (T4_CAP)
1183 
1184 static int
1185 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1186 {
1187 	struct ifnet *ifp;
1188 	struct sbuf *sb;
1189 
1190 	vi->xact_addr_filt = -1;
1191 	callout_init(&vi->tick, 1);
1192 
1193 	/* Allocate an ifnet and set it up */
1194 	ifp = if_alloc(IFT_ETHER);
1195 	if (ifp == NULL) {
1196 		device_printf(dev, "Cannot allocate ifnet\n");
1197 		return (ENOMEM);
1198 	}
1199 	vi->ifp = ifp;
1200 	ifp->if_softc = vi;
1201 
1202 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1203 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1204 
1205 	ifp->if_init = cxgbe_init;
1206 	ifp->if_ioctl = cxgbe_ioctl;
1207 	ifp->if_transmit = cxgbe_transmit;
1208 	ifp->if_qflush = cxgbe_qflush;
1209 	ifp->if_get_counter = cxgbe_get_counter;
1210 
1211 	ifp->if_capabilities = T4_CAP;
1212 #ifdef TCP_OFFLOAD
1213 	if (vi->nofldrxq != 0)
1214 		ifp->if_capabilities |= IFCAP_TOE;
1215 #endif
1216 	ifp->if_capenable = T4_CAP_ENABLE;
1217 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1218 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1219 
1220 	ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1221 	ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1222 	ifp->if_hw_tsomaxsegsize = 65536;
1223 
1224 	/* Initialize ifmedia for this VI */
1225 	ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1226 	    cxgbe_media_status);
1227 	build_medialist(vi->pi, &vi->media);
1228 
1229 	vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1230 	    EVENTHANDLER_PRI_ANY);
1231 
1232 	ether_ifattach(ifp, vi->hw_addr);
1233 
1234 	sb = sbuf_new_auto();
1235 	sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1236 #ifdef TCP_OFFLOAD
1237 	if (ifp->if_capabilities & IFCAP_TOE)
1238 		sbuf_printf(sb, "; %d txq, %d rxq (TOE)",
1239 		    vi->nofldtxq, vi->nofldrxq);
1240 #endif
1241 	sbuf_finish(sb);
1242 	device_printf(dev, "%s\n", sbuf_data(sb));
1243 	sbuf_delete(sb);
1244 
1245 	vi_sysctls(vi);
1246 
1247 	return (0);
1248 }
1249 
1250 static int
1251 cxgbe_attach(device_t dev)
1252 {
1253 	struct port_info *pi = device_get_softc(dev);
1254 	struct vi_info *vi;
1255 	int i, rc;
1256 
1257 	callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1258 
1259 	rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1260 	if (rc)
1261 		return (rc);
1262 
1263 	for_each_vi(pi, i, vi) {
1264 		if (i == 0)
1265 			continue;
1266 #ifdef DEV_NETMAP
1267 		if (vi->flags & VI_NETMAP) {
1268 			/*
1269 			 * media handled here to keep
1270 			 * implementation private to this file
1271 			 */
1272 			ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1273 			    cxgbe_media_status);
1274 			build_medialist(pi, &vi->media);
1275 			vi->dev = device_add_child(dev, is_t4(pi->adapter) ?
1276 			    "ncxgbe" : "ncxl", device_get_unit(dev));
1277 		} else
1278 #endif
1279 			vi->dev = device_add_child(dev, is_t4(pi->adapter) ?
1280 			    "vcxgbe" : "vcxl", -1);
1281 		if (vi->dev == NULL) {
1282 			device_printf(dev, "failed to add VI %d\n", i);
1283 			continue;
1284 		}
1285 		device_set_softc(vi->dev, vi);
1286 	}
1287 
1288 	cxgbe_sysctls(pi);
1289 
1290 	bus_generic_attach(dev);
1291 
1292 	return (0);
1293 }
1294 
1295 static void
1296 cxgbe_vi_detach(struct vi_info *vi)
1297 {
1298 	struct ifnet *ifp = vi->ifp;
1299 
1300 	ether_ifdetach(ifp);
1301 
1302 	if (vi->vlan_c)
1303 		EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c);
1304 
1305 	/* Let detach proceed even if these fail. */
1306 	cxgbe_uninit_synchronized(vi);
1307 	callout_drain(&vi->tick);
1308 	vi_full_uninit(vi);
1309 
1310 	ifmedia_removeall(&vi->media);
1311 	if_free(vi->ifp);
1312 	vi->ifp = NULL;
1313 }
1314 
1315 static int
1316 cxgbe_detach(device_t dev)
1317 {
1318 	struct port_info *pi = device_get_softc(dev);
1319 	struct adapter *sc = pi->adapter;
1320 	int rc;
1321 
1322 	/* Detach the extra VIs first. */
1323 	rc = bus_generic_detach(dev);
1324 	if (rc)
1325 		return (rc);
1326 	device_delete_children(dev);
1327 
1328 	doom_vi(sc, &pi->vi[0]);
1329 
1330 	if (pi->flags & HAS_TRACEQ) {
1331 		sc->traceq = -1;	/* cloner should not create ifnet */
1332 		t4_tracer_port_detach(sc);
1333 	}
1334 
1335 	cxgbe_vi_detach(&pi->vi[0]);
1336 	callout_drain(&pi->tick);
1337 
1338 	end_synchronized_op(sc, 0);
1339 
1340 	return (0);
1341 }
1342 
1343 static void
1344 cxgbe_init(void *arg)
1345 {
1346 	struct vi_info *vi = arg;
1347 	struct adapter *sc = vi->pi->adapter;
1348 
1349 	if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1350 		return;
1351 	cxgbe_init_synchronized(vi);
1352 	end_synchronized_op(sc, 0);
1353 }
1354 
1355 static int
1356 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1357 {
1358 	int rc = 0, mtu, flags, can_sleep;
1359 	struct vi_info *vi = ifp->if_softc;
1360 	struct adapter *sc = vi->pi->adapter;
1361 	struct ifreq *ifr = (struct ifreq *)data;
1362 	uint32_t mask;
1363 
1364 	switch (cmd) {
1365 	case SIOCSIFMTU:
1366 		mtu = ifr->ifr_mtu;
1367 		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1368 			return (EINVAL);
1369 
1370 		rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1371 		if (rc)
1372 			return (rc);
1373 		ifp->if_mtu = mtu;
1374 		if (vi->flags & VI_INIT_DONE) {
1375 			t4_update_fl_bufsize(ifp);
1376 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1377 				rc = update_mac_settings(ifp, XGMAC_MTU);
1378 		}
1379 		end_synchronized_op(sc, 0);
1380 		break;
1381 
1382 	case SIOCSIFFLAGS:
1383 		can_sleep = 0;
1384 redo_sifflags:
1385 		rc = begin_synchronized_op(sc, vi,
1386 		    can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1387 		if (rc)
1388 			return (rc);
1389 
1390 		if (ifp->if_flags & IFF_UP) {
1391 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1392 				flags = vi->if_flags;
1393 				if ((ifp->if_flags ^ flags) &
1394 				    (IFF_PROMISC | IFF_ALLMULTI)) {
1395 					if (can_sleep == 1) {
1396 						end_synchronized_op(sc, 0);
1397 						can_sleep = 0;
1398 						goto redo_sifflags;
1399 					}
1400 					rc = update_mac_settings(ifp,
1401 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
1402 				}
1403 			} else {
1404 				if (can_sleep == 0) {
1405 					end_synchronized_op(sc, LOCK_HELD);
1406 					can_sleep = 1;
1407 					goto redo_sifflags;
1408 				}
1409 				rc = cxgbe_init_synchronized(vi);
1410 			}
1411 			vi->if_flags = ifp->if_flags;
1412 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1413 			if (can_sleep == 0) {
1414 				end_synchronized_op(sc, LOCK_HELD);
1415 				can_sleep = 1;
1416 				goto redo_sifflags;
1417 			}
1418 			rc = cxgbe_uninit_synchronized(vi);
1419 		}
1420 		end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1421 		break;
1422 
1423 	case SIOCADDMULTI:
1424 	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1425 		rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi");
1426 		if (rc)
1427 			return (rc);
1428 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1429 			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1430 		end_synchronized_op(sc, LOCK_HELD);
1431 		break;
1432 
1433 	case SIOCSIFCAP:
1434 		rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1435 		if (rc)
1436 			return (rc);
1437 
1438 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1439 		if (mask & IFCAP_TXCSUM) {
1440 			ifp->if_capenable ^= IFCAP_TXCSUM;
1441 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1442 
1443 			if (IFCAP_TSO4 & ifp->if_capenable &&
1444 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1445 				ifp->if_capenable &= ~IFCAP_TSO4;
1446 				if_printf(ifp,
1447 				    "tso4 disabled due to -txcsum.\n");
1448 			}
1449 		}
1450 		if (mask & IFCAP_TXCSUM_IPV6) {
1451 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1452 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1453 
1454 			if (IFCAP_TSO6 & ifp->if_capenable &&
1455 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1456 				ifp->if_capenable &= ~IFCAP_TSO6;
1457 				if_printf(ifp,
1458 				    "tso6 disabled due to -txcsum6.\n");
1459 			}
1460 		}
1461 		if (mask & IFCAP_RXCSUM)
1462 			ifp->if_capenable ^= IFCAP_RXCSUM;
1463 		if (mask & IFCAP_RXCSUM_IPV6)
1464 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1465 
1466 		/*
1467 		 * Note that we leave CSUM_TSO alone (it is always set).  The
1468 		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1469 		 * sending a TSO request our way, so it's sufficient to toggle
1470 		 * IFCAP_TSOx only.
1471 		 */
1472 		if (mask & IFCAP_TSO4) {
1473 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1474 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1475 				if_printf(ifp, "enable txcsum first.\n");
1476 				rc = EAGAIN;
1477 				goto fail;
1478 			}
1479 			ifp->if_capenable ^= IFCAP_TSO4;
1480 		}
1481 		if (mask & IFCAP_TSO6) {
1482 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1483 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1484 				if_printf(ifp, "enable txcsum6 first.\n");
1485 				rc = EAGAIN;
1486 				goto fail;
1487 			}
1488 			ifp->if_capenable ^= IFCAP_TSO6;
1489 		}
1490 		if (mask & IFCAP_LRO) {
1491 #if defined(INET) || defined(INET6)
1492 			int i;
1493 			struct sge_rxq *rxq;
1494 
1495 			ifp->if_capenable ^= IFCAP_LRO;
1496 			for_each_rxq(vi, i, rxq) {
1497 				if (ifp->if_capenable & IFCAP_LRO)
1498 					rxq->iq.flags |= IQ_LRO_ENABLED;
1499 				else
1500 					rxq->iq.flags &= ~IQ_LRO_ENABLED;
1501 			}
1502 #endif
1503 		}
1504 #ifdef TCP_OFFLOAD
1505 		if (mask & IFCAP_TOE) {
1506 			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1507 
1508 			rc = toe_capability(vi, enable);
1509 			if (rc != 0)
1510 				goto fail;
1511 
1512 			ifp->if_capenable ^= mask;
1513 		}
1514 #endif
1515 		if (mask & IFCAP_VLAN_HWTAGGING) {
1516 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1517 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1518 				rc = update_mac_settings(ifp, XGMAC_VLANEX);
1519 		}
1520 		if (mask & IFCAP_VLAN_MTU) {
1521 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
1522 
1523 			/* Need to find out how to disable auto-mtu-inflation */
1524 		}
1525 		if (mask & IFCAP_VLAN_HWTSO)
1526 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1527 		if (mask & IFCAP_VLAN_HWCSUM)
1528 			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1529 
1530 #ifdef VLAN_CAPABILITIES
1531 		VLAN_CAPABILITIES(ifp);
1532 #endif
1533 fail:
1534 		end_synchronized_op(sc, 0);
1535 		break;
1536 
1537 	case SIOCSIFMEDIA:
1538 	case SIOCGIFMEDIA:
1539 		ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
1540 		break;
1541 
1542 	case SIOCGI2C: {
1543 		struct ifi2creq i2c;
1544 
1545 		rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1546 		if (rc != 0)
1547 			break;
1548 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1549 			rc = EPERM;
1550 			break;
1551 		}
1552 		if (i2c.len > sizeof(i2c.data)) {
1553 			rc = EINVAL;
1554 			break;
1555 		}
1556 		rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1557 		if (rc)
1558 			return (rc);
1559 		rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr,
1560 		    i2c.offset, i2c.len, &i2c.data[0]);
1561 		end_synchronized_op(sc, 0);
1562 		if (rc == 0)
1563 			rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1564 		break;
1565 	}
1566 
1567 	default:
1568 		rc = ether_ioctl(ifp, cmd, data);
1569 	}
1570 
1571 	return (rc);
1572 }
1573 
1574 static int
1575 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1576 {
1577 	struct vi_info *vi = ifp->if_softc;
1578 	struct port_info *pi = vi->pi;
1579 	struct adapter *sc = pi->adapter;
1580 	struct sge_txq *txq;
1581 	void *items[1];
1582 	int rc;
1583 
1584 	M_ASSERTPKTHDR(m);
1585 	MPASS(m->m_nextpkt == NULL);	/* not quite ready for this yet */
1586 
1587 	if (__predict_false(pi->link_cfg.link_ok == 0)) {
1588 		m_freem(m);
1589 		return (ENETDOWN);
1590 	}
1591 
1592 	rc = parse_pkt(&m);
1593 	if (__predict_false(rc != 0)) {
1594 		MPASS(m == NULL);			/* was freed already */
1595 		atomic_add_int(&pi->tx_parse_error, 1);	/* rare, atomic is ok */
1596 		return (rc);
1597 	}
1598 
1599 	/* Select a txq. */
1600 	txq = &sc->sge.txq[vi->first_txq];
1601 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1602 		txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1603 		    vi->rsrv_noflowq);
1604 
1605 	items[0] = m;
1606 	rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1607 	if (__predict_false(rc != 0))
1608 		m_freem(m);
1609 
1610 	return (rc);
1611 }
1612 
1613 static void
1614 cxgbe_qflush(struct ifnet *ifp)
1615 {
1616 	struct vi_info *vi = ifp->if_softc;
1617 	struct sge_txq *txq;
1618 	int i;
1619 
1620 	/* queues do not exist if !VI_INIT_DONE. */
1621 	if (vi->flags & VI_INIT_DONE) {
1622 		for_each_txq(vi, i, txq) {
1623 			TXQ_LOCK(txq);
1624 			txq->eq.flags &= ~EQ_ENABLED;
1625 			TXQ_UNLOCK(txq);
1626 			while (!mp_ring_is_idle(txq->r)) {
1627 				mp_ring_check_drainage(txq->r, 0);
1628 				pause("qflush", 1);
1629 			}
1630 		}
1631 	}
1632 	if_qflush(ifp);
1633 }
1634 
1635 static uint64_t
1636 vi_get_counter(struct ifnet *ifp, ift_counter c)
1637 {
1638 	struct vi_info *vi = ifp->if_softc;
1639 	struct fw_vi_stats_vf *s = &vi->stats;
1640 
1641 	vi_refresh_stats(vi->pi->adapter, vi);
1642 
1643 	switch (c) {
1644 	case IFCOUNTER_IPACKETS:
1645 		return (s->rx_bcast_frames + s->rx_mcast_frames +
1646 		    s->rx_ucast_frames);
1647 	case IFCOUNTER_IERRORS:
1648 		return (s->rx_err_frames);
1649 	case IFCOUNTER_OPACKETS:
1650 		return (s->tx_bcast_frames + s->tx_mcast_frames +
1651 		    s->tx_ucast_frames + s->tx_offload_frames);
1652 	case IFCOUNTER_OERRORS:
1653 		return (s->tx_drop_frames);
1654 	case IFCOUNTER_IBYTES:
1655 		return (s->rx_bcast_bytes + s->rx_mcast_bytes +
1656 		    s->rx_ucast_bytes);
1657 	case IFCOUNTER_OBYTES:
1658 		return (s->tx_bcast_bytes + s->tx_mcast_bytes +
1659 		    s->tx_ucast_bytes + s->tx_offload_bytes);
1660 	case IFCOUNTER_IMCASTS:
1661 		return (s->rx_mcast_frames);
1662 	case IFCOUNTER_OMCASTS:
1663 		return (s->tx_mcast_frames);
1664 	case IFCOUNTER_OQDROPS: {
1665 		uint64_t drops;
1666 
1667 		drops = 0;
1668 		if ((vi->flags & (VI_INIT_DONE | VI_NETMAP)) == VI_INIT_DONE) {
1669 			int i;
1670 			struct sge_txq *txq;
1671 
1672 			for_each_txq(vi, i, txq)
1673 				drops += counter_u64_fetch(txq->r->drops);
1674 		}
1675 
1676 		return (drops);
1677 
1678 	}
1679 
1680 	default:
1681 		return (if_get_counter_default(ifp, c));
1682 	}
1683 }
1684 
1685 uint64_t
1686 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1687 {
1688 	struct vi_info *vi = ifp->if_softc;
1689 	struct port_info *pi = vi->pi;
1690 	struct adapter *sc = pi->adapter;
1691 	struct port_stats *s = &pi->stats;
1692 
1693 	if (pi->nvi > 1)
1694 		return (vi_get_counter(ifp, c));
1695 
1696 	cxgbe_refresh_stats(sc, pi);
1697 
1698 	switch (c) {
1699 	case IFCOUNTER_IPACKETS:
1700 		return (s->rx_frames - s->rx_pause);
1701 
1702 	case IFCOUNTER_IERRORS:
1703 		return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1704 		    s->rx_fcs_err + s->rx_len_err);
1705 
1706 	case IFCOUNTER_OPACKETS:
1707 		return (s->tx_frames - s->tx_pause);
1708 
1709 	case IFCOUNTER_OERRORS:
1710 		return (s->tx_error_frames);
1711 
1712 	case IFCOUNTER_IBYTES:
1713 		return (s->rx_octets - s->rx_pause * 64);
1714 
1715 	case IFCOUNTER_OBYTES:
1716 		return (s->tx_octets - s->tx_pause * 64);
1717 
1718 	case IFCOUNTER_IMCASTS:
1719 		return (s->rx_mcast_frames - s->rx_pause);
1720 
1721 	case IFCOUNTER_OMCASTS:
1722 		return (s->tx_mcast_frames - s->tx_pause);
1723 
1724 	case IFCOUNTER_IQDROPS:
1725 		return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1726 		    s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1727 		    s->rx_trunc3 + pi->tnl_cong_drops);
1728 
1729 	case IFCOUNTER_OQDROPS: {
1730 		uint64_t drops;
1731 
1732 		drops = s->tx_drop;
1733 		if (vi->flags & VI_INIT_DONE) {
1734 			int i;
1735 			struct sge_txq *txq;
1736 
1737 			for_each_txq(vi, i, txq)
1738 				drops += counter_u64_fetch(txq->r->drops);
1739 		}
1740 
1741 		return (drops);
1742 
1743 	}
1744 
1745 	default:
1746 		return (if_get_counter_default(ifp, c));
1747 	}
1748 }
1749 
1750 static int
1751 cxgbe_media_change(struct ifnet *ifp)
1752 {
1753 	struct vi_info *vi = ifp->if_softc;
1754 
1755 	device_printf(vi->dev, "%s unimplemented.\n", __func__);
1756 
1757 	return (EOPNOTSUPP);
1758 }
1759 
1760 static void
1761 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1762 {
1763 	struct vi_info *vi = ifp->if_softc;
1764 	struct port_info *pi = vi->pi;
1765 	struct ifmedia_entry *cur;
1766 	int speed = pi->link_cfg.speed;
1767 
1768 	cur = vi->media.ifm_cur;
1769 
1770 	ifmr->ifm_status = IFM_AVALID;
1771 	if (!pi->link_cfg.link_ok)
1772 		return;
1773 
1774 	ifmr->ifm_status |= IFM_ACTIVE;
1775 
1776 	/* active and current will differ iff current media is autoselect. */
1777 	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1778 		return;
1779 
1780 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1781 	if (speed == SPEED_10000)
1782 		ifmr->ifm_active |= IFM_10G_T;
1783 	else if (speed == SPEED_1000)
1784 		ifmr->ifm_active |= IFM_1000_T;
1785 	else if (speed == SPEED_100)
1786 		ifmr->ifm_active |= IFM_100_TX;
1787 	else if (speed == SPEED_10)
1788 		ifmr->ifm_active |= IFM_10_T;
1789 	else
1790 		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1791 			    speed));
1792 }
1793 
1794 static int
1795 vcxgbe_probe(device_t dev)
1796 {
1797 	char buf[128];
1798 	struct vi_info *vi = device_get_softc(dev);
1799 
1800 	snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
1801 	    vi - vi->pi->vi);
1802 	device_set_desc_copy(dev, buf);
1803 
1804 	return (BUS_PROBE_DEFAULT);
1805 }
1806 
1807 static int
1808 vcxgbe_attach(device_t dev)
1809 {
1810 	struct vi_info *vi;
1811 	struct port_info *pi;
1812 	struct adapter *sc;
1813 	int func, index, rc;
1814 	u32 param, val;
1815 
1816 	vi = device_get_softc(dev);
1817 	pi = vi->pi;
1818 	sc = pi->adapter;
1819 
1820 	index = vi - pi->vi;
1821 	KASSERT(index < nitems(vi_mac_funcs),
1822 	    ("%s: VI %s doesn't have a MAC func", __func__,
1823 	    device_get_nameunit(dev)));
1824 	func = vi_mac_funcs[index];
1825 	rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
1826 	    vi->hw_addr, &vi->rss_size, func, 0);
1827 	if (rc < 0) {
1828 		device_printf(dev, "Failed to allocate virtual interface "
1829 		    "for port %d: %d\n", pi->port_id, -rc);
1830 		return (-rc);
1831 	}
1832 	vi->viid = rc;
1833 
1834 	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
1835 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
1836 	    V_FW_PARAMS_PARAM_YZ(vi->viid);
1837 	rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1838 	if (rc)
1839 		vi->rss_base = 0xffff;
1840 	else {
1841 		/* MPASS((val >> 16) == rss_size); */
1842 		vi->rss_base = val & 0xffff;
1843 	}
1844 
1845 	rc = cxgbe_vi_attach(dev, vi);
1846 	if (rc) {
1847 		t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1848 		return (rc);
1849 	}
1850 	return (0);
1851 }
1852 
1853 static int
1854 vcxgbe_detach(device_t dev)
1855 {
1856 	struct vi_info *vi;
1857 	struct adapter *sc;
1858 
1859 	vi = device_get_softc(dev);
1860 	sc = vi->pi->adapter;
1861 
1862 	doom_vi(sc, vi);
1863 
1864 	cxgbe_vi_detach(vi);
1865 	t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
1866 
1867 	end_synchronized_op(sc, 0);
1868 
1869 	return (0);
1870 }
1871 
1872 void
1873 t4_fatal_err(struct adapter *sc)
1874 {
1875 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1876 	t4_intr_disable(sc);
1877 	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1878 	    device_get_nameunit(sc->dev));
1879 }
1880 
1881 static int
1882 map_bars_0_and_4(struct adapter *sc)
1883 {
1884 	sc->regs_rid = PCIR_BAR(0);
1885 	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1886 	    &sc->regs_rid, RF_ACTIVE);
1887 	if (sc->regs_res == NULL) {
1888 		device_printf(sc->dev, "cannot map registers.\n");
1889 		return (ENXIO);
1890 	}
1891 	sc->bt = rman_get_bustag(sc->regs_res);
1892 	sc->bh = rman_get_bushandle(sc->regs_res);
1893 	sc->mmio_len = rman_get_size(sc->regs_res);
1894 	setbit(&sc->doorbells, DOORBELL_KDB);
1895 
1896 	sc->msix_rid = PCIR_BAR(4);
1897 	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1898 	    &sc->msix_rid, RF_ACTIVE);
1899 	if (sc->msix_res == NULL) {
1900 		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1901 		return (ENXIO);
1902 	}
1903 
1904 	return (0);
1905 }
1906 
1907 static int
1908 map_bar_2(struct adapter *sc)
1909 {
1910 
1911 	/*
1912 	 * T4: only iWARP driver uses the userspace doorbells.  There is no need
1913 	 * to map it if RDMA is disabled.
1914 	 */
1915 	if (is_t4(sc) && sc->rdmacaps == 0)
1916 		return (0);
1917 
1918 	sc->udbs_rid = PCIR_BAR(2);
1919 	sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1920 	    &sc->udbs_rid, RF_ACTIVE);
1921 	if (sc->udbs_res == NULL) {
1922 		device_printf(sc->dev, "cannot map doorbell BAR.\n");
1923 		return (ENXIO);
1924 	}
1925 	sc->udbs_base = rman_get_virtual(sc->udbs_res);
1926 
1927 	if (is_t5(sc)) {
1928 		setbit(&sc->doorbells, DOORBELL_UDB);
1929 #if defined(__i386__) || defined(__amd64__)
1930 		if (t5_write_combine) {
1931 			int rc;
1932 
1933 			/*
1934 			 * Enable write combining on BAR2.  This is the
1935 			 * userspace doorbell BAR and is split into 128B
1936 			 * (UDBS_SEG_SIZE) doorbell regions, each associated
1937 			 * with an egress queue.  The first 64B has the doorbell
1938 			 * and the second 64B can be used to submit a tx work
1939 			 * request with an implicit doorbell.
1940 			 */
1941 
1942 			rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1943 			    rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1944 			if (rc == 0) {
1945 				clrbit(&sc->doorbells, DOORBELL_UDB);
1946 				setbit(&sc->doorbells, DOORBELL_WCWR);
1947 				setbit(&sc->doorbells, DOORBELL_UDBWC);
1948 			} else {
1949 				device_printf(sc->dev,
1950 				    "couldn't enable write combining: %d\n",
1951 				    rc);
1952 			}
1953 
1954 			t4_write_reg(sc, A_SGE_STAT_CFG,
1955 			    V_STATSOURCE_T5(7) | V_STATMODE(0));
1956 		}
1957 #endif
1958 	}
1959 
1960 	return (0);
1961 }
1962 
1963 static const struct memwin t4_memwin[] = {
1964 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1965 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1966 	{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1967 };
1968 
1969 static const struct memwin t5_memwin[] = {
1970 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1971 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1972 	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1973 };
1974 
1975 static void
1976 setup_memwin(struct adapter *sc)
1977 {
1978 	const struct memwin *mw;
1979 	int i, n;
1980 	uint32_t bar0;
1981 
1982 	if (is_t4(sc)) {
1983 		/*
1984 		 * Read low 32b of bar0 indirectly via the hardware backdoor
1985 		 * mechanism.  Works from within PCI passthrough environments
1986 		 * too, where rman_get_start() can return a different value.  We
1987 		 * need to program the T4 memory window decoders with the actual
1988 		 * addresses that will be coming across the PCIe link.
1989 		 */
1990 		bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1991 		bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1992 
1993 		mw = &t4_memwin[0];
1994 		n = nitems(t4_memwin);
1995 	} else {
1996 		/* T5 uses the relative offset inside the PCIe BAR */
1997 		bar0 = 0;
1998 
1999 		mw = &t5_memwin[0];
2000 		n = nitems(t5_memwin);
2001 	}
2002 
2003 	for (i = 0; i < n; i++, mw++) {
2004 		t4_write_reg(sc,
2005 		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
2006 		    (mw->base + bar0) | V_BIR(0) |
2007 		    V_WINDOW(ilog2(mw->aperture) - 10));
2008 	}
2009 
2010 	/* flush */
2011 	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
2012 }
2013 
2014 /*
2015  * Verify that the memory range specified by the addr/len pair is valid and lies
2016  * entirely within a single region (EDCx or MCx).
2017  */
2018 static int
2019 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
2020 {
2021 	uint32_t em, addr_len, maddr, mlen;
2022 
2023 	/* Memory can only be accessed in naturally aligned 4 byte units */
2024 	if (addr & 3 || len & 3 || len == 0)
2025 		return (EINVAL);
2026 
2027 	/* Enabled memories */
2028 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2029 	if (em & F_EDRAM0_ENABLE) {
2030 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2031 		maddr = G_EDRAM0_BASE(addr_len) << 20;
2032 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
2033 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2034 		    addr + len <= maddr + mlen)
2035 			return (0);
2036 	}
2037 	if (em & F_EDRAM1_ENABLE) {
2038 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2039 		maddr = G_EDRAM1_BASE(addr_len) << 20;
2040 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
2041 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2042 		    addr + len <= maddr + mlen)
2043 			return (0);
2044 	}
2045 	if (em & F_EXT_MEM_ENABLE) {
2046 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2047 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
2048 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
2049 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2050 		    addr + len <= maddr + mlen)
2051 			return (0);
2052 	}
2053 	if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
2054 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2055 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2056 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
2057 		if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
2058 		    addr + len <= maddr + mlen)
2059 			return (0);
2060 	}
2061 
2062 	return (EFAULT);
2063 }
2064 
2065 static int
2066 fwmtype_to_hwmtype(int mtype)
2067 {
2068 
2069 	switch (mtype) {
2070 	case FW_MEMTYPE_EDC0:
2071 		return (MEM_EDC0);
2072 	case FW_MEMTYPE_EDC1:
2073 		return (MEM_EDC1);
2074 	case FW_MEMTYPE_EXTMEM:
2075 		return (MEM_MC0);
2076 	case FW_MEMTYPE_EXTMEM1:
2077 		return (MEM_MC1);
2078 	default:
2079 		panic("%s: cannot translate fw mtype %d.", __func__, mtype);
2080 	}
2081 }
2082 
2083 /*
2084  * Verify that the memory range specified by the memtype/offset/len pair is
2085  * valid and lies entirely within the memtype specified.  The global address of
2086  * the start of the range is returned in addr.
2087  */
2088 static int
2089 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
2090     uint32_t *addr)
2091 {
2092 	uint32_t em, addr_len, maddr, mlen;
2093 
2094 	/* Memory can only be accessed in naturally aligned 4 byte units */
2095 	if (off & 3 || len & 3 || len == 0)
2096 		return (EINVAL);
2097 
2098 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2099 	switch (fwmtype_to_hwmtype(mtype)) {
2100 	case MEM_EDC0:
2101 		if (!(em & F_EDRAM0_ENABLE))
2102 			return (EINVAL);
2103 		addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2104 		maddr = G_EDRAM0_BASE(addr_len) << 20;
2105 		mlen = G_EDRAM0_SIZE(addr_len) << 20;
2106 		break;
2107 	case MEM_EDC1:
2108 		if (!(em & F_EDRAM1_ENABLE))
2109 			return (EINVAL);
2110 		addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2111 		maddr = G_EDRAM1_BASE(addr_len) << 20;
2112 		mlen = G_EDRAM1_SIZE(addr_len) << 20;
2113 		break;
2114 	case MEM_MC:
2115 		if (!(em & F_EXT_MEM_ENABLE))
2116 			return (EINVAL);
2117 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2118 		maddr = G_EXT_MEM_BASE(addr_len) << 20;
2119 		mlen = G_EXT_MEM_SIZE(addr_len) << 20;
2120 		break;
2121 	case MEM_MC1:
2122 		if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
2123 			return (EINVAL);
2124 		addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2125 		maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2126 		mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
2127 		break;
2128 	default:
2129 		return (EINVAL);
2130 	}
2131 
2132 	if (mlen > 0 && off < mlen && off + len <= mlen) {
2133 		*addr = maddr + off;	/* global address */
2134 		return (0);
2135 	}
2136 
2137 	return (EFAULT);
2138 }
2139 
2140 static void
2141 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
2142 {
2143 	const struct memwin *mw;
2144 
2145 	if (is_t4(sc)) {
2146 		KASSERT(win >= 0 && win < nitems(t4_memwin),
2147 		    ("%s: incorrect memwin# (%d)", __func__, win));
2148 		mw = &t4_memwin[win];
2149 	} else {
2150 		KASSERT(win >= 0 && win < nitems(t5_memwin),
2151 		    ("%s: incorrect memwin# (%d)", __func__, win));
2152 		mw = &t5_memwin[win];
2153 	}
2154 
2155 	if (base != NULL)
2156 		*base = mw->base;
2157 	if (aperture != NULL)
2158 		*aperture = mw->aperture;
2159 }
2160 
2161 /*
2162  * Positions the memory window such that it can be used to access the specified
2163  * address in the chip's address space.  The return value is the offset of addr
2164  * from the start of the window.
2165  */
2166 static uint32_t
2167 position_memwin(struct adapter *sc, int n, uint32_t addr)
2168 {
2169 	uint32_t start, pf;
2170 	uint32_t reg;
2171 
2172 	KASSERT(n >= 0 && n <= 3,
2173 	    ("%s: invalid window %d.", __func__, n));
2174 	KASSERT((addr & 3) == 0,
2175 	    ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
2176 
2177 	if (is_t4(sc)) {
2178 		pf = 0;
2179 		start = addr & ~0xf;	/* start must be 16B aligned */
2180 	} else {
2181 		pf = V_PFNUM(sc->pf);
2182 		start = addr & ~0x7f;	/* start must be 128B aligned */
2183 	}
2184 	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
2185 
2186 	t4_write_reg(sc, reg, start | pf);
2187 	t4_read_reg(sc, reg);
2188 
2189 	return (addr - start);
2190 }
2191 
2192 static int
2193 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis,
2194     struct intrs_and_queues *iaq)
2195 {
2196 	int rc, itype, navail, nrxq10g, nrxq1g, n;
2197 	int nofldrxq10g = 0, nofldrxq1g = 0;
2198 	int nnmrxq10g = 0, nnmrxq1g = 0;
2199 
2200 	bzero(iaq, sizeof(*iaq));
2201 
2202 	iaq->ntxq10g = t4_ntxq10g;
2203 	iaq->ntxq1g = t4_ntxq1g;
2204 	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
2205 	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
2206 	iaq->rsrv_noflowq = t4_rsrv_noflowq;
2207 #ifdef TCP_OFFLOAD
2208 	if (is_offload(sc)) {
2209 		iaq->nofldtxq10g = t4_nofldtxq10g;
2210 		iaq->nofldtxq1g = t4_nofldtxq1g;
2211 		iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
2212 		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
2213 	}
2214 #endif
2215 #ifdef DEV_NETMAP
2216 	iaq->nnmtxq10g = t4_nnmtxq10g;
2217 	iaq->nnmtxq1g = t4_nnmtxq1g;
2218 	iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
2219 	iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
2220 #endif
2221 
2222 	for (itype = INTR_MSIX; itype; itype >>= 1) {
2223 
2224 		if ((itype & t4_intr_types) == 0)
2225 			continue;	/* not allowed */
2226 
2227 		if (itype == INTR_MSIX)
2228 			navail = pci_msix_count(sc->dev);
2229 		else if (itype == INTR_MSI)
2230 			navail = pci_msi_count(sc->dev);
2231 		else
2232 			navail = 1;
2233 restart:
2234 		if (navail == 0)
2235 			continue;
2236 
2237 		iaq->intr_type = itype;
2238 		iaq->intr_flags_10g = 0;
2239 		iaq->intr_flags_1g = 0;
2240 
2241 		/*
2242 		 * Best option: an interrupt vector for errors, one for the
2243 		 * firmware event queue, and one for every rxq (NIC, TOE, and
2244 		 * netmap).
2245 		 */
2246 		iaq->nirq = T4_EXTRA_INTR;
2247 		iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
2248 		iaq->nirq += n10g * 2 * (num_vis - 1);
2249 		iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
2250 		iaq->nirq += n1g * 2 * (num_vis - 1);
2251 		if (iaq->nirq <= navail &&
2252 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
2253 			iaq->intr_flags_10g = INTR_ALL;
2254 			iaq->intr_flags_1g = INTR_ALL;
2255 			goto allocate;
2256 		}
2257 
2258 		/*
2259 		 * Second best option: a vector for errors, one for the firmware
2260 		 * event queue, and vectors for either all the NIC rx queues or
2261 		 * all the TOE rx queues.  The queues that don't get vectors
2262 		 * will forward their interrupts to those that do.
2263 		 *
2264 		 * Note: netmap rx queues cannot be created early and so they
2265 		 * can't be setup to receive forwarded interrupts for others.
2266 		 */
2267 		iaq->nirq = T4_EXTRA_INTR;
2268 		if (nrxq10g >= nofldrxq10g) {
2269 			iaq->intr_flags_10g = INTR_RXQ;
2270 			iaq->nirq += n10g * nrxq10g;
2271 			iaq->nirq += n10g * (num_vis - 1);
2272 #ifdef DEV_NETMAP
2273 			iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2274 #endif
2275 		} else {
2276 			iaq->intr_flags_10g = INTR_OFLD_RXQ;
2277 			iaq->nirq += n10g * nofldrxq10g;
2278 #ifdef DEV_NETMAP
2279 			iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2280 #endif
2281 		}
2282 		if (nrxq1g >= nofldrxq1g) {
2283 			iaq->intr_flags_1g = INTR_RXQ;
2284 			iaq->nirq += n1g * nrxq1g;
2285 			iaq->nirq += n1g * (num_vis - 1);
2286 #ifdef DEV_NETMAP
2287 			iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2288 #endif
2289 		} else {
2290 			iaq->intr_flags_1g = INTR_OFLD_RXQ;
2291 			iaq->nirq += n1g * nofldrxq1g;
2292 #ifdef DEV_NETMAP
2293 			iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2294 #endif
2295 		}
2296 		if (iaq->nirq <= navail &&
2297 		    (itype != INTR_MSI || powerof2(iaq->nirq)))
2298 			goto allocate;
2299 
2300 		/*
2301 		 * Next best option: an interrupt vector for errors, one for the
2302 		 * firmware event queue, and at least one per VI.  At this
2303 		 * point we know we'll have to downsize nrxq and/or nofldrxq
2304 		 * and/or nnmrxq to fit what's available to us.
2305 		 */
2306 		iaq->nirq = T4_EXTRA_INTR;
2307 		iaq->nirq += (n10g + n1g) * num_vis;
2308 		if (iaq->nirq <= navail) {
2309 			int leftover = navail - iaq->nirq;
2310 
2311 			if (n10g > 0) {
2312 				int target = max(nrxq10g, nofldrxq10g);
2313 
2314 				iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2315 				    INTR_RXQ : INTR_OFLD_RXQ;
2316 
2317 				n = 1;
2318 				while (n < target && leftover >= n10g) {
2319 					leftover -= n10g;
2320 					iaq->nirq += n10g;
2321 					n++;
2322 				}
2323 				iaq->nrxq10g = min(n, nrxq10g);
2324 #ifdef TCP_OFFLOAD
2325 				iaq->nofldrxq10g = min(n, nofldrxq10g);
2326 #endif
2327 #ifdef DEV_NETMAP
2328 				iaq->nnmrxq10g = min(n, nnmrxq10g);
2329 #endif
2330 			}
2331 
2332 			if (n1g > 0) {
2333 				int target = max(nrxq1g, nofldrxq1g);
2334 
2335 				iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2336 				    INTR_RXQ : INTR_OFLD_RXQ;
2337 
2338 				n = 1;
2339 				while (n < target && leftover >= n1g) {
2340 					leftover -= n1g;
2341 					iaq->nirq += n1g;
2342 					n++;
2343 				}
2344 				iaq->nrxq1g = min(n, nrxq1g);
2345 #ifdef TCP_OFFLOAD
2346 				iaq->nofldrxq1g = min(n, nofldrxq1g);
2347 #endif
2348 #ifdef DEV_NETMAP
2349 				iaq->nnmrxq1g = min(n, nnmrxq1g);
2350 #endif
2351 			}
2352 
2353 			if (itype != INTR_MSI || powerof2(iaq->nirq))
2354 				goto allocate;
2355 		}
2356 
2357 		/*
2358 		 * Least desirable option: one interrupt vector for everything.
2359 		 */
2360 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2361 		iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2362 #ifdef TCP_OFFLOAD
2363 		if (is_offload(sc))
2364 			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2365 #endif
2366 #ifdef DEV_NETMAP
2367 		iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2368 #endif
2369 
2370 allocate:
2371 		navail = iaq->nirq;
2372 		rc = 0;
2373 		if (itype == INTR_MSIX)
2374 			rc = pci_alloc_msix(sc->dev, &navail);
2375 		else if (itype == INTR_MSI)
2376 			rc = pci_alloc_msi(sc->dev, &navail);
2377 
2378 		if (rc == 0) {
2379 			if (navail == iaq->nirq)
2380 				return (0);
2381 
2382 			/*
2383 			 * Didn't get the number requested.  Use whatever number
2384 			 * the kernel is willing to allocate (it's in navail).
2385 			 */
2386 			device_printf(sc->dev, "fewer vectors than requested, "
2387 			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2388 			    itype, iaq->nirq, navail);
2389 			pci_release_msi(sc->dev);
2390 			goto restart;
2391 		}
2392 
2393 		device_printf(sc->dev,
2394 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2395 		    itype, rc, iaq->nirq, navail);
2396 	}
2397 
2398 	device_printf(sc->dev,
2399 	    "failed to find a usable interrupt type.  "
2400 	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2401 	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2402 
2403 	return (ENXIO);
2404 }
2405 
2406 #define FW_VERSION(chip) ( \
2407     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2408     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2409     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2410     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2411 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2412 
2413 struct fw_info {
2414 	uint8_t chip;
2415 	char *kld_name;
2416 	char *fw_mod_name;
2417 	struct fw_hdr fw_hdr;	/* XXX: waste of space, need a sparse struct */
2418 } fw_info[] = {
2419 	{
2420 		.chip = CHELSIO_T4,
2421 		.kld_name = "t4fw_cfg",
2422 		.fw_mod_name = "t4fw",
2423 		.fw_hdr = {
2424 			.chip = FW_HDR_CHIP_T4,
2425 			.fw_ver = htobe32_const(FW_VERSION(T4)),
2426 			.intfver_nic = FW_INTFVER(T4, NIC),
2427 			.intfver_vnic = FW_INTFVER(T4, VNIC),
2428 			.intfver_ofld = FW_INTFVER(T4, OFLD),
2429 			.intfver_ri = FW_INTFVER(T4, RI),
2430 			.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2431 			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
2432 			.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2433 			.intfver_fcoe = FW_INTFVER(T4, FCOE),
2434 		},
2435 	}, {
2436 		.chip = CHELSIO_T5,
2437 		.kld_name = "t5fw_cfg",
2438 		.fw_mod_name = "t5fw",
2439 		.fw_hdr = {
2440 			.chip = FW_HDR_CHIP_T5,
2441 			.fw_ver = htobe32_const(FW_VERSION(T5)),
2442 			.intfver_nic = FW_INTFVER(T5, NIC),
2443 			.intfver_vnic = FW_INTFVER(T5, VNIC),
2444 			.intfver_ofld = FW_INTFVER(T5, OFLD),
2445 			.intfver_ri = FW_INTFVER(T5, RI),
2446 			.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2447 			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
2448 			.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2449 			.intfver_fcoe = FW_INTFVER(T5, FCOE),
2450 		},
2451 	}
2452 };
2453 
2454 static struct fw_info *
2455 find_fw_info(int chip)
2456 {
2457 	int i;
2458 
2459 	for (i = 0; i < nitems(fw_info); i++) {
2460 		if (fw_info[i].chip == chip)
2461 			return (&fw_info[i]);
2462 	}
2463 	return (NULL);
2464 }
2465 
2466 /*
2467  * Is the given firmware API compatible with the one the driver was compiled
2468  * with?
2469  */
2470 static int
2471 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2472 {
2473 
2474 	/* short circuit if it's the exact same firmware version */
2475 	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2476 		return (1);
2477 
2478 	/*
2479 	 * XXX: Is this too conservative?  Perhaps I should limit this to the
2480 	 * features that are supported in the driver.
2481 	 */
2482 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2483 	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2484 	    SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2485 	    SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2486 		return (1);
2487 #undef SAME_INTF
2488 
2489 	return (0);
2490 }
2491 
2492 /*
2493  * The firmware in the KLD is usable, but should it be installed?  This routine
2494  * explains itself in detail if it indicates the KLD firmware should be
2495  * installed.
2496  */
2497 static int
2498 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2499 {
2500 	const char *reason;
2501 
2502 	if (!card_fw_usable) {
2503 		reason = "incompatible or unusable";
2504 		goto install;
2505 	}
2506 
2507 	if (k > c) {
2508 		reason = "older than the version bundled with this driver";
2509 		goto install;
2510 	}
2511 
2512 	if (t4_fw_install == 2 && k != c) {
2513 		reason = "different than the version bundled with this driver";
2514 		goto install;
2515 	}
2516 
2517 	return (0);
2518 
2519 install:
2520 	if (t4_fw_install == 0) {
2521 		device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2522 		    "but the driver is prohibited from installing a different "
2523 		    "firmware on the card.\n",
2524 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2525 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2526 
2527 		return (0);
2528 	}
2529 
2530 	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2531 	    "installing firmware %u.%u.%u.%u on card.\n",
2532 	    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2533 	    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2534 	    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2535 	    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2536 
2537 	return (1);
2538 }
2539 /*
2540  * Establish contact with the firmware and determine if we are the master driver
2541  * or not, and whether we are responsible for chip initialization.
2542  */
2543 static int
2544 prep_firmware(struct adapter *sc)
2545 {
2546 	const struct firmware *fw = NULL, *default_cfg;
2547 	int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2548 	enum dev_state state;
2549 	struct fw_info *fw_info;
2550 	struct fw_hdr *card_fw;		/* fw on the card */
2551 	const struct fw_hdr *kld_fw;	/* fw in the KLD */
2552 	const struct fw_hdr *drv_fw;	/* fw header the driver was compiled
2553 					   against */
2554 
2555 	/* Contact firmware. */
2556 	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2557 	if (rc < 0 || state == DEV_STATE_ERR) {
2558 		rc = -rc;
2559 		device_printf(sc->dev,
2560 		    "failed to connect to the firmware: %d, %d.\n", rc, state);
2561 		return (rc);
2562 	}
2563 	pf = rc;
2564 	if (pf == sc->mbox)
2565 		sc->flags |= MASTER_PF;
2566 	else if (state == DEV_STATE_UNINIT) {
2567 		/*
2568 		 * We didn't get to be the master so we definitely won't be
2569 		 * configuring the chip.  It's a bug if someone else hasn't
2570 		 * configured it already.
2571 		 */
2572 		device_printf(sc->dev, "couldn't be master(%d), "
2573 		    "device not already initialized either(%d).\n", rc, state);
2574 		return (EDOOFUS);
2575 	}
2576 
2577 	/* This is the firmware whose headers the driver was compiled against */
2578 	fw_info = find_fw_info(chip_id(sc));
2579 	if (fw_info == NULL) {
2580 		device_printf(sc->dev,
2581 		    "unable to look up firmware information for chip %d.\n",
2582 		    chip_id(sc));
2583 		return (EINVAL);
2584 	}
2585 	drv_fw = &fw_info->fw_hdr;
2586 
2587 	/*
2588 	 * The firmware KLD contains many modules.  The KLD name is also the
2589 	 * name of the module that contains the default config file.
2590 	 */
2591 	default_cfg = firmware_get(fw_info->kld_name);
2592 
2593 	/* Read the header of the firmware on the card */
2594 	card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2595 	rc = -t4_read_flash(sc, FLASH_FW_START,
2596 	    sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2597 	if (rc == 0)
2598 		card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2599 	else {
2600 		device_printf(sc->dev,
2601 		    "Unable to read card's firmware header: %d\n", rc);
2602 		card_fw_usable = 0;
2603 	}
2604 
2605 	/* This is the firmware in the KLD */
2606 	fw = firmware_get(fw_info->fw_mod_name);
2607 	if (fw != NULL) {
2608 		kld_fw = (const void *)fw->data;
2609 		kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2610 	} else {
2611 		kld_fw = NULL;
2612 		kld_fw_usable = 0;
2613 	}
2614 
2615 	if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2616 	    (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2617 		/*
2618 		 * Common case: the firmware on the card is an exact match and
2619 		 * the KLD is an exact match too, or the KLD is
2620 		 * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2621 		 * here -- use cxgbetool loadfw if you want to reinstall the
2622 		 * same firmware as the one on the card.
2623 		 */
2624 	} else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2625 	    should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2626 	    be32toh(card_fw->fw_ver))) {
2627 
2628 		rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2629 		if (rc != 0) {
2630 			device_printf(sc->dev,
2631 			    "failed to install firmware: %d\n", rc);
2632 			goto done;
2633 		}
2634 
2635 		/* Installed successfully, update the cached header too. */
2636 		memcpy(card_fw, kld_fw, sizeof(*card_fw));
2637 		card_fw_usable = 1;
2638 		need_fw_reset = 0;	/* already reset as part of load_fw */
2639 	}
2640 
2641 	if (!card_fw_usable) {
2642 		uint32_t d, c, k;
2643 
2644 		d = ntohl(drv_fw->fw_ver);
2645 		c = ntohl(card_fw->fw_ver);
2646 		k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2647 
2648 		device_printf(sc->dev, "Cannot find a usable firmware: "
2649 		    "fw_install %d, chip state %d, "
2650 		    "driver compiled with %d.%d.%d.%d, "
2651 		    "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2652 		    t4_fw_install, state,
2653 		    G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2654 		    G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2655 		    G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2656 		    G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2657 		    G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2658 		    G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2659 		rc = EINVAL;
2660 		goto done;
2661 	}
2662 
2663 	/* We're using whatever's on the card and it's known to be good. */
2664 	sc->params.fw_vers = ntohl(card_fw->fw_ver);
2665 	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2666 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2667 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2668 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2669 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2670 	t4_get_tp_version(sc, &sc->params.tp_vers);
2671 
2672 	/* Reset device */
2673 	if (need_fw_reset &&
2674 	    (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2675 		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2676 		if (rc != ETIMEDOUT && rc != EIO)
2677 			t4_fw_bye(sc, sc->mbox);
2678 		goto done;
2679 	}
2680 	sc->flags |= FW_OK;
2681 
2682 	rc = get_params__pre_init(sc);
2683 	if (rc != 0)
2684 		goto done; /* error message displayed already */
2685 
2686 	/* Partition adapter resources as specified in the config file. */
2687 	if (state == DEV_STATE_UNINIT) {
2688 
2689 		KASSERT(sc->flags & MASTER_PF,
2690 		    ("%s: trying to change chip settings when not master.",
2691 		    __func__));
2692 
2693 		rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2694 		if (rc != 0)
2695 			goto done;	/* error message displayed already */
2696 
2697 		t4_tweak_chip_settings(sc);
2698 
2699 		/* get basic stuff going */
2700 		rc = -t4_fw_initialize(sc, sc->mbox);
2701 		if (rc != 0) {
2702 			device_printf(sc->dev, "fw init failed: %d.\n", rc);
2703 			goto done;
2704 		}
2705 	} else {
2706 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2707 		sc->cfcsum = 0;
2708 	}
2709 
2710 done:
2711 	free(card_fw, M_CXGBE);
2712 	if (fw != NULL)
2713 		firmware_put(fw, FIRMWARE_UNLOAD);
2714 	if (default_cfg != NULL)
2715 		firmware_put(default_cfg, FIRMWARE_UNLOAD);
2716 
2717 	return (rc);
2718 }
2719 
2720 #define FW_PARAM_DEV(param) \
2721 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2722 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2723 #define FW_PARAM_PFVF(param) \
2724 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2725 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2726 
2727 /*
2728  * Partition chip resources for use between various PFs, VFs, etc.
2729  */
2730 static int
2731 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2732     const char *name_prefix)
2733 {
2734 	const struct firmware *cfg = NULL;
2735 	int rc = 0;
2736 	struct fw_caps_config_cmd caps;
2737 	uint32_t mtype, moff, finicsum, cfcsum;
2738 
2739 	/*
2740 	 * Figure out what configuration file to use.  Pick the default config
2741 	 * file for the card if the user hasn't specified one explicitly.
2742 	 */
2743 	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2744 	if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2745 		/* Card specific overrides go here. */
2746 		if (pci_get_device(sc->dev) == 0x440a)
2747 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2748 		if (is_fpga(sc))
2749 			snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2750 	}
2751 
2752 	/*
2753 	 * We need to load another module if the profile is anything except
2754 	 * "default" or "flash".
2755 	 */
2756 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2757 	    strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2758 		char s[32];
2759 
2760 		snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2761 		cfg = firmware_get(s);
2762 		if (cfg == NULL) {
2763 			if (default_cfg != NULL) {
2764 				device_printf(sc->dev,
2765 				    "unable to load module \"%s\" for "
2766 				    "configuration profile \"%s\", will use "
2767 				    "the default config file instead.\n",
2768 				    s, sc->cfg_file);
2769 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2770 				    "%s", DEFAULT_CF);
2771 			} else {
2772 				device_printf(sc->dev,
2773 				    "unable to load module \"%s\" for "
2774 				    "configuration profile \"%s\", will use "
2775 				    "the config file on the card's flash "
2776 				    "instead.\n", s, sc->cfg_file);
2777 				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2778 				    "%s", FLASH_CF);
2779 			}
2780 		}
2781 	}
2782 
2783 	if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2784 	    default_cfg == NULL) {
2785 		device_printf(sc->dev,
2786 		    "default config file not available, will use the config "
2787 		    "file on the card's flash instead.\n");
2788 		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2789 	}
2790 
2791 	if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2792 		u_int cflen, i, n;
2793 		const uint32_t *cfdata;
2794 		uint32_t param, val, addr, off, mw_base, mw_aperture;
2795 
2796 		KASSERT(cfg != NULL || default_cfg != NULL,
2797 		    ("%s: no config to upload", __func__));
2798 
2799 		/*
2800 		 * Ask the firmware where it wants us to upload the config file.
2801 		 */
2802 		param = FW_PARAM_DEV(CF);
2803 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2804 		if (rc != 0) {
2805 			/* No support for config file?  Shouldn't happen. */
2806 			device_printf(sc->dev,
2807 			    "failed to query config file location: %d.\n", rc);
2808 			goto done;
2809 		}
2810 		mtype = G_FW_PARAMS_PARAM_Y(val);
2811 		moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2812 
2813 		/*
2814 		 * XXX: sheer laziness.  We deliberately added 4 bytes of
2815 		 * useless stuffing/comments at the end of the config file so
2816 		 * it's ok to simply throw away the last remaining bytes when
2817 		 * the config file is not an exact multiple of 4.  This also
2818 		 * helps with the validate_mt_off_len check.
2819 		 */
2820 		if (cfg != NULL) {
2821 			cflen = cfg->datasize & ~3;
2822 			cfdata = cfg->data;
2823 		} else {
2824 			cflen = default_cfg->datasize & ~3;
2825 			cfdata = default_cfg->data;
2826 		}
2827 
2828 		if (cflen > FLASH_CFG_MAX_SIZE) {
2829 			device_printf(sc->dev,
2830 			    "config file too long (%d, max allowed is %d).  "
2831 			    "Will try to use the config on the card, if any.\n",
2832 			    cflen, FLASH_CFG_MAX_SIZE);
2833 			goto use_config_on_flash;
2834 		}
2835 
2836 		rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2837 		if (rc != 0) {
2838 			device_printf(sc->dev,
2839 			    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2840 			    "Will try to use the config on the card, if any.\n",
2841 			    __func__, mtype, moff, cflen, rc);
2842 			goto use_config_on_flash;
2843 		}
2844 
2845 		memwin_info(sc, 2, &mw_base, &mw_aperture);
2846 		while (cflen) {
2847 			off = position_memwin(sc, 2, addr);
2848 			n = min(cflen, mw_aperture - off);
2849 			for (i = 0; i < n; i += 4)
2850 				t4_write_reg(sc, mw_base + off + i, *cfdata++);
2851 			cflen -= n;
2852 			addr += n;
2853 		}
2854 	} else {
2855 use_config_on_flash:
2856 		mtype = FW_MEMTYPE_FLASH;
2857 		moff = t4_flash_cfg_addr(sc);
2858 	}
2859 
2860 	bzero(&caps, sizeof(caps));
2861 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2862 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2863 	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2864 	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2865 	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2866 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2867 	if (rc != 0) {
2868 		device_printf(sc->dev,
2869 		    "failed to pre-process config file: %d "
2870 		    "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2871 		goto done;
2872 	}
2873 
2874 	finicsum = be32toh(caps.finicsum);
2875 	cfcsum = be32toh(caps.cfcsum);
2876 	if (finicsum != cfcsum) {
2877 		device_printf(sc->dev,
2878 		    "WARNING: config file checksum mismatch: %08x %08x\n",
2879 		    finicsum, cfcsum);
2880 	}
2881 	sc->cfcsum = cfcsum;
2882 
2883 #define LIMIT_CAPS(x) do { \
2884 	caps.x &= htobe16(t4_##x##_allowed); \
2885 } while (0)
2886 
2887 	/*
2888 	 * Let the firmware know what features will (not) be used so it can tune
2889 	 * things accordingly.
2890 	 */
2891 	LIMIT_CAPS(linkcaps);
2892 	LIMIT_CAPS(niccaps);
2893 	LIMIT_CAPS(toecaps);
2894 	LIMIT_CAPS(rdmacaps);
2895 	LIMIT_CAPS(iscsicaps);
2896 	LIMIT_CAPS(fcoecaps);
2897 #undef LIMIT_CAPS
2898 
2899 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2900 	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2901 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2902 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2903 	if (rc != 0) {
2904 		device_printf(sc->dev,
2905 		    "failed to process config file: %d.\n", rc);
2906 	}
2907 done:
2908 	if (cfg != NULL)
2909 		firmware_put(cfg, FIRMWARE_UNLOAD);
2910 	return (rc);
2911 }
2912 
2913 /*
2914  * Retrieve parameters that are needed (or nice to have) very early.
2915  */
2916 static int
2917 get_params__pre_init(struct adapter *sc)
2918 {
2919 	int rc;
2920 	uint32_t param[2], val[2];
2921 	struct fw_devlog_cmd cmd;
2922 	struct devlog_params *dlog = &sc->params.devlog;
2923 
2924 	param[0] = FW_PARAM_DEV(PORTVEC);
2925 	param[1] = FW_PARAM_DEV(CCLK);
2926 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2927 	if (rc != 0) {
2928 		device_printf(sc->dev,
2929 		    "failed to query parameters (pre_init): %d.\n", rc);
2930 		return (rc);
2931 	}
2932 
2933 	sc->params.portvec = val[0];
2934 	sc->params.nports = bitcount32(val[0]);
2935 	sc->params.vpd.cclk = val[1];
2936 
2937 	/* Read device log parameters. */
2938 	bzero(&cmd, sizeof(cmd));
2939 	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2940 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2941 	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2942 	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2943 	if (rc != 0) {
2944 		device_printf(sc->dev,
2945 		    "failed to get devlog parameters: %d.\n", rc);
2946 		bzero(dlog, sizeof (*dlog));
2947 		rc = 0;	/* devlog isn't critical for device operation */
2948 	} else {
2949 		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2950 		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2951 		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2952 		dlog->size = be32toh(cmd.memsize_devlog);
2953 	}
2954 
2955 	return (rc);
2956 }
2957 
2958 /*
2959  * Retrieve various parameters that are of interest to the driver.  The device
2960  * has been initialized by the firmware at this point.
2961  */
2962 static int
2963 get_params__post_init(struct adapter *sc)
2964 {
2965 	int rc;
2966 	uint32_t param[7], val[7];
2967 	struct fw_caps_config_cmd caps;
2968 
2969 	param[0] = FW_PARAM_PFVF(IQFLINT_START);
2970 	param[1] = FW_PARAM_PFVF(EQ_START);
2971 	param[2] = FW_PARAM_PFVF(FILTER_START);
2972 	param[3] = FW_PARAM_PFVF(FILTER_END);
2973 	param[4] = FW_PARAM_PFVF(L2T_START);
2974 	param[5] = FW_PARAM_PFVF(L2T_END);
2975 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2976 	if (rc != 0) {
2977 		device_printf(sc->dev,
2978 		    "failed to query parameters (post_init): %d.\n", rc);
2979 		return (rc);
2980 	}
2981 
2982 	sc->sge.iq_start = val[0];
2983 	sc->sge.eq_start = val[1];
2984 	sc->tids.ftid_base = val[2];
2985 	sc->tids.nftids = val[3] - val[2] + 1;
2986 	sc->params.ftid_min = val[2];
2987 	sc->params.ftid_max = val[3];
2988 	sc->vres.l2t.start = val[4];
2989 	sc->vres.l2t.size = val[5] - val[4] + 1;
2990 	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2991 	    ("%s: L2 table size (%u) larger than expected (%u)",
2992 	    __func__, sc->vres.l2t.size, L2T_SIZE));
2993 
2994 	/* get capabilites */
2995 	bzero(&caps, sizeof(caps));
2996 	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2997 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
2998 	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2999 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3000 	if (rc != 0) {
3001 		device_printf(sc->dev,
3002 		    "failed to get card capabilities: %d.\n", rc);
3003 		return (rc);
3004 	}
3005 
3006 #define READ_CAPS(x) do { \
3007 	sc->x = htobe16(caps.x); \
3008 } while (0)
3009 	READ_CAPS(linkcaps);
3010 	READ_CAPS(niccaps);
3011 	READ_CAPS(toecaps);
3012 	READ_CAPS(rdmacaps);
3013 	READ_CAPS(iscsicaps);
3014 	READ_CAPS(fcoecaps);
3015 
3016 	if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
3017 		param[0] = FW_PARAM_PFVF(ETHOFLD_START);
3018 		param[1] = FW_PARAM_PFVF(ETHOFLD_END);
3019 		param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3020 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
3021 		if (rc != 0) {
3022 			device_printf(sc->dev,
3023 			    "failed to query NIC parameters: %d.\n", rc);
3024 			return (rc);
3025 		}
3026 		sc->tids.etid_base = val[0];
3027 		sc->params.etid_min = val[0];
3028 		sc->tids.netids = val[1] - val[0] + 1;
3029 		sc->params.netids = sc->tids.netids;
3030 		sc->params.eo_wr_cred = val[2];
3031 		sc->params.ethoffload = 1;
3032 	}
3033 
3034 	if (sc->toecaps) {
3035 		/* query offload-related parameters */
3036 		param[0] = FW_PARAM_DEV(NTID);
3037 		param[1] = FW_PARAM_PFVF(SERVER_START);
3038 		param[2] = FW_PARAM_PFVF(SERVER_END);
3039 		param[3] = FW_PARAM_PFVF(TDDP_START);
3040 		param[4] = FW_PARAM_PFVF(TDDP_END);
3041 		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3042 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3043 		if (rc != 0) {
3044 			device_printf(sc->dev,
3045 			    "failed to query TOE parameters: %d.\n", rc);
3046 			return (rc);
3047 		}
3048 		sc->tids.ntids = val[0];
3049 		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
3050 		sc->tids.stid_base = val[1];
3051 		sc->tids.nstids = val[2] - val[1] + 1;
3052 		sc->vres.ddp.start = val[3];
3053 		sc->vres.ddp.size = val[4] - val[3] + 1;
3054 		sc->params.ofldq_wr_cred = val[5];
3055 		sc->params.offload = 1;
3056 	}
3057 	if (sc->rdmacaps) {
3058 		param[0] = FW_PARAM_PFVF(STAG_START);
3059 		param[1] = FW_PARAM_PFVF(STAG_END);
3060 		param[2] = FW_PARAM_PFVF(RQ_START);
3061 		param[3] = FW_PARAM_PFVF(RQ_END);
3062 		param[4] = FW_PARAM_PFVF(PBL_START);
3063 		param[5] = FW_PARAM_PFVF(PBL_END);
3064 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3065 		if (rc != 0) {
3066 			device_printf(sc->dev,
3067 			    "failed to query RDMA parameters(1): %d.\n", rc);
3068 			return (rc);
3069 		}
3070 		sc->vres.stag.start = val[0];
3071 		sc->vres.stag.size = val[1] - val[0] + 1;
3072 		sc->vres.rq.start = val[2];
3073 		sc->vres.rq.size = val[3] - val[2] + 1;
3074 		sc->vres.pbl.start = val[4];
3075 		sc->vres.pbl.size = val[5] - val[4] + 1;
3076 
3077 		param[0] = FW_PARAM_PFVF(SQRQ_START);
3078 		param[1] = FW_PARAM_PFVF(SQRQ_END);
3079 		param[2] = FW_PARAM_PFVF(CQ_START);
3080 		param[3] = FW_PARAM_PFVF(CQ_END);
3081 		param[4] = FW_PARAM_PFVF(OCQ_START);
3082 		param[5] = FW_PARAM_PFVF(OCQ_END);
3083 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3084 		if (rc != 0) {
3085 			device_printf(sc->dev,
3086 			    "failed to query RDMA parameters(2): %d.\n", rc);
3087 			return (rc);
3088 		}
3089 		sc->vres.qp.start = val[0];
3090 		sc->vres.qp.size = val[1] - val[0] + 1;
3091 		sc->vres.cq.start = val[2];
3092 		sc->vres.cq.size = val[3] - val[2] + 1;
3093 		sc->vres.ocq.start = val[4];
3094 		sc->vres.ocq.size = val[5] - val[4] + 1;
3095 	}
3096 	if (sc->iscsicaps) {
3097 		param[0] = FW_PARAM_PFVF(ISCSI_START);
3098 		param[1] = FW_PARAM_PFVF(ISCSI_END);
3099 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3100 		if (rc != 0) {
3101 			device_printf(sc->dev,
3102 			    "failed to query iSCSI parameters: %d.\n", rc);
3103 			return (rc);
3104 		}
3105 		sc->vres.iscsi.start = val[0];
3106 		sc->vres.iscsi.size = val[1] - val[0] + 1;
3107 	}
3108 
3109 	/*
3110 	 * We've got the params we wanted to query via the firmware.  Now grab
3111 	 * some others directly from the chip.
3112 	 */
3113 	rc = t4_read_chip_settings(sc);
3114 
3115 	return (rc);
3116 }
3117 
3118 static int
3119 set_params__post_init(struct adapter *sc)
3120 {
3121 	uint32_t param, val;
3122 
3123 	/* ask for encapsulated CPLs */
3124 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3125 	val = 1;
3126 	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3127 
3128 	return (0);
3129 }
3130 
3131 #undef FW_PARAM_PFVF
3132 #undef FW_PARAM_DEV
3133 
3134 static void
3135 t4_set_desc(struct adapter *sc)
3136 {
3137 	char buf[128];
3138 	struct adapter_params *p = &sc->params;
3139 
3140 	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
3141 	    "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
3142 	    chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
3143 
3144 	device_set_desc_copy(sc->dev, buf);
3145 }
3146 
3147 static void
3148 build_medialist(struct port_info *pi, struct ifmedia *media)
3149 {
3150 	int m;
3151 
3152 	PORT_LOCK(pi);
3153 
3154 	ifmedia_removeall(media);
3155 
3156 	m = IFM_ETHER | IFM_FDX;
3157 
3158 	switch(pi->port_type) {
3159 	case FW_PORT_TYPE_BT_XFI:
3160 	case FW_PORT_TYPE_BT_XAUI:
3161 		ifmedia_add(media, m | IFM_10G_T, 0, NULL);
3162 		/* fall through */
3163 
3164 	case FW_PORT_TYPE_BT_SGMII:
3165 		ifmedia_add(media, m | IFM_1000_T, 0, NULL);
3166 		ifmedia_add(media, m | IFM_100_TX, 0, NULL);
3167 		ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
3168 		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
3169 		break;
3170 
3171 	case FW_PORT_TYPE_CX4:
3172 		ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
3173 		ifmedia_set(media, m | IFM_10G_CX4);
3174 		break;
3175 
3176 	case FW_PORT_TYPE_QSFP_10G:
3177 	case FW_PORT_TYPE_SFP:
3178 	case FW_PORT_TYPE_FIBER_XFI:
3179 	case FW_PORT_TYPE_FIBER_XAUI:
3180 		switch (pi->mod_type) {
3181 
3182 		case FW_PORT_MOD_TYPE_LR:
3183 			ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
3184 			ifmedia_set(media, m | IFM_10G_LR);
3185 			break;
3186 
3187 		case FW_PORT_MOD_TYPE_SR:
3188 			ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
3189 			ifmedia_set(media, m | IFM_10G_SR);
3190 			break;
3191 
3192 		case FW_PORT_MOD_TYPE_LRM:
3193 			ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
3194 			ifmedia_set(media, m | IFM_10G_LRM);
3195 			break;
3196 
3197 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3198 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3199 			ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
3200 			ifmedia_set(media, m | IFM_10G_TWINAX);
3201 			break;
3202 
3203 		case FW_PORT_MOD_TYPE_NONE:
3204 			m &= ~IFM_FDX;
3205 			ifmedia_add(media, m | IFM_NONE, 0, NULL);
3206 			ifmedia_set(media, m | IFM_NONE);
3207 			break;
3208 
3209 		case FW_PORT_MOD_TYPE_NA:
3210 		case FW_PORT_MOD_TYPE_ER:
3211 		default:
3212 			device_printf(pi->dev,
3213 			    "unknown port_type (%d), mod_type (%d)\n",
3214 			    pi->port_type, pi->mod_type);
3215 			ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3216 			ifmedia_set(media, m | IFM_UNKNOWN);
3217 			break;
3218 		}
3219 		break;
3220 
3221 	case FW_PORT_TYPE_QSFP:
3222 		switch (pi->mod_type) {
3223 
3224 		case FW_PORT_MOD_TYPE_LR:
3225 			ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
3226 			ifmedia_set(media, m | IFM_40G_LR4);
3227 			break;
3228 
3229 		case FW_PORT_MOD_TYPE_SR:
3230 			ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
3231 			ifmedia_set(media, m | IFM_40G_SR4);
3232 			break;
3233 
3234 		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3235 		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3236 			ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
3237 			ifmedia_set(media, m | IFM_40G_CR4);
3238 			break;
3239 
3240 		case FW_PORT_MOD_TYPE_NONE:
3241 			m &= ~IFM_FDX;
3242 			ifmedia_add(media, m | IFM_NONE, 0, NULL);
3243 			ifmedia_set(media, m | IFM_NONE);
3244 			break;
3245 
3246 		default:
3247 			device_printf(pi->dev,
3248 			    "unknown port_type (%d), mod_type (%d)\n",
3249 			    pi->port_type, pi->mod_type);
3250 			ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3251 			ifmedia_set(media, m | IFM_UNKNOWN);
3252 			break;
3253 		}
3254 		break;
3255 
3256 	default:
3257 		device_printf(pi->dev,
3258 		    "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3259 		    pi->mod_type);
3260 		ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3261 		ifmedia_set(media, m | IFM_UNKNOWN);
3262 		break;
3263 	}
3264 
3265 	PORT_UNLOCK(pi);
3266 }
3267 
3268 #define FW_MAC_EXACT_CHUNK	7
3269 
3270 /*
3271  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3272  * indicates which parameters should be programmed (the rest are left alone).
3273  */
3274 int
3275 update_mac_settings(struct ifnet *ifp, int flags)
3276 {
3277 	int rc = 0;
3278 	struct vi_info *vi = ifp->if_softc;
3279 	struct port_info *pi = vi->pi;
3280 	struct adapter *sc = pi->adapter;
3281 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3282 
3283 	ASSERT_SYNCHRONIZED_OP(sc);
3284 	KASSERT(flags, ("%s: not told what to update.", __func__));
3285 
3286 	if (flags & XGMAC_MTU)
3287 		mtu = ifp->if_mtu;
3288 
3289 	if (flags & XGMAC_PROMISC)
3290 		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3291 
3292 	if (flags & XGMAC_ALLMULTI)
3293 		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3294 
3295 	if (flags & XGMAC_VLANEX)
3296 		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3297 
3298 	if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3299 		rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
3300 		    allmulti, 1, vlanex, false);
3301 		if (rc) {
3302 			if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3303 			    rc);
3304 			return (rc);
3305 		}
3306 	}
3307 
3308 	if (flags & XGMAC_UCADDR) {
3309 		uint8_t ucaddr[ETHER_ADDR_LEN];
3310 
3311 		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3312 		rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
3313 		    ucaddr, true, true);
3314 		if (rc < 0) {
3315 			rc = -rc;
3316 			if_printf(ifp, "change_mac failed: %d\n", rc);
3317 			return (rc);
3318 		} else {
3319 			vi->xact_addr_filt = rc;
3320 			rc = 0;
3321 		}
3322 	}
3323 
3324 	if (flags & XGMAC_MCADDRS) {
3325 		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3326 		int del = 1;
3327 		uint64_t hash = 0;
3328 		struct ifmultiaddr *ifma;
3329 		int i = 0, j;
3330 
3331 		if_maddr_rlock(ifp);
3332 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3333 			if (ifma->ifma_addr->sa_family != AF_LINK)
3334 				continue;
3335 			mcaddr[i] =
3336 			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3337 			MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3338 			i++;
3339 
3340 			if (i == FW_MAC_EXACT_CHUNK) {
3341 				rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
3342 				    del, i, mcaddr, NULL, &hash, 0);
3343 				if (rc < 0) {
3344 					rc = -rc;
3345 					for (j = 0; j < i; j++) {
3346 						if_printf(ifp,
3347 						    "failed to add mc address"
3348 						    " %02x:%02x:%02x:"
3349 						    "%02x:%02x:%02x rc=%d\n",
3350 						    mcaddr[j][0], mcaddr[j][1],
3351 						    mcaddr[j][2], mcaddr[j][3],
3352 						    mcaddr[j][4], mcaddr[j][5],
3353 						    rc);
3354 					}
3355 					goto mcfail;
3356 				}
3357 				del = 0;
3358 				i = 0;
3359 			}
3360 		}
3361 		if (i > 0) {
3362 			rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
3363 			    mcaddr, NULL, &hash, 0);
3364 			if (rc < 0) {
3365 				rc = -rc;
3366 				for (j = 0; j < i; j++) {
3367 					if_printf(ifp,
3368 					    "failed to add mc address"
3369 					    " %02x:%02x:%02x:"
3370 					    "%02x:%02x:%02x rc=%d\n",
3371 					    mcaddr[j][0], mcaddr[j][1],
3372 					    mcaddr[j][2], mcaddr[j][3],
3373 					    mcaddr[j][4], mcaddr[j][5],
3374 					    rc);
3375 				}
3376 				goto mcfail;
3377 			}
3378 		}
3379 
3380 		rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
3381 		if (rc != 0)
3382 			if_printf(ifp, "failed to set mc address hash: %d", rc);
3383 mcfail:
3384 		if_maddr_runlock(ifp);
3385 	}
3386 
3387 	return (rc);
3388 }
3389 
3390 /*
3391  * {begin|end}_synchronized_op must be called from the same thread.
3392  */
3393 int
3394 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
3395     char *wmesg)
3396 {
3397 	int rc, pri;
3398 
3399 #ifdef WITNESS
3400 	/* the caller thinks it's ok to sleep, but is it really? */
3401 	if (flags & SLEEP_OK)
3402 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
3403 		    "begin_synchronized_op");
3404 #endif
3405 
3406 	if (INTR_OK)
3407 		pri = PCATCH;
3408 	else
3409 		pri = 0;
3410 
3411 	ADAPTER_LOCK(sc);
3412 	for (;;) {
3413 
3414 		if (vi && IS_DOOMED(vi)) {
3415 			rc = ENXIO;
3416 			goto done;
3417 		}
3418 
3419 		if (!IS_BUSY(sc)) {
3420 			rc = 0;
3421 			break;
3422 		}
3423 
3424 		if (!(flags & SLEEP_OK)) {
3425 			rc = EBUSY;
3426 			goto done;
3427 		}
3428 
3429 		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3430 			rc = EINTR;
3431 			goto done;
3432 		}
3433 	}
3434 
3435 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3436 	SET_BUSY(sc);
3437 #ifdef INVARIANTS
3438 	sc->last_op = wmesg;
3439 	sc->last_op_thr = curthread;
3440 	sc->last_op_flags = flags;
3441 #endif
3442 
3443 done:
3444 	if (!(flags & HOLD_LOCK) || rc)
3445 		ADAPTER_UNLOCK(sc);
3446 
3447 	return (rc);
3448 }
3449 
3450 /*
3451  * Tell if_ioctl and if_init that the VI is going away.  This is
3452  * special variant of begin_synchronized_op and must be paired with a
3453  * call to end_synchronized_op.
3454  */
3455 void
3456 doom_vi(struct adapter *sc, struct vi_info *vi)
3457 {
3458 
3459 	ADAPTER_LOCK(sc);
3460 	SET_DOOMED(vi);
3461 	wakeup(&sc->flags);
3462 	while (IS_BUSY(sc))
3463 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
3464 	SET_BUSY(sc);
3465 #ifdef INVARIANTS
3466 	sc->last_op = "t4detach";
3467 	sc->last_op_thr = curthread;
3468 	sc->last_op_flags = 0;
3469 #endif
3470 	ADAPTER_UNLOCK(sc);
3471 }
3472 
3473 /*
3474  * {begin|end}_synchronized_op must be called from the same thread.
3475  */
3476 void
3477 end_synchronized_op(struct adapter *sc, int flags)
3478 {
3479 
3480 	if (flags & LOCK_HELD)
3481 		ADAPTER_LOCK_ASSERT_OWNED(sc);
3482 	else
3483 		ADAPTER_LOCK(sc);
3484 
3485 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3486 	CLR_BUSY(sc);
3487 	wakeup(&sc->flags);
3488 	ADAPTER_UNLOCK(sc);
3489 }
3490 
3491 static int
3492 cxgbe_init_synchronized(struct vi_info *vi)
3493 {
3494 	struct port_info *pi = vi->pi;
3495 	struct adapter *sc = pi->adapter;
3496 	struct ifnet *ifp = vi->ifp;
3497 	int rc = 0, i;
3498 	struct sge_txq *txq;
3499 
3500 	ASSERT_SYNCHRONIZED_OP(sc);
3501 
3502 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3503 		return (0);	/* already running */
3504 
3505 	if (!(sc->flags & FULL_INIT_DONE) &&
3506 	    ((rc = adapter_full_init(sc)) != 0))
3507 		return (rc);	/* error message displayed already */
3508 
3509 	if (!(vi->flags & VI_INIT_DONE) &&
3510 	    ((rc = vi_full_init(vi)) != 0))
3511 		return (rc); /* error message displayed already */
3512 
3513 	rc = update_mac_settings(ifp, XGMAC_ALL);
3514 	if (rc)
3515 		goto done;	/* error message displayed already */
3516 
3517 	rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
3518 	if (rc != 0) {
3519 		if_printf(ifp, "enable_vi failed: %d\n", rc);
3520 		goto done;
3521 	}
3522 
3523 	/*
3524 	 * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3525 	 * if this changes.
3526 	 */
3527 
3528 	for_each_txq(vi, i, txq) {
3529 		TXQ_LOCK(txq);
3530 		txq->eq.flags |= EQ_ENABLED;
3531 		TXQ_UNLOCK(txq);
3532 	}
3533 
3534 	/*
3535 	 * The first iq of the first port to come up is used for tracing.
3536 	 */
3537 	if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
3538 		sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
3539 		t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3540 		    A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3541 		    V_QUEUENUMBER(sc->traceq));
3542 		pi->flags |= HAS_TRACEQ;
3543 	}
3544 
3545 	/* all ok */
3546 	PORT_LOCK(pi);
3547 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
3548 	pi->up_vis++;
3549 
3550 	if (pi->nvi > 1)
3551 		callout_reset(&vi->tick, hz, vi_tick, vi);
3552 	else
3553 		callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3554 	PORT_UNLOCK(pi);
3555 done:
3556 	if (rc != 0)
3557 		cxgbe_uninit_synchronized(vi);
3558 
3559 	return (rc);
3560 }
3561 
3562 /*
3563  * Idempotent.
3564  */
3565 static int
3566 cxgbe_uninit_synchronized(struct vi_info *vi)
3567 {
3568 	struct port_info *pi = vi->pi;
3569 	struct adapter *sc = pi->adapter;
3570 	struct ifnet *ifp = vi->ifp;
3571 	int rc, i;
3572 	struct sge_txq *txq;
3573 
3574 	ASSERT_SYNCHRONIZED_OP(sc);
3575 
3576 	if (!(vi->flags & VI_INIT_DONE)) {
3577 		KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3578 		    ("uninited VI is running"));
3579 		return (0);
3580 	}
3581 
3582 	/*
3583 	 * Disable the VI so that all its data in either direction is discarded
3584 	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3585 	 * tick) intact as the TP can deliver negative advice or data that it's
3586 	 * holding in its RAM (for an offloaded connection) even after the VI is
3587 	 * disabled.
3588 	 */
3589 	rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
3590 	if (rc) {
3591 		if_printf(ifp, "disable_vi failed: %d\n", rc);
3592 		return (rc);
3593 	}
3594 
3595 	for_each_txq(vi, i, txq) {
3596 		TXQ_LOCK(txq);
3597 		txq->eq.flags &= ~EQ_ENABLED;
3598 		TXQ_UNLOCK(txq);
3599 	}
3600 
3601 	PORT_LOCK(pi);
3602 	if (pi->nvi == 1)
3603 		callout_stop(&pi->tick);
3604 	else
3605 		callout_stop(&vi->tick);
3606 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3607 		PORT_UNLOCK(pi);
3608 		return (0);
3609 	}
3610 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3611 	pi->up_vis--;
3612 	if (pi->up_vis > 0) {
3613 		PORT_UNLOCK(pi);
3614 		return (0);
3615 	}
3616 	PORT_UNLOCK(pi);
3617 
3618 	pi->link_cfg.link_ok = 0;
3619 	pi->link_cfg.speed = 0;
3620 	pi->linkdnrc = -1;
3621 	t4_os_link_changed(sc, pi->port_id, 0, -1);
3622 
3623 	return (0);
3624 }
3625 
3626 /*
3627  * It is ok for this function to fail midway and return right away.  t4_detach
3628  * will walk the entire sc->irq list and clean up whatever is valid.
3629  */
3630 static int
3631 setup_intr_handlers(struct adapter *sc)
3632 {
3633 	int rc, rid, p, q, v;
3634 	char s[8];
3635 	struct irq *irq;
3636 	struct port_info *pi;
3637 	struct vi_info *vi;
3638 	struct sge_rxq *rxq;
3639 #ifdef TCP_OFFLOAD
3640 	struct sge_ofld_rxq *ofld_rxq;
3641 #endif
3642 #ifdef DEV_NETMAP
3643 	struct sge_nm_rxq *nm_rxq;
3644 #endif
3645 #ifdef RSS
3646 	int nbuckets = rss_getnumbuckets();
3647 #endif
3648 
3649 	/*
3650 	 * Setup interrupts.
3651 	 */
3652 	irq = &sc->irq[0];
3653 	rid = sc->intr_type == INTR_INTX ? 0 : 1;
3654 	if (sc->intr_count == 1)
3655 		return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3656 
3657 	/* Multiple interrupts. */
3658 	KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3659 	    ("%s: too few intr.", __func__));
3660 
3661 	/* The first one is always error intr */
3662 	rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3663 	if (rc != 0)
3664 		return (rc);
3665 	irq++;
3666 	rid++;
3667 
3668 	/* The second one is always the firmware event queue */
3669 	rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3670 	if (rc != 0)
3671 		return (rc);
3672 	irq++;
3673 	rid++;
3674 
3675 	for_each_port(sc, p) {
3676 		pi = sc->port[p];
3677 		for_each_vi(pi, v, vi) {
3678 			vi->first_intr = rid - 1;
3679 #ifdef DEV_NETMAP
3680 			if (vi->flags & VI_NETMAP) {
3681 				for_each_nm_rxq(vi, q, nm_rxq) {
3682 					snprintf(s, sizeof(s), "%d-%d", p, q);
3683 					rc = t4_alloc_irq(sc, irq, rid,
3684 					    t4_nm_intr, nm_rxq, s);
3685 					if (rc != 0)
3686 						return (rc);
3687 					irq++;
3688 					rid++;
3689 					vi->nintr++;
3690 				}
3691 				continue;
3692 			}
3693 #endif
3694 			if (vi->flags & INTR_RXQ) {
3695 				for_each_rxq(vi, q, rxq) {
3696 					if (v == 0)
3697 						snprintf(s, sizeof(s), "%d.%d",
3698 						    p, q);
3699 					else
3700 						snprintf(s, sizeof(s),
3701 						    "%d(%d).%d", p, v, q);
3702 					rc = t4_alloc_irq(sc, irq, rid,
3703 					    t4_intr, rxq, s);
3704 					if (rc != 0)
3705 						return (rc);
3706 #ifdef RSS
3707 					bus_bind_intr(sc->dev, irq->res,
3708 					    rss_getcpu(q % nbuckets));
3709 #endif
3710 					irq++;
3711 					rid++;
3712 					vi->nintr++;
3713 				}
3714 			}
3715 #ifdef TCP_OFFLOAD
3716 			if (vi->flags & INTR_OFLD_RXQ) {
3717 				for_each_ofld_rxq(vi, q, ofld_rxq) {
3718 					snprintf(s, sizeof(s), "%d,%d", p, q);
3719 					rc = t4_alloc_irq(sc, irq, rid,
3720 					    t4_intr, ofld_rxq, s);
3721 					if (rc != 0)
3722 						return (rc);
3723 					irq++;
3724 					rid++;
3725 					vi->nintr++;
3726 				}
3727 			}
3728 #endif
3729 		}
3730 	}
3731 	MPASS(irq == &sc->irq[sc->intr_count]);
3732 
3733 	return (0);
3734 }
3735 
3736 int
3737 adapter_full_init(struct adapter *sc)
3738 {
3739 	int rc, i;
3740 
3741 	ASSERT_SYNCHRONIZED_OP(sc);
3742 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3743 	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3744 	    ("%s: FULL_INIT_DONE already", __func__));
3745 
3746 	/*
3747 	 * queues that belong to the adapter (not any particular port).
3748 	 */
3749 	rc = t4_setup_adapter_queues(sc);
3750 	if (rc != 0)
3751 		goto done;
3752 
3753 	for (i = 0; i < nitems(sc->tq); i++) {
3754 		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3755 		    taskqueue_thread_enqueue, &sc->tq[i]);
3756 		if (sc->tq[i] == NULL) {
3757 			device_printf(sc->dev,
3758 			    "failed to allocate task queue %d\n", i);
3759 			rc = ENOMEM;
3760 			goto done;
3761 		}
3762 		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3763 		    device_get_nameunit(sc->dev), i);
3764 	}
3765 
3766 	t4_intr_enable(sc);
3767 	sc->flags |= FULL_INIT_DONE;
3768 done:
3769 	if (rc != 0)
3770 		adapter_full_uninit(sc);
3771 
3772 	return (rc);
3773 }
3774 
3775 int
3776 adapter_full_uninit(struct adapter *sc)
3777 {
3778 	int i;
3779 
3780 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3781 
3782 	t4_teardown_adapter_queues(sc);
3783 
3784 	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3785 		taskqueue_free(sc->tq[i]);
3786 		sc->tq[i] = NULL;
3787 	}
3788 
3789 	sc->flags &= ~FULL_INIT_DONE;
3790 
3791 	return (0);
3792 }
3793 
3794 #ifdef RSS
3795 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
3796     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
3797     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
3798     RSS_HASHTYPE_RSS_UDP_IPV6)
3799 
3800 /* Translates kernel hash types to hardware. */
3801 static int
3802 hashconfig_to_hashen(int hashconfig)
3803 {
3804 	int hashen = 0;
3805 
3806 	if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
3807 		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
3808 	if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
3809 		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
3810 	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
3811 		hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3812 		    F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3813 	}
3814 	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
3815 		hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3816 		    F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3817 	}
3818 	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
3819 		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3820 	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
3821 		hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3822 
3823 	return (hashen);
3824 }
3825 
3826 /* Translates hardware hash types to kernel. */
3827 static int
3828 hashen_to_hashconfig(int hashen)
3829 {
3830 	int hashconfig = 0;
3831 
3832 	if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
3833 		/*
3834 		 * If UDP hashing was enabled it must have been enabled for
3835 		 * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
3836 		 * enabling any 4-tuple hash is nonsense configuration.
3837 		 */
3838 		MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3839 		    F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
3840 
3841 		if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3842 			hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
3843 		if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3844 			hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
3845 	}
3846 	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3847 		hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
3848 	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3849 		hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
3850 	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
3851 		hashconfig |= RSS_HASHTYPE_RSS_IPV4;
3852 	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
3853 		hashconfig |= RSS_HASHTYPE_RSS_IPV6;
3854 
3855 	return (hashconfig);
3856 }
3857 #endif
3858 
3859 int
3860 vi_full_init(struct vi_info *vi)
3861 {
3862 	struct adapter *sc = vi->pi->adapter;
3863 	struct ifnet *ifp = vi->ifp;
3864 	uint16_t *rss;
3865 	struct sge_rxq *rxq;
3866 	int rc, i, j, hashen;
3867 #ifdef RSS
3868 	int nbuckets = rss_getnumbuckets();
3869 	int hashconfig = rss_gethashconfig();
3870 	int extra;
3871 	uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3872 	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3873 #endif
3874 
3875 	ASSERT_SYNCHRONIZED_OP(sc);
3876 	KASSERT((vi->flags & VI_INIT_DONE) == 0,
3877 	    ("%s: VI_INIT_DONE already", __func__));
3878 
3879 	sysctl_ctx_init(&vi->ctx);
3880 	vi->flags |= VI_SYSCTL_CTX;
3881 
3882 	/*
3883 	 * Allocate tx/rx/fl queues for this VI.
3884 	 */
3885 	rc = t4_setup_vi_queues(vi);
3886 	if (rc != 0)
3887 		goto done;	/* error message displayed already */
3888 
3889 #ifdef DEV_NETMAP
3890 	/* Netmap VIs configure RSS when netmap is enabled. */
3891 	if (vi->flags & VI_NETMAP) {
3892 		vi->flags |= VI_INIT_DONE;
3893 		return (0);
3894 	}
3895 #endif
3896 
3897 	/*
3898 	 * Setup RSS for this VI.  Save a copy of the RSS table for later use.
3899 	 */
3900 	if (vi->nrxq > vi->rss_size) {
3901 		if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
3902 		    "some queues will never receive traffic.\n", vi->nrxq,
3903 		    vi->rss_size);
3904 	} else if (vi->rss_size % vi->nrxq) {
3905 		if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
3906 		    "expect uneven traffic distribution.\n", vi->nrxq,
3907 		    vi->rss_size);
3908 	}
3909 #ifdef RSS
3910 	MPASS(RSS_KEYSIZE == 40);
3911 	if (vi->nrxq != nbuckets) {
3912 		if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
3913 		    "performance will be impacted.\n", vi->nrxq, nbuckets);
3914 	}
3915 
3916 	rss_getkey((void *)&raw_rss_key[0]);
3917 	for (i = 0; i < nitems(rss_key); i++) {
3918 		rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
3919 	}
3920 	t4_write_rss_key(sc, (void *)&rss_key[0], -1);
3921 #endif
3922 	rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3923 	for (i = 0; i < vi->rss_size;) {
3924 #ifdef RSS
3925 		j = rss_get_indirection_to_bucket(i);
3926 		j %= vi->nrxq;
3927 		rxq = &sc->sge.rxq[vi->first_rxq + j];
3928 		rss[i++] = rxq->iq.abs_id;
3929 #else
3930 		for_each_rxq(vi, j, rxq) {
3931 			rss[i++] = rxq->iq.abs_id;
3932 			if (i == vi->rss_size)
3933 				break;
3934 		}
3935 #endif
3936 	}
3937 
3938 	rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
3939 	    vi->rss_size);
3940 	if (rc != 0) {
3941 		if_printf(ifp, "rss_config failed: %d\n", rc);
3942 		goto done;
3943 	}
3944 
3945 #ifdef RSS
3946 	hashen = hashconfig_to_hashen(hashconfig);
3947 
3948 	/*
3949 	 * We may have had to enable some hashes even though the global config
3950 	 * wants them disabled.  This is a potential problem that must be
3951 	 * reported to the user.
3952 	 */
3953 	extra = hashen_to_hashconfig(hashen) ^ hashconfig;
3954 
3955 	/*
3956 	 * If we consider only the supported hash types, then the enabled hashes
3957 	 * are a superset of the requested hashes.  In other words, there cannot
3958 	 * be any supported hash that was requested but not enabled, but there
3959 	 * can be hashes that were not requested but had to be enabled.
3960 	 */
3961 	extra &= SUPPORTED_RSS_HASHTYPES;
3962 	MPASS((extra & hashconfig) == 0);
3963 
3964 	if (extra) {
3965 		if_printf(ifp,
3966 		    "global RSS config (0x%x) cannot be accomodated.\n",
3967 		    hashconfig);
3968 	}
3969 	if (extra & RSS_HASHTYPE_RSS_IPV4)
3970 		if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
3971 	if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
3972 		if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
3973 	if (extra & RSS_HASHTYPE_RSS_IPV6)
3974 		if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
3975 	if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
3976 		if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
3977 	if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
3978 		if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
3979 	if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
3980 		if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
3981 #else
3982 	hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
3983 	    F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
3984 	    F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3985 	    F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
3986 #endif
3987 	rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]);
3988 	if (rc != 0) {
3989 		if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
3990 		goto done;
3991 	}
3992 
3993 	vi->rss = rss;
3994 	vi->flags |= VI_INIT_DONE;
3995 done:
3996 	if (rc != 0)
3997 		vi_full_uninit(vi);
3998 
3999 	return (rc);
4000 }
4001 
4002 /*
4003  * Idempotent.
4004  */
4005 int
4006 vi_full_uninit(struct vi_info *vi)
4007 {
4008 	struct port_info *pi = vi->pi;
4009 	struct adapter *sc = pi->adapter;
4010 	int i;
4011 	struct sge_rxq *rxq;
4012 	struct sge_txq *txq;
4013 #ifdef TCP_OFFLOAD
4014 	struct sge_ofld_rxq *ofld_rxq;
4015 	struct sge_wrq *ofld_txq;
4016 #endif
4017 
4018 	if (vi->flags & VI_INIT_DONE) {
4019 
4020 		/* Need to quiesce queues.  */
4021 #ifdef DEV_NETMAP
4022 		if (vi->flags & VI_NETMAP)
4023 			goto skip;
4024 #endif
4025 
4026 		/* XXX: Only for the first VI? */
4027 		if (IS_MAIN_VI(vi))
4028 			quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
4029 
4030 		for_each_txq(vi, i, txq) {
4031 			quiesce_txq(sc, txq);
4032 		}
4033 
4034 #ifdef TCP_OFFLOAD
4035 		for_each_ofld_txq(vi, i, ofld_txq) {
4036 			quiesce_wrq(sc, ofld_txq);
4037 		}
4038 #endif
4039 
4040 		for_each_rxq(vi, i, rxq) {
4041 			quiesce_iq(sc, &rxq->iq);
4042 			quiesce_fl(sc, &rxq->fl);
4043 		}
4044 
4045 #ifdef TCP_OFFLOAD
4046 		for_each_ofld_rxq(vi, i, ofld_rxq) {
4047 			quiesce_iq(sc, &ofld_rxq->iq);
4048 			quiesce_fl(sc, &ofld_rxq->fl);
4049 		}
4050 #endif
4051 		free(vi->rss, M_CXGBE);
4052 	}
4053 #ifdef DEV_NETMAP
4054 skip:
4055 #endif
4056 
4057 	t4_teardown_vi_queues(vi);
4058 	vi->flags &= ~VI_INIT_DONE;
4059 
4060 	return (0);
4061 }
4062 
4063 static void
4064 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
4065 {
4066 	struct sge_eq *eq = &txq->eq;
4067 	struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4068 
4069 	(void) sc;	/* unused */
4070 
4071 #ifdef INVARIANTS
4072 	TXQ_LOCK(txq);
4073 	MPASS((eq->flags & EQ_ENABLED) == 0);
4074 	TXQ_UNLOCK(txq);
4075 #endif
4076 
4077 	/* Wait for the mp_ring to empty. */
4078 	while (!mp_ring_is_idle(txq->r)) {
4079 		mp_ring_check_drainage(txq->r, 0);
4080 		pause("rquiesce", 1);
4081 	}
4082 
4083 	/* Then wait for the hardware to finish. */
4084 	while (spg->cidx != htobe16(eq->pidx))
4085 		pause("equiesce", 1);
4086 
4087 	/* Finally, wait for the driver to reclaim all descriptors. */
4088 	while (eq->cidx != eq->pidx)
4089 		pause("dquiesce", 1);
4090 }
4091 
4092 static void
4093 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
4094 {
4095 
4096 	/* XXXTX */
4097 }
4098 
4099 static void
4100 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
4101 {
4102 	(void) sc;	/* unused */
4103 
4104 	/* Synchronize with the interrupt handler */
4105 	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
4106 		pause("iqfree", 1);
4107 }
4108 
4109 static void
4110 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
4111 {
4112 	mtx_lock(&sc->sfl_lock);
4113 	FL_LOCK(fl);
4114 	fl->flags |= FL_DOOMED;
4115 	FL_UNLOCK(fl);
4116 	callout_stop(&sc->sfl_callout);
4117 	mtx_unlock(&sc->sfl_lock);
4118 
4119 	KASSERT((fl->flags & FL_STARVING) == 0,
4120 	    ("%s: still starving", __func__));
4121 }
4122 
4123 static int
4124 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
4125     driver_intr_t *handler, void *arg, char *name)
4126 {
4127 	int rc;
4128 
4129 	irq->rid = rid;
4130 	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
4131 	    RF_SHAREABLE | RF_ACTIVE);
4132 	if (irq->res == NULL) {
4133 		device_printf(sc->dev,
4134 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
4135 		return (ENOMEM);
4136 	}
4137 
4138 	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
4139 	    NULL, handler, arg, &irq->tag);
4140 	if (rc != 0) {
4141 		device_printf(sc->dev,
4142 		    "failed to setup interrupt for rid %d, name %s: %d\n",
4143 		    rid, name, rc);
4144 	} else if (name)
4145 		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
4146 
4147 	return (rc);
4148 }
4149 
4150 static int
4151 t4_free_irq(struct adapter *sc, struct irq *irq)
4152 {
4153 	if (irq->tag)
4154 		bus_teardown_intr(sc->dev, irq->res, irq->tag);
4155 	if (irq->res)
4156 		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
4157 
4158 	bzero(irq, sizeof(*irq));
4159 
4160 	return (0);
4161 }
4162 
4163 static void
4164 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
4165     unsigned int end)
4166 {
4167 	uint32_t *p = (uint32_t *)(buf + start);
4168 
4169 	for ( ; start <= end; start += sizeof(uint32_t))
4170 		*p++ = t4_read_reg(sc, start);
4171 }
4172 
4173 static void
4174 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
4175 {
4176 	int i, n;
4177 	const unsigned int *reg_ranges;
4178 	static const unsigned int t4_reg_ranges[] = {
4179 		0x1008, 0x1108,
4180 		0x1180, 0x11b4,
4181 		0x11fc, 0x123c,
4182 		0x1300, 0x173c,
4183 		0x1800, 0x18fc,
4184 		0x3000, 0x30d8,
4185 		0x30e0, 0x5924,
4186 		0x5960, 0x59d4,
4187 		0x5a00, 0x5af8,
4188 		0x6000, 0x6098,
4189 		0x6100, 0x6150,
4190 		0x6200, 0x6208,
4191 		0x6240, 0x6248,
4192 		0x6280, 0x6338,
4193 		0x6370, 0x638c,
4194 		0x6400, 0x643c,
4195 		0x6500, 0x6524,
4196 		0x6a00, 0x6a38,
4197 		0x6a60, 0x6a78,
4198 		0x6b00, 0x6b84,
4199 		0x6bf0, 0x6c84,
4200 		0x6cf0, 0x6d84,
4201 		0x6df0, 0x6e84,
4202 		0x6ef0, 0x6f84,
4203 		0x6ff0, 0x7084,
4204 		0x70f0, 0x7184,
4205 		0x71f0, 0x7284,
4206 		0x72f0, 0x7384,
4207 		0x73f0, 0x7450,
4208 		0x7500, 0x7530,
4209 		0x7600, 0x761c,
4210 		0x7680, 0x76cc,
4211 		0x7700, 0x7798,
4212 		0x77c0, 0x77fc,
4213 		0x7900, 0x79fc,
4214 		0x7b00, 0x7c38,
4215 		0x7d00, 0x7efc,
4216 		0x8dc0, 0x8e1c,
4217 		0x8e30, 0x8e78,
4218 		0x8ea0, 0x8f6c,
4219 		0x8fc0, 0x9074,
4220 		0x90fc, 0x90fc,
4221 		0x9400, 0x9458,
4222 		0x9600, 0x96bc,
4223 		0x9800, 0x9808,
4224 		0x9820, 0x983c,
4225 		0x9850, 0x9864,
4226 		0x9c00, 0x9c6c,
4227 		0x9c80, 0x9cec,
4228 		0x9d00, 0x9d6c,
4229 		0x9d80, 0x9dec,
4230 		0x9e00, 0x9e6c,
4231 		0x9e80, 0x9eec,
4232 		0x9f00, 0x9f6c,
4233 		0x9f80, 0x9fec,
4234 		0xd004, 0xd03c,
4235 		0xdfc0, 0xdfe0,
4236 		0xe000, 0xea7c,
4237 		0xf000, 0x11110,
4238 		0x11118, 0x11190,
4239 		0x19040, 0x1906c,
4240 		0x19078, 0x19080,
4241 		0x1908c, 0x19124,
4242 		0x19150, 0x191b0,
4243 		0x191d0, 0x191e8,
4244 		0x19238, 0x1924c,
4245 		0x193f8, 0x19474,
4246 		0x19490, 0x194f8,
4247 		0x19800, 0x19f30,
4248 		0x1a000, 0x1a06c,
4249 		0x1a0b0, 0x1a120,
4250 		0x1a128, 0x1a138,
4251 		0x1a190, 0x1a1c4,
4252 		0x1a1fc, 0x1a1fc,
4253 		0x1e040, 0x1e04c,
4254 		0x1e284, 0x1e28c,
4255 		0x1e2c0, 0x1e2c0,
4256 		0x1e2e0, 0x1e2e0,
4257 		0x1e300, 0x1e384,
4258 		0x1e3c0, 0x1e3c8,
4259 		0x1e440, 0x1e44c,
4260 		0x1e684, 0x1e68c,
4261 		0x1e6c0, 0x1e6c0,
4262 		0x1e6e0, 0x1e6e0,
4263 		0x1e700, 0x1e784,
4264 		0x1e7c0, 0x1e7c8,
4265 		0x1e840, 0x1e84c,
4266 		0x1ea84, 0x1ea8c,
4267 		0x1eac0, 0x1eac0,
4268 		0x1eae0, 0x1eae0,
4269 		0x1eb00, 0x1eb84,
4270 		0x1ebc0, 0x1ebc8,
4271 		0x1ec40, 0x1ec4c,
4272 		0x1ee84, 0x1ee8c,
4273 		0x1eec0, 0x1eec0,
4274 		0x1eee0, 0x1eee0,
4275 		0x1ef00, 0x1ef84,
4276 		0x1efc0, 0x1efc8,
4277 		0x1f040, 0x1f04c,
4278 		0x1f284, 0x1f28c,
4279 		0x1f2c0, 0x1f2c0,
4280 		0x1f2e0, 0x1f2e0,
4281 		0x1f300, 0x1f384,
4282 		0x1f3c0, 0x1f3c8,
4283 		0x1f440, 0x1f44c,
4284 		0x1f684, 0x1f68c,
4285 		0x1f6c0, 0x1f6c0,
4286 		0x1f6e0, 0x1f6e0,
4287 		0x1f700, 0x1f784,
4288 		0x1f7c0, 0x1f7c8,
4289 		0x1f840, 0x1f84c,
4290 		0x1fa84, 0x1fa8c,
4291 		0x1fac0, 0x1fac0,
4292 		0x1fae0, 0x1fae0,
4293 		0x1fb00, 0x1fb84,
4294 		0x1fbc0, 0x1fbc8,
4295 		0x1fc40, 0x1fc4c,
4296 		0x1fe84, 0x1fe8c,
4297 		0x1fec0, 0x1fec0,
4298 		0x1fee0, 0x1fee0,
4299 		0x1ff00, 0x1ff84,
4300 		0x1ffc0, 0x1ffc8,
4301 		0x20000, 0x2002c,
4302 		0x20100, 0x2013c,
4303 		0x20190, 0x201c8,
4304 		0x20200, 0x20318,
4305 		0x20400, 0x20528,
4306 		0x20540, 0x20614,
4307 		0x21000, 0x21040,
4308 		0x2104c, 0x21060,
4309 		0x210c0, 0x210ec,
4310 		0x21200, 0x21268,
4311 		0x21270, 0x21284,
4312 		0x212fc, 0x21388,
4313 		0x21400, 0x21404,
4314 		0x21500, 0x21518,
4315 		0x2152c, 0x2153c,
4316 		0x21550, 0x21554,
4317 		0x21600, 0x21600,
4318 		0x21608, 0x21628,
4319 		0x21630, 0x2163c,
4320 		0x21700, 0x2171c,
4321 		0x21780, 0x2178c,
4322 		0x21800, 0x21c38,
4323 		0x21c80, 0x21d7c,
4324 		0x21e00, 0x21e04,
4325 		0x22000, 0x2202c,
4326 		0x22100, 0x2213c,
4327 		0x22190, 0x221c8,
4328 		0x22200, 0x22318,
4329 		0x22400, 0x22528,
4330 		0x22540, 0x22614,
4331 		0x23000, 0x23040,
4332 		0x2304c, 0x23060,
4333 		0x230c0, 0x230ec,
4334 		0x23200, 0x23268,
4335 		0x23270, 0x23284,
4336 		0x232fc, 0x23388,
4337 		0x23400, 0x23404,
4338 		0x23500, 0x23518,
4339 		0x2352c, 0x2353c,
4340 		0x23550, 0x23554,
4341 		0x23600, 0x23600,
4342 		0x23608, 0x23628,
4343 		0x23630, 0x2363c,
4344 		0x23700, 0x2371c,
4345 		0x23780, 0x2378c,
4346 		0x23800, 0x23c38,
4347 		0x23c80, 0x23d7c,
4348 		0x23e00, 0x23e04,
4349 		0x24000, 0x2402c,
4350 		0x24100, 0x2413c,
4351 		0x24190, 0x241c8,
4352 		0x24200, 0x24318,
4353 		0x24400, 0x24528,
4354 		0x24540, 0x24614,
4355 		0x25000, 0x25040,
4356 		0x2504c, 0x25060,
4357 		0x250c0, 0x250ec,
4358 		0x25200, 0x25268,
4359 		0x25270, 0x25284,
4360 		0x252fc, 0x25388,
4361 		0x25400, 0x25404,
4362 		0x25500, 0x25518,
4363 		0x2552c, 0x2553c,
4364 		0x25550, 0x25554,
4365 		0x25600, 0x25600,
4366 		0x25608, 0x25628,
4367 		0x25630, 0x2563c,
4368 		0x25700, 0x2571c,
4369 		0x25780, 0x2578c,
4370 		0x25800, 0x25c38,
4371 		0x25c80, 0x25d7c,
4372 		0x25e00, 0x25e04,
4373 		0x26000, 0x2602c,
4374 		0x26100, 0x2613c,
4375 		0x26190, 0x261c8,
4376 		0x26200, 0x26318,
4377 		0x26400, 0x26528,
4378 		0x26540, 0x26614,
4379 		0x27000, 0x27040,
4380 		0x2704c, 0x27060,
4381 		0x270c0, 0x270ec,
4382 		0x27200, 0x27268,
4383 		0x27270, 0x27284,
4384 		0x272fc, 0x27388,
4385 		0x27400, 0x27404,
4386 		0x27500, 0x27518,
4387 		0x2752c, 0x2753c,
4388 		0x27550, 0x27554,
4389 		0x27600, 0x27600,
4390 		0x27608, 0x27628,
4391 		0x27630, 0x2763c,
4392 		0x27700, 0x2771c,
4393 		0x27780, 0x2778c,
4394 		0x27800, 0x27c38,
4395 		0x27c80, 0x27d7c,
4396 		0x27e00, 0x27e04
4397 	};
4398 	static const unsigned int t5_reg_ranges[] = {
4399 		0x1008, 0x1148,
4400 		0x1180, 0x11b4,
4401 		0x11fc, 0x123c,
4402 		0x1280, 0x173c,
4403 		0x1800, 0x18fc,
4404 		0x3000, 0x3028,
4405 		0x3060, 0x30d8,
4406 		0x30e0, 0x30fc,
4407 		0x3140, 0x357c,
4408 		0x35a8, 0x35cc,
4409 		0x35ec, 0x35ec,
4410 		0x3600, 0x5624,
4411 		0x56cc, 0x575c,
4412 		0x580c, 0x5814,
4413 		0x5890, 0x58bc,
4414 		0x5940, 0x59dc,
4415 		0x59fc, 0x5a18,
4416 		0x5a60, 0x5a9c,
4417 		0x5b94, 0x5bfc,
4418 		0x6000, 0x6040,
4419 		0x6058, 0x614c,
4420 		0x7700, 0x7798,
4421 		0x77c0, 0x78fc,
4422 		0x7b00, 0x7c54,
4423 		0x7d00, 0x7efc,
4424 		0x8dc0, 0x8de0,
4425 		0x8df8, 0x8e84,
4426 		0x8ea0, 0x8f84,
4427 		0x8fc0, 0x90f8,
4428 		0x9400, 0x9470,
4429 		0x9600, 0x96f4,
4430 		0x9800, 0x9808,
4431 		0x9820, 0x983c,
4432 		0x9850, 0x9864,
4433 		0x9c00, 0x9c6c,
4434 		0x9c80, 0x9cec,
4435 		0x9d00, 0x9d6c,
4436 		0x9d80, 0x9dec,
4437 		0x9e00, 0x9e6c,
4438 		0x9e80, 0x9eec,
4439 		0x9f00, 0x9f6c,
4440 		0x9f80, 0xa020,
4441 		0xd004, 0xd03c,
4442 		0xdfc0, 0xdfe0,
4443 		0xe000, 0x11088,
4444 		0x1109c, 0x11110,
4445 		0x11118, 0x1117c,
4446 		0x11190, 0x11204,
4447 		0x19040, 0x1906c,
4448 		0x19078, 0x19080,
4449 		0x1908c, 0x19124,
4450 		0x19150, 0x191b0,
4451 		0x191d0, 0x191e8,
4452 		0x19238, 0x19290,
4453 		0x193f8, 0x19474,
4454 		0x19490, 0x194cc,
4455 		0x194f0, 0x194f8,
4456 		0x19c00, 0x19c60,
4457 		0x19c94, 0x19e10,
4458 		0x19e50, 0x19f34,
4459 		0x19f40, 0x19f50,
4460 		0x19f90, 0x19fe4,
4461 		0x1a000, 0x1a06c,
4462 		0x1a0b0, 0x1a120,
4463 		0x1a128, 0x1a138,
4464 		0x1a190, 0x1a1c4,
4465 		0x1a1fc, 0x1a1fc,
4466 		0x1e008, 0x1e00c,
4467 		0x1e040, 0x1e04c,
4468 		0x1e284, 0x1e290,
4469 		0x1e2c0, 0x1e2c0,
4470 		0x1e2e0, 0x1e2e0,
4471 		0x1e300, 0x1e384,
4472 		0x1e3c0, 0x1e3c8,
4473 		0x1e408, 0x1e40c,
4474 		0x1e440, 0x1e44c,
4475 		0x1e684, 0x1e690,
4476 		0x1e6c0, 0x1e6c0,
4477 		0x1e6e0, 0x1e6e0,
4478 		0x1e700, 0x1e784,
4479 		0x1e7c0, 0x1e7c8,
4480 		0x1e808, 0x1e80c,
4481 		0x1e840, 0x1e84c,
4482 		0x1ea84, 0x1ea90,
4483 		0x1eac0, 0x1eac0,
4484 		0x1eae0, 0x1eae0,
4485 		0x1eb00, 0x1eb84,
4486 		0x1ebc0, 0x1ebc8,
4487 		0x1ec08, 0x1ec0c,
4488 		0x1ec40, 0x1ec4c,
4489 		0x1ee84, 0x1ee90,
4490 		0x1eec0, 0x1eec0,
4491 		0x1eee0, 0x1eee0,
4492 		0x1ef00, 0x1ef84,
4493 		0x1efc0, 0x1efc8,
4494 		0x1f008, 0x1f00c,
4495 		0x1f040, 0x1f04c,
4496 		0x1f284, 0x1f290,
4497 		0x1f2c0, 0x1f2c0,
4498 		0x1f2e0, 0x1f2e0,
4499 		0x1f300, 0x1f384,
4500 		0x1f3c0, 0x1f3c8,
4501 		0x1f408, 0x1f40c,
4502 		0x1f440, 0x1f44c,
4503 		0x1f684, 0x1f690,
4504 		0x1f6c0, 0x1f6c0,
4505 		0x1f6e0, 0x1f6e0,
4506 		0x1f700, 0x1f784,
4507 		0x1f7c0, 0x1f7c8,
4508 		0x1f808, 0x1f80c,
4509 		0x1f840, 0x1f84c,
4510 		0x1fa84, 0x1fa90,
4511 		0x1fac0, 0x1fac0,
4512 		0x1fae0, 0x1fae0,
4513 		0x1fb00, 0x1fb84,
4514 		0x1fbc0, 0x1fbc8,
4515 		0x1fc08, 0x1fc0c,
4516 		0x1fc40, 0x1fc4c,
4517 		0x1fe84, 0x1fe90,
4518 		0x1fec0, 0x1fec0,
4519 		0x1fee0, 0x1fee0,
4520 		0x1ff00, 0x1ff84,
4521 		0x1ffc0, 0x1ffc8,
4522 		0x30000, 0x30030,
4523 		0x30100, 0x30144,
4524 		0x30190, 0x301d0,
4525 		0x30200, 0x30318,
4526 		0x30400, 0x3052c,
4527 		0x30540, 0x3061c,
4528 		0x30800, 0x30834,
4529 		0x308c0, 0x30908,
4530 		0x30910, 0x309ac,
4531 		0x30a00, 0x30a2c,
4532 		0x30a44, 0x30a50,
4533 		0x30a74, 0x30c24,
4534 		0x30d00, 0x30d00,
4535 		0x30d08, 0x30d14,
4536 		0x30d1c, 0x30d20,
4537 		0x30d3c, 0x30d50,
4538 		0x31200, 0x3120c,
4539 		0x31220, 0x31220,
4540 		0x31240, 0x31240,
4541 		0x31600, 0x3160c,
4542 		0x31a00, 0x31a1c,
4543 		0x31e00, 0x31e20,
4544 		0x31e38, 0x31e3c,
4545 		0x31e80, 0x31e80,
4546 		0x31e88, 0x31ea8,
4547 		0x31eb0, 0x31eb4,
4548 		0x31ec8, 0x31ed4,
4549 		0x31fb8, 0x32004,
4550 		0x32200, 0x32200,
4551 		0x32208, 0x32240,
4552 		0x32248, 0x32280,
4553 		0x32288, 0x322c0,
4554 		0x322c8, 0x322fc,
4555 		0x32600, 0x32630,
4556 		0x32a00, 0x32abc,
4557 		0x32b00, 0x32b70,
4558 		0x33000, 0x33048,
4559 		0x33060, 0x3309c,
4560 		0x330f0, 0x33148,
4561 		0x33160, 0x3319c,
4562 		0x331f0, 0x332e4,
4563 		0x332f8, 0x333e4,
4564 		0x333f8, 0x33448,
4565 		0x33460, 0x3349c,
4566 		0x334f0, 0x33548,
4567 		0x33560, 0x3359c,
4568 		0x335f0, 0x336e4,
4569 		0x336f8, 0x337e4,
4570 		0x337f8, 0x337fc,
4571 		0x33814, 0x33814,
4572 		0x3382c, 0x3382c,
4573 		0x33880, 0x3388c,
4574 		0x338e8, 0x338ec,
4575 		0x33900, 0x33948,
4576 		0x33960, 0x3399c,
4577 		0x339f0, 0x33ae4,
4578 		0x33af8, 0x33b10,
4579 		0x33b28, 0x33b28,
4580 		0x33b3c, 0x33b50,
4581 		0x33bf0, 0x33c10,
4582 		0x33c28, 0x33c28,
4583 		0x33c3c, 0x33c50,
4584 		0x33cf0, 0x33cfc,
4585 		0x34000, 0x34030,
4586 		0x34100, 0x34144,
4587 		0x34190, 0x341d0,
4588 		0x34200, 0x34318,
4589 		0x34400, 0x3452c,
4590 		0x34540, 0x3461c,
4591 		0x34800, 0x34834,
4592 		0x348c0, 0x34908,
4593 		0x34910, 0x349ac,
4594 		0x34a00, 0x34a2c,
4595 		0x34a44, 0x34a50,
4596 		0x34a74, 0x34c24,
4597 		0x34d00, 0x34d00,
4598 		0x34d08, 0x34d14,
4599 		0x34d1c, 0x34d20,
4600 		0x34d3c, 0x34d50,
4601 		0x35200, 0x3520c,
4602 		0x35220, 0x35220,
4603 		0x35240, 0x35240,
4604 		0x35600, 0x3560c,
4605 		0x35a00, 0x35a1c,
4606 		0x35e00, 0x35e20,
4607 		0x35e38, 0x35e3c,
4608 		0x35e80, 0x35e80,
4609 		0x35e88, 0x35ea8,
4610 		0x35eb0, 0x35eb4,
4611 		0x35ec8, 0x35ed4,
4612 		0x35fb8, 0x36004,
4613 		0x36200, 0x36200,
4614 		0x36208, 0x36240,
4615 		0x36248, 0x36280,
4616 		0x36288, 0x362c0,
4617 		0x362c8, 0x362fc,
4618 		0x36600, 0x36630,
4619 		0x36a00, 0x36abc,
4620 		0x36b00, 0x36b70,
4621 		0x37000, 0x37048,
4622 		0x37060, 0x3709c,
4623 		0x370f0, 0x37148,
4624 		0x37160, 0x3719c,
4625 		0x371f0, 0x372e4,
4626 		0x372f8, 0x373e4,
4627 		0x373f8, 0x37448,
4628 		0x37460, 0x3749c,
4629 		0x374f0, 0x37548,
4630 		0x37560, 0x3759c,
4631 		0x375f0, 0x376e4,
4632 		0x376f8, 0x377e4,
4633 		0x377f8, 0x377fc,
4634 		0x37814, 0x37814,
4635 		0x3782c, 0x3782c,
4636 		0x37880, 0x3788c,
4637 		0x378e8, 0x378ec,
4638 		0x37900, 0x37948,
4639 		0x37960, 0x3799c,
4640 		0x379f0, 0x37ae4,
4641 		0x37af8, 0x37b10,
4642 		0x37b28, 0x37b28,
4643 		0x37b3c, 0x37b50,
4644 		0x37bf0, 0x37c10,
4645 		0x37c28, 0x37c28,
4646 		0x37c3c, 0x37c50,
4647 		0x37cf0, 0x37cfc,
4648 		0x38000, 0x38030,
4649 		0x38100, 0x38144,
4650 		0x38190, 0x381d0,
4651 		0x38200, 0x38318,
4652 		0x38400, 0x3852c,
4653 		0x38540, 0x3861c,
4654 		0x38800, 0x38834,
4655 		0x388c0, 0x38908,
4656 		0x38910, 0x389ac,
4657 		0x38a00, 0x38a2c,
4658 		0x38a44, 0x38a50,
4659 		0x38a74, 0x38c24,
4660 		0x38d00, 0x38d00,
4661 		0x38d08, 0x38d14,
4662 		0x38d1c, 0x38d20,
4663 		0x38d3c, 0x38d50,
4664 		0x39200, 0x3920c,
4665 		0x39220, 0x39220,
4666 		0x39240, 0x39240,
4667 		0x39600, 0x3960c,
4668 		0x39a00, 0x39a1c,
4669 		0x39e00, 0x39e20,
4670 		0x39e38, 0x39e3c,
4671 		0x39e80, 0x39e80,
4672 		0x39e88, 0x39ea8,
4673 		0x39eb0, 0x39eb4,
4674 		0x39ec8, 0x39ed4,
4675 		0x39fb8, 0x3a004,
4676 		0x3a200, 0x3a200,
4677 		0x3a208, 0x3a240,
4678 		0x3a248, 0x3a280,
4679 		0x3a288, 0x3a2c0,
4680 		0x3a2c8, 0x3a2fc,
4681 		0x3a600, 0x3a630,
4682 		0x3aa00, 0x3aabc,
4683 		0x3ab00, 0x3ab70,
4684 		0x3b000, 0x3b048,
4685 		0x3b060, 0x3b09c,
4686 		0x3b0f0, 0x3b148,
4687 		0x3b160, 0x3b19c,
4688 		0x3b1f0, 0x3b2e4,
4689 		0x3b2f8, 0x3b3e4,
4690 		0x3b3f8, 0x3b448,
4691 		0x3b460, 0x3b49c,
4692 		0x3b4f0, 0x3b548,
4693 		0x3b560, 0x3b59c,
4694 		0x3b5f0, 0x3b6e4,
4695 		0x3b6f8, 0x3b7e4,
4696 		0x3b7f8, 0x3b7fc,
4697 		0x3b814, 0x3b814,
4698 		0x3b82c, 0x3b82c,
4699 		0x3b880, 0x3b88c,
4700 		0x3b8e8, 0x3b8ec,
4701 		0x3b900, 0x3b948,
4702 		0x3b960, 0x3b99c,
4703 		0x3b9f0, 0x3bae4,
4704 		0x3baf8, 0x3bb10,
4705 		0x3bb28, 0x3bb28,
4706 		0x3bb3c, 0x3bb50,
4707 		0x3bbf0, 0x3bc10,
4708 		0x3bc28, 0x3bc28,
4709 		0x3bc3c, 0x3bc50,
4710 		0x3bcf0, 0x3bcfc,
4711 		0x3c000, 0x3c030,
4712 		0x3c100, 0x3c144,
4713 		0x3c190, 0x3c1d0,
4714 		0x3c200, 0x3c318,
4715 		0x3c400, 0x3c52c,
4716 		0x3c540, 0x3c61c,
4717 		0x3c800, 0x3c834,
4718 		0x3c8c0, 0x3c908,
4719 		0x3c910, 0x3c9ac,
4720 		0x3ca00, 0x3ca2c,
4721 		0x3ca44, 0x3ca50,
4722 		0x3ca74, 0x3cc24,
4723 		0x3cd00, 0x3cd00,
4724 		0x3cd08, 0x3cd14,
4725 		0x3cd1c, 0x3cd20,
4726 		0x3cd3c, 0x3cd50,
4727 		0x3d200, 0x3d20c,
4728 		0x3d220, 0x3d220,
4729 		0x3d240, 0x3d240,
4730 		0x3d600, 0x3d60c,
4731 		0x3da00, 0x3da1c,
4732 		0x3de00, 0x3de20,
4733 		0x3de38, 0x3de3c,
4734 		0x3de80, 0x3de80,
4735 		0x3de88, 0x3dea8,
4736 		0x3deb0, 0x3deb4,
4737 		0x3dec8, 0x3ded4,
4738 		0x3dfb8, 0x3e004,
4739 		0x3e200, 0x3e200,
4740 		0x3e208, 0x3e240,
4741 		0x3e248, 0x3e280,
4742 		0x3e288, 0x3e2c0,
4743 		0x3e2c8, 0x3e2fc,
4744 		0x3e600, 0x3e630,
4745 		0x3ea00, 0x3eabc,
4746 		0x3eb00, 0x3eb70,
4747 		0x3f000, 0x3f048,
4748 		0x3f060, 0x3f09c,
4749 		0x3f0f0, 0x3f148,
4750 		0x3f160, 0x3f19c,
4751 		0x3f1f0, 0x3f2e4,
4752 		0x3f2f8, 0x3f3e4,
4753 		0x3f3f8, 0x3f448,
4754 		0x3f460, 0x3f49c,
4755 		0x3f4f0, 0x3f548,
4756 		0x3f560, 0x3f59c,
4757 		0x3f5f0, 0x3f6e4,
4758 		0x3f6f8, 0x3f7e4,
4759 		0x3f7f8, 0x3f7fc,
4760 		0x3f814, 0x3f814,
4761 		0x3f82c, 0x3f82c,
4762 		0x3f880, 0x3f88c,
4763 		0x3f8e8, 0x3f8ec,
4764 		0x3f900, 0x3f948,
4765 		0x3f960, 0x3f99c,
4766 		0x3f9f0, 0x3fae4,
4767 		0x3faf8, 0x3fb10,
4768 		0x3fb28, 0x3fb28,
4769 		0x3fb3c, 0x3fb50,
4770 		0x3fbf0, 0x3fc10,
4771 		0x3fc28, 0x3fc28,
4772 		0x3fc3c, 0x3fc50,
4773 		0x3fcf0, 0x3fcfc,
4774 		0x40000, 0x4000c,
4775 		0x40040, 0x40068,
4776 		0x4007c, 0x40144,
4777 		0x40180, 0x4018c,
4778 		0x40200, 0x40298,
4779 		0x402ac, 0x4033c,
4780 		0x403f8, 0x403fc,
4781 		0x41304, 0x413c4,
4782 		0x41400, 0x4141c,
4783 		0x41480, 0x414d0,
4784 		0x44000, 0x44078,
4785 		0x440c0, 0x44278,
4786 		0x442c0, 0x44478,
4787 		0x444c0, 0x44678,
4788 		0x446c0, 0x44878,
4789 		0x448c0, 0x449fc,
4790 		0x45000, 0x45068,
4791 		0x45080, 0x45084,
4792 		0x450a0, 0x450b0,
4793 		0x45200, 0x45268,
4794 		0x45280, 0x45284,
4795 		0x452a0, 0x452b0,
4796 		0x460c0, 0x460e4,
4797 		0x47000, 0x4708c,
4798 		0x47200, 0x47250,
4799 		0x47400, 0x47420,
4800 		0x47600, 0x47618,
4801 		0x47800, 0x47814,
4802 		0x48000, 0x4800c,
4803 		0x48040, 0x48068,
4804 		0x4807c, 0x48144,
4805 		0x48180, 0x4818c,
4806 		0x48200, 0x48298,
4807 		0x482ac, 0x4833c,
4808 		0x483f8, 0x483fc,
4809 		0x49304, 0x493c4,
4810 		0x49400, 0x4941c,
4811 		0x49480, 0x494d0,
4812 		0x4c000, 0x4c078,
4813 		0x4c0c0, 0x4c278,
4814 		0x4c2c0, 0x4c478,
4815 		0x4c4c0, 0x4c678,
4816 		0x4c6c0, 0x4c878,
4817 		0x4c8c0, 0x4c9fc,
4818 		0x4d000, 0x4d068,
4819 		0x4d080, 0x4d084,
4820 		0x4d0a0, 0x4d0b0,
4821 		0x4d200, 0x4d268,
4822 		0x4d280, 0x4d284,
4823 		0x4d2a0, 0x4d2b0,
4824 		0x4e0c0, 0x4e0e4,
4825 		0x4f000, 0x4f08c,
4826 		0x4f200, 0x4f250,
4827 		0x4f400, 0x4f420,
4828 		0x4f600, 0x4f618,
4829 		0x4f800, 0x4f814,
4830 		0x50000, 0x500cc,
4831 		0x50400, 0x50400,
4832 		0x50800, 0x508cc,
4833 		0x50c00, 0x50c00,
4834 		0x51000, 0x5101c,
4835 		0x51300, 0x51308,
4836 	};
4837 
4838 	if (is_t4(sc)) {
4839 		reg_ranges = &t4_reg_ranges[0];
4840 		n = nitems(t4_reg_ranges);
4841 	} else {
4842 		reg_ranges = &t5_reg_ranges[0];
4843 		n = nitems(t5_reg_ranges);
4844 	}
4845 
4846 	regs->version = chip_id(sc) | chip_rev(sc) << 10;
4847 	for (i = 0; i < n; i += 2)
4848 		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4849 }
4850 
4851 #define	A_PL_INDIR_CMD	0x1f8
4852 
4853 #define	S_PL_AUTOINC	31
4854 #define	M_PL_AUTOINC	0x1U
4855 #define	V_PL_AUTOINC(x)	((x) << S_PL_AUTOINC)
4856 #define	G_PL_AUTOINC(x)	(((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
4857 
4858 #define	S_PL_VFID	20
4859 #define	M_PL_VFID	0xffU
4860 #define	V_PL_VFID(x)	((x) << S_PL_VFID)
4861 #define	G_PL_VFID(x)	(((x) >> S_PL_VFID) & M_PL_VFID)
4862 
4863 #define	S_PL_ADDR	0
4864 #define	M_PL_ADDR	0xfffffU
4865 #define	V_PL_ADDR(x)	((x) << S_PL_ADDR)
4866 #define	G_PL_ADDR(x)	(((x) >> S_PL_ADDR) & M_PL_ADDR)
4867 
4868 #define	A_PL_INDIR_DATA	0x1fc
4869 
4870 static uint64_t
4871 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
4872 {
4873 	u32 stats[2];
4874 
4875 	mtx_assert(&sc->regwin_lock, MA_OWNED);
4876 	t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4877 	    V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg)));
4878 	stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
4879 	stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
4880 	return (((uint64_t)stats[1]) << 32 | stats[0]);
4881 }
4882 
4883 static void
4884 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
4885     struct fw_vi_stats_vf *stats)
4886 {
4887 
4888 #define GET_STAT(name) \
4889 	read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
4890 
4891 	stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
4892 	stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
4893 	stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
4894 	stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
4895 	stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
4896 	stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
4897 	stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
4898 	stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
4899 	stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
4900 	stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
4901 	stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
4902 	stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
4903 	stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
4904 	stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
4905 	stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
4906 	stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
4907 
4908 #undef GET_STAT
4909 }
4910 
4911 static void
4912 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
4913 {
4914 	int reg;
4915 
4916 	t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4917 	    V_PL_VFID(G_FW_VIID_VIN(viid)) |
4918 	    V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
4919 	for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
4920 	     reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
4921 		t4_write_reg(sc, A_PL_INDIR_DATA, 0);
4922 }
4923 
4924 static void
4925 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
4926 {
4927 	struct timeval tv;
4928 	const struct timeval interval = {0, 250000};	/* 250ms */
4929 
4930 	if (!(vi->flags & VI_INIT_DONE))
4931 		return;
4932 
4933 	getmicrotime(&tv);
4934 	timevalsub(&tv, &interval);
4935 	if (timevalcmp(&tv, &vi->last_refreshed, <))
4936 		return;
4937 
4938 	mtx_lock(&sc->regwin_lock);
4939 	t4_get_vi_stats(sc, vi->viid, &vi->stats);
4940 	getmicrotime(&vi->last_refreshed);
4941 	mtx_unlock(&sc->regwin_lock);
4942 }
4943 
4944 static void
4945 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4946 {
4947 	int i;
4948 	u_int v, tnl_cong_drops;
4949 	struct timeval tv;
4950 	const struct timeval interval = {0, 250000};	/* 250ms */
4951 
4952 	getmicrotime(&tv);
4953 	timevalsub(&tv, &interval);
4954 	if (timevalcmp(&tv, &pi->last_refreshed, <))
4955 		return;
4956 
4957 	tnl_cong_drops = 0;
4958 	t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4959 	for (i = 0; i < NCHAN; i++) {
4960 		if (pi->rx_chan_map & (1 << i)) {
4961 			mtx_lock(&sc->regwin_lock);
4962 			t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4963 			    1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4964 			mtx_unlock(&sc->regwin_lock);
4965 			tnl_cong_drops += v;
4966 		}
4967 	}
4968 	pi->tnl_cong_drops = tnl_cong_drops;
4969 	getmicrotime(&pi->last_refreshed);
4970 }
4971 
4972 static void
4973 cxgbe_tick(void *arg)
4974 {
4975 	struct port_info *pi = arg;
4976 	struct adapter *sc = pi->adapter;
4977 
4978 	PORT_LOCK_ASSERT_OWNED(pi);
4979 	cxgbe_refresh_stats(sc, pi);
4980 
4981 	callout_schedule(&pi->tick, hz);
4982 }
4983 
4984 void
4985 vi_tick(void *arg)
4986 {
4987 	struct vi_info *vi = arg;
4988 	struct adapter *sc = vi->pi->adapter;
4989 
4990 	vi_refresh_stats(sc, vi);
4991 
4992 	callout_schedule(&vi->tick, hz);
4993 }
4994 
4995 static void
4996 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4997 {
4998 	struct ifnet *vlan;
4999 
5000 	if (arg != ifp || ifp->if_type != IFT_ETHER)
5001 		return;
5002 
5003 	vlan = VLAN_DEVAT(ifp, vid);
5004 	VLAN_SETCOOKIE(vlan, ifp);
5005 }
5006 
5007 static int
5008 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5009 {
5010 
5011 #ifdef INVARIANTS
5012 	panic("%s: opcode 0x%02x on iq %p with payload %p",
5013 	    __func__, rss->opcode, iq, m);
5014 #else
5015 	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
5016 	    __func__, rss->opcode, iq, m);
5017 	m_freem(m);
5018 #endif
5019 	return (EDOOFUS);
5020 }
5021 
5022 int
5023 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
5024 {
5025 	uintptr_t *loc, new;
5026 
5027 	if (opcode >= nitems(sc->cpl_handler))
5028 		return (EINVAL);
5029 
5030 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
5031 	loc = (uintptr_t *) &sc->cpl_handler[opcode];
5032 	atomic_store_rel_ptr(loc, new);
5033 
5034 	return (0);
5035 }
5036 
5037 static int
5038 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
5039 {
5040 
5041 #ifdef INVARIANTS
5042 	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
5043 #else
5044 	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
5045 	    __func__, iq, ctrl);
5046 #endif
5047 	return (EDOOFUS);
5048 }
5049 
5050 int
5051 t4_register_an_handler(struct adapter *sc, an_handler_t h)
5052 {
5053 	uintptr_t *loc, new;
5054 
5055 	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
5056 	loc = (uintptr_t *) &sc->an_handler;
5057 	atomic_store_rel_ptr(loc, new);
5058 
5059 	return (0);
5060 }
5061 
5062 static int
5063 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
5064 {
5065 	const struct cpl_fw6_msg *cpl =
5066 	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
5067 
5068 #ifdef INVARIANTS
5069 	panic("%s: fw_msg type %d", __func__, cpl->type);
5070 #else
5071 	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
5072 #endif
5073 	return (EDOOFUS);
5074 }
5075 
5076 int
5077 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
5078 {
5079 	uintptr_t *loc, new;
5080 
5081 	if (type >= nitems(sc->fw_msg_handler))
5082 		return (EINVAL);
5083 
5084 	/*
5085 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
5086 	 * handler dispatch table.  Reject any attempt to install a handler for
5087 	 * this subtype.
5088 	 */
5089 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
5090 		return (EINVAL);
5091 
5092 	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
5093 	loc = (uintptr_t *) &sc->fw_msg_handler[type];
5094 	atomic_store_rel_ptr(loc, new);
5095 
5096 	return (0);
5097 }
5098 
5099 static void
5100 t4_sysctls(struct adapter *sc)
5101 {
5102 	struct sysctl_ctx_list *ctx;
5103 	struct sysctl_oid *oid;
5104 	struct sysctl_oid_list *children, *c0;
5105 	static char *caps[] = {
5106 		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
5107 		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"	/* caps[1] niccaps */
5108 		    "\6HASHFILTER\7ETHOFLD",
5109 		"\20\1TOE",				/* caps[2] toecaps */
5110 		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
5111 		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
5112 		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
5113 		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
5114 		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
5115 		    "\4PO_INITIAOR\5PO_TARGET"
5116 	};
5117 	static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
5118 
5119 	ctx = device_get_sysctl_ctx(sc->dev);
5120 
5121 	/*
5122 	 * dev.t4nex.X.
5123 	 */
5124 	oid = device_get_sysctl_tree(sc->dev);
5125 	c0 = children = SYSCTL_CHILDREN(oid);
5126 
5127 	sc->sc_do_rxcopy = 1;
5128 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
5129 	    &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
5130 
5131 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
5132 	    sc->params.nports, "# of ports");
5133 
5134 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
5135 	    NULL, chip_rev(sc), "chip hardware revision");
5136 
5137 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
5138 	    CTLFLAG_RD, sc->fw_version, 0, "firmware version");
5139 
5140 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
5141 	    CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
5142 
5143 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
5144 	    sc->cfcsum, "config file checksum");
5145 
5146 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
5147 	    CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
5148 	    sysctl_bitfield, "A", "available doorbells");
5149 
5150 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
5151 	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
5152 	    sysctl_bitfield, "A", "available link capabilities");
5153 
5154 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
5155 	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
5156 	    sysctl_bitfield, "A", "available NIC capabilities");
5157 
5158 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
5159 	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
5160 	    sysctl_bitfield, "A", "available TCP offload capabilities");
5161 
5162 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
5163 	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
5164 	    sysctl_bitfield, "A", "available RDMA capabilities");
5165 
5166 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
5167 	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
5168 	    sysctl_bitfield, "A", "available iSCSI capabilities");
5169 
5170 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
5171 	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
5172 	    sysctl_bitfield, "A", "available FCoE capabilities");
5173 
5174 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
5175 	    sc->params.vpd.cclk, "core clock frequency (in KHz)");
5176 
5177 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
5178 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
5179 	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
5180 	    "interrupt holdoff timer values (us)");
5181 
5182 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
5183 	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
5184 	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
5185 	    "interrupt holdoff packet counter values");
5186 
5187 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
5188 	    NULL, sc->tids.nftids, "number of filters");
5189 
5190 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
5191 	    CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
5192 	    "chip temperature (in Celsius)");
5193 
5194 	t4_sge_sysctls(sc, ctx, children);
5195 
5196 	sc->lro_timeout = 100;
5197 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
5198 	    &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
5199 
5200 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
5201 	    &sc->debug_flags, 0, "flags to enable runtime debugging");
5202 
5203 #ifdef SBUF_DRAIN
5204 	/*
5205 	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
5206 	 */
5207 	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
5208 	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
5209 	    "logs and miscellaneous information");
5210 	children = SYSCTL_CHILDREN(oid);
5211 
5212 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
5213 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5214 	    sysctl_cctrl, "A", "congestion control");
5215 
5216 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
5217 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5218 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
5219 
5220 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
5221 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
5222 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
5223 
5224 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
5225 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
5226 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
5227 
5228 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
5229 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
5230 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
5231 
5232 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
5233 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
5234 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
5235 
5236 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
5237 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
5238 	    sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
5239 
5240 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
5241 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5242 	    sysctl_cim_la, "A", "CIM logic analyzer");
5243 
5244 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
5245 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5246 	    sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
5247 
5248 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
5249 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
5250 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
5251 
5252 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
5253 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
5254 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
5255 
5256 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
5257 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
5258 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
5259 
5260 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
5261 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
5262 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
5263 
5264 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
5265 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
5266 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
5267 
5268 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
5269 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
5270 	    sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
5271 
5272 	if (is_t5(sc)) {
5273 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
5274 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
5275 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
5276 
5277 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
5278 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
5279 		    sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
5280 	}
5281 
5282 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
5283 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5284 	    sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
5285 
5286 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
5287 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5288 	    sysctl_cim_qcfg, "A", "CIM queue configuration");
5289 
5290 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5291 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5292 	    sysctl_cpl_stats, "A", "CPL statistics");
5293 
5294 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5295 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5296 	    sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5297 
5298 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5299 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5300 	    sysctl_devlog, "A", "firmware's device log");
5301 
5302 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5303 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5304 	    sysctl_fcoe_stats, "A", "FCoE statistics");
5305 
5306 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5307 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5308 	    sysctl_hw_sched, "A", "hardware scheduler ");
5309 
5310 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5311 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5312 	    sysctl_l2t, "A", "hardware L2 table");
5313 
5314 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5315 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5316 	    sysctl_lb_stats, "A", "loopback statistics");
5317 
5318 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5319 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5320 	    sysctl_meminfo, "A", "memory regions");
5321 
5322 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5323 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5324 	    sysctl_mps_tcam, "A", "MPS TCAM entries");
5325 
5326 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5327 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5328 	    sysctl_path_mtus, "A", "path MTUs");
5329 
5330 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5331 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5332 	    sysctl_pm_stats, "A", "PM statistics");
5333 
5334 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5335 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5336 	    sysctl_rdma_stats, "A", "RDMA statistics");
5337 
5338 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5339 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5340 	    sysctl_tcp_stats, "A", "TCP statistics");
5341 
5342 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5343 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5344 	    sysctl_tids, "A", "TID information");
5345 
5346 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5347 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5348 	    sysctl_tp_err_stats, "A", "TP error statistics");
5349 
5350 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5351 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5352 	    sysctl_tp_la, "A", "TP logic analyzer");
5353 
5354 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5355 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5356 	    sysctl_tx_rate, "A", "Tx rate");
5357 
5358 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5359 	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5360 	    sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5361 
5362 	if (is_t5(sc)) {
5363 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5364 		    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5365 		    sysctl_wcwr_stats, "A", "write combined work requests");
5366 	}
5367 #endif
5368 
5369 #ifdef TCP_OFFLOAD
5370 	if (is_offload(sc)) {
5371 		/*
5372 		 * dev.t4nex.X.toe.
5373 		 */
5374 		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5375 		    NULL, "TOE parameters");
5376 		children = SYSCTL_CHILDREN(oid);
5377 
5378 		sc->tt.sndbuf = 256 * 1024;
5379 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5380 		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
5381 
5382 		sc->tt.ddp = 0;
5383 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5384 		    &sc->tt.ddp, 0, "DDP allowed");
5385 
5386 		sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
5387 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
5388 		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
5389 
5390 		sc->tt.ddp_thres =
5391 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
5392 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
5393 		    &sc->tt.ddp_thres, 0, "DDP threshold");
5394 
5395 		sc->tt.rx_coalesce = 1;
5396 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5397 		    CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5398 
5399 		sc->tt.tx_align = 1;
5400 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5401 		    CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5402 	}
5403 #endif
5404 }
5405 
5406 void
5407 vi_sysctls(struct vi_info *vi)
5408 {
5409 	struct sysctl_ctx_list *ctx;
5410 	struct sysctl_oid *oid;
5411 	struct sysctl_oid_list *children;
5412 
5413 	ctx = device_get_sysctl_ctx(vi->dev);
5414 
5415 	/*
5416 	 * dev.[nv](cxgbe|cxl).X.
5417 	 */
5418 	oid = device_get_sysctl_tree(vi->dev);
5419 	children = SYSCTL_CHILDREN(oid);
5420 
5421 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
5422 	    vi->viid, "VI identifer");
5423 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5424 	    &vi->nrxq, 0, "# of rx queues");
5425 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5426 	    &vi->ntxq, 0, "# of tx queues");
5427 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5428 	    &vi->first_rxq, 0, "index of first rx queue");
5429 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5430 	    &vi->first_txq, 0, "index of first tx queue");
5431 
5432 	if (vi->flags & VI_NETMAP)
5433 		return;
5434 
5435 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
5436 	    CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
5437 	    "Reserve queue 0 for non-flowid packets");
5438 
5439 #ifdef TCP_OFFLOAD
5440 	if (vi->nofldrxq != 0) {
5441 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5442 		    &vi->nofldrxq, 0,
5443 		    "# of rx queues for offloaded TCP connections");
5444 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5445 		    &vi->nofldtxq, 0,
5446 		    "# of tx queues for offloaded TCP connections");
5447 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5448 		    CTLFLAG_RD, &vi->first_ofld_rxq, 0,
5449 		    "index of first TOE rx queue");
5450 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5451 		    CTLFLAG_RD, &vi->first_ofld_txq, 0,
5452 		    "index of first TOE tx queue");
5453 	}
5454 #endif
5455 
5456 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5457 	    CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
5458 	    "holdoff timer index");
5459 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5460 	    CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
5461 	    "holdoff packet counter index");
5462 
5463 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5464 	    CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
5465 	    "rx queue size");
5466 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5467 	    CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
5468 	    "tx queue size");
5469 }
5470 
5471 static void
5472 cxgbe_sysctls(struct port_info *pi)
5473 {
5474 	struct sysctl_ctx_list *ctx;
5475 	struct sysctl_oid *oid;
5476 	struct sysctl_oid_list *children;
5477 	struct adapter *sc = pi->adapter;
5478 
5479 	ctx = device_get_sysctl_ctx(pi->dev);
5480 
5481 	/*
5482 	 * dev.cxgbe.X.
5483 	 */
5484 	oid = device_get_sysctl_tree(pi->dev);
5485 	children = SYSCTL_CHILDREN(oid);
5486 
5487 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5488 	   CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5489 	if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5490 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5491 		    CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5492 		    "PHY temperature (in Celsius)");
5493 		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5494 		    CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5495 		    "PHY firmware version");
5496 	}
5497 
5498 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5499 	    CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
5500 	    "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5501 
5502 	/*
5503 	 * dev.cxgbe.X.stats.
5504 	 */
5505 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5506 	    NULL, "port statistics");
5507 	children = SYSCTL_CHILDREN(oid);
5508 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5509 	    &pi->tx_parse_error, 0,
5510 	    "# of tx packets with invalid length or # of segments");
5511 
5512 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5513 	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5514 	    CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5515 	    sysctl_handle_t4_reg64, "QU", desc)
5516 
5517 	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5518 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5519 	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5520 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5521 	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5522 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5523 	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5524 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5525 	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5526 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5527 	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5528 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5529 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5530 	    "# of tx frames in this range",
5531 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5532 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5533 	    "# of tx frames in this range",
5534 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5535 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5536 	    "# of tx frames in this range",
5537 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5538 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5539 	    "# of tx frames in this range",
5540 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5541 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5542 	    "# of tx frames in this range",
5543 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5544 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5545 	    "# of tx frames in this range",
5546 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5547 	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5548 	    "# of tx frames in this range",
5549 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5550 	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5551 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5552 	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5553 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5554 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5555 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5556 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5557 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5558 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5559 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5560 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5561 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5562 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5563 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5564 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5565 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5566 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5567 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5568 	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5569 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5570 
5571 	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5572 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5573 	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5574 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5575 	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5576 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5577 	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5578 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5579 	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5580 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5581 	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5582 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5583 	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5584 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5585 	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5586 	    "# of frames received with bad FCS",
5587 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5588 	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5589 	    "# of frames received with length error",
5590 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5591 	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5592 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5593 	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5594 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5595 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5596 	    "# of rx frames in this range",
5597 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5598 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5599 	    "# of rx frames in this range",
5600 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5601 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5602 	    "# of rx frames in this range",
5603 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5604 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5605 	    "# of rx frames in this range",
5606 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5607 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5608 	    "# of rx frames in this range",
5609 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5610 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5611 	    "# of rx frames in this range",
5612 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5613 	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5614 	    "# of rx frames in this range",
5615 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5616 	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5617 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5618 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5619 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5620 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5621 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5622 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5623 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5624 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5625 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5626 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5627 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5628 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5629 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5630 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5631 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5632 	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5633 	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5634 
5635 #undef SYSCTL_ADD_T4_REG64
5636 
5637 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5638 	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5639 	    &pi->stats.name, desc)
5640 
5641 	/* We get these from port_stats and they may be stale by upto 1s */
5642 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5643 	    "# drops due to buffer-group 0 overflows");
5644 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5645 	    "# drops due to buffer-group 1 overflows");
5646 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5647 	    "# drops due to buffer-group 2 overflows");
5648 	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5649 	    "# drops due to buffer-group 3 overflows");
5650 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5651 	    "# of buffer-group 0 truncated packets");
5652 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5653 	    "# of buffer-group 1 truncated packets");
5654 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5655 	    "# of buffer-group 2 truncated packets");
5656 	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5657 	    "# of buffer-group 3 truncated packets");
5658 
5659 #undef SYSCTL_ADD_T4_PORTSTAT
5660 }
5661 
5662 static int
5663 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5664 {
5665 	int rc, *i, space = 0;
5666 	struct sbuf sb;
5667 
5668 	sbuf_new_for_sysctl(&sb, NULL, 64, req);
5669 	for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5670 		if (space)
5671 			sbuf_printf(&sb, " ");
5672 		sbuf_printf(&sb, "%d", *i);
5673 		space = 1;
5674 	}
5675 	rc = sbuf_finish(&sb);
5676 	sbuf_delete(&sb);
5677 	return (rc);
5678 }
5679 
5680 static int
5681 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5682 {
5683 	int rc;
5684 	struct sbuf *sb;
5685 
5686 	rc = sysctl_wire_old_buffer(req, 0);
5687 	if (rc != 0)
5688 		return(rc);
5689 
5690 	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5691 	if (sb == NULL)
5692 		return (ENOMEM);
5693 
5694 	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5695 	rc = sbuf_finish(sb);
5696 	sbuf_delete(sb);
5697 
5698 	return (rc);
5699 }
5700 
5701 static int
5702 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5703 {
5704 	struct port_info *pi = arg1;
5705 	int op = arg2;
5706 	struct adapter *sc = pi->adapter;
5707 	u_int v;
5708 	int rc;
5709 
5710 	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
5711 	if (rc)
5712 		return (rc);
5713 	/* XXX: magic numbers */
5714 	rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5715 	    &v);
5716 	end_synchronized_op(sc, 0);
5717 	if (rc)
5718 		return (rc);
5719 	if (op == 0)
5720 		v /= 256;
5721 
5722 	rc = sysctl_handle_int(oidp, &v, 0, req);
5723 	return (rc);
5724 }
5725 
5726 static int
5727 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5728 {
5729 	struct vi_info *vi = arg1;
5730 	int rc, val;
5731 
5732 	val = vi->rsrv_noflowq;
5733 	rc = sysctl_handle_int(oidp, &val, 0, req);
5734 	if (rc != 0 || req->newptr == NULL)
5735 		return (rc);
5736 
5737 	if ((val >= 1) && (vi->ntxq > 1))
5738 		vi->rsrv_noflowq = 1;
5739 	else
5740 		vi->rsrv_noflowq = 0;
5741 
5742 	return (rc);
5743 }
5744 
5745 static int
5746 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5747 {
5748 	struct vi_info *vi = arg1;
5749 	struct adapter *sc = vi->pi->adapter;
5750 	int idx, rc, i;
5751 	struct sge_rxq *rxq;
5752 #ifdef TCP_OFFLOAD
5753 	struct sge_ofld_rxq *ofld_rxq;
5754 #endif
5755 	uint8_t v;
5756 
5757 	idx = vi->tmr_idx;
5758 
5759 	rc = sysctl_handle_int(oidp, &idx, 0, req);
5760 	if (rc != 0 || req->newptr == NULL)
5761 		return (rc);
5762 
5763 	if (idx < 0 || idx >= SGE_NTIMERS)
5764 		return (EINVAL);
5765 
5766 	rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5767 	    "t4tmr");
5768 	if (rc)
5769 		return (rc);
5770 
5771 	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
5772 	for_each_rxq(vi, i, rxq) {
5773 #ifdef atomic_store_rel_8
5774 		atomic_store_rel_8(&rxq->iq.intr_params, v);
5775 #else
5776 		rxq->iq.intr_params = v;
5777 #endif
5778 	}
5779 #ifdef TCP_OFFLOAD
5780 	for_each_ofld_rxq(vi, i, ofld_rxq) {
5781 #ifdef atomic_store_rel_8
5782 		atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5783 #else
5784 		ofld_rxq->iq.intr_params = v;
5785 #endif
5786 	}
5787 #endif
5788 	vi->tmr_idx = idx;
5789 
5790 	end_synchronized_op(sc, LOCK_HELD);
5791 	return (0);
5792 }
5793 
5794 static int
5795 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5796 {
5797 	struct vi_info *vi = arg1;
5798 	struct adapter *sc = vi->pi->adapter;
5799 	int idx, rc;
5800 
5801 	idx = vi->pktc_idx;
5802 
5803 	rc = sysctl_handle_int(oidp, &idx, 0, req);
5804 	if (rc != 0 || req->newptr == NULL)
5805 		return (rc);
5806 
5807 	if (idx < -1 || idx >= SGE_NCOUNTERS)
5808 		return (EINVAL);
5809 
5810 	rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5811 	    "t4pktc");
5812 	if (rc)
5813 		return (rc);
5814 
5815 	if (vi->flags & VI_INIT_DONE)
5816 		rc = EBUSY; /* cannot be changed once the queues are created */
5817 	else
5818 		vi->pktc_idx = idx;
5819 
5820 	end_synchronized_op(sc, LOCK_HELD);
5821 	return (rc);
5822 }
5823 
5824 static int
5825 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5826 {
5827 	struct vi_info *vi = arg1;
5828 	struct adapter *sc = vi->pi->adapter;
5829 	int qsize, rc;
5830 
5831 	qsize = vi->qsize_rxq;
5832 
5833 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
5834 	if (rc != 0 || req->newptr == NULL)
5835 		return (rc);
5836 
5837 	if (qsize < 128 || (qsize & 7))
5838 		return (EINVAL);
5839 
5840 	rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5841 	    "t4rxqs");
5842 	if (rc)
5843 		return (rc);
5844 
5845 	if (vi->flags & VI_INIT_DONE)
5846 		rc = EBUSY; /* cannot be changed once the queues are created */
5847 	else
5848 		vi->qsize_rxq = qsize;
5849 
5850 	end_synchronized_op(sc, LOCK_HELD);
5851 	return (rc);
5852 }
5853 
5854 static int
5855 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5856 {
5857 	struct vi_info *vi = arg1;
5858 	struct adapter *sc = vi->pi->adapter;
5859 	int qsize, rc;
5860 
5861 	qsize = vi->qsize_txq;
5862 
5863 	rc = sysctl_handle_int(oidp, &qsize, 0, req);
5864 	if (rc != 0 || req->newptr == NULL)
5865 		return (rc);
5866 
5867 	if (qsize < 128 || qsize > 65536)
5868 		return (EINVAL);
5869 
5870 	rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5871 	    "t4txqs");
5872 	if (rc)
5873 		return (rc);
5874 
5875 	if (vi->flags & VI_INIT_DONE)
5876 		rc = EBUSY; /* cannot be changed once the queues are created */
5877 	else
5878 		vi->qsize_txq = qsize;
5879 
5880 	end_synchronized_op(sc, LOCK_HELD);
5881 	return (rc);
5882 }
5883 
5884 static int
5885 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5886 {
5887 	struct port_info *pi = arg1;
5888 	struct adapter *sc = pi->adapter;
5889 	struct link_config *lc = &pi->link_cfg;
5890 	int rc;
5891 
5892 	if (req->newptr == NULL) {
5893 		struct sbuf *sb;
5894 		static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5895 
5896 		rc = sysctl_wire_old_buffer(req, 0);
5897 		if (rc != 0)
5898 			return(rc);
5899 
5900 		sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5901 		if (sb == NULL)
5902 			return (ENOMEM);
5903 
5904 		sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5905 		rc = sbuf_finish(sb);
5906 		sbuf_delete(sb);
5907 	} else {
5908 		char s[2];
5909 		int n;
5910 
5911 		s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5912 		s[1] = 0;
5913 
5914 		rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5915 		if (rc != 0)
5916 			return(rc);
5917 
5918 		if (s[1] != 0)
5919 			return (EINVAL);
5920 		if (s[0] < '0' || s[0] > '9')
5921 			return (EINVAL);	/* not a number */
5922 		n = s[0] - '0';
5923 		if (n & ~(PAUSE_TX | PAUSE_RX))
5924 			return (EINVAL);	/* some other bit is set too */
5925 
5926 		rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5927 		    "t4PAUSE");
5928 		if (rc)
5929 			return (rc);
5930 		if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5931 			int link_ok = lc->link_ok;
5932 
5933 			lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5934 			lc->requested_fc |= n;
5935 			rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5936 			lc->link_ok = link_ok;	/* restore */
5937 		}
5938 		end_synchronized_op(sc, 0);
5939 	}
5940 
5941 	return (rc);
5942 }
5943 
5944 static int
5945 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5946 {
5947 	struct adapter *sc = arg1;
5948 	int reg = arg2;
5949 	uint64_t val;
5950 
5951 	val = t4_read_reg64(sc, reg);
5952 
5953 	return (sysctl_handle_64(oidp, &val, 0, req));
5954 }
5955 
5956 static int
5957 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5958 {
5959 	struct adapter *sc = arg1;
5960 	int rc, t;
5961 	uint32_t param, val;
5962 
5963 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5964 	if (rc)
5965 		return (rc);
5966 	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5967 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5968 	    V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5969 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5970 	end_synchronized_op(sc, 0);
5971 	if (rc)
5972 		return (rc);
5973 
5974 	/* unknown is returned as 0 but we display -1 in that case */
5975 	t = val == 0 ? -1 : val;
5976 
5977 	rc = sysctl_handle_int(oidp, &t, 0, req);
5978 	return (rc);
5979 }
5980 
5981 #ifdef SBUF_DRAIN
5982 static int
5983 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5984 {
5985 	struct adapter *sc = arg1;
5986 	struct sbuf *sb;
5987 	int rc, i;
5988 	uint16_t incr[NMTUS][NCCTRL_WIN];
5989 	static const char *dec_fac[] = {
5990 		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5991 		"0.9375"
5992 	};
5993 
5994 	rc = sysctl_wire_old_buffer(req, 0);
5995 	if (rc != 0)
5996 		return (rc);
5997 
5998 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5999 	if (sb == NULL)
6000 		return (ENOMEM);
6001 
6002 	t4_read_cong_tbl(sc, incr);
6003 
6004 	for (i = 0; i < NCCTRL_WIN; ++i) {
6005 		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
6006 		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
6007 		    incr[5][i], incr[6][i], incr[7][i]);
6008 		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
6009 		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
6010 		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
6011 		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
6012 	}
6013 
6014 	rc = sbuf_finish(sb);
6015 	sbuf_delete(sb);
6016 
6017 	return (rc);
6018 }
6019 
6020 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
6021 	"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",	/* ibq's */
6022 	"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI",	/* obq's */
6023 	"SGE0-RX", "SGE1-RX"	/* additional obq's (T5 onwards) */
6024 };
6025 
6026 static int
6027 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
6028 {
6029 	struct adapter *sc = arg1;
6030 	struct sbuf *sb;
6031 	int rc, i, n, qid = arg2;
6032 	uint32_t *buf, *p;
6033 	char *qtype;
6034 	u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
6035 
6036 	KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
6037 	    ("%s: bad qid %d\n", __func__, qid));
6038 
6039 	if (qid < CIM_NUM_IBQ) {
6040 		/* inbound queue */
6041 		qtype = "IBQ";
6042 		n = 4 * CIM_IBQ_SIZE;
6043 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
6044 		rc = t4_read_cim_ibq(sc, qid, buf, n);
6045 	} else {
6046 		/* outbound queue */
6047 		qtype = "OBQ";
6048 		qid -= CIM_NUM_IBQ;
6049 		n = 4 * cim_num_obq * CIM_OBQ_SIZE;
6050 		buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
6051 		rc = t4_read_cim_obq(sc, qid, buf, n);
6052 	}
6053 
6054 	if (rc < 0) {
6055 		rc = -rc;
6056 		goto done;
6057 	}
6058 	n = rc * sizeof(uint32_t);	/* rc has # of words actually read */
6059 
6060 	rc = sysctl_wire_old_buffer(req, 0);
6061 	if (rc != 0)
6062 		goto done;
6063 
6064 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6065 	if (sb == NULL) {
6066 		rc = ENOMEM;
6067 		goto done;
6068 	}
6069 
6070 	sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
6071 	for (i = 0, p = buf; i < n; i += 16, p += 4)
6072 		sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
6073 		    p[2], p[3]);
6074 
6075 	rc = sbuf_finish(sb);
6076 	sbuf_delete(sb);
6077 done:
6078 	free(buf, M_CXGBE);
6079 	return (rc);
6080 }
6081 
6082 static int
6083 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
6084 {
6085 	struct adapter *sc = arg1;
6086 	u_int cfg;
6087 	struct sbuf *sb;
6088 	uint32_t *buf, *p;
6089 	int rc;
6090 
6091 	rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
6092 	if (rc != 0)
6093 		return (rc);
6094 
6095 	rc = sysctl_wire_old_buffer(req, 0);
6096 	if (rc != 0)
6097 		return (rc);
6098 
6099 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6100 	if (sb == NULL)
6101 		return (ENOMEM);
6102 
6103 	buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
6104 	    M_ZERO | M_WAITOK);
6105 
6106 	rc = -t4_cim_read_la(sc, buf, NULL);
6107 	if (rc != 0)
6108 		goto done;
6109 
6110 	sbuf_printf(sb, "Status   Data      PC%s",
6111 	    cfg & F_UPDBGLACAPTPCONLY ? "" :
6112 	    "     LS0Stat  LS0Addr             LS0Data");
6113 
6114 	KASSERT((sc->params.cim_la_size & 7) == 0,
6115 	    ("%s: p will walk off the end of buf", __func__));
6116 
6117 	for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
6118 		if (cfg & F_UPDBGLACAPTPCONLY) {
6119 			sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
6120 			    p[6], p[7]);
6121 			sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
6122 			    (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
6123 			    p[4] & 0xff, p[5] >> 8);
6124 			sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
6125 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6126 			    p[1] & 0xf, p[2] >> 4);
6127 		} else {
6128 			sbuf_printf(sb,
6129 			    "\n  %02x   %x%07x %x%07x %08x %08x "
6130 			    "%08x%08x%08x%08x",
6131 			    (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6132 			    p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
6133 			    p[6], p[7]);
6134 		}
6135 	}
6136 
6137 	rc = sbuf_finish(sb);
6138 	sbuf_delete(sb);
6139 done:
6140 	free(buf, M_CXGBE);
6141 	return (rc);
6142 }
6143 
6144 static int
6145 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
6146 {
6147 	struct adapter *sc = arg1;
6148 	u_int i;
6149 	struct sbuf *sb;
6150 	uint32_t *buf, *p;
6151 	int rc;
6152 
6153 	rc = sysctl_wire_old_buffer(req, 0);
6154 	if (rc != 0)
6155 		return (rc);
6156 
6157 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6158 	if (sb == NULL)
6159 		return (ENOMEM);
6160 
6161 	buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
6162 	    M_ZERO | M_WAITOK);
6163 
6164 	t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
6165 	p = buf;
6166 
6167 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6168 		sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
6169 		    p[1], p[0]);
6170 	}
6171 
6172 	sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
6173 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6174 		sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
6175 		    (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
6176 		    (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
6177 		    (p[1] >> 2) | ((p[2] & 3) << 30),
6178 		    (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
6179 		    p[0] & 1);
6180 	}
6181 
6182 	rc = sbuf_finish(sb);
6183 	sbuf_delete(sb);
6184 	free(buf, M_CXGBE);
6185 	return (rc);
6186 }
6187 
6188 static int
6189 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
6190 {
6191 	struct adapter *sc = arg1;
6192 	u_int i;
6193 	struct sbuf *sb;
6194 	uint32_t *buf, *p;
6195 	int rc;
6196 
6197 	rc = sysctl_wire_old_buffer(req, 0);
6198 	if (rc != 0)
6199 		return (rc);
6200 
6201 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6202 	if (sb == NULL)
6203 		return (ENOMEM);
6204 
6205 	buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
6206 	    M_ZERO | M_WAITOK);
6207 
6208 	t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
6209 	p = buf;
6210 
6211 	sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
6212 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6213 		sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
6214 		    (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
6215 		    p[4], p[3], p[2], p[1], p[0]);
6216 	}
6217 
6218 	sbuf_printf(sb, "\n\nCntl ID               Data");
6219 	for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
6220 		sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
6221 		    (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
6222 	}
6223 
6224 	rc = sbuf_finish(sb);
6225 	sbuf_delete(sb);
6226 	free(buf, M_CXGBE);
6227 	return (rc);
6228 }
6229 
6230 static int
6231 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
6232 {
6233 	struct adapter *sc = arg1;
6234 	struct sbuf *sb;
6235 	int rc, i;
6236 	uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6237 	uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6238 	uint16_t thres[CIM_NUM_IBQ];
6239 	uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
6240 	uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
6241 	u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
6242 
6243 	if (is_t4(sc)) {
6244 		cim_num_obq = CIM_NUM_OBQ;
6245 		ibq_rdaddr = A_UP_IBQ_0_RDADDR;
6246 		obq_rdaddr = A_UP_OBQ_0_REALADDR;
6247 	} else {
6248 		cim_num_obq = CIM_NUM_OBQ_T5;
6249 		ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
6250 		obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
6251 	}
6252 	nq = CIM_NUM_IBQ + cim_num_obq;
6253 
6254 	rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
6255 	if (rc == 0)
6256 		rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
6257 	if (rc != 0)
6258 		return (rc);
6259 
6260 	t4_read_cimq_cfg(sc, base, size, thres);
6261 
6262 	rc = sysctl_wire_old_buffer(req, 0);
6263 	if (rc != 0)
6264 		return (rc);
6265 
6266 	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6267 	if (sb == NULL)
6268 		return (ENOMEM);
6269 
6270 	sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
6271 
6272 	for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
6273 		sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
6274 		    qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
6275 		    G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6276 		    G_QUEREMFLITS(p[2]) * 16);
6277 	for ( ; i < nq; i++, p += 4, wr += 2)
6278 		sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
6279 		    base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
6280 		    wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6281 		    G_QUEREMFLITS(p[2]) * 16);
6282 
6283 	rc = sbuf_finish(sb);
6284 	sbuf_delete(sb);
6285 
6286 	return (rc);
6287 }
6288 
6289 static int
6290 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
6291 {
6292 	struct adapter *sc = arg1;
6293 	struct sbuf *sb;
6294 	int rc;
6295 	struct tp_cpl_stats stats;
6296 
6297 	rc = sysctl_wire_old_buffer(req, 0);
6298 	if (rc != 0)
6299 		return (rc);
6300 
6301 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6302 	if (sb == NULL)
6303 		return (ENOMEM);
6304 
6305 	t4_tp_get_cpl_stats(sc, &stats);
6306 
6307 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6308 	    "channel 3\n");
6309 	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
6310 		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
6311 	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
6312 		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
6313 
6314 	rc = sbuf_finish(sb);
6315 	sbuf_delete(sb);
6316 
6317 	return (rc);
6318 }
6319 
6320 static int
6321 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
6322 {
6323 	struct adapter *sc = arg1;
6324 	struct sbuf *sb;
6325 	int rc;
6326 	struct tp_usm_stats stats;
6327 
6328 	rc = sysctl_wire_old_buffer(req, 0);
6329 	if (rc != 0)
6330 		return(rc);
6331 
6332 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6333 	if (sb == NULL)
6334 		return (ENOMEM);
6335 
6336 	t4_get_usm_stats(sc, &stats);
6337 
6338 	sbuf_printf(sb, "Frames: %u\n", stats.frames);
6339 	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
6340 	sbuf_printf(sb, "Drops:  %u", stats.drops);
6341 
6342 	rc = sbuf_finish(sb);
6343 	sbuf_delete(sb);
6344 
6345 	return (rc);
6346 }
6347 
6348 const char *devlog_level_strings[] = {
6349 	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
6350 	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
6351 	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
6352 	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
6353 	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
6354 	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
6355 };
6356 
6357 const char *devlog_facility_strings[] = {
6358 	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
6359 	[FW_DEVLOG_FACILITY_CF]		= "CF",
6360 	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
6361 	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
6362 	[FW_DEVLOG_FACILITY_RES]	= "RES",
6363 	[FW_DEVLOG_FACILITY_HW]		= "HW",
6364 	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
6365 	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
6366 	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
6367 	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
6368 	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
6369 	[FW_DEVLOG_FACILITY_VI]		= "VI",
6370 	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
6371 	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
6372 	[FW_DEVLOG_FACILITY_TM]		= "TM",
6373 	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
6374 	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
6375 	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
6376 	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
6377 	[FW_DEVLOG_FACILITY_RI]		= "RI",
6378 	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
6379 	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
6380 	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
6381 	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
6382 };
6383 
6384 static int
6385 sysctl_devlog(SYSCTL_HANDLER_ARGS)
6386 {
6387 	struct adapter *sc = arg1;
6388 	struct devlog_params *dparams = &sc->params.devlog;
6389 	struct fw_devlog_e *buf, *e;
6390 	int i, j, rc, nentries, first = 0, m;
6391 	struct sbuf *sb;
6392 	uint64_t ftstamp = UINT64_MAX;
6393 
6394 	if (dparams->start == 0) {
6395 		dparams->memtype = FW_MEMTYPE_EDC0;
6396 		dparams->start = 0x84000;
6397 		dparams->size = 32768;
6398 	}
6399 
6400 	nentries = dparams->size / sizeof(struct fw_devlog_e);
6401 
6402 	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
6403 	if (buf == NULL)
6404 		return (ENOMEM);
6405 
6406 	m = fwmtype_to_hwmtype(dparams->memtype);
6407 	rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
6408 	if (rc != 0)
6409 		goto done;
6410 
6411 	for (i = 0; i < nentries; i++) {
6412 		e = &buf[i];
6413 
6414 		if (e->timestamp == 0)
6415 			break;	/* end */
6416 
6417 		e->timestamp = be64toh(e->timestamp);
6418 		e->seqno = be32toh(e->seqno);
6419 		for (j = 0; j < 8; j++)
6420 			e->params[j] = be32toh(e->params[j]);
6421 
6422 		if (e->timestamp < ftstamp) {
6423 			ftstamp = e->timestamp;
6424 			first = i;
6425 		}
6426 	}
6427 
6428 	if (buf[first].timestamp == 0)
6429 		goto done;	/* nothing in the log */
6430 
6431 	rc = sysctl_wire_old_buffer(req, 0);
6432 	if (rc != 0)
6433 		goto done;
6434 
6435 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6436 	if (sb == NULL) {
6437 		rc = ENOMEM;
6438 		goto done;
6439 	}
6440 	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6441 	    "Seq#", "Tstamp", "Level", "Facility", "Message");
6442 
6443 	i = first;
6444 	do {
6445 		e = &buf[i];
6446 		if (e->timestamp == 0)
6447 			break;	/* end */
6448 
6449 		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6450 		    e->seqno, e->timestamp,
6451 		    (e->level < nitems(devlog_level_strings) ?
6452 			devlog_level_strings[e->level] : "UNKNOWN"),
6453 		    (e->facility < nitems(devlog_facility_strings) ?
6454 			devlog_facility_strings[e->facility] : "UNKNOWN"));
6455 		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6456 		    e->params[2], e->params[3], e->params[4],
6457 		    e->params[5], e->params[6], e->params[7]);
6458 
6459 		if (++i == nentries)
6460 			i = 0;
6461 	} while (i != first);
6462 
6463 	rc = sbuf_finish(sb);
6464 	sbuf_delete(sb);
6465 done:
6466 	free(buf, M_CXGBE);
6467 	return (rc);
6468 }
6469 
6470 static int
6471 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6472 {
6473 	struct adapter *sc = arg1;
6474 	struct sbuf *sb;
6475 	int rc;
6476 	struct tp_fcoe_stats stats[4];
6477 
6478 	rc = sysctl_wire_old_buffer(req, 0);
6479 	if (rc != 0)
6480 		return (rc);
6481 
6482 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6483 	if (sb == NULL)
6484 		return (ENOMEM);
6485 
6486 	t4_get_fcoe_stats(sc, 0, &stats[0]);
6487 	t4_get_fcoe_stats(sc, 1, &stats[1]);
6488 	t4_get_fcoe_stats(sc, 2, &stats[2]);
6489 	t4_get_fcoe_stats(sc, 3, &stats[3]);
6490 
6491 	sbuf_printf(sb, "                   channel 0        channel 1        "
6492 	    "channel 2        channel 3\n");
6493 	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
6494 	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
6495 	    stats[3].octetsDDP);
6496 	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
6497 	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
6498 	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
6499 	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
6500 	    stats[3].framesDrop);
6501 
6502 	rc = sbuf_finish(sb);
6503 	sbuf_delete(sb);
6504 
6505 	return (rc);
6506 }
6507 
6508 static int
6509 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6510 {
6511 	struct adapter *sc = arg1;
6512 	struct sbuf *sb;
6513 	int rc, i;
6514 	unsigned int map, kbps, ipg, mode;
6515 	unsigned int pace_tab[NTX_SCHED];
6516 
6517 	rc = sysctl_wire_old_buffer(req, 0);
6518 	if (rc != 0)
6519 		return (rc);
6520 
6521 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6522 	if (sb == NULL)
6523 		return (ENOMEM);
6524 
6525 	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6526 	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6527 	t4_read_pace_tbl(sc, pace_tab);
6528 
6529 	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6530 	    "Class IPG (0.1 ns)   Flow IPG (us)");
6531 
6532 	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6533 		t4_get_tx_sched(sc, i, &kbps, &ipg);
6534 		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6535 		    (mode & (1 << i)) ? "flow" : "class", map & 3);
6536 		if (kbps)
6537 			sbuf_printf(sb, "%9u     ", kbps);
6538 		else
6539 			sbuf_printf(sb, " disabled     ");
6540 
6541 		if (ipg)
6542 			sbuf_printf(sb, "%13u        ", ipg);
6543 		else
6544 			sbuf_printf(sb, "     disabled        ");
6545 
6546 		if (pace_tab[i])
6547 			sbuf_printf(sb, "%10u", pace_tab[i]);
6548 		else
6549 			sbuf_printf(sb, "  disabled");
6550 	}
6551 
6552 	rc = sbuf_finish(sb);
6553 	sbuf_delete(sb);
6554 
6555 	return (rc);
6556 }
6557 
6558 static int
6559 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6560 {
6561 	struct adapter *sc = arg1;
6562 	struct sbuf *sb;
6563 	int rc, i, j;
6564 	uint64_t *p0, *p1;
6565 	struct lb_port_stats s[2];
6566 	static const char *stat_name[] = {
6567 		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6568 		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6569 		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
6570 		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6571 		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6572 		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6573 		"BG2FramesTrunc:", "BG3FramesTrunc:"
6574 	};
6575 
6576 	rc = sysctl_wire_old_buffer(req, 0);
6577 	if (rc != 0)
6578 		return (rc);
6579 
6580 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6581 	if (sb == NULL)
6582 		return (ENOMEM);
6583 
6584 	memset(s, 0, sizeof(s));
6585 
6586 	for (i = 0; i < 4; i += 2) {
6587 		t4_get_lb_stats(sc, i, &s[0]);
6588 		t4_get_lb_stats(sc, i + 1, &s[1]);
6589 
6590 		p0 = &s[0].octets;
6591 		p1 = &s[1].octets;
6592 		sbuf_printf(sb, "%s                       Loopback %u"
6593 		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6594 
6595 		for (j = 0; j < nitems(stat_name); j++)
6596 			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6597 				   *p0++, *p1++);
6598 	}
6599 
6600 	rc = sbuf_finish(sb);
6601 	sbuf_delete(sb);
6602 
6603 	return (rc);
6604 }
6605 
6606 static int
6607 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6608 {
6609 	int rc = 0;
6610 	struct port_info *pi = arg1;
6611 	struct sbuf *sb;
6612 	static const char *linkdnreasons[] = {
6613 		"non-specific", "remote fault", "autoneg failed", "reserved3",
6614 		"PHY overheated", "unknown", "rx los", "reserved7"
6615 	};
6616 
6617 	rc = sysctl_wire_old_buffer(req, 0);
6618 	if (rc != 0)
6619 		return(rc);
6620 	sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6621 	if (sb == NULL)
6622 		return (ENOMEM);
6623 
6624 	if (pi->linkdnrc < 0)
6625 		sbuf_printf(sb, "n/a");
6626 	else if (pi->linkdnrc < nitems(linkdnreasons))
6627 		sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6628 	else
6629 		sbuf_printf(sb, "%d", pi->linkdnrc);
6630 
6631 	rc = sbuf_finish(sb);
6632 	sbuf_delete(sb);
6633 
6634 	return (rc);
6635 }
6636 
6637 struct mem_desc {
6638 	unsigned int base;
6639 	unsigned int limit;
6640 	unsigned int idx;
6641 };
6642 
6643 static int
6644 mem_desc_cmp(const void *a, const void *b)
6645 {
6646 	return ((const struct mem_desc *)a)->base -
6647 	       ((const struct mem_desc *)b)->base;
6648 }
6649 
6650 static void
6651 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6652     unsigned int to)
6653 {
6654 	unsigned int size;
6655 
6656 	size = to - from + 1;
6657 	if (size == 0)
6658 		return;
6659 
6660 	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6661 	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6662 }
6663 
6664 static int
6665 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6666 {
6667 	struct adapter *sc = arg1;
6668 	struct sbuf *sb;
6669 	int rc, i, n;
6670 	uint32_t lo, hi, used, alloc;
6671 	static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6672 	static const char *region[] = {
6673 		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6674 		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6675 		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6676 		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6677 		"RQUDP region:", "PBL region:", "TXPBL region:",
6678 		"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6679 		"On-chip queues:"
6680 	};
6681 	struct mem_desc avail[4];
6682 	struct mem_desc mem[nitems(region) + 3];	/* up to 3 holes */
6683 	struct mem_desc *md = mem;
6684 
6685 	rc = sysctl_wire_old_buffer(req, 0);
6686 	if (rc != 0)
6687 		return (rc);
6688 
6689 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6690 	if (sb == NULL)
6691 		return (ENOMEM);
6692 
6693 	for (i = 0; i < nitems(mem); i++) {
6694 		mem[i].limit = 0;
6695 		mem[i].idx = i;
6696 	}
6697 
6698 	/* Find and sort the populated memory ranges */
6699 	i = 0;
6700 	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6701 	if (lo & F_EDRAM0_ENABLE) {
6702 		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6703 		avail[i].base = G_EDRAM0_BASE(hi) << 20;
6704 		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6705 		avail[i].idx = 0;
6706 		i++;
6707 	}
6708 	if (lo & F_EDRAM1_ENABLE) {
6709 		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6710 		avail[i].base = G_EDRAM1_BASE(hi) << 20;
6711 		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6712 		avail[i].idx = 1;
6713 		i++;
6714 	}
6715 	if (lo & F_EXT_MEM_ENABLE) {
6716 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6717 		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6718 		avail[i].limit = avail[i].base +
6719 		    (G_EXT_MEM_SIZE(hi) << 20);
6720 		avail[i].idx = is_t4(sc) ? 2 : 3;	/* Call it MC for T4 */
6721 		i++;
6722 	}
6723 	if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6724 		hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6725 		avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6726 		avail[i].limit = avail[i].base +
6727 		    (G_EXT_MEM1_SIZE(hi) << 20);
6728 		avail[i].idx = 4;
6729 		i++;
6730 	}
6731 	if (!i)                                    /* no memory available */
6732 		return 0;
6733 	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6734 
6735 	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6736 	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6737 	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6738 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6739 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6740 	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6741 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6742 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6743 	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6744 
6745 	/* the next few have explicit upper bounds */
6746 	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6747 	md->limit = md->base - 1 +
6748 		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6749 		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6750 	md++;
6751 
6752 	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6753 	md->limit = md->base - 1 +
6754 		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6755 		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6756 	md++;
6757 
6758 	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6759 		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6760 		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6761 		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6762 	} else {
6763 		md->base = 0;
6764 		md->idx = nitems(region);  /* hide it */
6765 	}
6766 	md++;
6767 
6768 #define ulp_region(reg) \
6769 	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6770 	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6771 
6772 	ulp_region(RX_ISCSI);
6773 	ulp_region(RX_TDDP);
6774 	ulp_region(TX_TPT);
6775 	ulp_region(RX_STAG);
6776 	ulp_region(RX_RQ);
6777 	ulp_region(RX_RQUDP);
6778 	ulp_region(RX_PBL);
6779 	ulp_region(TX_PBL);
6780 #undef ulp_region
6781 
6782 	md->base = 0;
6783 	md->idx = nitems(region);
6784 	if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6785 		md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6786 		md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6787 		    A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6788 	}
6789 	md++;
6790 
6791 	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6792 	md->limit = md->base + sc->tids.ntids - 1;
6793 	md++;
6794 	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6795 	md->limit = md->base + sc->tids.ntids - 1;
6796 	md++;
6797 
6798 	md->base = sc->vres.ocq.start;
6799 	if (sc->vres.ocq.size)
6800 		md->limit = md->base + sc->vres.ocq.size - 1;
6801 	else
6802 		md->idx = nitems(region);  /* hide it */
6803 	md++;
6804 
6805 	/* add any address-space holes, there can be up to 3 */
6806 	for (n = 0; n < i - 1; n++)
6807 		if (avail[n].limit < avail[n + 1].base)
6808 			(md++)->base = avail[n].limit;
6809 	if (avail[n].limit)
6810 		(md++)->base = avail[n].limit;
6811 
6812 	n = md - mem;
6813 	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6814 
6815 	for (lo = 0; lo < i; lo++)
6816 		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6817 				avail[lo].limit - 1);
6818 
6819 	sbuf_printf(sb, "\n");
6820 	for (i = 0; i < n; i++) {
6821 		if (mem[i].idx >= nitems(region))
6822 			continue;                        /* skip holes */
6823 		if (!mem[i].limit)
6824 			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6825 		mem_region_show(sb, region[mem[i].idx], mem[i].base,
6826 				mem[i].limit);
6827 	}
6828 
6829 	sbuf_printf(sb, "\n");
6830 	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6831 	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6832 	mem_region_show(sb, "uP RAM:", lo, hi);
6833 
6834 	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6835 	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6836 	mem_region_show(sb, "uP Extmem2:", lo, hi);
6837 
6838 	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6839 	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6840 		   G_PMRXMAXPAGE(lo),
6841 		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6842 		   (lo & F_PMRXNUMCHN) ? 2 : 1);
6843 
6844 	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6845 	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6846 	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6847 		   G_PMTXMAXPAGE(lo),
6848 		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6849 		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6850 	sbuf_printf(sb, "%u p-structs\n",
6851 		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6852 
6853 	for (i = 0; i < 4; i++) {
6854 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6855 		if (is_t4(sc)) {
6856 			used = G_USED(lo);
6857 			alloc = G_ALLOC(lo);
6858 		} else {
6859 			used = G_T5_USED(lo);
6860 			alloc = G_T5_ALLOC(lo);
6861 		}
6862 		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6863 			   i, used, alloc);
6864 	}
6865 	for (i = 0; i < 4; i++) {
6866 		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6867 		if (is_t4(sc)) {
6868 			used = G_USED(lo);
6869 			alloc = G_ALLOC(lo);
6870 		} else {
6871 			used = G_T5_USED(lo);
6872 			alloc = G_T5_ALLOC(lo);
6873 		}
6874 		sbuf_printf(sb,
6875 			   "\nLoopback %d using %u pages out of %u allocated",
6876 			   i, used, alloc);
6877 	}
6878 
6879 	rc = sbuf_finish(sb);
6880 	sbuf_delete(sb);
6881 
6882 	return (rc);
6883 }
6884 
6885 static inline void
6886 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6887 {
6888 	*mask = x | y;
6889 	y = htobe64(y);
6890 	memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6891 }
6892 
6893 static int
6894 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6895 {
6896 	struct adapter *sc = arg1;
6897 	struct sbuf *sb;
6898 	int rc, i, n;
6899 
6900 	rc = sysctl_wire_old_buffer(req, 0);
6901 	if (rc != 0)
6902 		return (rc);
6903 
6904 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6905 	if (sb == NULL)
6906 		return (ENOMEM);
6907 
6908 	sbuf_printf(sb,
6909 	    "Idx  Ethernet address     Mask     Vld Ports PF"
6910 	    "  VF              Replication             P0 P1 P2 P3  ML");
6911 	n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6912 	    NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6913 	for (i = 0; i < n; i++) {
6914 		uint64_t tcamx, tcamy, mask;
6915 		uint32_t cls_lo, cls_hi;
6916 		uint8_t addr[ETHER_ADDR_LEN];
6917 
6918 		tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6919 		tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6920 		cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6921 		cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6922 
6923 		if (tcamx & tcamy)
6924 			continue;
6925 
6926 		tcamxy2valmask(tcamx, tcamy, addr, &mask);
6927 		sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6928 			   "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6929 			   addr[3], addr[4], addr[5], (uintmax_t)mask,
6930 			   (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6931 			   G_PORTMAP(cls_hi), G_PF(cls_lo),
6932 			   (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6933 
6934 		if (cls_lo & F_REPLICATE) {
6935 			struct fw_ldst_cmd ldst_cmd;
6936 
6937 			memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6938 			ldst_cmd.op_to_addrspace =
6939 			    htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6940 				F_FW_CMD_REQUEST | F_FW_CMD_READ |
6941 				V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6942 			ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6943 			ldst_cmd.u.mps.rplc.fid_idx =
6944 			    htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6945 				V_FW_LDST_CMD_IDX(i));
6946 
6947 			rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6948 			    "t4mps");
6949 			if (rc)
6950 				break;
6951 			rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6952 			    sizeof(ldst_cmd), &ldst_cmd);
6953 			end_synchronized_op(sc, 0);
6954 
6955 			if (rc != 0) {
6956 				sbuf_printf(sb,
6957 				    " ------------ error %3u ------------", rc);
6958 				rc = 0;
6959 			} else {
6960 				sbuf_printf(sb, " %08x %08x %08x %08x",
6961 				    be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6962 				    be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6963 				    be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6964 				    be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6965 			}
6966 		} else
6967 			sbuf_printf(sb, "%36s", "");
6968 
6969 		sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6970 		    G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6971 		    G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6972 	}
6973 
6974 	if (rc)
6975 		(void) sbuf_finish(sb);
6976 	else
6977 		rc = sbuf_finish(sb);
6978 	sbuf_delete(sb);
6979 
6980 	return (rc);
6981 }
6982 
6983 static int
6984 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6985 {
6986 	struct adapter *sc = arg1;
6987 	struct sbuf *sb;
6988 	int rc;
6989 	uint16_t mtus[NMTUS];
6990 
6991 	rc = sysctl_wire_old_buffer(req, 0);
6992 	if (rc != 0)
6993 		return (rc);
6994 
6995 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6996 	if (sb == NULL)
6997 		return (ENOMEM);
6998 
6999 	t4_read_mtu_tbl(sc, mtus, NULL);
7000 
7001 	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
7002 	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
7003 	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
7004 	    mtus[14], mtus[15]);
7005 
7006 	rc = sbuf_finish(sb);
7007 	sbuf_delete(sb);
7008 
7009 	return (rc);
7010 }
7011 
7012 static int
7013 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
7014 {
7015 	struct adapter *sc = arg1;
7016 	struct sbuf *sb;
7017 	int rc, i;
7018 	uint32_t cnt[PM_NSTATS];
7019 	uint64_t cyc[PM_NSTATS];
7020 	static const char *rx_stats[] = {
7021 		"Read:", "Write bypass:", "Write mem:", "Flush:"
7022 	};
7023 	static const char *tx_stats[] = {
7024 		"Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
7025 	};
7026 
7027 	rc = sysctl_wire_old_buffer(req, 0);
7028 	if (rc != 0)
7029 		return (rc);
7030 
7031 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7032 	if (sb == NULL)
7033 		return (ENOMEM);
7034 
7035 	t4_pmtx_get_stats(sc, cnt, cyc);
7036 	sbuf_printf(sb, "                Tx pcmds             Tx bytes");
7037 	for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
7038 		sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
7039 		    cyc[i]);
7040 
7041 	t4_pmrx_get_stats(sc, cnt, cyc);
7042 	sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
7043 	for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
7044 		sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
7045 		    cyc[i]);
7046 
7047 	rc = sbuf_finish(sb);
7048 	sbuf_delete(sb);
7049 
7050 	return (rc);
7051 }
7052 
7053 static int
7054 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
7055 {
7056 	struct adapter *sc = arg1;
7057 	struct sbuf *sb;
7058 	int rc;
7059 	struct tp_rdma_stats stats;
7060 
7061 	rc = sysctl_wire_old_buffer(req, 0);
7062 	if (rc != 0)
7063 		return (rc);
7064 
7065 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7066 	if (sb == NULL)
7067 		return (ENOMEM);
7068 
7069 	t4_tp_get_rdma_stats(sc, &stats);
7070 	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
7071 	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
7072 
7073 	rc = sbuf_finish(sb);
7074 	sbuf_delete(sb);
7075 
7076 	return (rc);
7077 }
7078 
7079 static int
7080 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
7081 {
7082 	struct adapter *sc = arg1;
7083 	struct sbuf *sb;
7084 	int rc;
7085 	struct tp_tcp_stats v4, v6;
7086 
7087 	rc = sysctl_wire_old_buffer(req, 0);
7088 	if (rc != 0)
7089 		return (rc);
7090 
7091 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7092 	if (sb == NULL)
7093 		return (ENOMEM);
7094 
7095 	t4_tp_get_tcp_stats(sc, &v4, &v6);
7096 	sbuf_printf(sb,
7097 	    "                                IP                 IPv6\n");
7098 	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
7099 	    v4.tcpOutRsts, v6.tcpOutRsts);
7100 	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
7101 	    v4.tcpInSegs, v6.tcpInSegs);
7102 	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
7103 	    v4.tcpOutSegs, v6.tcpOutSegs);
7104 	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
7105 	    v4.tcpRetransSegs, v6.tcpRetransSegs);
7106 
7107 	rc = sbuf_finish(sb);
7108 	sbuf_delete(sb);
7109 
7110 	return (rc);
7111 }
7112 
7113 static int
7114 sysctl_tids(SYSCTL_HANDLER_ARGS)
7115 {
7116 	struct adapter *sc = arg1;
7117 	struct sbuf *sb;
7118 	int rc;
7119 	struct tid_info *t = &sc->tids;
7120 
7121 	rc = sysctl_wire_old_buffer(req, 0);
7122 	if (rc != 0)
7123 		return (rc);
7124 
7125 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7126 	if (sb == NULL)
7127 		return (ENOMEM);
7128 
7129 	if (t->natids) {
7130 		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
7131 		    t->atids_in_use);
7132 	}
7133 
7134 	if (t->ntids) {
7135 		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7136 			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
7137 
7138 			if (b) {
7139 				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
7140 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7141 				    t->ntids - 1);
7142 			} else {
7143 				sbuf_printf(sb, "TID range: %u-%u",
7144 				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
7145 				    t->ntids - 1);
7146 			}
7147 		} else
7148 			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
7149 		sbuf_printf(sb, ", in use: %u\n",
7150 		    atomic_load_acq_int(&t->tids_in_use));
7151 	}
7152 
7153 	if (t->nstids) {
7154 		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
7155 		    t->stid_base + t->nstids - 1, t->stids_in_use);
7156 	}
7157 
7158 	if (t->nftids) {
7159 		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
7160 		    t->ftid_base + t->nftids - 1);
7161 	}
7162 
7163 	if (t->netids) {
7164 		sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
7165 		    t->etid_base + t->netids - 1);
7166 	}
7167 
7168 	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
7169 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
7170 	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
7171 
7172 	rc = sbuf_finish(sb);
7173 	sbuf_delete(sb);
7174 
7175 	return (rc);
7176 }
7177 
7178 static int
7179 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
7180 {
7181 	struct adapter *sc = arg1;
7182 	struct sbuf *sb;
7183 	int rc;
7184 	struct tp_err_stats stats;
7185 
7186 	rc = sysctl_wire_old_buffer(req, 0);
7187 	if (rc != 0)
7188 		return (rc);
7189 
7190 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7191 	if (sb == NULL)
7192 		return (ENOMEM);
7193 
7194 	t4_tp_get_err_stats(sc, &stats);
7195 
7196 	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
7197 		      "channel 3\n");
7198 	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
7199 	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
7200 	    stats.macInErrs[3]);
7201 	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
7202 	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
7203 	    stats.hdrInErrs[3]);
7204 	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
7205 	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
7206 	    stats.tcpInErrs[3]);
7207 	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
7208 	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
7209 	    stats.tcp6InErrs[3]);
7210 	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
7211 	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
7212 	    stats.tnlCongDrops[3]);
7213 	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
7214 	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
7215 	    stats.tnlTxDrops[3]);
7216 	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
7217 	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
7218 	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
7219 	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
7220 	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
7221 	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
7222 	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
7223 	    stats.ofldNoNeigh, stats.ofldCongDefer);
7224 
7225 	rc = sbuf_finish(sb);
7226 	sbuf_delete(sb);
7227 
7228 	return (rc);
7229 }
7230 
7231 struct field_desc {
7232 	const char *name;
7233 	u_int start;
7234 	u_int width;
7235 };
7236 
7237 static void
7238 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
7239 {
7240 	char buf[32];
7241 	int line_size = 0;
7242 
7243 	while (f->name) {
7244 		uint64_t mask = (1ULL << f->width) - 1;
7245 		int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
7246 		    ((uintmax_t)v >> f->start) & mask);
7247 
7248 		if (line_size + len >= 79) {
7249 			line_size = 8;
7250 			sbuf_printf(sb, "\n        ");
7251 		}
7252 		sbuf_printf(sb, "%s ", buf);
7253 		line_size += len + 1;
7254 		f++;
7255 	}
7256 	sbuf_printf(sb, "\n");
7257 }
7258 
7259 static struct field_desc tp_la0[] = {
7260 	{ "RcfOpCodeOut", 60, 4 },
7261 	{ "State", 56, 4 },
7262 	{ "WcfState", 52, 4 },
7263 	{ "RcfOpcSrcOut", 50, 2 },
7264 	{ "CRxError", 49, 1 },
7265 	{ "ERxError", 48, 1 },
7266 	{ "SanityFailed", 47, 1 },
7267 	{ "SpuriousMsg", 46, 1 },
7268 	{ "FlushInputMsg", 45, 1 },
7269 	{ "FlushInputCpl", 44, 1 },
7270 	{ "RssUpBit", 43, 1 },
7271 	{ "RssFilterHit", 42, 1 },
7272 	{ "Tid", 32, 10 },
7273 	{ "InitTcb", 31, 1 },
7274 	{ "LineNumber", 24, 7 },
7275 	{ "Emsg", 23, 1 },
7276 	{ "EdataOut", 22, 1 },
7277 	{ "Cmsg", 21, 1 },
7278 	{ "CdataOut", 20, 1 },
7279 	{ "EreadPdu", 19, 1 },
7280 	{ "CreadPdu", 18, 1 },
7281 	{ "TunnelPkt", 17, 1 },
7282 	{ "RcfPeerFin", 16, 1 },
7283 	{ "RcfReasonOut", 12, 4 },
7284 	{ "TxCchannel", 10, 2 },
7285 	{ "RcfTxChannel", 8, 2 },
7286 	{ "RxEchannel", 6, 2 },
7287 	{ "RcfRxChannel", 5, 1 },
7288 	{ "RcfDataOutSrdy", 4, 1 },
7289 	{ "RxDvld", 3, 1 },
7290 	{ "RxOoDvld", 2, 1 },
7291 	{ "RxCongestion", 1, 1 },
7292 	{ "TxCongestion", 0, 1 },
7293 	{ NULL }
7294 };
7295 
7296 static struct field_desc tp_la1[] = {
7297 	{ "CplCmdIn", 56, 8 },
7298 	{ "CplCmdOut", 48, 8 },
7299 	{ "ESynOut", 47, 1 },
7300 	{ "EAckOut", 46, 1 },
7301 	{ "EFinOut", 45, 1 },
7302 	{ "ERstOut", 44, 1 },
7303 	{ "SynIn", 43, 1 },
7304 	{ "AckIn", 42, 1 },
7305 	{ "FinIn", 41, 1 },
7306 	{ "RstIn", 40, 1 },
7307 	{ "DataIn", 39, 1 },
7308 	{ "DataInVld", 38, 1 },
7309 	{ "PadIn", 37, 1 },
7310 	{ "RxBufEmpty", 36, 1 },
7311 	{ "RxDdp", 35, 1 },
7312 	{ "RxFbCongestion", 34, 1 },
7313 	{ "TxFbCongestion", 33, 1 },
7314 	{ "TxPktSumSrdy", 32, 1 },
7315 	{ "RcfUlpType", 28, 4 },
7316 	{ "Eread", 27, 1 },
7317 	{ "Ebypass", 26, 1 },
7318 	{ "Esave", 25, 1 },
7319 	{ "Static0", 24, 1 },
7320 	{ "Cread", 23, 1 },
7321 	{ "Cbypass", 22, 1 },
7322 	{ "Csave", 21, 1 },
7323 	{ "CPktOut", 20, 1 },
7324 	{ "RxPagePoolFull", 18, 2 },
7325 	{ "RxLpbkPkt", 17, 1 },
7326 	{ "TxLpbkPkt", 16, 1 },
7327 	{ "RxVfValid", 15, 1 },
7328 	{ "SynLearned", 14, 1 },
7329 	{ "SetDelEntry", 13, 1 },
7330 	{ "SetInvEntry", 12, 1 },
7331 	{ "CpcmdDvld", 11, 1 },
7332 	{ "CpcmdSave", 10, 1 },
7333 	{ "RxPstructsFull", 8, 2 },
7334 	{ "EpcmdDvld", 7, 1 },
7335 	{ "EpcmdFlush", 6, 1 },
7336 	{ "EpcmdTrimPrefix", 5, 1 },
7337 	{ "EpcmdTrimPostfix", 4, 1 },
7338 	{ "ERssIp4Pkt", 3, 1 },
7339 	{ "ERssIp6Pkt", 2, 1 },
7340 	{ "ERssTcpUdpPkt", 1, 1 },
7341 	{ "ERssFceFipPkt", 0, 1 },
7342 	{ NULL }
7343 };
7344 
7345 static struct field_desc tp_la2[] = {
7346 	{ "CplCmdIn", 56, 8 },
7347 	{ "MpsVfVld", 55, 1 },
7348 	{ "MpsPf", 52, 3 },
7349 	{ "MpsVf", 44, 8 },
7350 	{ "SynIn", 43, 1 },
7351 	{ "AckIn", 42, 1 },
7352 	{ "FinIn", 41, 1 },
7353 	{ "RstIn", 40, 1 },
7354 	{ "DataIn", 39, 1 },
7355 	{ "DataInVld", 38, 1 },
7356 	{ "PadIn", 37, 1 },
7357 	{ "RxBufEmpty", 36, 1 },
7358 	{ "RxDdp", 35, 1 },
7359 	{ "RxFbCongestion", 34, 1 },
7360 	{ "TxFbCongestion", 33, 1 },
7361 	{ "TxPktSumSrdy", 32, 1 },
7362 	{ "RcfUlpType", 28, 4 },
7363 	{ "Eread", 27, 1 },
7364 	{ "Ebypass", 26, 1 },
7365 	{ "Esave", 25, 1 },
7366 	{ "Static0", 24, 1 },
7367 	{ "Cread", 23, 1 },
7368 	{ "Cbypass", 22, 1 },
7369 	{ "Csave", 21, 1 },
7370 	{ "CPktOut", 20, 1 },
7371 	{ "RxPagePoolFull", 18, 2 },
7372 	{ "RxLpbkPkt", 17, 1 },
7373 	{ "TxLpbkPkt", 16, 1 },
7374 	{ "RxVfValid", 15, 1 },
7375 	{ "SynLearned", 14, 1 },
7376 	{ "SetDelEntry", 13, 1 },
7377 	{ "SetInvEntry", 12, 1 },
7378 	{ "CpcmdDvld", 11, 1 },
7379 	{ "CpcmdSave", 10, 1 },
7380 	{ "RxPstructsFull", 8, 2 },
7381 	{ "EpcmdDvld", 7, 1 },
7382 	{ "EpcmdFlush", 6, 1 },
7383 	{ "EpcmdTrimPrefix", 5, 1 },
7384 	{ "EpcmdTrimPostfix", 4, 1 },
7385 	{ "ERssIp4Pkt", 3, 1 },
7386 	{ "ERssIp6Pkt", 2, 1 },
7387 	{ "ERssTcpUdpPkt", 1, 1 },
7388 	{ "ERssFceFipPkt", 0, 1 },
7389 	{ NULL }
7390 };
7391 
7392 static void
7393 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
7394 {
7395 
7396 	field_desc_show(sb, *p, tp_la0);
7397 }
7398 
7399 static void
7400 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
7401 {
7402 
7403 	if (idx)
7404 		sbuf_printf(sb, "\n");
7405 	field_desc_show(sb, p[0], tp_la0);
7406 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7407 		field_desc_show(sb, p[1], tp_la0);
7408 }
7409 
7410 static void
7411 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
7412 {
7413 
7414 	if (idx)
7415 		sbuf_printf(sb, "\n");
7416 	field_desc_show(sb, p[0], tp_la0);
7417 	if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7418 		field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7419 }
7420 
7421 static int
7422 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7423 {
7424 	struct adapter *sc = arg1;
7425 	struct sbuf *sb;
7426 	uint64_t *buf, *p;
7427 	int rc;
7428 	u_int i, inc;
7429 	void (*show_func)(struct sbuf *, uint64_t *, int);
7430 
7431 	rc = sysctl_wire_old_buffer(req, 0);
7432 	if (rc != 0)
7433 		return (rc);
7434 
7435 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7436 	if (sb == NULL)
7437 		return (ENOMEM);
7438 
7439 	buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7440 
7441 	t4_tp_read_la(sc, buf, NULL);
7442 	p = buf;
7443 
7444 	switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7445 	case 2:
7446 		inc = 2;
7447 		show_func = tp_la_show2;
7448 		break;
7449 	case 3:
7450 		inc = 2;
7451 		show_func = tp_la_show3;
7452 		break;
7453 	default:
7454 		inc = 1;
7455 		show_func = tp_la_show;
7456 	}
7457 
7458 	for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7459 		(*show_func)(sb, p, i);
7460 
7461 	rc = sbuf_finish(sb);
7462 	sbuf_delete(sb);
7463 	free(buf, M_CXGBE);
7464 	return (rc);
7465 }
7466 
7467 static int
7468 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7469 {
7470 	struct adapter *sc = arg1;
7471 	struct sbuf *sb;
7472 	int rc;
7473 	u64 nrate[NCHAN], orate[NCHAN];
7474 
7475 	rc = sysctl_wire_old_buffer(req, 0);
7476 	if (rc != 0)
7477 		return (rc);
7478 
7479 	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7480 	if (sb == NULL)
7481 		return (ENOMEM);
7482 
7483 	t4_get_chan_txrate(sc, nrate, orate);
7484 	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
7485 		 "channel 3\n");
7486 	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7487 	    nrate[0], nrate[1], nrate[2], nrate[3]);
7488 	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7489 	    orate[0], orate[1], orate[2], orate[3]);
7490 
7491 	rc = sbuf_finish(sb);
7492 	sbuf_delete(sb);
7493 
7494 	return (rc);
7495 }
7496 
7497 static int
7498 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7499 {
7500 	struct adapter *sc = arg1;
7501 	struct sbuf *sb;
7502 	uint32_t *buf, *p;
7503 	int rc, i;
7504 
7505 	rc = sysctl_wire_old_buffer(req, 0);
7506 	if (rc != 0)
7507 		return (rc);
7508 
7509 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7510 	if (sb == NULL)
7511 		return (ENOMEM);
7512 
7513 	buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7514 	    M_ZERO | M_WAITOK);
7515 
7516 	t4_ulprx_read_la(sc, buf);
7517 	p = buf;
7518 
7519 	sbuf_printf(sb, "      Pcmd        Type   Message"
7520 	    "                Data");
7521 	for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7522 		sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7523 		    p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7524 	}
7525 
7526 	rc = sbuf_finish(sb);
7527 	sbuf_delete(sb);
7528 	free(buf, M_CXGBE);
7529 	return (rc);
7530 }
7531 
7532 static int
7533 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7534 {
7535 	struct adapter *sc = arg1;
7536 	struct sbuf *sb;
7537 	int rc, v;
7538 
7539 	rc = sysctl_wire_old_buffer(req, 0);
7540 	if (rc != 0)
7541 		return (rc);
7542 
7543 	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7544 	if (sb == NULL)
7545 		return (ENOMEM);
7546 
7547 	v = t4_read_reg(sc, A_SGE_STAT_CFG);
7548 	if (G_STATSOURCE_T5(v) == 7) {
7549 		if (G_STATMODE(v) == 0) {
7550 			sbuf_printf(sb, "total %d, incomplete %d",
7551 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
7552 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
7553 		} else if (G_STATMODE(v) == 1) {
7554 			sbuf_printf(sb, "total %d, data overflow %d",
7555 			    t4_read_reg(sc, A_SGE_STAT_TOTAL),
7556 			    t4_read_reg(sc, A_SGE_STAT_MATCH));
7557 		}
7558 	}
7559 	rc = sbuf_finish(sb);
7560 	sbuf_delete(sb);
7561 
7562 	return (rc);
7563 }
7564 #endif
7565 
7566 static uint32_t
7567 fconf_to_mode(uint32_t fconf)
7568 {
7569 	uint32_t mode;
7570 
7571 	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7572 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7573 
7574 	if (fconf & F_FRAGMENTATION)
7575 		mode |= T4_FILTER_IP_FRAGMENT;
7576 
7577 	if (fconf & F_MPSHITTYPE)
7578 		mode |= T4_FILTER_MPS_HIT_TYPE;
7579 
7580 	if (fconf & F_MACMATCH)
7581 		mode |= T4_FILTER_MAC_IDX;
7582 
7583 	if (fconf & F_ETHERTYPE)
7584 		mode |= T4_FILTER_ETH_TYPE;
7585 
7586 	if (fconf & F_PROTOCOL)
7587 		mode |= T4_FILTER_IP_PROTO;
7588 
7589 	if (fconf & F_TOS)
7590 		mode |= T4_FILTER_IP_TOS;
7591 
7592 	if (fconf & F_VLAN)
7593 		mode |= T4_FILTER_VLAN;
7594 
7595 	if (fconf & F_VNIC_ID)
7596 		mode |= T4_FILTER_VNIC;
7597 
7598 	if (fconf & F_PORT)
7599 		mode |= T4_FILTER_PORT;
7600 
7601 	if (fconf & F_FCOE)
7602 		mode |= T4_FILTER_FCoE;
7603 
7604 	return (mode);
7605 }
7606 
7607 static uint32_t
7608 mode_to_fconf(uint32_t mode)
7609 {
7610 	uint32_t fconf = 0;
7611 
7612 	if (mode & T4_FILTER_IP_FRAGMENT)
7613 		fconf |= F_FRAGMENTATION;
7614 
7615 	if (mode & T4_FILTER_MPS_HIT_TYPE)
7616 		fconf |= F_MPSHITTYPE;
7617 
7618 	if (mode & T4_FILTER_MAC_IDX)
7619 		fconf |= F_MACMATCH;
7620 
7621 	if (mode & T4_FILTER_ETH_TYPE)
7622 		fconf |= F_ETHERTYPE;
7623 
7624 	if (mode & T4_FILTER_IP_PROTO)
7625 		fconf |= F_PROTOCOL;
7626 
7627 	if (mode & T4_FILTER_IP_TOS)
7628 		fconf |= F_TOS;
7629 
7630 	if (mode & T4_FILTER_VLAN)
7631 		fconf |= F_VLAN;
7632 
7633 	if (mode & T4_FILTER_VNIC)
7634 		fconf |= F_VNIC_ID;
7635 
7636 	if (mode & T4_FILTER_PORT)
7637 		fconf |= F_PORT;
7638 
7639 	if (mode & T4_FILTER_FCoE)
7640 		fconf |= F_FCOE;
7641 
7642 	return (fconf);
7643 }
7644 
7645 static uint32_t
7646 fspec_to_fconf(struct t4_filter_specification *fs)
7647 {
7648 	uint32_t fconf = 0;
7649 
7650 	if (fs->val.frag || fs->mask.frag)
7651 		fconf |= F_FRAGMENTATION;
7652 
7653 	if (fs->val.matchtype || fs->mask.matchtype)
7654 		fconf |= F_MPSHITTYPE;
7655 
7656 	if (fs->val.macidx || fs->mask.macidx)
7657 		fconf |= F_MACMATCH;
7658 
7659 	if (fs->val.ethtype || fs->mask.ethtype)
7660 		fconf |= F_ETHERTYPE;
7661 
7662 	if (fs->val.proto || fs->mask.proto)
7663 		fconf |= F_PROTOCOL;
7664 
7665 	if (fs->val.tos || fs->mask.tos)
7666 		fconf |= F_TOS;
7667 
7668 	if (fs->val.vlan_vld || fs->mask.vlan_vld)
7669 		fconf |= F_VLAN;
7670 
7671 	if (fs->val.vnic_vld || fs->mask.vnic_vld)
7672 		fconf |= F_VNIC_ID;
7673 
7674 	if (fs->val.iport || fs->mask.iport)
7675 		fconf |= F_PORT;
7676 
7677 	if (fs->val.fcoe || fs->mask.fcoe)
7678 		fconf |= F_FCOE;
7679 
7680 	return (fconf);
7681 }
7682 
7683 static int
7684 get_filter_mode(struct adapter *sc, uint32_t *mode)
7685 {
7686 	int rc;
7687 	uint32_t fconf;
7688 
7689 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7690 	    "t4getfm");
7691 	if (rc)
7692 		return (rc);
7693 
7694 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7695 	    A_TP_VLAN_PRI_MAP);
7696 
7697 	if (sc->params.tp.vlan_pri_map != fconf) {
7698 		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7699 		    device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7700 		    fconf);
7701 	}
7702 
7703 	*mode = fconf_to_mode(fconf);
7704 
7705 	end_synchronized_op(sc, LOCK_HELD);
7706 	return (0);
7707 }
7708 
7709 static int
7710 set_filter_mode(struct adapter *sc, uint32_t mode)
7711 {
7712 	uint32_t fconf;
7713 	int rc;
7714 
7715 	fconf = mode_to_fconf(mode);
7716 
7717 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7718 	    "t4setfm");
7719 	if (rc)
7720 		return (rc);
7721 
7722 	if (sc->tids.ftids_in_use > 0) {
7723 		rc = EBUSY;
7724 		goto done;
7725 	}
7726 
7727 #ifdef TCP_OFFLOAD
7728 	if (uld_active(sc, ULD_TOM)) {
7729 		rc = EBUSY;
7730 		goto done;
7731 	}
7732 #endif
7733 
7734 	rc = -t4_set_filter_mode(sc, fconf);
7735 done:
7736 	end_synchronized_op(sc, LOCK_HELD);
7737 	return (rc);
7738 }
7739 
7740 static inline uint64_t
7741 get_filter_hits(struct adapter *sc, uint32_t fid)
7742 {
7743 	uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7744 	uint64_t hits;
7745 
7746 	memwin_info(sc, 0, &mw_base, NULL);
7747 	off = position_memwin(sc, 0,
7748 	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7749 	if (is_t4(sc)) {
7750 		hits = t4_read_reg64(sc, mw_base + off + 16);
7751 		hits = be64toh(hits);
7752 	} else {
7753 		hits = t4_read_reg(sc, mw_base + off + 24);
7754 		hits = be32toh(hits);
7755 	}
7756 
7757 	return (hits);
7758 }
7759 
7760 static int
7761 get_filter(struct adapter *sc, struct t4_filter *t)
7762 {
7763 	int i, rc, nfilters = sc->tids.nftids;
7764 	struct filter_entry *f;
7765 
7766 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7767 	    "t4getf");
7768 	if (rc)
7769 		return (rc);
7770 
7771 	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7772 	    t->idx >= nfilters) {
7773 		t->idx = 0xffffffff;
7774 		goto done;
7775 	}
7776 
7777 	f = &sc->tids.ftid_tab[t->idx];
7778 	for (i = t->idx; i < nfilters; i++, f++) {
7779 		if (f->valid) {
7780 			t->idx = i;
7781 			t->l2tidx = f->l2t ? f->l2t->idx : 0;
7782 			t->smtidx = f->smtidx;
7783 			if (f->fs.hitcnts)
7784 				t->hits = get_filter_hits(sc, t->idx);
7785 			else
7786 				t->hits = UINT64_MAX;
7787 			t->fs = f->fs;
7788 
7789 			goto done;
7790 		}
7791 	}
7792 
7793 	t->idx = 0xffffffff;
7794 done:
7795 	end_synchronized_op(sc, LOCK_HELD);
7796 	return (0);
7797 }
7798 
7799 static int
7800 set_filter(struct adapter *sc, struct t4_filter *t)
7801 {
7802 	unsigned int nfilters, nports;
7803 	struct filter_entry *f;
7804 	int i, rc;
7805 
7806 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7807 	if (rc)
7808 		return (rc);
7809 
7810 	nfilters = sc->tids.nftids;
7811 	nports = sc->params.nports;
7812 
7813 	if (nfilters == 0) {
7814 		rc = ENOTSUP;
7815 		goto done;
7816 	}
7817 
7818 	if (!(sc->flags & FULL_INIT_DONE)) {
7819 		rc = EAGAIN;
7820 		goto done;
7821 	}
7822 
7823 	if (t->idx >= nfilters) {
7824 		rc = EINVAL;
7825 		goto done;
7826 	}
7827 
7828 	/* Validate against the global filter mode */
7829 	if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7830 	    sc->params.tp.vlan_pri_map) {
7831 		rc = E2BIG;
7832 		goto done;
7833 	}
7834 
7835 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7836 		rc = EINVAL;
7837 		goto done;
7838 	}
7839 
7840 	if (t->fs.val.iport >= nports) {
7841 		rc = EINVAL;
7842 		goto done;
7843 	}
7844 
7845 	/* Can't specify an iq if not steering to it */
7846 	if (!t->fs.dirsteer && t->fs.iq) {
7847 		rc = EINVAL;
7848 		goto done;
7849 	}
7850 
7851 	/* IPv6 filter idx must be 4 aligned */
7852 	if (t->fs.type == 1 &&
7853 	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7854 		rc = EINVAL;
7855 		goto done;
7856 	}
7857 
7858 	if (sc->tids.ftid_tab == NULL) {
7859 		KASSERT(sc->tids.ftids_in_use == 0,
7860 		    ("%s: no memory allocated but filters_in_use > 0",
7861 		    __func__));
7862 
7863 		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7864 		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7865 		if (sc->tids.ftid_tab == NULL) {
7866 			rc = ENOMEM;
7867 			goto done;
7868 		}
7869 		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7870 	}
7871 
7872 	for (i = 0; i < 4; i++) {
7873 		f = &sc->tids.ftid_tab[t->idx + i];
7874 
7875 		if (f->pending || f->valid) {
7876 			rc = EBUSY;
7877 			goto done;
7878 		}
7879 		if (f->locked) {
7880 			rc = EPERM;
7881 			goto done;
7882 		}
7883 
7884 		if (t->fs.type == 0)
7885 			break;
7886 	}
7887 
7888 	f = &sc->tids.ftid_tab[t->idx];
7889 	f->fs = t->fs;
7890 
7891 	rc = set_filter_wr(sc, t->idx);
7892 done:
7893 	end_synchronized_op(sc, 0);
7894 
7895 	if (rc == 0) {
7896 		mtx_lock(&sc->tids.ftid_lock);
7897 		for (;;) {
7898 			if (f->pending == 0) {
7899 				rc = f->valid ? 0 : EIO;
7900 				break;
7901 			}
7902 
7903 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7904 			    PCATCH, "t4setfw", 0)) {
7905 				rc = EINPROGRESS;
7906 				break;
7907 			}
7908 		}
7909 		mtx_unlock(&sc->tids.ftid_lock);
7910 	}
7911 	return (rc);
7912 }
7913 
7914 static int
7915 del_filter(struct adapter *sc, struct t4_filter *t)
7916 {
7917 	unsigned int nfilters;
7918 	struct filter_entry *f;
7919 	int rc;
7920 
7921 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7922 	if (rc)
7923 		return (rc);
7924 
7925 	nfilters = sc->tids.nftids;
7926 
7927 	if (nfilters == 0) {
7928 		rc = ENOTSUP;
7929 		goto done;
7930 	}
7931 
7932 	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7933 	    t->idx >= nfilters) {
7934 		rc = EINVAL;
7935 		goto done;
7936 	}
7937 
7938 	if (!(sc->flags & FULL_INIT_DONE)) {
7939 		rc = EAGAIN;
7940 		goto done;
7941 	}
7942 
7943 	f = &sc->tids.ftid_tab[t->idx];
7944 
7945 	if (f->pending) {
7946 		rc = EBUSY;
7947 		goto done;
7948 	}
7949 	if (f->locked) {
7950 		rc = EPERM;
7951 		goto done;
7952 	}
7953 
7954 	if (f->valid) {
7955 		t->fs = f->fs;	/* extra info for the caller */
7956 		rc = del_filter_wr(sc, t->idx);
7957 	}
7958 
7959 done:
7960 	end_synchronized_op(sc, 0);
7961 
7962 	if (rc == 0) {
7963 		mtx_lock(&sc->tids.ftid_lock);
7964 		for (;;) {
7965 			if (f->pending == 0) {
7966 				rc = f->valid ? EIO : 0;
7967 				break;
7968 			}
7969 
7970 			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7971 			    PCATCH, "t4delfw", 0)) {
7972 				rc = EINPROGRESS;
7973 				break;
7974 			}
7975 		}
7976 		mtx_unlock(&sc->tids.ftid_lock);
7977 	}
7978 
7979 	return (rc);
7980 }
7981 
7982 static void
7983 clear_filter(struct filter_entry *f)
7984 {
7985 	if (f->l2t)
7986 		t4_l2t_release(f->l2t);
7987 
7988 	bzero(f, sizeof (*f));
7989 }
7990 
7991 static int
7992 set_filter_wr(struct adapter *sc, int fidx)
7993 {
7994 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7995 	struct fw_filter_wr *fwr;
7996 	unsigned int ftid;
7997 	struct wrq_cookie cookie;
7998 
7999 	ASSERT_SYNCHRONIZED_OP(sc);
8000 
8001 	if (f->fs.newdmac || f->fs.newvlan) {
8002 		/* This filter needs an L2T entry; allocate one. */
8003 		f->l2t = t4_l2t_alloc_switching(sc->l2t);
8004 		if (f->l2t == NULL)
8005 			return (EAGAIN);
8006 		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
8007 		    f->fs.dmac)) {
8008 			t4_l2t_release(f->l2t);
8009 			f->l2t = NULL;
8010 			return (ENOMEM);
8011 		}
8012 	}
8013 
8014 	ftid = sc->tids.ftid_base + fidx;
8015 
8016 	fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8017 	if (fwr == NULL)
8018 		return (ENOMEM);
8019 	bzero(fwr, sizeof(*fwr));
8020 
8021 	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
8022 	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
8023 	fwr->tid_to_iq =
8024 	    htobe32(V_FW_FILTER_WR_TID(ftid) |
8025 		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
8026 		V_FW_FILTER_WR_NOREPLY(0) |
8027 		V_FW_FILTER_WR_IQ(f->fs.iq));
8028 	fwr->del_filter_to_l2tix =
8029 	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
8030 		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
8031 		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
8032 		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
8033 		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
8034 		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
8035 		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
8036 		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
8037 		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
8038 		    f->fs.newvlan == VLAN_REWRITE) |
8039 		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
8040 		    f->fs.newvlan == VLAN_REWRITE) |
8041 		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
8042 		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
8043 		V_FW_FILTER_WR_PRIO(f->fs.prio) |
8044 		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
8045 	fwr->ethtype = htobe16(f->fs.val.ethtype);
8046 	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
8047 	fwr->frag_to_ovlan_vldm =
8048 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
8049 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
8050 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
8051 		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
8052 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
8053 		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
8054 	fwr->smac_sel = 0;
8055 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
8056 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
8057 	fwr->maci_to_matchtypem =
8058 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
8059 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
8060 		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
8061 		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
8062 		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
8063 		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
8064 		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
8065 		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
8066 	fwr->ptcl = f->fs.val.proto;
8067 	fwr->ptclm = f->fs.mask.proto;
8068 	fwr->ttyp = f->fs.val.tos;
8069 	fwr->ttypm = f->fs.mask.tos;
8070 	fwr->ivlan = htobe16(f->fs.val.vlan);
8071 	fwr->ivlanm = htobe16(f->fs.mask.vlan);
8072 	fwr->ovlan = htobe16(f->fs.val.vnic);
8073 	fwr->ovlanm = htobe16(f->fs.mask.vnic);
8074 	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
8075 	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
8076 	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
8077 	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
8078 	fwr->lp = htobe16(f->fs.val.dport);
8079 	fwr->lpm = htobe16(f->fs.mask.dport);
8080 	fwr->fp = htobe16(f->fs.val.sport);
8081 	fwr->fpm = htobe16(f->fs.mask.sport);
8082 	if (f->fs.newsmac)
8083 		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
8084 
8085 	f->pending = 1;
8086 	sc->tids.ftids_in_use++;
8087 
8088 	commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8089 	return (0);
8090 }
8091 
8092 static int
8093 del_filter_wr(struct adapter *sc, int fidx)
8094 {
8095 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8096 	struct fw_filter_wr *fwr;
8097 	unsigned int ftid;
8098 	struct wrq_cookie cookie;
8099 
8100 	ftid = sc->tids.ftid_base + fidx;
8101 
8102 	fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8103 	if (fwr == NULL)
8104 		return (ENOMEM);
8105 	bzero(fwr, sizeof (*fwr));
8106 
8107 	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
8108 
8109 	f->pending = 1;
8110 	commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8111 	return (0);
8112 }
8113 
8114 int
8115 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8116 {
8117 	struct adapter *sc = iq->adapter;
8118 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
8119 	unsigned int idx = GET_TID(rpl);
8120 	unsigned int rc;
8121 	struct filter_entry *f;
8122 
8123 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
8124 	    rss->opcode));
8125 
8126 	if (is_ftid(sc, idx)) {
8127 
8128 		idx -= sc->tids.ftid_base;
8129 		f = &sc->tids.ftid_tab[idx];
8130 		rc = G_COOKIE(rpl->cookie);
8131 
8132 		mtx_lock(&sc->tids.ftid_lock);
8133 		if (rc == FW_FILTER_WR_FLT_ADDED) {
8134 			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
8135 			    __func__, idx));
8136 			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
8137 			f->pending = 0;  /* asynchronous setup completed */
8138 			f->valid = 1;
8139 		} else {
8140 			if (rc != FW_FILTER_WR_FLT_DELETED) {
8141 				/* Add or delete failed, display an error */
8142 				log(LOG_ERR,
8143 				    "filter %u setup failed with error %u\n",
8144 				    idx, rc);
8145 			}
8146 
8147 			clear_filter(f);
8148 			sc->tids.ftids_in_use--;
8149 		}
8150 		wakeup(&sc->tids.ftid_tab);
8151 		mtx_unlock(&sc->tids.ftid_lock);
8152 	}
8153 
8154 	return (0);
8155 }
8156 
8157 static int
8158 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
8159 {
8160 	int rc;
8161 
8162 	if (cntxt->cid > M_CTXTQID)
8163 		return (EINVAL);
8164 
8165 	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
8166 	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
8167 		return (EINVAL);
8168 
8169 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
8170 	if (rc)
8171 		return (rc);
8172 
8173 	if (sc->flags & FW_OK) {
8174 		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
8175 		    &cntxt->data[0]);
8176 		if (rc == 0)
8177 			goto done;
8178 	}
8179 
8180 	/*
8181 	 * Read via firmware failed or wasn't even attempted.  Read directly via
8182 	 * the backdoor.
8183 	 */
8184 	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
8185 done:
8186 	end_synchronized_op(sc, 0);
8187 	return (rc);
8188 }
8189 
8190 static int
8191 load_fw(struct adapter *sc, struct t4_data *fw)
8192 {
8193 	int rc;
8194 	uint8_t *fw_data;
8195 
8196 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
8197 	if (rc)
8198 		return (rc);
8199 
8200 	if (sc->flags & FULL_INIT_DONE) {
8201 		rc = EBUSY;
8202 		goto done;
8203 	}
8204 
8205 	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
8206 	if (fw_data == NULL) {
8207 		rc = ENOMEM;
8208 		goto done;
8209 	}
8210 
8211 	rc = copyin(fw->data, fw_data, fw->len);
8212 	if (rc == 0)
8213 		rc = -t4_load_fw(sc, fw_data, fw->len);
8214 
8215 	free(fw_data, M_CXGBE);
8216 done:
8217 	end_synchronized_op(sc, 0);
8218 	return (rc);
8219 }
8220 
8221 static int
8222 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
8223 {
8224 	uint32_t addr, off, remaining, i, n;
8225 	uint32_t *buf, *b;
8226 	uint32_t mw_base, mw_aperture;
8227 	int rc;
8228 	uint8_t *dst;
8229 
8230 	rc = validate_mem_range(sc, mr->addr, mr->len);
8231 	if (rc != 0)
8232 		return (rc);
8233 
8234 	memwin_info(sc, win, &mw_base, &mw_aperture);
8235 	buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
8236 	addr = mr->addr;
8237 	remaining = mr->len;
8238 	dst = (void *)mr->data;
8239 
8240 	while (remaining) {
8241 		off = position_memwin(sc, win, addr);
8242 
8243 		/* number of bytes that we'll copy in the inner loop */
8244 		n = min(remaining, mw_aperture - off);
8245 		for (i = 0; i < n; i += 4)
8246 			*b++ = t4_read_reg(sc, mw_base + off + i);
8247 
8248 		rc = copyout(buf, dst, n);
8249 		if (rc != 0)
8250 			break;
8251 
8252 		b = buf;
8253 		dst += n;
8254 		remaining -= n;
8255 		addr += n;
8256 	}
8257 
8258 	free(buf, M_CXGBE);
8259 	return (rc);
8260 }
8261 
8262 static int
8263 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
8264 {
8265 	int rc;
8266 
8267 	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
8268 		return (EINVAL);
8269 
8270 	if (i2cd->len > sizeof(i2cd->data))
8271 		return (EFBIG);
8272 
8273 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
8274 	if (rc)
8275 		return (rc);
8276 	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
8277 	    i2cd->offset, i2cd->len, &i2cd->data[0]);
8278 	end_synchronized_op(sc, 0);
8279 
8280 	return (rc);
8281 }
8282 
8283 static int
8284 in_range(int val, int lo, int hi)
8285 {
8286 
8287 	return (val < 0 || (val <= hi && val >= lo));
8288 }
8289 
8290 static int
8291 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
8292 {
8293 	int fw_subcmd, fw_type, rc;
8294 
8295 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
8296 	if (rc)
8297 		return (rc);
8298 
8299 	if (!(sc->flags & FULL_INIT_DONE)) {
8300 		rc = EAGAIN;
8301 		goto done;
8302 	}
8303 
8304 	/*
8305 	 * Translate the cxgbetool parameters into T4 firmware parameters.  (The
8306 	 * sub-command and type are in common locations.)
8307 	 */
8308 	if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
8309 		fw_subcmd = FW_SCHED_SC_CONFIG;
8310 	else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
8311 		fw_subcmd = FW_SCHED_SC_PARAMS;
8312 	else {
8313 		rc = EINVAL;
8314 		goto done;
8315 	}
8316 	if (p->type == SCHED_CLASS_TYPE_PACKET)
8317 		fw_type = FW_SCHED_TYPE_PKTSCHED;
8318 	else {
8319 		rc = EINVAL;
8320 		goto done;
8321 	}
8322 
8323 	if (fw_subcmd == FW_SCHED_SC_CONFIG) {
8324 		/* Vet our parameters ..*/
8325 		if (p->u.config.minmax < 0) {
8326 			rc = EINVAL;
8327 			goto done;
8328 		}
8329 
8330 		/* And pass the request to the firmware ...*/
8331 		rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
8332 		goto done;
8333 	}
8334 
8335 	if (fw_subcmd == FW_SCHED_SC_PARAMS) {
8336 		int fw_level;
8337 		int fw_mode;
8338 		int fw_rateunit;
8339 		int fw_ratemode;
8340 
8341 		if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
8342 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
8343 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
8344 			fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
8345 		else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
8346 			fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
8347 		else {
8348 			rc = EINVAL;
8349 			goto done;
8350 		}
8351 
8352 		if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
8353 			fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
8354 		else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
8355 			fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
8356 		else {
8357 			rc = EINVAL;
8358 			goto done;
8359 		}
8360 
8361 		if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
8362 			fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
8363 		else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
8364 			fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
8365 		else {
8366 			rc = EINVAL;
8367 			goto done;
8368 		}
8369 
8370 		if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
8371 			fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
8372 		else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
8373 			fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
8374 		else {
8375 			rc = EINVAL;
8376 			goto done;
8377 		}
8378 
8379 		/* Vet our parameters ... */
8380 		if (!in_range(p->u.params.channel, 0, 3) ||
8381 		    !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
8382 		    !in_range(p->u.params.minrate, 0, 10000000) ||
8383 		    !in_range(p->u.params.maxrate, 0, 10000000) ||
8384 		    !in_range(p->u.params.weight, 0, 100)) {
8385 			rc = ERANGE;
8386 			goto done;
8387 		}
8388 
8389 		/*
8390 		 * Translate any unset parameters into the firmware's
8391 		 * nomenclature and/or fail the call if the parameters
8392 		 * are required ...
8393 		 */
8394 		if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
8395 		    p->u.params.channel < 0 || p->u.params.cl < 0) {
8396 			rc = EINVAL;
8397 			goto done;
8398 		}
8399 		if (p->u.params.minrate < 0)
8400 			p->u.params.minrate = 0;
8401 		if (p->u.params.maxrate < 0) {
8402 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8403 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8404 				rc = EINVAL;
8405 				goto done;
8406 			} else
8407 				p->u.params.maxrate = 0;
8408 		}
8409 		if (p->u.params.weight < 0) {
8410 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
8411 				rc = EINVAL;
8412 				goto done;
8413 			} else
8414 				p->u.params.weight = 0;
8415 		}
8416 		if (p->u.params.pktsize < 0) {
8417 			if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8418 			    p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8419 				rc = EINVAL;
8420 				goto done;
8421 			} else
8422 				p->u.params.pktsize = 0;
8423 		}
8424 
8425 		/* See what the firmware thinks of the request ... */
8426 		rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
8427 		    fw_rateunit, fw_ratemode, p->u.params.channel,
8428 		    p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
8429 		    p->u.params.weight, p->u.params.pktsize, 1);
8430 		goto done;
8431 	}
8432 
8433 	rc = EINVAL;
8434 done:
8435 	end_synchronized_op(sc, 0);
8436 	return (rc);
8437 }
8438 
8439 static int
8440 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8441 {
8442 	struct port_info *pi = NULL;
8443 	struct vi_info *vi;
8444 	struct sge_txq *txq;
8445 	uint32_t fw_mnem, fw_queue, fw_class;
8446 	int i, rc;
8447 
8448 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8449 	if (rc)
8450 		return (rc);
8451 
8452 	if (!(sc->flags & FULL_INIT_DONE)) {
8453 		rc = EAGAIN;
8454 		goto done;
8455 	}
8456 
8457 	if (p->port >= sc->params.nports) {
8458 		rc = EINVAL;
8459 		goto done;
8460 	}
8461 
8462 	/* XXX: Only supported for the main VI. */
8463 	pi = sc->port[p->port];
8464 	vi = &pi->vi[0];
8465 	if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
8466 		rc = EINVAL;
8467 		goto done;
8468 	}
8469 
8470 	/*
8471 	 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8472 	 * Scheduling Class in this case).
8473 	 */
8474 	fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8475 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
8476 	fw_class = p->cl < 0 ? 0xffffffff : p->cl;
8477 
8478 	/*
8479 	 * If op.queue is non-negative, then we're only changing the scheduling
8480 	 * on a single specified TX queue.
8481 	 */
8482 	if (p->queue >= 0) {
8483 		txq = &sc->sge.txq[vi->first_txq + p->queue];
8484 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8485 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8486 		    &fw_class);
8487 		goto done;
8488 	}
8489 
8490 	/*
8491 	 * Change the scheduling on all the TX queues for the
8492 	 * interface.
8493 	 */
8494 	for_each_txq(vi, i, txq) {
8495 		fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8496 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8497 		    &fw_class);
8498 		if (rc)
8499 			goto done;
8500 	}
8501 
8502 	rc = 0;
8503 done:
8504 	end_synchronized_op(sc, 0);
8505 	return (rc);
8506 }
8507 
8508 int
8509 t4_os_find_pci_capability(struct adapter *sc, int cap)
8510 {
8511 	int i;
8512 
8513 	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
8514 }
8515 
8516 int
8517 t4_os_pci_save_state(struct adapter *sc)
8518 {
8519 	device_t dev;
8520 	struct pci_devinfo *dinfo;
8521 
8522 	dev = sc->dev;
8523 	dinfo = device_get_ivars(dev);
8524 
8525 	pci_cfg_save(dev, dinfo, 0);
8526 	return (0);
8527 }
8528 
8529 int
8530 t4_os_pci_restore_state(struct adapter *sc)
8531 {
8532 	device_t dev;
8533 	struct pci_devinfo *dinfo;
8534 
8535 	dev = sc->dev;
8536 	dinfo = device_get_ivars(dev);
8537 
8538 	pci_cfg_restore(dev, dinfo);
8539 	return (0);
8540 }
8541 
8542 void
8543 t4_os_portmod_changed(const struct adapter *sc, int idx)
8544 {
8545 	struct port_info *pi = sc->port[idx];
8546 	struct vi_info *vi;
8547 	struct ifnet *ifp;
8548 	int v;
8549 	static const char *mod_str[] = {
8550 		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8551 	};
8552 
8553 	for_each_vi(pi, v, vi) {
8554 		build_medialist(pi, &vi->media);
8555 	}
8556 
8557 	ifp = pi->vi[0].ifp;
8558 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8559 		if_printf(ifp, "transceiver unplugged.\n");
8560 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8561 		if_printf(ifp, "unknown transceiver inserted.\n");
8562 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8563 		if_printf(ifp, "unsupported transceiver inserted.\n");
8564 	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8565 		if_printf(ifp, "%s transceiver inserted.\n",
8566 		    mod_str[pi->mod_type]);
8567 	} else {
8568 		if_printf(ifp, "transceiver (type %d) inserted.\n",
8569 		    pi->mod_type);
8570 	}
8571 }
8572 
8573 void
8574 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8575 {
8576 	struct port_info *pi = sc->port[idx];
8577 	struct vi_info *vi;
8578 	struct ifnet *ifp;
8579 	int v;
8580 
8581 	if (link_stat)
8582 		pi->linkdnrc = -1;
8583 	else {
8584 		if (reason >= 0)
8585 			pi->linkdnrc = reason;
8586 	}
8587 	for_each_vi(pi, v, vi) {
8588 		ifp = vi->ifp;
8589 		if (ifp == NULL)
8590 			continue;
8591 
8592 		if (link_stat) {
8593 			ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8594 			if_link_state_change(ifp, LINK_STATE_UP);
8595 		} else {
8596 			if_link_state_change(ifp, LINK_STATE_DOWN);
8597 		}
8598 	}
8599 }
8600 
8601 void
8602 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8603 {
8604 	struct adapter *sc;
8605 
8606 	sx_slock(&t4_list_lock);
8607 	SLIST_FOREACH(sc, &t4_list, link) {
8608 		/*
8609 		 * func should not make any assumptions about what state sc is
8610 		 * in - the only guarantee is that sc->sc_lock is a valid lock.
8611 		 */
8612 		func(sc, arg);
8613 	}
8614 	sx_sunlock(&t4_list_lock);
8615 }
8616 
8617 static int
8618 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8619 {
8620        return (0);
8621 }
8622 
8623 static int
8624 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8625 {
8626        return (0);
8627 }
8628 
8629 static int
8630 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8631     struct thread *td)
8632 {
8633 	int rc;
8634 	struct adapter *sc = dev->si_drv1;
8635 
8636 	rc = priv_check(td, PRIV_DRIVER);
8637 	if (rc != 0)
8638 		return (rc);
8639 
8640 	switch (cmd) {
8641 	case CHELSIO_T4_GETREG: {
8642 		struct t4_reg *edata = (struct t4_reg *)data;
8643 
8644 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8645 			return (EFAULT);
8646 
8647 		if (edata->size == 4)
8648 			edata->val = t4_read_reg(sc, edata->addr);
8649 		else if (edata->size == 8)
8650 			edata->val = t4_read_reg64(sc, edata->addr);
8651 		else
8652 			return (EINVAL);
8653 
8654 		break;
8655 	}
8656 	case CHELSIO_T4_SETREG: {
8657 		struct t4_reg *edata = (struct t4_reg *)data;
8658 
8659 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8660 			return (EFAULT);
8661 
8662 		if (edata->size == 4) {
8663 			if (edata->val & 0xffffffff00000000)
8664 				return (EINVAL);
8665 			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8666 		} else if (edata->size == 8)
8667 			t4_write_reg64(sc, edata->addr, edata->val);
8668 		else
8669 			return (EINVAL);
8670 		break;
8671 	}
8672 	case CHELSIO_T4_REGDUMP: {
8673 		struct t4_regdump *regs = (struct t4_regdump *)data;
8674 		int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8675 		uint8_t *buf;
8676 
8677 		if (regs->len < reglen) {
8678 			regs->len = reglen; /* hint to the caller */
8679 			return (ENOBUFS);
8680 		}
8681 
8682 		regs->len = reglen;
8683 		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8684 		t4_get_regs(sc, regs, buf);
8685 		rc = copyout(buf, regs->data, reglen);
8686 		free(buf, M_CXGBE);
8687 		break;
8688 	}
8689 	case CHELSIO_T4_GET_FILTER_MODE:
8690 		rc = get_filter_mode(sc, (uint32_t *)data);
8691 		break;
8692 	case CHELSIO_T4_SET_FILTER_MODE:
8693 		rc = set_filter_mode(sc, *(uint32_t *)data);
8694 		break;
8695 	case CHELSIO_T4_GET_FILTER:
8696 		rc = get_filter(sc, (struct t4_filter *)data);
8697 		break;
8698 	case CHELSIO_T4_SET_FILTER:
8699 		rc = set_filter(sc, (struct t4_filter *)data);
8700 		break;
8701 	case CHELSIO_T4_DEL_FILTER:
8702 		rc = del_filter(sc, (struct t4_filter *)data);
8703 		break;
8704 	case CHELSIO_T4_GET_SGE_CONTEXT:
8705 		rc = get_sge_context(sc, (struct t4_sge_context *)data);
8706 		break;
8707 	case CHELSIO_T4_LOAD_FW:
8708 		rc = load_fw(sc, (struct t4_data *)data);
8709 		break;
8710 	case CHELSIO_T4_GET_MEM:
8711 		rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8712 		break;
8713 	case CHELSIO_T4_GET_I2C:
8714 		rc = read_i2c(sc, (struct t4_i2c_data *)data);
8715 		break;
8716 	case CHELSIO_T4_CLEAR_STATS: {
8717 		int i, v;
8718 		u_int port_id = *(uint32_t *)data;
8719 		struct port_info *pi;
8720 		struct vi_info *vi;
8721 
8722 		if (port_id >= sc->params.nports)
8723 			return (EINVAL);
8724 		pi = sc->port[port_id];
8725 
8726 		/* MAC stats */
8727 		t4_clr_port_stats(sc, pi->tx_chan);
8728 		pi->tx_parse_error = 0;
8729 		mtx_lock(&sc->regwin_lock);
8730 		for_each_vi(pi, v, vi) {
8731 			if (vi->flags & VI_INIT_DONE)
8732 				t4_clr_vi_stats(sc, vi->viid);
8733 		}
8734 		mtx_unlock(&sc->regwin_lock);
8735 
8736 		/*
8737 		 * Since this command accepts a port, clear stats for
8738 		 * all VIs on this port.
8739 		 */
8740 		for_each_vi(pi, v, vi) {
8741 			if (vi->flags & VI_INIT_DONE) {
8742 				struct sge_rxq *rxq;
8743 				struct sge_txq *txq;
8744 				struct sge_wrq *wrq;
8745 
8746 				if (vi->flags & VI_NETMAP)
8747 					continue;
8748 
8749 				for_each_rxq(vi, i, rxq) {
8750 #if defined(INET) || defined(INET6)
8751 					rxq->lro.lro_queued = 0;
8752 					rxq->lro.lro_flushed = 0;
8753 #endif
8754 					rxq->rxcsum = 0;
8755 					rxq->vlan_extraction = 0;
8756 				}
8757 
8758 				for_each_txq(vi, i, txq) {
8759 					txq->txcsum = 0;
8760 					txq->tso_wrs = 0;
8761 					txq->vlan_insertion = 0;
8762 					txq->imm_wrs = 0;
8763 					txq->sgl_wrs = 0;
8764 					txq->txpkt_wrs = 0;
8765 					txq->txpkts0_wrs = 0;
8766 					txq->txpkts1_wrs = 0;
8767 					txq->txpkts0_pkts = 0;
8768 					txq->txpkts1_pkts = 0;
8769 					mp_ring_reset_stats(txq->r);
8770 				}
8771 
8772 #ifdef TCP_OFFLOAD
8773 				/* nothing to clear for each ofld_rxq */
8774 
8775 				for_each_ofld_txq(vi, i, wrq) {
8776 					wrq->tx_wrs_direct = 0;
8777 					wrq->tx_wrs_copied = 0;
8778 				}
8779 #endif
8780 
8781 				if (IS_MAIN_VI(vi)) {
8782 					wrq = &sc->sge.ctrlq[pi->port_id];
8783 					wrq->tx_wrs_direct = 0;
8784 					wrq->tx_wrs_copied = 0;
8785 				}
8786 			}
8787 		}
8788 		break;
8789 	}
8790 	case CHELSIO_T4_SCHED_CLASS:
8791 		rc = set_sched_class(sc, (struct t4_sched_params *)data);
8792 		break;
8793 	case CHELSIO_T4_SCHED_QUEUE:
8794 		rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8795 		break;
8796 	case CHELSIO_T4_GET_TRACER:
8797 		rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8798 		break;
8799 	case CHELSIO_T4_SET_TRACER:
8800 		rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8801 		break;
8802 	default:
8803 		rc = EINVAL;
8804 	}
8805 
8806 	return (rc);
8807 }
8808 
8809 #ifdef TCP_OFFLOAD
8810 void
8811 t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
8812 {
8813 
8814 	t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8815 	t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8816 		V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8817 		V_HPZ3(pgsz_order[3]));
8818 }
8819 
8820 static int
8821 toe_capability(struct vi_info *vi, int enable)
8822 {
8823 	int rc;
8824 	struct port_info *pi = vi->pi;
8825 	struct adapter *sc = pi->adapter;
8826 
8827 	ASSERT_SYNCHRONIZED_OP(sc);
8828 
8829 	if (!is_offload(sc))
8830 		return (ENODEV);
8831 
8832 	if (enable) {
8833 		if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
8834 			/* TOE is already enabled. */
8835 			return (0);
8836 		}
8837 
8838 		/*
8839 		 * We need the port's queues around so that we're able to send
8840 		 * and receive CPLs to/from the TOE even if the ifnet for this
8841 		 * port has never been UP'd administratively.
8842 		 */
8843 		if (!(vi->flags & VI_INIT_DONE)) {
8844 			rc = cxgbe_init_synchronized(vi);
8845 			if (rc)
8846 				return (rc);
8847 		}
8848 		if (!(pi->vi[0].flags & VI_INIT_DONE)) {
8849 			rc = cxgbe_init_synchronized(&pi->vi[0]);
8850 			if (rc)
8851 				return (rc);
8852 		}
8853 
8854 		if (isset(&sc->offload_map, pi->port_id)) {
8855 			/* TOE is enabled on another VI of this port. */
8856 			pi->uld_vis++;
8857 			return (0);
8858 		}
8859 
8860 		if (!uld_active(sc, ULD_TOM)) {
8861 			rc = t4_activate_uld(sc, ULD_TOM);
8862 			if (rc == EAGAIN) {
8863 				log(LOG_WARNING,
8864 				    "You must kldload t4_tom.ko before trying "
8865 				    "to enable TOE on a cxgbe interface.\n");
8866 			}
8867 			if (rc != 0)
8868 				return (rc);
8869 			KASSERT(sc->tom_softc != NULL,
8870 			    ("%s: TOM activated but softc NULL", __func__));
8871 			KASSERT(uld_active(sc, ULD_TOM),
8872 			    ("%s: TOM activated but flag not set", __func__));
8873 		}
8874 
8875 		/* Activate iWARP and iSCSI too, if the modules are loaded. */
8876 		if (!uld_active(sc, ULD_IWARP))
8877 			(void) t4_activate_uld(sc, ULD_IWARP);
8878 		if (!uld_active(sc, ULD_ISCSI))
8879 			(void) t4_activate_uld(sc, ULD_ISCSI);
8880 
8881 		pi->uld_vis++;
8882 		setbit(&sc->offload_map, pi->port_id);
8883 	} else {
8884 		pi->uld_vis--;
8885 
8886 		if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
8887 			return (0);
8888 
8889 		KASSERT(uld_active(sc, ULD_TOM),
8890 		    ("%s: TOM never initialized?", __func__));
8891 		clrbit(&sc->offload_map, pi->port_id);
8892 	}
8893 
8894 	return (0);
8895 }
8896 
8897 /*
8898  * Add an upper layer driver to the global list.
8899  */
8900 int
8901 t4_register_uld(struct uld_info *ui)
8902 {
8903 	int rc = 0;
8904 	struct uld_info *u;
8905 
8906 	sx_xlock(&t4_uld_list_lock);
8907 	SLIST_FOREACH(u, &t4_uld_list, link) {
8908 	    if (u->uld_id == ui->uld_id) {
8909 		    rc = EEXIST;
8910 		    goto done;
8911 	    }
8912 	}
8913 
8914 	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8915 	ui->refcount = 0;
8916 done:
8917 	sx_xunlock(&t4_uld_list_lock);
8918 	return (rc);
8919 }
8920 
8921 int
8922 t4_unregister_uld(struct uld_info *ui)
8923 {
8924 	int rc = EINVAL;
8925 	struct uld_info *u;
8926 
8927 	sx_xlock(&t4_uld_list_lock);
8928 
8929 	SLIST_FOREACH(u, &t4_uld_list, link) {
8930 	    if (u == ui) {
8931 		    if (ui->refcount > 0) {
8932 			    rc = EBUSY;
8933 			    goto done;
8934 		    }
8935 
8936 		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8937 		    rc = 0;
8938 		    goto done;
8939 	    }
8940 	}
8941 done:
8942 	sx_xunlock(&t4_uld_list_lock);
8943 	return (rc);
8944 }
8945 
8946 int
8947 t4_activate_uld(struct adapter *sc, int id)
8948 {
8949 	int rc;
8950 	struct uld_info *ui;
8951 
8952 	ASSERT_SYNCHRONIZED_OP(sc);
8953 
8954 	if (id < 0 || id > ULD_MAX)
8955 		return (EINVAL);
8956 	rc = EAGAIN;	/* kldoad the module with this ULD and try again. */
8957 
8958 	sx_slock(&t4_uld_list_lock);
8959 
8960 	SLIST_FOREACH(ui, &t4_uld_list, link) {
8961 		if (ui->uld_id == id) {
8962 			if (!(sc->flags & FULL_INIT_DONE)) {
8963 				rc = adapter_full_init(sc);
8964 				if (rc != 0)
8965 					break;
8966 			}
8967 
8968 			rc = ui->activate(sc);
8969 			if (rc == 0) {
8970 				setbit(&sc->active_ulds, id);
8971 				ui->refcount++;
8972 			}
8973 			break;
8974 		}
8975 	}
8976 
8977 	sx_sunlock(&t4_uld_list_lock);
8978 
8979 	return (rc);
8980 }
8981 
8982 int
8983 t4_deactivate_uld(struct adapter *sc, int id)
8984 {
8985 	int rc;
8986 	struct uld_info *ui;
8987 
8988 	ASSERT_SYNCHRONIZED_OP(sc);
8989 
8990 	if (id < 0 || id > ULD_MAX)
8991 		return (EINVAL);
8992 	rc = ENXIO;
8993 
8994 	sx_slock(&t4_uld_list_lock);
8995 
8996 	SLIST_FOREACH(ui, &t4_uld_list, link) {
8997 		if (ui->uld_id == id) {
8998 			rc = ui->deactivate(sc);
8999 			if (rc == 0) {
9000 				clrbit(&sc->active_ulds, id);
9001 				ui->refcount--;
9002 			}
9003 			break;
9004 		}
9005 	}
9006 
9007 	sx_sunlock(&t4_uld_list_lock);
9008 
9009 	return (rc);
9010 }
9011 
9012 int
9013 uld_active(struct adapter *sc, int uld_id)
9014 {
9015 
9016 	MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
9017 
9018 	return (isset(&sc->active_ulds, uld_id));
9019 }
9020 #endif
9021 
9022 /*
9023  * Come up with reasonable defaults for some of the tunables, provided they're
9024  * not set by the user (in which case we'll use the values as is).
9025  */
9026 static void
9027 tweak_tunables(void)
9028 {
9029 	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
9030 
9031 	if (t4_ntxq10g < 1) {
9032 #ifdef RSS
9033 		t4_ntxq10g = rss_getnumbuckets();
9034 #else
9035 		t4_ntxq10g = min(nc, NTXQ_10G);
9036 #endif
9037 	}
9038 
9039 	if (t4_ntxq1g < 1) {
9040 #ifdef RSS
9041 		/* XXX: way too many for 1GbE? */
9042 		t4_ntxq1g = rss_getnumbuckets();
9043 #else
9044 		t4_ntxq1g = min(nc, NTXQ_1G);
9045 #endif
9046 	}
9047 
9048 	if (t4_nrxq10g < 1) {
9049 #ifdef RSS
9050 		t4_nrxq10g = rss_getnumbuckets();
9051 #else
9052 		t4_nrxq10g = min(nc, NRXQ_10G);
9053 #endif
9054 	}
9055 
9056 	if (t4_nrxq1g < 1) {
9057 #ifdef RSS
9058 		/* XXX: way too many for 1GbE? */
9059 		t4_nrxq1g = rss_getnumbuckets();
9060 #else
9061 		t4_nrxq1g = min(nc, NRXQ_1G);
9062 #endif
9063 	}
9064 
9065 #ifdef TCP_OFFLOAD
9066 	if (t4_nofldtxq10g < 1)
9067 		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
9068 
9069 	if (t4_nofldtxq1g < 1)
9070 		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
9071 
9072 	if (t4_nofldrxq10g < 1)
9073 		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
9074 
9075 	if (t4_nofldrxq1g < 1)
9076 		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
9077 
9078 	if (t4_toecaps_allowed == -1)
9079 		t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
9080 #else
9081 	if (t4_toecaps_allowed == -1)
9082 		t4_toecaps_allowed = 0;
9083 #endif
9084 
9085 #ifdef DEV_NETMAP
9086 	if (t4_nnmtxq10g < 1)
9087 		t4_nnmtxq10g = min(nc, NNMTXQ_10G);
9088 
9089 	if (t4_nnmtxq1g < 1)
9090 		t4_nnmtxq1g = min(nc, NNMTXQ_1G);
9091 
9092 	if (t4_nnmrxq10g < 1)
9093 		t4_nnmrxq10g = min(nc, NNMRXQ_10G);
9094 
9095 	if (t4_nnmrxq1g < 1)
9096 		t4_nnmrxq1g = min(nc, NNMRXQ_1G);
9097 #endif
9098 
9099 	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
9100 		t4_tmr_idx_10g = TMR_IDX_10G;
9101 
9102 	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
9103 		t4_pktc_idx_10g = PKTC_IDX_10G;
9104 
9105 	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
9106 		t4_tmr_idx_1g = TMR_IDX_1G;
9107 
9108 	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
9109 		t4_pktc_idx_1g = PKTC_IDX_1G;
9110 
9111 	if (t4_qsize_txq < 128)
9112 		t4_qsize_txq = 128;
9113 
9114 	if (t4_qsize_rxq < 128)
9115 		t4_qsize_rxq = 128;
9116 	while (t4_qsize_rxq & 7)
9117 		t4_qsize_rxq++;
9118 
9119 	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
9120 }
9121 
9122 static struct sx mlu;	/* mod load unload */
9123 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
9124 
9125 static int
9126 mod_event(module_t mod, int cmd, void *arg)
9127 {
9128 	int rc = 0;
9129 	static int loaded = 0;
9130 
9131 	switch (cmd) {
9132 	case MOD_LOAD:
9133 		sx_xlock(&mlu);
9134 		if (loaded++ == 0) {
9135 			t4_sge_modload();
9136 			sx_init(&t4_list_lock, "T4/T5 adapters");
9137 			SLIST_INIT(&t4_list);
9138 #ifdef TCP_OFFLOAD
9139 			sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
9140 			SLIST_INIT(&t4_uld_list);
9141 #endif
9142 			t4_tracer_modload();
9143 			tweak_tunables();
9144 		}
9145 		sx_xunlock(&mlu);
9146 		break;
9147 
9148 	case MOD_UNLOAD:
9149 		sx_xlock(&mlu);
9150 		if (--loaded == 0) {
9151 			int tries;
9152 
9153 			sx_slock(&t4_list_lock);
9154 			if (!SLIST_EMPTY(&t4_list)) {
9155 				rc = EBUSY;
9156 				sx_sunlock(&t4_list_lock);
9157 				goto done_unload;
9158 			}
9159 #ifdef TCP_OFFLOAD
9160 			sx_slock(&t4_uld_list_lock);
9161 			if (!SLIST_EMPTY(&t4_uld_list)) {
9162 				rc = EBUSY;
9163 				sx_sunlock(&t4_uld_list_lock);
9164 				sx_sunlock(&t4_list_lock);
9165 				goto done_unload;
9166 			}
9167 #endif
9168 			tries = 0;
9169 			while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
9170 				uprintf("%ju clusters with custom free routine "
9171 				    "still is use.\n", t4_sge_extfree_refs());
9172 				pause("t4unload", 2 * hz);
9173 			}
9174 #ifdef TCP_OFFLOAD
9175 			sx_sunlock(&t4_uld_list_lock);
9176 #endif
9177 			sx_sunlock(&t4_list_lock);
9178 
9179 			if (t4_sge_extfree_refs() == 0) {
9180 				t4_tracer_modunload();
9181 #ifdef TCP_OFFLOAD
9182 				sx_destroy(&t4_uld_list_lock);
9183 #endif
9184 				sx_destroy(&t4_list_lock);
9185 				t4_sge_modunload();
9186 				loaded = 0;
9187 			} else {
9188 				rc = EBUSY;
9189 				loaded++;	/* undo earlier decrement */
9190 			}
9191 		}
9192 done_unload:
9193 		sx_xunlock(&mlu);
9194 		break;
9195 	}
9196 
9197 	return (rc);
9198 }
9199 
9200 static devclass_t t4_devclass, t5_devclass;
9201 static devclass_t cxgbe_devclass, cxl_devclass;
9202 static devclass_t vcxgbe_devclass, vcxl_devclass;
9203 
9204 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
9205 MODULE_VERSION(t4nex, 1);
9206 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
9207 #ifdef DEV_NETMAP
9208 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
9209 #endif /* DEV_NETMAP */
9210 
9211 
9212 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
9213 MODULE_VERSION(t5nex, 1);
9214 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
9215 #ifdef DEV_NETMAP
9216 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
9217 #endif /* DEV_NETMAP */
9218 
9219 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
9220 MODULE_VERSION(cxgbe, 1);
9221 
9222 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
9223 MODULE_VERSION(cxl, 1);
9224 
9225 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
9226 MODULE_VERSION(vcxgbe, 1);
9227 
9228 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
9229 MODULE_VERSION(vcxl, 1);
9230