1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 586 sc = device_get_softc(dev); 587 sc->dev = dev; 588 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 589 590 pci_enable_busmaster(dev); 591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 592 uint32_t v; 593 594 pci_set_max_read_req(dev, 4096); 595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 598 599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 600 } 601 602 sc->traceq = -1; 603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 605 device_get_nameunit(dev)); 606 607 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 608 device_get_nameunit(dev)); 609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 610 sx_xlock(&t4_list_lock); 611 SLIST_INSERT_HEAD(&t4_list, sc, link); 612 sx_xunlock(&t4_list_lock); 613 614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 615 TAILQ_INIT(&sc->sfl); 616 callout_init(&sc->sfl_callout, 1); 617 618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 619 620 rc = map_bars_0_and_4(sc); 621 if (rc != 0) 622 goto done; /* error message displayed already */ 623 624 /* 625 * This is the real PF# to which we're attaching. Works from within PCI 626 * passthrough environments too, where pci_get_function() could return a 627 * different PF# depending on the passthrough configuration. We need to 628 * use the real PF# in all our communication with the firmware. 629 */ 630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 631 sc->mbox = sc->pf; 632 633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 634 sc->an_handler = an_not_handled; 635 for (i = 0; i < nitems(sc->cpl_handler); i++) 636 sc->cpl_handler[i] = cpl_not_handled; 637 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 638 sc->fw_msg_handler[i] = fw_msg_not_handled; 639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 642 t4_init_sge_cpl_handlers(sc); 643 644 /* Prepare the adapter for operation */ 645 rc = -t4_prep_adapter(sc); 646 if (rc != 0) { 647 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 648 goto done; 649 } 650 651 /* 652 * Do this really early, with the memory windows set up even before the 653 * character device. The userland tool's register i/o and mem read 654 * will work even in "recovery mode". 655 */ 656 setup_memwin(sc); 657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 659 device_get_nameunit(dev)); 660 if (sc->cdev == NULL) 661 device_printf(dev, "failed to create nexus char device.\n"); 662 else 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 671 #if defined(__i386__) 672 if ((cpu_feature & CPUID_CX8) == 0) { 673 device_printf(dev, "64 bit atomics not available.\n"); 674 rc = ENOTSUP; 675 goto done; 676 } 677 #endif 678 679 /* Prepare the firmware for operation */ 680 rc = prep_firmware(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = get_params__post_init(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = set_params__post_init(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 rc = map_bar_2(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 rc = t4_create_dma_tag(sc); 697 if (rc != 0) 698 goto done; /* error message displayed already */ 699 700 /* 701 * First pass over all the ports - allocate VIs and initialize some 702 * basic parameters like mac address, port type, etc. We also figure 703 * out whether a port is 10G or 1G and use that information when 704 * calculating how many interrupts to attempt to allocate. 705 */ 706 n10g = n1g = 0; 707 for_each_port(sc, i) { 708 struct port_info *pi; 709 710 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 711 sc->port[i] = pi; 712 713 /* These must be set before t4_port_init */ 714 pi->adapter = sc; 715 pi->port_id = i; 716 717 /* Allocate the vi and initialize parameters like mac addr */ 718 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 719 if (rc != 0) { 720 device_printf(dev, "unable to initialize port %d: %d\n", 721 i, rc); 722 free(pi, M_CXGBE); 723 sc->port[i] = NULL; 724 goto done; 725 } 726 727 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 728 pi->link_cfg.requested_fc |= t4_pause_settings; 729 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 730 pi->link_cfg.fc |= t4_pause_settings; 731 732 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 733 if (rc != 0) { 734 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 735 free(pi, M_CXGBE); 736 sc->port[i] = NULL; 737 goto done; 738 } 739 740 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 741 device_get_nameunit(dev), i); 742 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 743 sc->chan_map[pi->tx_chan] = i; 744 745 if (is_10G_port(pi) || is_40G_port(pi)) { 746 n10g++; 747 pi->tmr_idx = t4_tmr_idx_10g; 748 pi->pktc_idx = t4_pktc_idx_10g; 749 } else { 750 n1g++; 751 pi->tmr_idx = t4_tmr_idx_1g; 752 pi->pktc_idx = t4_pktc_idx_1g; 753 } 754 755 pi->xact_addr_filt = -1; 756 pi->linkdnrc = -1; 757 758 pi->qsize_rxq = t4_qsize_rxq; 759 pi->qsize_txq = t4_qsize_txq; 760 761 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 762 if (pi->dev == NULL) { 763 device_printf(dev, 764 "failed to add device for port %d.\n", i); 765 rc = ENXIO; 766 goto done; 767 } 768 device_set_softc(pi->dev, pi); 769 } 770 771 /* 772 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 773 */ 774 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 775 if (rc != 0) 776 goto done; /* error message displayed already */ 777 778 sc->intr_type = iaq.intr_type; 779 sc->intr_count = iaq.nirq; 780 781 s = &sc->sge; 782 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 783 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 784 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 785 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 786 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 787 #ifdef TCP_OFFLOAD 788 if (is_offload(sc)) { 789 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 790 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 791 s->neq += s->nofldtxq + s->nofldrxq; 792 s->niq += s->nofldrxq; 793 794 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 797 M_CXGBE, M_ZERO | M_WAITOK); 798 } 799 #endif 800 #ifdef DEV_NETMAP 801 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 802 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 803 s->neq += s->nnmtxq + s->nnmrxq; 804 s->niq += s->nnmrxq; 805 806 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 807 M_CXGBE, M_ZERO | M_WAITOK); 808 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 809 M_CXGBE, M_ZERO | M_WAITOK); 810 #endif 811 812 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 823 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 824 M_ZERO | M_WAITOK); 825 826 t4_init_l2t(sc, M_WAITOK); 827 828 /* 829 * Second pass over the ports. This time we know the number of rx and 830 * tx queues that each port should get. 831 */ 832 rqidx = tqidx = 0; 833 #ifdef TCP_OFFLOAD 834 ofld_rqidx = ofld_tqidx = 0; 835 #endif 836 #ifdef DEV_NETMAP 837 nm_rqidx = nm_tqidx = 0; 838 #endif 839 for_each_port(sc, i) { 840 struct port_info *pi = sc->port[i]; 841 842 if (pi == NULL) 843 continue; 844 845 pi->first_rxq = rqidx; 846 pi->first_txq = tqidx; 847 if (is_10G_port(pi) || is_40G_port(pi)) { 848 pi->flags |= iaq.intr_flags_10g; 849 pi->nrxq = iaq.nrxq10g; 850 pi->ntxq = iaq.ntxq10g; 851 } else { 852 pi->flags |= iaq.intr_flags_1g; 853 pi->nrxq = iaq.nrxq1g; 854 pi->ntxq = iaq.ntxq1g; 855 } 856 857 if (pi->ntxq > 1) 858 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 859 else 860 pi->rsrv_noflowq = 0; 861 862 rqidx += pi->nrxq; 863 tqidx += pi->ntxq; 864 #ifdef TCP_OFFLOAD 865 if (is_offload(sc)) { 866 pi->first_ofld_rxq = ofld_rqidx; 867 pi->first_ofld_txq = ofld_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nofldrxq = iaq.nofldrxq10g; 870 pi->nofldtxq = iaq.nofldtxq10g; 871 } else { 872 pi->nofldrxq = iaq.nofldrxq1g; 873 pi->nofldtxq = iaq.nofldtxq1g; 874 } 875 ofld_rqidx += pi->nofldrxq; 876 ofld_tqidx += pi->nofldtxq; 877 } 878 #endif 879 #ifdef DEV_NETMAP 880 pi->first_nm_rxq = nm_rqidx; 881 pi->first_nm_txq = nm_tqidx; 882 if (is_10G_port(pi) || is_40G_port(pi)) { 883 pi->nnmrxq = iaq.nnmrxq10g; 884 pi->nnmtxq = iaq.nnmtxq10g; 885 } else { 886 pi->nnmrxq = iaq.nnmrxq1g; 887 pi->nnmtxq = iaq.nnmtxq1g; 888 } 889 nm_rqidx += pi->nnmrxq; 890 nm_tqidx += pi->nnmtxq; 891 #endif 892 } 893 894 rc = setup_intr_handlers(sc); 895 if (rc != 0) { 896 device_printf(dev, 897 "failed to setup interrupt handlers: %d\n", rc); 898 goto done; 899 } 900 901 rc = bus_generic_attach(dev); 902 if (rc != 0) { 903 device_printf(dev, 904 "failed to attach all child ports: %d\n", rc); 905 goto done; 906 } 907 908 device_printf(dev, 909 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 910 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 911 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 912 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 913 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 914 915 t4_set_desc(sc); 916 917 done: 918 if (rc != 0 && sc->cdev) { 919 /* cdev was created and so cxgbetool works; recover that way. */ 920 device_printf(dev, 921 "error during attach, adapter is now in recovery mode.\n"); 922 rc = 0; 923 } 924 925 if (rc != 0) 926 t4_detach(dev); 927 else 928 t4_sysctls(sc); 929 930 return (rc); 931 } 932 933 /* 934 * Idempotent 935 */ 936 static int 937 t4_detach(device_t dev) 938 { 939 struct adapter *sc; 940 struct port_info *pi; 941 int i, rc; 942 943 sc = device_get_softc(dev); 944 945 if (sc->flags & FULL_INIT_DONE) 946 t4_intr_disable(sc); 947 948 if (sc->cdev) { 949 destroy_dev(sc->cdev); 950 sc->cdev = NULL; 951 } 952 953 rc = bus_generic_detach(dev); 954 if (rc) { 955 device_printf(dev, 956 "failed to detach child devices: %d\n", rc); 957 return (rc); 958 } 959 960 for (i = 0; i < sc->intr_count; i++) 961 t4_free_irq(sc, &sc->irq[i]); 962 963 for (i = 0; i < MAX_NPORTS; i++) { 964 pi = sc->port[i]; 965 if (pi) { 966 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 967 if (pi->dev) 968 device_delete_child(dev, pi->dev); 969 970 mtx_destroy(&pi->pi_lock); 971 free(pi, M_CXGBE); 972 } 973 } 974 975 if (sc->flags & FULL_INIT_DONE) 976 adapter_full_uninit(sc); 977 978 if (sc->flags & FW_OK) 979 t4_fw_bye(sc, sc->mbox); 980 981 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 982 pci_release_msi(dev); 983 984 if (sc->regs_res) 985 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 986 sc->regs_res); 987 988 if (sc->udbs_res) 989 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 990 sc->udbs_res); 991 992 if (sc->msix_res) 993 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 994 sc->msix_res); 995 996 if (sc->l2t) 997 t4_free_l2t(sc->l2t); 998 999 #ifdef TCP_OFFLOAD 1000 free(sc->sge.ofld_rxq, M_CXGBE); 1001 free(sc->sge.ofld_txq, M_CXGBE); 1002 #endif 1003 #ifdef DEV_NETMAP 1004 free(sc->sge.nm_rxq, M_CXGBE); 1005 free(sc->sge.nm_txq, M_CXGBE); 1006 #endif 1007 free(sc->irq, M_CXGBE); 1008 free(sc->sge.rxq, M_CXGBE); 1009 free(sc->sge.txq, M_CXGBE); 1010 free(sc->sge.ctrlq, M_CXGBE); 1011 free(sc->sge.iqmap, M_CXGBE); 1012 free(sc->sge.eqmap, M_CXGBE); 1013 free(sc->tids.ftid_tab, M_CXGBE); 1014 t4_destroy_dma_tag(sc); 1015 if (mtx_initialized(&sc->sc_lock)) { 1016 sx_xlock(&t4_list_lock); 1017 SLIST_REMOVE(&t4_list, sc, adapter, link); 1018 sx_xunlock(&t4_list_lock); 1019 mtx_destroy(&sc->sc_lock); 1020 } 1021 1022 if (mtx_initialized(&sc->tids.ftid_lock)) 1023 mtx_destroy(&sc->tids.ftid_lock); 1024 if (mtx_initialized(&sc->sfl_lock)) 1025 mtx_destroy(&sc->sfl_lock); 1026 if (mtx_initialized(&sc->ifp_lock)) 1027 mtx_destroy(&sc->ifp_lock); 1028 if (mtx_initialized(&sc->regwin_lock)) 1029 mtx_destroy(&sc->regwin_lock); 1030 1031 bzero(sc, sizeof(*sc)); 1032 1033 return (0); 1034 } 1035 1036 static int 1037 cxgbe_probe(device_t dev) 1038 { 1039 char buf[128]; 1040 struct port_info *pi = device_get_softc(dev); 1041 1042 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1043 device_set_desc_copy(dev, buf); 1044 1045 return (BUS_PROBE_DEFAULT); 1046 } 1047 1048 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1049 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1050 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1051 #define T4_CAP_ENABLE (T4_CAP) 1052 1053 static int 1054 cxgbe_attach(device_t dev) 1055 { 1056 struct port_info *pi = device_get_softc(dev); 1057 struct ifnet *ifp; 1058 char *s; 1059 int n, o; 1060 1061 /* Allocate an ifnet and set it up */ 1062 ifp = if_alloc(IFT_ETHER); 1063 if (ifp == NULL) { 1064 device_printf(dev, "Cannot allocate ifnet\n"); 1065 return (ENOMEM); 1066 } 1067 pi->ifp = ifp; 1068 ifp->if_softc = pi; 1069 1070 callout_init(&pi->tick, 1); 1071 1072 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1073 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1074 1075 ifp->if_init = cxgbe_init; 1076 ifp->if_ioctl = cxgbe_ioctl; 1077 ifp->if_transmit = cxgbe_transmit; 1078 ifp->if_qflush = cxgbe_qflush; 1079 ifp->if_get_counter = cxgbe_get_counter; 1080 1081 ifp->if_capabilities = T4_CAP; 1082 #ifdef TCP_OFFLOAD 1083 if (is_offload(pi->adapter)) 1084 ifp->if_capabilities |= IFCAP_TOE; 1085 #endif 1086 ifp->if_capenable = T4_CAP_ENABLE; 1087 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1088 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1089 1090 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1091 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1092 ifp->if_hw_tsomaxsegsize = 65536; 1093 1094 /* Initialize ifmedia for this port */ 1095 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1096 cxgbe_media_status); 1097 build_medialist(pi, &pi->media); 1098 1099 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1100 EVENTHANDLER_PRI_ANY); 1101 1102 ether_ifattach(ifp, pi->hw_addr); 1103 1104 n = 128; 1105 s = malloc(n, M_CXGBE, M_WAITOK); 1106 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1107 MPASS(n > o); 1108 #ifdef TCP_OFFLOAD 1109 if (is_offload(pi->adapter)) { 1110 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1111 pi->nofldtxq, pi->nofldrxq); 1112 MPASS(n > o); 1113 } 1114 #endif 1115 #ifdef DEV_NETMAP 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1117 pi->nnmrxq); 1118 MPASS(n > o); 1119 #endif 1120 device_printf(dev, "%s\n", s); 1121 free(s, M_CXGBE); 1122 1123 #ifdef DEV_NETMAP 1124 /* nm_media handled here to keep implementation private to this file */ 1125 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1126 cxgbe_media_status); 1127 build_medialist(pi, &pi->nm_media); 1128 create_netmap_ifnet(pi); /* logs errors it something fails */ 1129 #endif 1130 cxgbe_sysctls(pi); 1131 1132 return (0); 1133 } 1134 1135 static int 1136 cxgbe_detach(device_t dev) 1137 { 1138 struct port_info *pi = device_get_softc(dev); 1139 struct adapter *sc = pi->adapter; 1140 struct ifnet *ifp = pi->ifp; 1141 1142 /* Tell if_ioctl and if_init that the port is going away */ 1143 ADAPTER_LOCK(sc); 1144 SET_DOOMED(pi); 1145 wakeup(&sc->flags); 1146 while (IS_BUSY(sc)) 1147 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1148 SET_BUSY(sc); 1149 #ifdef INVARIANTS 1150 sc->last_op = "t4detach"; 1151 sc->last_op_thr = curthread; 1152 sc->last_op_flags = 0; 1153 #endif 1154 ADAPTER_UNLOCK(sc); 1155 1156 if (pi->flags & HAS_TRACEQ) { 1157 sc->traceq = -1; /* cloner should not create ifnet */ 1158 t4_tracer_port_detach(sc); 1159 } 1160 1161 if (pi->vlan_c) 1162 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1163 1164 PORT_LOCK(pi); 1165 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1166 callout_stop(&pi->tick); 1167 PORT_UNLOCK(pi); 1168 callout_drain(&pi->tick); 1169 1170 /* Let detach proceed even if these fail. */ 1171 cxgbe_uninit_synchronized(pi); 1172 port_full_uninit(pi); 1173 1174 ifmedia_removeall(&pi->media); 1175 ether_ifdetach(pi->ifp); 1176 if_free(pi->ifp); 1177 1178 #ifdef DEV_NETMAP 1179 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1180 destroy_netmap_ifnet(pi); 1181 #endif 1182 1183 ADAPTER_LOCK(sc); 1184 CLR_BUSY(sc); 1185 wakeup(&sc->flags); 1186 ADAPTER_UNLOCK(sc); 1187 1188 return (0); 1189 } 1190 1191 static void 1192 cxgbe_init(void *arg) 1193 { 1194 struct port_info *pi = arg; 1195 struct adapter *sc = pi->adapter; 1196 1197 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1198 return; 1199 cxgbe_init_synchronized(pi); 1200 end_synchronized_op(sc, 0); 1201 } 1202 1203 static int 1204 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1205 { 1206 int rc = 0, mtu, flags, can_sleep; 1207 struct port_info *pi = ifp->if_softc; 1208 struct adapter *sc = pi->adapter; 1209 struct ifreq *ifr = (struct ifreq *)data; 1210 uint32_t mask; 1211 1212 switch (cmd) { 1213 case SIOCSIFMTU: 1214 mtu = ifr->ifr_mtu; 1215 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1216 return (EINVAL); 1217 1218 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1219 if (rc) 1220 return (rc); 1221 ifp->if_mtu = mtu; 1222 if (pi->flags & PORT_INIT_DONE) { 1223 t4_update_fl_bufsize(ifp); 1224 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1225 rc = update_mac_settings(ifp, XGMAC_MTU); 1226 } 1227 end_synchronized_op(sc, 0); 1228 break; 1229 1230 case SIOCSIFFLAGS: 1231 can_sleep = 0; 1232 redo_sifflags: 1233 rc = begin_synchronized_op(sc, pi, 1234 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1235 if (rc) 1236 return (rc); 1237 1238 if (ifp->if_flags & IFF_UP) { 1239 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1240 flags = pi->if_flags; 1241 if ((ifp->if_flags ^ flags) & 1242 (IFF_PROMISC | IFF_ALLMULTI)) { 1243 if (can_sleep == 1) { 1244 end_synchronized_op(sc, 0); 1245 can_sleep = 0; 1246 goto redo_sifflags; 1247 } 1248 rc = update_mac_settings(ifp, 1249 XGMAC_PROMISC | XGMAC_ALLMULTI); 1250 } 1251 } else { 1252 if (can_sleep == 0) { 1253 end_synchronized_op(sc, LOCK_HELD); 1254 can_sleep = 1; 1255 goto redo_sifflags; 1256 } 1257 rc = cxgbe_init_synchronized(pi); 1258 } 1259 pi->if_flags = ifp->if_flags; 1260 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1261 if (can_sleep == 0) { 1262 end_synchronized_op(sc, LOCK_HELD); 1263 can_sleep = 1; 1264 goto redo_sifflags; 1265 } 1266 rc = cxgbe_uninit_synchronized(pi); 1267 } 1268 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1269 break; 1270 1271 case SIOCADDMULTI: 1272 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1273 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1274 if (rc) 1275 return (rc); 1276 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1277 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1278 end_synchronized_op(sc, LOCK_HELD); 1279 break; 1280 1281 case SIOCSIFCAP: 1282 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1283 if (rc) 1284 return (rc); 1285 1286 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1287 if (mask & IFCAP_TXCSUM) { 1288 ifp->if_capenable ^= IFCAP_TXCSUM; 1289 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1290 1291 if (IFCAP_TSO4 & ifp->if_capenable && 1292 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1293 ifp->if_capenable &= ~IFCAP_TSO4; 1294 if_printf(ifp, 1295 "tso4 disabled due to -txcsum.\n"); 1296 } 1297 } 1298 if (mask & IFCAP_TXCSUM_IPV6) { 1299 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1300 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1301 1302 if (IFCAP_TSO6 & ifp->if_capenable && 1303 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1304 ifp->if_capenable &= ~IFCAP_TSO6; 1305 if_printf(ifp, 1306 "tso6 disabled due to -txcsum6.\n"); 1307 } 1308 } 1309 if (mask & IFCAP_RXCSUM) 1310 ifp->if_capenable ^= IFCAP_RXCSUM; 1311 if (mask & IFCAP_RXCSUM_IPV6) 1312 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1313 1314 /* 1315 * Note that we leave CSUM_TSO alone (it is always set). The 1316 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1317 * sending a TSO request our way, so it's sufficient to toggle 1318 * IFCAP_TSOx only. 1319 */ 1320 if (mask & IFCAP_TSO4) { 1321 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1322 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1323 if_printf(ifp, "enable txcsum first.\n"); 1324 rc = EAGAIN; 1325 goto fail; 1326 } 1327 ifp->if_capenable ^= IFCAP_TSO4; 1328 } 1329 if (mask & IFCAP_TSO6) { 1330 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1331 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1332 if_printf(ifp, "enable txcsum6 first.\n"); 1333 rc = EAGAIN; 1334 goto fail; 1335 } 1336 ifp->if_capenable ^= IFCAP_TSO6; 1337 } 1338 if (mask & IFCAP_LRO) { 1339 #if defined(INET) || defined(INET6) 1340 int i; 1341 struct sge_rxq *rxq; 1342 1343 ifp->if_capenable ^= IFCAP_LRO; 1344 for_each_rxq(pi, i, rxq) { 1345 if (ifp->if_capenable & IFCAP_LRO) 1346 rxq->iq.flags |= IQ_LRO_ENABLED; 1347 else 1348 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1349 } 1350 #endif 1351 } 1352 #ifdef TCP_OFFLOAD 1353 if (mask & IFCAP_TOE) { 1354 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1355 1356 rc = toe_capability(pi, enable); 1357 if (rc != 0) 1358 goto fail; 1359 1360 ifp->if_capenable ^= mask; 1361 } 1362 #endif 1363 if (mask & IFCAP_VLAN_HWTAGGING) { 1364 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1365 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1366 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1367 } 1368 if (mask & IFCAP_VLAN_MTU) { 1369 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1370 1371 /* Need to find out how to disable auto-mtu-inflation */ 1372 } 1373 if (mask & IFCAP_VLAN_HWTSO) 1374 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1375 if (mask & IFCAP_VLAN_HWCSUM) 1376 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1377 1378 #ifdef VLAN_CAPABILITIES 1379 VLAN_CAPABILITIES(ifp); 1380 #endif 1381 fail: 1382 end_synchronized_op(sc, 0); 1383 break; 1384 1385 case SIOCSIFMEDIA: 1386 case SIOCGIFMEDIA: 1387 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1388 break; 1389 1390 case SIOCGI2C: { 1391 struct ifi2creq i2c; 1392 1393 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1394 if (rc != 0) 1395 break; 1396 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1397 rc = EPERM; 1398 break; 1399 } 1400 if (i2c.len > sizeof(i2c.data)) { 1401 rc = EINVAL; 1402 break; 1403 } 1404 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1405 if (rc) 1406 return (rc); 1407 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1408 i2c.offset, i2c.len, &i2c.data[0]); 1409 end_synchronized_op(sc, 0); 1410 if (rc == 0) 1411 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1412 break; 1413 } 1414 1415 default: 1416 rc = ether_ioctl(ifp, cmd, data); 1417 } 1418 1419 return (rc); 1420 } 1421 1422 static int 1423 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1424 { 1425 struct port_info *pi = ifp->if_softc; 1426 struct adapter *sc = pi->adapter; 1427 struct sge_txq *txq; 1428 void *items[1]; 1429 int rc; 1430 1431 M_ASSERTPKTHDR(m); 1432 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1433 1434 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1435 m_freem(m); 1436 return (ENETDOWN); 1437 } 1438 1439 rc = parse_pkt(&m); 1440 if (__predict_false(rc != 0)) { 1441 MPASS(m == NULL); /* was freed already */ 1442 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1443 return (rc); 1444 } 1445 1446 /* Select a txq. */ 1447 txq = &sc->sge.txq[pi->first_txq]; 1448 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1449 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1450 pi->rsrv_noflowq); 1451 1452 items[0] = m; 1453 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1454 if (__predict_false(rc != 0)) 1455 m_freem(m); 1456 1457 return (rc); 1458 } 1459 1460 static void 1461 cxgbe_qflush(struct ifnet *ifp) 1462 { 1463 struct port_info *pi = ifp->if_softc; 1464 struct sge_txq *txq; 1465 int i; 1466 1467 /* queues do not exist if !PORT_INIT_DONE. */ 1468 if (pi->flags & PORT_INIT_DONE) { 1469 for_each_txq(pi, i, txq) { 1470 TXQ_LOCK(txq); 1471 txq->eq.flags &= ~EQ_ENABLED; 1472 TXQ_UNLOCK(txq); 1473 while (!mp_ring_is_idle(txq->r)) { 1474 mp_ring_check_drainage(txq->r, 0); 1475 pause("qflush", 1); 1476 } 1477 } 1478 } 1479 if_qflush(ifp); 1480 } 1481 1482 static uint64_t 1483 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1484 { 1485 struct port_info *pi = ifp->if_softc; 1486 struct adapter *sc = pi->adapter; 1487 struct port_stats *s = &pi->stats; 1488 1489 cxgbe_refresh_stats(sc, pi); 1490 1491 switch (c) { 1492 case IFCOUNTER_IPACKETS: 1493 return (s->rx_frames - s->rx_pause); 1494 1495 case IFCOUNTER_IERRORS: 1496 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1497 s->rx_fcs_err + s->rx_len_err); 1498 1499 case IFCOUNTER_OPACKETS: 1500 return (s->tx_frames - s->tx_pause); 1501 1502 case IFCOUNTER_OERRORS: 1503 return (s->tx_error_frames); 1504 1505 case IFCOUNTER_IBYTES: 1506 return (s->rx_octets - s->rx_pause * 64); 1507 1508 case IFCOUNTER_OBYTES: 1509 return (s->tx_octets - s->tx_pause * 64); 1510 1511 case IFCOUNTER_IMCASTS: 1512 return (s->rx_mcast_frames - s->rx_pause); 1513 1514 case IFCOUNTER_OMCASTS: 1515 return (s->tx_mcast_frames - s->tx_pause); 1516 1517 case IFCOUNTER_IQDROPS: 1518 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1519 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1520 s->rx_trunc3 + pi->tnl_cong_drops); 1521 1522 case IFCOUNTER_OQDROPS: { 1523 uint64_t drops; 1524 1525 drops = s->tx_drop; 1526 if (pi->flags & PORT_INIT_DONE) { 1527 int i; 1528 struct sge_txq *txq; 1529 1530 for_each_txq(pi, i, txq) 1531 drops += counter_u64_fetch(txq->r->drops); 1532 } 1533 1534 return (drops); 1535 1536 } 1537 1538 default: 1539 return (if_get_counter_default(ifp, c)); 1540 } 1541 } 1542 1543 static int 1544 cxgbe_media_change(struct ifnet *ifp) 1545 { 1546 struct port_info *pi = ifp->if_softc; 1547 1548 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1549 1550 return (EOPNOTSUPP); 1551 } 1552 1553 static void 1554 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1555 { 1556 struct port_info *pi = ifp->if_softc; 1557 struct ifmedia *media = NULL; 1558 struct ifmedia_entry *cur; 1559 int speed = pi->link_cfg.speed; 1560 1561 if (ifp == pi->ifp) 1562 media = &pi->media; 1563 #ifdef DEV_NETMAP 1564 else if (ifp == pi->nm_ifp) 1565 media = &pi->nm_media; 1566 #endif 1567 MPASS(media != NULL); 1568 1569 cur = media->ifm_cur; 1570 1571 ifmr->ifm_status = IFM_AVALID; 1572 if (!pi->link_cfg.link_ok) 1573 return; 1574 1575 ifmr->ifm_status |= IFM_ACTIVE; 1576 1577 /* active and current will differ iff current media is autoselect. */ 1578 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1579 return; 1580 1581 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1582 if (speed == SPEED_10000) 1583 ifmr->ifm_active |= IFM_10G_T; 1584 else if (speed == SPEED_1000) 1585 ifmr->ifm_active |= IFM_1000_T; 1586 else if (speed == SPEED_100) 1587 ifmr->ifm_active |= IFM_100_TX; 1588 else if (speed == SPEED_10) 1589 ifmr->ifm_active |= IFM_10_T; 1590 else 1591 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1592 speed)); 1593 } 1594 1595 void 1596 t4_fatal_err(struct adapter *sc) 1597 { 1598 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1599 t4_intr_disable(sc); 1600 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1601 device_get_nameunit(sc->dev)); 1602 } 1603 1604 static int 1605 map_bars_0_and_4(struct adapter *sc) 1606 { 1607 sc->regs_rid = PCIR_BAR(0); 1608 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1609 &sc->regs_rid, RF_ACTIVE); 1610 if (sc->regs_res == NULL) { 1611 device_printf(sc->dev, "cannot map registers.\n"); 1612 return (ENXIO); 1613 } 1614 sc->bt = rman_get_bustag(sc->regs_res); 1615 sc->bh = rman_get_bushandle(sc->regs_res); 1616 sc->mmio_len = rman_get_size(sc->regs_res); 1617 setbit(&sc->doorbells, DOORBELL_KDB); 1618 1619 sc->msix_rid = PCIR_BAR(4); 1620 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1621 &sc->msix_rid, RF_ACTIVE); 1622 if (sc->msix_res == NULL) { 1623 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1624 return (ENXIO); 1625 } 1626 1627 return (0); 1628 } 1629 1630 static int 1631 map_bar_2(struct adapter *sc) 1632 { 1633 1634 /* 1635 * T4: only iWARP driver uses the userspace doorbells. There is no need 1636 * to map it if RDMA is disabled. 1637 */ 1638 if (is_t4(sc) && sc->rdmacaps == 0) 1639 return (0); 1640 1641 sc->udbs_rid = PCIR_BAR(2); 1642 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1643 &sc->udbs_rid, RF_ACTIVE); 1644 if (sc->udbs_res == NULL) { 1645 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1646 return (ENXIO); 1647 } 1648 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1649 1650 if (is_t5(sc)) { 1651 setbit(&sc->doorbells, DOORBELL_UDB); 1652 #if defined(__i386__) || defined(__amd64__) 1653 if (t5_write_combine) { 1654 int rc; 1655 1656 /* 1657 * Enable write combining on BAR2. This is the 1658 * userspace doorbell BAR and is split into 128B 1659 * (UDBS_SEG_SIZE) doorbell regions, each associated 1660 * with an egress queue. The first 64B has the doorbell 1661 * and the second 64B can be used to submit a tx work 1662 * request with an implicit doorbell. 1663 */ 1664 1665 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1666 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1667 if (rc == 0) { 1668 clrbit(&sc->doorbells, DOORBELL_UDB); 1669 setbit(&sc->doorbells, DOORBELL_WCWR); 1670 setbit(&sc->doorbells, DOORBELL_UDBWC); 1671 } else { 1672 device_printf(sc->dev, 1673 "couldn't enable write combining: %d\n", 1674 rc); 1675 } 1676 1677 t4_write_reg(sc, A_SGE_STAT_CFG, 1678 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1679 } 1680 #endif 1681 } 1682 1683 return (0); 1684 } 1685 1686 static const struct memwin t4_memwin[] = { 1687 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1688 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1689 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1690 }; 1691 1692 static const struct memwin t5_memwin[] = { 1693 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1694 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1695 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1696 }; 1697 1698 static void 1699 setup_memwin(struct adapter *sc) 1700 { 1701 const struct memwin *mw; 1702 int i, n; 1703 uint32_t bar0; 1704 1705 if (is_t4(sc)) { 1706 /* 1707 * Read low 32b of bar0 indirectly via the hardware backdoor 1708 * mechanism. Works from within PCI passthrough environments 1709 * too, where rman_get_start() can return a different value. We 1710 * need to program the T4 memory window decoders with the actual 1711 * addresses that will be coming across the PCIe link. 1712 */ 1713 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1714 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1715 1716 mw = &t4_memwin[0]; 1717 n = nitems(t4_memwin); 1718 } else { 1719 /* T5 uses the relative offset inside the PCIe BAR */ 1720 bar0 = 0; 1721 1722 mw = &t5_memwin[0]; 1723 n = nitems(t5_memwin); 1724 } 1725 1726 for (i = 0; i < n; i++, mw++) { 1727 t4_write_reg(sc, 1728 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1729 (mw->base + bar0) | V_BIR(0) | 1730 V_WINDOW(ilog2(mw->aperture) - 10)); 1731 } 1732 1733 /* flush */ 1734 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1735 } 1736 1737 /* 1738 * Verify that the memory range specified by the addr/len pair is valid and lies 1739 * entirely within a single region (EDCx or MCx). 1740 */ 1741 static int 1742 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1743 { 1744 uint32_t em, addr_len, maddr, mlen; 1745 1746 /* Memory can only be accessed in naturally aligned 4 byte units */ 1747 if (addr & 3 || len & 3 || len == 0) 1748 return (EINVAL); 1749 1750 /* Enabled memories */ 1751 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1752 if (em & F_EDRAM0_ENABLE) { 1753 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1754 maddr = G_EDRAM0_BASE(addr_len) << 20; 1755 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1756 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1757 addr + len <= maddr + mlen) 1758 return (0); 1759 } 1760 if (em & F_EDRAM1_ENABLE) { 1761 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1762 maddr = G_EDRAM1_BASE(addr_len) << 20; 1763 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1764 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1765 addr + len <= maddr + mlen) 1766 return (0); 1767 } 1768 if (em & F_EXT_MEM_ENABLE) { 1769 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1770 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1771 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1772 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1773 addr + len <= maddr + mlen) 1774 return (0); 1775 } 1776 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1777 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1778 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1779 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1780 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1781 addr + len <= maddr + mlen) 1782 return (0); 1783 } 1784 1785 return (EFAULT); 1786 } 1787 1788 static int 1789 fwmtype_to_hwmtype(int mtype) 1790 { 1791 1792 switch (mtype) { 1793 case FW_MEMTYPE_EDC0: 1794 return (MEM_EDC0); 1795 case FW_MEMTYPE_EDC1: 1796 return (MEM_EDC1); 1797 case FW_MEMTYPE_EXTMEM: 1798 return (MEM_MC0); 1799 case FW_MEMTYPE_EXTMEM1: 1800 return (MEM_MC1); 1801 default: 1802 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1803 } 1804 } 1805 1806 /* 1807 * Verify that the memory range specified by the memtype/offset/len pair is 1808 * valid and lies entirely within the memtype specified. The global address of 1809 * the start of the range is returned in addr. 1810 */ 1811 static int 1812 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1813 uint32_t *addr) 1814 { 1815 uint32_t em, addr_len, maddr, mlen; 1816 1817 /* Memory can only be accessed in naturally aligned 4 byte units */ 1818 if (off & 3 || len & 3 || len == 0) 1819 return (EINVAL); 1820 1821 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1822 switch (fwmtype_to_hwmtype(mtype)) { 1823 case MEM_EDC0: 1824 if (!(em & F_EDRAM0_ENABLE)) 1825 return (EINVAL); 1826 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1827 maddr = G_EDRAM0_BASE(addr_len) << 20; 1828 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1829 break; 1830 case MEM_EDC1: 1831 if (!(em & F_EDRAM1_ENABLE)) 1832 return (EINVAL); 1833 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1834 maddr = G_EDRAM1_BASE(addr_len) << 20; 1835 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1836 break; 1837 case MEM_MC: 1838 if (!(em & F_EXT_MEM_ENABLE)) 1839 return (EINVAL); 1840 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1841 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1842 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1843 break; 1844 case MEM_MC1: 1845 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1846 return (EINVAL); 1847 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1848 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1849 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1850 break; 1851 default: 1852 return (EINVAL); 1853 } 1854 1855 if (mlen > 0 && off < mlen && off + len <= mlen) { 1856 *addr = maddr + off; /* global address */ 1857 return (0); 1858 } 1859 1860 return (EFAULT); 1861 } 1862 1863 static void 1864 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1865 { 1866 const struct memwin *mw; 1867 1868 if (is_t4(sc)) { 1869 KASSERT(win >= 0 && win < nitems(t4_memwin), 1870 ("%s: incorrect memwin# (%d)", __func__, win)); 1871 mw = &t4_memwin[win]; 1872 } else { 1873 KASSERT(win >= 0 && win < nitems(t5_memwin), 1874 ("%s: incorrect memwin# (%d)", __func__, win)); 1875 mw = &t5_memwin[win]; 1876 } 1877 1878 if (base != NULL) 1879 *base = mw->base; 1880 if (aperture != NULL) 1881 *aperture = mw->aperture; 1882 } 1883 1884 /* 1885 * Positions the memory window such that it can be used to access the specified 1886 * address in the chip's address space. The return value is the offset of addr 1887 * from the start of the window. 1888 */ 1889 static uint32_t 1890 position_memwin(struct adapter *sc, int n, uint32_t addr) 1891 { 1892 uint32_t start, pf; 1893 uint32_t reg; 1894 1895 KASSERT(n >= 0 && n <= 3, 1896 ("%s: invalid window %d.", __func__, n)); 1897 KASSERT((addr & 3) == 0, 1898 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1899 1900 if (is_t4(sc)) { 1901 pf = 0; 1902 start = addr & ~0xf; /* start must be 16B aligned */ 1903 } else { 1904 pf = V_PFNUM(sc->pf); 1905 start = addr & ~0x7f; /* start must be 128B aligned */ 1906 } 1907 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1908 1909 t4_write_reg(sc, reg, start | pf); 1910 t4_read_reg(sc, reg); 1911 1912 return (addr - start); 1913 } 1914 1915 static int 1916 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1917 struct intrs_and_queues *iaq) 1918 { 1919 int rc, itype, navail, nrxq10g, nrxq1g, n; 1920 int nofldrxq10g = 0, nofldrxq1g = 0; 1921 int nnmrxq10g = 0, nnmrxq1g = 0; 1922 1923 bzero(iaq, sizeof(*iaq)); 1924 1925 iaq->ntxq10g = t4_ntxq10g; 1926 iaq->ntxq1g = t4_ntxq1g; 1927 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1928 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1929 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1930 #ifdef TCP_OFFLOAD 1931 if (is_offload(sc)) { 1932 iaq->nofldtxq10g = t4_nofldtxq10g; 1933 iaq->nofldtxq1g = t4_nofldtxq1g; 1934 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1935 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1936 } 1937 #endif 1938 #ifdef DEV_NETMAP 1939 iaq->nnmtxq10g = t4_nnmtxq10g; 1940 iaq->nnmtxq1g = t4_nnmtxq1g; 1941 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1942 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1943 #endif 1944 1945 for (itype = INTR_MSIX; itype; itype >>= 1) { 1946 1947 if ((itype & t4_intr_types) == 0) 1948 continue; /* not allowed */ 1949 1950 if (itype == INTR_MSIX) 1951 navail = pci_msix_count(sc->dev); 1952 else if (itype == INTR_MSI) 1953 navail = pci_msi_count(sc->dev); 1954 else 1955 navail = 1; 1956 restart: 1957 if (navail == 0) 1958 continue; 1959 1960 iaq->intr_type = itype; 1961 iaq->intr_flags_10g = 0; 1962 iaq->intr_flags_1g = 0; 1963 1964 /* 1965 * Best option: an interrupt vector for errors, one for the 1966 * firmware event queue, and one for every rxq (NIC, TOE, and 1967 * netmap). 1968 */ 1969 iaq->nirq = T4_EXTRA_INTR; 1970 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1971 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1972 if (iaq->nirq <= navail && 1973 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1974 iaq->intr_flags_10g = INTR_ALL; 1975 iaq->intr_flags_1g = INTR_ALL; 1976 goto allocate; 1977 } 1978 1979 /* 1980 * Second best option: a vector for errors, one for the firmware 1981 * event queue, and vectors for either all the NIC rx queues or 1982 * all the TOE rx queues. The queues that don't get vectors 1983 * will forward their interrupts to those that do. 1984 * 1985 * Note: netmap rx queues cannot be created early and so they 1986 * can't be setup to receive forwarded interrupts for others. 1987 */ 1988 iaq->nirq = T4_EXTRA_INTR; 1989 if (nrxq10g >= nofldrxq10g) { 1990 iaq->intr_flags_10g = INTR_RXQ; 1991 iaq->nirq += n10g * nrxq10g; 1992 #ifdef DEV_NETMAP 1993 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1994 #endif 1995 } else { 1996 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1997 iaq->nirq += n10g * nofldrxq10g; 1998 #ifdef DEV_NETMAP 1999 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2000 #endif 2001 } 2002 if (nrxq1g >= nofldrxq1g) { 2003 iaq->intr_flags_1g = INTR_RXQ; 2004 iaq->nirq += n1g * nrxq1g; 2005 #ifdef DEV_NETMAP 2006 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2007 #endif 2008 } else { 2009 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2010 iaq->nirq += n1g * nofldrxq1g; 2011 #ifdef DEV_NETMAP 2012 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2013 #endif 2014 } 2015 if (iaq->nirq <= navail && 2016 (itype != INTR_MSI || powerof2(iaq->nirq))) 2017 goto allocate; 2018 2019 /* 2020 * Next best option: an interrupt vector for errors, one for the 2021 * firmware event queue, and at least one per port. At this 2022 * point we know we'll have to downsize nrxq and/or nofldrxq 2023 * and/or nnmrxq to fit what's available to us. 2024 */ 2025 iaq->nirq = T4_EXTRA_INTR; 2026 iaq->nirq += n10g + n1g; 2027 if (iaq->nirq <= navail) { 2028 int leftover = navail - iaq->nirq; 2029 2030 if (n10g > 0) { 2031 int target = max(nrxq10g, nofldrxq10g); 2032 2033 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2034 INTR_RXQ : INTR_OFLD_RXQ; 2035 2036 n = 1; 2037 while (n < target && leftover >= n10g) { 2038 leftover -= n10g; 2039 iaq->nirq += n10g; 2040 n++; 2041 } 2042 iaq->nrxq10g = min(n, nrxq10g); 2043 #ifdef TCP_OFFLOAD 2044 iaq->nofldrxq10g = min(n, nofldrxq10g); 2045 #endif 2046 #ifdef DEV_NETMAP 2047 iaq->nnmrxq10g = min(n, nnmrxq10g); 2048 #endif 2049 } 2050 2051 if (n1g > 0) { 2052 int target = max(nrxq1g, nofldrxq1g); 2053 2054 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2055 INTR_RXQ : INTR_OFLD_RXQ; 2056 2057 n = 1; 2058 while (n < target && leftover >= n1g) { 2059 leftover -= n1g; 2060 iaq->nirq += n1g; 2061 n++; 2062 } 2063 iaq->nrxq1g = min(n, nrxq1g); 2064 #ifdef TCP_OFFLOAD 2065 iaq->nofldrxq1g = min(n, nofldrxq1g); 2066 #endif 2067 #ifdef DEV_NETMAP 2068 iaq->nnmrxq1g = min(n, nnmrxq1g); 2069 #endif 2070 } 2071 2072 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2073 goto allocate; 2074 } 2075 2076 /* 2077 * Least desirable option: one interrupt vector for everything. 2078 */ 2079 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2080 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2081 #ifdef TCP_OFFLOAD 2082 if (is_offload(sc)) 2083 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2084 #endif 2085 #ifdef DEV_NETMAP 2086 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2087 #endif 2088 2089 allocate: 2090 navail = iaq->nirq; 2091 rc = 0; 2092 if (itype == INTR_MSIX) 2093 rc = pci_alloc_msix(sc->dev, &navail); 2094 else if (itype == INTR_MSI) 2095 rc = pci_alloc_msi(sc->dev, &navail); 2096 2097 if (rc == 0) { 2098 if (navail == iaq->nirq) 2099 return (0); 2100 2101 /* 2102 * Didn't get the number requested. Use whatever number 2103 * the kernel is willing to allocate (it's in navail). 2104 */ 2105 device_printf(sc->dev, "fewer vectors than requested, " 2106 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2107 itype, iaq->nirq, navail); 2108 pci_release_msi(sc->dev); 2109 goto restart; 2110 } 2111 2112 device_printf(sc->dev, 2113 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2114 itype, rc, iaq->nirq, navail); 2115 } 2116 2117 device_printf(sc->dev, 2118 "failed to find a usable interrupt type. " 2119 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2120 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2121 2122 return (ENXIO); 2123 } 2124 2125 #define FW_VERSION(chip) ( \ 2126 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2127 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2128 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2129 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2130 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2131 2132 struct fw_info { 2133 uint8_t chip; 2134 char *kld_name; 2135 char *fw_mod_name; 2136 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2137 } fw_info[] = { 2138 { 2139 .chip = CHELSIO_T4, 2140 .kld_name = "t4fw_cfg", 2141 .fw_mod_name = "t4fw", 2142 .fw_hdr = { 2143 .chip = FW_HDR_CHIP_T4, 2144 .fw_ver = htobe32_const(FW_VERSION(T4)), 2145 .intfver_nic = FW_INTFVER(T4, NIC), 2146 .intfver_vnic = FW_INTFVER(T4, VNIC), 2147 .intfver_ofld = FW_INTFVER(T4, OFLD), 2148 .intfver_ri = FW_INTFVER(T4, RI), 2149 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2150 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2151 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2152 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2153 }, 2154 }, { 2155 .chip = CHELSIO_T5, 2156 .kld_name = "t5fw_cfg", 2157 .fw_mod_name = "t5fw", 2158 .fw_hdr = { 2159 .chip = FW_HDR_CHIP_T5, 2160 .fw_ver = htobe32_const(FW_VERSION(T5)), 2161 .intfver_nic = FW_INTFVER(T5, NIC), 2162 .intfver_vnic = FW_INTFVER(T5, VNIC), 2163 .intfver_ofld = FW_INTFVER(T5, OFLD), 2164 .intfver_ri = FW_INTFVER(T5, RI), 2165 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2166 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2167 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2168 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2169 }, 2170 } 2171 }; 2172 2173 static struct fw_info * 2174 find_fw_info(int chip) 2175 { 2176 int i; 2177 2178 for (i = 0; i < nitems(fw_info); i++) { 2179 if (fw_info[i].chip == chip) 2180 return (&fw_info[i]); 2181 } 2182 return (NULL); 2183 } 2184 2185 /* 2186 * Is the given firmware API compatible with the one the driver was compiled 2187 * with? 2188 */ 2189 static int 2190 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2191 { 2192 2193 /* short circuit if it's the exact same firmware version */ 2194 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2195 return (1); 2196 2197 /* 2198 * XXX: Is this too conservative? Perhaps I should limit this to the 2199 * features that are supported in the driver. 2200 */ 2201 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2202 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2203 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2204 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2205 return (1); 2206 #undef SAME_INTF 2207 2208 return (0); 2209 } 2210 2211 /* 2212 * The firmware in the KLD is usable, but should it be installed? This routine 2213 * explains itself in detail if it indicates the KLD firmware should be 2214 * installed. 2215 */ 2216 static int 2217 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2218 { 2219 const char *reason; 2220 2221 if (!card_fw_usable) { 2222 reason = "incompatible or unusable"; 2223 goto install; 2224 } 2225 2226 if (k > c) { 2227 reason = "older than the version bundled with this driver"; 2228 goto install; 2229 } 2230 2231 if (t4_fw_install == 2 && k != c) { 2232 reason = "different than the version bundled with this driver"; 2233 goto install; 2234 } 2235 2236 return (0); 2237 2238 install: 2239 if (t4_fw_install == 0) { 2240 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2241 "but the driver is prohibited from installing a different " 2242 "firmware on the card.\n", 2243 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2244 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2245 2246 return (0); 2247 } 2248 2249 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2250 "installing firmware %u.%u.%u.%u on card.\n", 2251 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2252 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2253 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2254 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2255 2256 return (1); 2257 } 2258 /* 2259 * Establish contact with the firmware and determine if we are the master driver 2260 * or not, and whether we are responsible for chip initialization. 2261 */ 2262 static int 2263 prep_firmware(struct adapter *sc) 2264 { 2265 const struct firmware *fw = NULL, *default_cfg; 2266 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2267 enum dev_state state; 2268 struct fw_info *fw_info; 2269 struct fw_hdr *card_fw; /* fw on the card */ 2270 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2271 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2272 against */ 2273 2274 /* Contact firmware. */ 2275 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2276 if (rc < 0 || state == DEV_STATE_ERR) { 2277 rc = -rc; 2278 device_printf(sc->dev, 2279 "failed to connect to the firmware: %d, %d.\n", rc, state); 2280 return (rc); 2281 } 2282 pf = rc; 2283 if (pf == sc->mbox) 2284 sc->flags |= MASTER_PF; 2285 else if (state == DEV_STATE_UNINIT) { 2286 /* 2287 * We didn't get to be the master so we definitely won't be 2288 * configuring the chip. It's a bug if someone else hasn't 2289 * configured it already. 2290 */ 2291 device_printf(sc->dev, "couldn't be master(%d), " 2292 "device not already initialized either(%d).\n", rc, state); 2293 return (EDOOFUS); 2294 } 2295 2296 /* This is the firmware whose headers the driver was compiled against */ 2297 fw_info = find_fw_info(chip_id(sc)); 2298 if (fw_info == NULL) { 2299 device_printf(sc->dev, 2300 "unable to look up firmware information for chip %d.\n", 2301 chip_id(sc)); 2302 return (EINVAL); 2303 } 2304 drv_fw = &fw_info->fw_hdr; 2305 2306 /* 2307 * The firmware KLD contains many modules. The KLD name is also the 2308 * name of the module that contains the default config file. 2309 */ 2310 default_cfg = firmware_get(fw_info->kld_name); 2311 2312 /* Read the header of the firmware on the card */ 2313 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2314 rc = -t4_read_flash(sc, FLASH_FW_START, 2315 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2316 if (rc == 0) 2317 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2318 else { 2319 device_printf(sc->dev, 2320 "Unable to read card's firmware header: %d\n", rc); 2321 card_fw_usable = 0; 2322 } 2323 2324 /* This is the firmware in the KLD */ 2325 fw = firmware_get(fw_info->fw_mod_name); 2326 if (fw != NULL) { 2327 kld_fw = (const void *)fw->data; 2328 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2329 } else { 2330 kld_fw = NULL; 2331 kld_fw_usable = 0; 2332 } 2333 2334 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2335 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2336 /* 2337 * Common case: the firmware on the card is an exact match and 2338 * the KLD is an exact match too, or the KLD is 2339 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2340 * here -- use cxgbetool loadfw if you want to reinstall the 2341 * same firmware as the one on the card. 2342 */ 2343 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2344 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2345 be32toh(card_fw->fw_ver))) { 2346 2347 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2348 if (rc != 0) { 2349 device_printf(sc->dev, 2350 "failed to install firmware: %d\n", rc); 2351 goto done; 2352 } 2353 2354 /* Installed successfully, update the cached header too. */ 2355 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2356 card_fw_usable = 1; 2357 need_fw_reset = 0; /* already reset as part of load_fw */ 2358 } 2359 2360 if (!card_fw_usable) { 2361 uint32_t d, c, k; 2362 2363 d = ntohl(drv_fw->fw_ver); 2364 c = ntohl(card_fw->fw_ver); 2365 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2366 2367 device_printf(sc->dev, "Cannot find a usable firmware: " 2368 "fw_install %d, chip state %d, " 2369 "driver compiled with %d.%d.%d.%d, " 2370 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2371 t4_fw_install, state, 2372 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2373 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2374 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2375 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2376 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2377 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2378 rc = EINVAL; 2379 goto done; 2380 } 2381 2382 /* We're using whatever's on the card and it's known to be good. */ 2383 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2384 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2385 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2386 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2387 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2388 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2389 t4_get_tp_version(sc, &sc->params.tp_vers); 2390 2391 /* Reset device */ 2392 if (need_fw_reset && 2393 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2394 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2395 if (rc != ETIMEDOUT && rc != EIO) 2396 t4_fw_bye(sc, sc->mbox); 2397 goto done; 2398 } 2399 sc->flags |= FW_OK; 2400 2401 rc = get_params__pre_init(sc); 2402 if (rc != 0) 2403 goto done; /* error message displayed already */ 2404 2405 /* Partition adapter resources as specified in the config file. */ 2406 if (state == DEV_STATE_UNINIT) { 2407 2408 KASSERT(sc->flags & MASTER_PF, 2409 ("%s: trying to change chip settings when not master.", 2410 __func__)); 2411 2412 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2413 if (rc != 0) 2414 goto done; /* error message displayed already */ 2415 2416 t4_tweak_chip_settings(sc); 2417 2418 /* get basic stuff going */ 2419 rc = -t4_fw_initialize(sc, sc->mbox); 2420 if (rc != 0) { 2421 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2422 goto done; 2423 } 2424 } else { 2425 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2426 sc->cfcsum = 0; 2427 } 2428 2429 done: 2430 free(card_fw, M_CXGBE); 2431 if (fw != NULL) 2432 firmware_put(fw, FIRMWARE_UNLOAD); 2433 if (default_cfg != NULL) 2434 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2435 2436 return (rc); 2437 } 2438 2439 #define FW_PARAM_DEV(param) \ 2440 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2441 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2442 #define FW_PARAM_PFVF(param) \ 2443 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2444 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2445 2446 /* 2447 * Partition chip resources for use between various PFs, VFs, etc. 2448 */ 2449 static int 2450 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2451 const char *name_prefix) 2452 { 2453 const struct firmware *cfg = NULL; 2454 int rc = 0; 2455 struct fw_caps_config_cmd caps; 2456 uint32_t mtype, moff, finicsum, cfcsum; 2457 2458 /* 2459 * Figure out what configuration file to use. Pick the default config 2460 * file for the card if the user hasn't specified one explicitly. 2461 */ 2462 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2463 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2464 /* Card specific overrides go here. */ 2465 if (pci_get_device(sc->dev) == 0x440a) 2466 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2467 if (is_fpga(sc)) 2468 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2469 } 2470 2471 /* 2472 * We need to load another module if the profile is anything except 2473 * "default" or "flash". 2474 */ 2475 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2476 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2477 char s[32]; 2478 2479 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2480 cfg = firmware_get(s); 2481 if (cfg == NULL) { 2482 if (default_cfg != NULL) { 2483 device_printf(sc->dev, 2484 "unable to load module \"%s\" for " 2485 "configuration profile \"%s\", will use " 2486 "the default config file instead.\n", 2487 s, sc->cfg_file); 2488 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2489 "%s", DEFAULT_CF); 2490 } else { 2491 device_printf(sc->dev, 2492 "unable to load module \"%s\" for " 2493 "configuration profile \"%s\", will use " 2494 "the config file on the card's flash " 2495 "instead.\n", s, sc->cfg_file); 2496 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2497 "%s", FLASH_CF); 2498 } 2499 } 2500 } 2501 2502 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2503 default_cfg == NULL) { 2504 device_printf(sc->dev, 2505 "default config file not available, will use the config " 2506 "file on the card's flash instead.\n"); 2507 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2508 } 2509 2510 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2511 u_int cflen, i, n; 2512 const uint32_t *cfdata; 2513 uint32_t param, val, addr, off, mw_base, mw_aperture; 2514 2515 KASSERT(cfg != NULL || default_cfg != NULL, 2516 ("%s: no config to upload", __func__)); 2517 2518 /* 2519 * Ask the firmware where it wants us to upload the config file. 2520 */ 2521 param = FW_PARAM_DEV(CF); 2522 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2523 if (rc != 0) { 2524 /* No support for config file? Shouldn't happen. */ 2525 device_printf(sc->dev, 2526 "failed to query config file location: %d.\n", rc); 2527 goto done; 2528 } 2529 mtype = G_FW_PARAMS_PARAM_Y(val); 2530 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2531 2532 /* 2533 * XXX: sheer laziness. We deliberately added 4 bytes of 2534 * useless stuffing/comments at the end of the config file so 2535 * it's ok to simply throw away the last remaining bytes when 2536 * the config file is not an exact multiple of 4. This also 2537 * helps with the validate_mt_off_len check. 2538 */ 2539 if (cfg != NULL) { 2540 cflen = cfg->datasize & ~3; 2541 cfdata = cfg->data; 2542 } else { 2543 cflen = default_cfg->datasize & ~3; 2544 cfdata = default_cfg->data; 2545 } 2546 2547 if (cflen > FLASH_CFG_MAX_SIZE) { 2548 device_printf(sc->dev, 2549 "config file too long (%d, max allowed is %d). " 2550 "Will try to use the config on the card, if any.\n", 2551 cflen, FLASH_CFG_MAX_SIZE); 2552 goto use_config_on_flash; 2553 } 2554 2555 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2556 if (rc != 0) { 2557 device_printf(sc->dev, 2558 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2559 "Will try to use the config on the card, if any.\n", 2560 __func__, mtype, moff, cflen, rc); 2561 goto use_config_on_flash; 2562 } 2563 2564 memwin_info(sc, 2, &mw_base, &mw_aperture); 2565 while (cflen) { 2566 off = position_memwin(sc, 2, addr); 2567 n = min(cflen, mw_aperture - off); 2568 for (i = 0; i < n; i += 4) 2569 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2570 cflen -= n; 2571 addr += n; 2572 } 2573 } else { 2574 use_config_on_flash: 2575 mtype = FW_MEMTYPE_FLASH; 2576 moff = t4_flash_cfg_addr(sc); 2577 } 2578 2579 bzero(&caps, sizeof(caps)); 2580 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2581 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2582 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2583 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2584 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2585 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2586 if (rc != 0) { 2587 device_printf(sc->dev, 2588 "failed to pre-process config file: %d " 2589 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2590 goto done; 2591 } 2592 2593 finicsum = be32toh(caps.finicsum); 2594 cfcsum = be32toh(caps.cfcsum); 2595 if (finicsum != cfcsum) { 2596 device_printf(sc->dev, 2597 "WARNING: config file checksum mismatch: %08x %08x\n", 2598 finicsum, cfcsum); 2599 } 2600 sc->cfcsum = cfcsum; 2601 2602 #define LIMIT_CAPS(x) do { \ 2603 caps.x &= htobe16(t4_##x##_allowed); \ 2604 } while (0) 2605 2606 /* 2607 * Let the firmware know what features will (not) be used so it can tune 2608 * things accordingly. 2609 */ 2610 LIMIT_CAPS(linkcaps); 2611 LIMIT_CAPS(niccaps); 2612 LIMIT_CAPS(toecaps); 2613 LIMIT_CAPS(rdmacaps); 2614 LIMIT_CAPS(iscsicaps); 2615 LIMIT_CAPS(fcoecaps); 2616 #undef LIMIT_CAPS 2617 2618 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2619 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2620 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2621 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2622 if (rc != 0) { 2623 device_printf(sc->dev, 2624 "failed to process config file: %d.\n", rc); 2625 } 2626 done: 2627 if (cfg != NULL) 2628 firmware_put(cfg, FIRMWARE_UNLOAD); 2629 return (rc); 2630 } 2631 2632 /* 2633 * Retrieve parameters that are needed (or nice to have) very early. 2634 */ 2635 static int 2636 get_params__pre_init(struct adapter *sc) 2637 { 2638 int rc; 2639 uint32_t param[2], val[2]; 2640 struct fw_devlog_cmd cmd; 2641 struct devlog_params *dlog = &sc->params.devlog; 2642 2643 param[0] = FW_PARAM_DEV(PORTVEC); 2644 param[1] = FW_PARAM_DEV(CCLK); 2645 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2646 if (rc != 0) { 2647 device_printf(sc->dev, 2648 "failed to query parameters (pre_init): %d.\n", rc); 2649 return (rc); 2650 } 2651 2652 sc->params.portvec = val[0]; 2653 sc->params.nports = bitcount32(val[0]); 2654 sc->params.vpd.cclk = val[1]; 2655 2656 /* Read device log parameters. */ 2657 bzero(&cmd, sizeof(cmd)); 2658 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2659 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2660 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2661 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2662 if (rc != 0) { 2663 device_printf(sc->dev, 2664 "failed to get devlog parameters: %d.\n", rc); 2665 bzero(dlog, sizeof (*dlog)); 2666 rc = 0; /* devlog isn't critical for device operation */ 2667 } else { 2668 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2669 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2670 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2671 dlog->size = be32toh(cmd.memsize_devlog); 2672 } 2673 2674 return (rc); 2675 } 2676 2677 /* 2678 * Retrieve various parameters that are of interest to the driver. The device 2679 * has been initialized by the firmware at this point. 2680 */ 2681 static int 2682 get_params__post_init(struct adapter *sc) 2683 { 2684 int rc; 2685 uint32_t param[7], val[7]; 2686 struct fw_caps_config_cmd caps; 2687 2688 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2689 param[1] = FW_PARAM_PFVF(EQ_START); 2690 param[2] = FW_PARAM_PFVF(FILTER_START); 2691 param[3] = FW_PARAM_PFVF(FILTER_END); 2692 param[4] = FW_PARAM_PFVF(L2T_START); 2693 param[5] = FW_PARAM_PFVF(L2T_END); 2694 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2695 if (rc != 0) { 2696 device_printf(sc->dev, 2697 "failed to query parameters (post_init): %d.\n", rc); 2698 return (rc); 2699 } 2700 2701 sc->sge.iq_start = val[0]; 2702 sc->sge.eq_start = val[1]; 2703 sc->tids.ftid_base = val[2]; 2704 sc->tids.nftids = val[3] - val[2] + 1; 2705 sc->params.ftid_min = val[2]; 2706 sc->params.ftid_max = val[3]; 2707 sc->vres.l2t.start = val[4]; 2708 sc->vres.l2t.size = val[5] - val[4] + 1; 2709 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2710 ("%s: L2 table size (%u) larger than expected (%u)", 2711 __func__, sc->vres.l2t.size, L2T_SIZE)); 2712 2713 /* get capabilites */ 2714 bzero(&caps, sizeof(caps)); 2715 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2716 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2717 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2718 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2719 if (rc != 0) { 2720 device_printf(sc->dev, 2721 "failed to get card capabilities: %d.\n", rc); 2722 return (rc); 2723 } 2724 2725 #define READ_CAPS(x) do { \ 2726 sc->x = htobe16(caps.x); \ 2727 } while (0) 2728 READ_CAPS(linkcaps); 2729 READ_CAPS(niccaps); 2730 READ_CAPS(toecaps); 2731 READ_CAPS(rdmacaps); 2732 READ_CAPS(iscsicaps); 2733 READ_CAPS(fcoecaps); 2734 2735 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2736 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2737 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2738 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2739 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2740 if (rc != 0) { 2741 device_printf(sc->dev, 2742 "failed to query NIC parameters: %d.\n", rc); 2743 return (rc); 2744 } 2745 sc->tids.etid_base = val[0]; 2746 sc->params.etid_min = val[0]; 2747 sc->tids.netids = val[1] - val[0] + 1; 2748 sc->params.netids = sc->tids.netids; 2749 sc->params.eo_wr_cred = val[2]; 2750 sc->params.ethoffload = 1; 2751 } 2752 2753 if (sc->toecaps) { 2754 /* query offload-related parameters */ 2755 param[0] = FW_PARAM_DEV(NTID); 2756 param[1] = FW_PARAM_PFVF(SERVER_START); 2757 param[2] = FW_PARAM_PFVF(SERVER_END); 2758 param[3] = FW_PARAM_PFVF(TDDP_START); 2759 param[4] = FW_PARAM_PFVF(TDDP_END); 2760 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2761 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2762 if (rc != 0) { 2763 device_printf(sc->dev, 2764 "failed to query TOE parameters: %d.\n", rc); 2765 return (rc); 2766 } 2767 sc->tids.ntids = val[0]; 2768 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2769 sc->tids.stid_base = val[1]; 2770 sc->tids.nstids = val[2] - val[1] + 1; 2771 sc->vres.ddp.start = val[3]; 2772 sc->vres.ddp.size = val[4] - val[3] + 1; 2773 sc->params.ofldq_wr_cred = val[5]; 2774 sc->params.offload = 1; 2775 } 2776 if (sc->rdmacaps) { 2777 param[0] = FW_PARAM_PFVF(STAG_START); 2778 param[1] = FW_PARAM_PFVF(STAG_END); 2779 param[2] = FW_PARAM_PFVF(RQ_START); 2780 param[3] = FW_PARAM_PFVF(RQ_END); 2781 param[4] = FW_PARAM_PFVF(PBL_START); 2782 param[5] = FW_PARAM_PFVF(PBL_END); 2783 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2784 if (rc != 0) { 2785 device_printf(sc->dev, 2786 "failed to query RDMA parameters(1): %d.\n", rc); 2787 return (rc); 2788 } 2789 sc->vres.stag.start = val[0]; 2790 sc->vres.stag.size = val[1] - val[0] + 1; 2791 sc->vres.rq.start = val[2]; 2792 sc->vres.rq.size = val[3] - val[2] + 1; 2793 sc->vres.pbl.start = val[4]; 2794 sc->vres.pbl.size = val[5] - val[4] + 1; 2795 2796 param[0] = FW_PARAM_PFVF(SQRQ_START); 2797 param[1] = FW_PARAM_PFVF(SQRQ_END); 2798 param[2] = FW_PARAM_PFVF(CQ_START); 2799 param[3] = FW_PARAM_PFVF(CQ_END); 2800 param[4] = FW_PARAM_PFVF(OCQ_START); 2801 param[5] = FW_PARAM_PFVF(OCQ_END); 2802 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2803 if (rc != 0) { 2804 device_printf(sc->dev, 2805 "failed to query RDMA parameters(2): %d.\n", rc); 2806 return (rc); 2807 } 2808 sc->vres.qp.start = val[0]; 2809 sc->vres.qp.size = val[1] - val[0] + 1; 2810 sc->vres.cq.start = val[2]; 2811 sc->vres.cq.size = val[3] - val[2] + 1; 2812 sc->vres.ocq.start = val[4]; 2813 sc->vres.ocq.size = val[5] - val[4] + 1; 2814 } 2815 if (sc->iscsicaps) { 2816 param[0] = FW_PARAM_PFVF(ISCSI_START); 2817 param[1] = FW_PARAM_PFVF(ISCSI_END); 2818 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2819 if (rc != 0) { 2820 device_printf(sc->dev, 2821 "failed to query iSCSI parameters: %d.\n", rc); 2822 return (rc); 2823 } 2824 sc->vres.iscsi.start = val[0]; 2825 sc->vres.iscsi.size = val[1] - val[0] + 1; 2826 } 2827 2828 /* 2829 * We've got the params we wanted to query via the firmware. Now grab 2830 * some others directly from the chip. 2831 */ 2832 rc = t4_read_chip_settings(sc); 2833 2834 return (rc); 2835 } 2836 2837 static int 2838 set_params__post_init(struct adapter *sc) 2839 { 2840 uint32_t param, val; 2841 2842 /* ask for encapsulated CPLs */ 2843 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2844 val = 1; 2845 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2846 2847 return (0); 2848 } 2849 2850 #undef FW_PARAM_PFVF 2851 #undef FW_PARAM_DEV 2852 2853 static void 2854 t4_set_desc(struct adapter *sc) 2855 { 2856 char buf[128]; 2857 struct adapter_params *p = &sc->params; 2858 2859 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2860 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2861 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2862 2863 device_set_desc_copy(sc->dev, buf); 2864 } 2865 2866 static void 2867 build_medialist(struct port_info *pi, struct ifmedia *media) 2868 { 2869 int m; 2870 2871 PORT_LOCK(pi); 2872 2873 ifmedia_removeall(media); 2874 2875 m = IFM_ETHER | IFM_FDX; 2876 2877 switch(pi->port_type) { 2878 case FW_PORT_TYPE_BT_XFI: 2879 case FW_PORT_TYPE_BT_XAUI: 2880 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2881 /* fall through */ 2882 2883 case FW_PORT_TYPE_BT_SGMII: 2884 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2885 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2886 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2887 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2888 break; 2889 2890 case FW_PORT_TYPE_CX4: 2891 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2892 ifmedia_set(media, m | IFM_10G_CX4); 2893 break; 2894 2895 case FW_PORT_TYPE_QSFP_10G: 2896 case FW_PORT_TYPE_SFP: 2897 case FW_PORT_TYPE_FIBER_XFI: 2898 case FW_PORT_TYPE_FIBER_XAUI: 2899 switch (pi->mod_type) { 2900 2901 case FW_PORT_MOD_TYPE_LR: 2902 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2903 ifmedia_set(media, m | IFM_10G_LR); 2904 break; 2905 2906 case FW_PORT_MOD_TYPE_SR: 2907 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2908 ifmedia_set(media, m | IFM_10G_SR); 2909 break; 2910 2911 case FW_PORT_MOD_TYPE_LRM: 2912 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2913 ifmedia_set(media, m | IFM_10G_LRM); 2914 break; 2915 2916 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2917 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2918 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2919 ifmedia_set(media, m | IFM_10G_TWINAX); 2920 break; 2921 2922 case FW_PORT_MOD_TYPE_NONE: 2923 m &= ~IFM_FDX; 2924 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2925 ifmedia_set(media, m | IFM_NONE); 2926 break; 2927 2928 case FW_PORT_MOD_TYPE_NA: 2929 case FW_PORT_MOD_TYPE_ER: 2930 default: 2931 device_printf(pi->dev, 2932 "unknown port_type (%d), mod_type (%d)\n", 2933 pi->port_type, pi->mod_type); 2934 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2935 ifmedia_set(media, m | IFM_UNKNOWN); 2936 break; 2937 } 2938 break; 2939 2940 case FW_PORT_TYPE_QSFP: 2941 switch (pi->mod_type) { 2942 2943 case FW_PORT_MOD_TYPE_LR: 2944 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2945 ifmedia_set(media, m | IFM_40G_LR4); 2946 break; 2947 2948 case FW_PORT_MOD_TYPE_SR: 2949 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2950 ifmedia_set(media, m | IFM_40G_SR4); 2951 break; 2952 2953 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2954 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2955 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2956 ifmedia_set(media, m | IFM_40G_CR4); 2957 break; 2958 2959 case FW_PORT_MOD_TYPE_NONE: 2960 m &= ~IFM_FDX; 2961 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2962 ifmedia_set(media, m | IFM_NONE); 2963 break; 2964 2965 default: 2966 device_printf(pi->dev, 2967 "unknown port_type (%d), mod_type (%d)\n", 2968 pi->port_type, pi->mod_type); 2969 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2970 ifmedia_set(media, m | IFM_UNKNOWN); 2971 break; 2972 } 2973 break; 2974 2975 default: 2976 device_printf(pi->dev, 2977 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2978 pi->mod_type); 2979 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2980 ifmedia_set(media, m | IFM_UNKNOWN); 2981 break; 2982 } 2983 2984 PORT_UNLOCK(pi); 2985 } 2986 2987 #define FW_MAC_EXACT_CHUNK 7 2988 2989 /* 2990 * Program the port's XGMAC based on parameters in ifnet. The caller also 2991 * indicates which parameters should be programmed (the rest are left alone). 2992 */ 2993 int 2994 update_mac_settings(struct ifnet *ifp, int flags) 2995 { 2996 int rc = 0; 2997 struct port_info *pi = ifp->if_softc; 2998 struct adapter *sc = pi->adapter; 2999 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3000 uint16_t viid = 0xffff; 3001 int16_t *xact_addr_filt = NULL; 3002 3003 ASSERT_SYNCHRONIZED_OP(sc); 3004 KASSERT(flags, ("%s: not told what to update.", __func__)); 3005 3006 if (ifp == pi->ifp) { 3007 viid = pi->viid; 3008 xact_addr_filt = &pi->xact_addr_filt; 3009 } 3010 #ifdef DEV_NETMAP 3011 else if (ifp == pi->nm_ifp) { 3012 viid = pi->nm_viid; 3013 xact_addr_filt = &pi->nm_xact_addr_filt; 3014 } 3015 #endif 3016 if (flags & XGMAC_MTU) 3017 mtu = ifp->if_mtu; 3018 3019 if (flags & XGMAC_PROMISC) 3020 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3021 3022 if (flags & XGMAC_ALLMULTI) 3023 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3024 3025 if (flags & XGMAC_VLANEX) 3026 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3027 3028 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3029 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3030 1, vlanex, false); 3031 if (rc) { 3032 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3033 rc); 3034 return (rc); 3035 } 3036 } 3037 3038 if (flags & XGMAC_UCADDR) { 3039 uint8_t ucaddr[ETHER_ADDR_LEN]; 3040 3041 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3042 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3043 true, true); 3044 if (rc < 0) { 3045 rc = -rc; 3046 if_printf(ifp, "change_mac failed: %d\n", rc); 3047 return (rc); 3048 } else { 3049 *xact_addr_filt = rc; 3050 rc = 0; 3051 } 3052 } 3053 3054 if (flags & XGMAC_MCADDRS) { 3055 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3056 int del = 1; 3057 uint64_t hash = 0; 3058 struct ifmultiaddr *ifma; 3059 int i = 0, j; 3060 3061 if_maddr_rlock(ifp); 3062 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3063 if (ifma->ifma_addr->sa_family != AF_LINK) 3064 continue; 3065 mcaddr[i] = 3066 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3067 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3068 i++; 3069 3070 if (i == FW_MAC_EXACT_CHUNK) { 3071 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3072 i, mcaddr, NULL, &hash, 0); 3073 if (rc < 0) { 3074 rc = -rc; 3075 for (j = 0; j < i; j++) { 3076 if_printf(ifp, 3077 "failed to add mc address" 3078 " %02x:%02x:%02x:" 3079 "%02x:%02x:%02x rc=%d\n", 3080 mcaddr[j][0], mcaddr[j][1], 3081 mcaddr[j][2], mcaddr[j][3], 3082 mcaddr[j][4], mcaddr[j][5], 3083 rc); 3084 } 3085 goto mcfail; 3086 } 3087 del = 0; 3088 i = 0; 3089 } 3090 } 3091 if (i > 0) { 3092 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3093 mcaddr, NULL, &hash, 0); 3094 if (rc < 0) { 3095 rc = -rc; 3096 for (j = 0; j < i; j++) { 3097 if_printf(ifp, 3098 "failed to add mc address" 3099 " %02x:%02x:%02x:" 3100 "%02x:%02x:%02x rc=%d\n", 3101 mcaddr[j][0], mcaddr[j][1], 3102 mcaddr[j][2], mcaddr[j][3], 3103 mcaddr[j][4], mcaddr[j][5], 3104 rc); 3105 } 3106 goto mcfail; 3107 } 3108 } 3109 3110 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3111 if (rc != 0) 3112 if_printf(ifp, "failed to set mc address hash: %d", rc); 3113 mcfail: 3114 if_maddr_runlock(ifp); 3115 } 3116 3117 return (rc); 3118 } 3119 3120 /* 3121 * {begin|end}_synchronized_op must be called from the same thread. 3122 */ 3123 int 3124 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3125 char *wmesg) 3126 { 3127 int rc, pri; 3128 3129 #ifdef WITNESS 3130 /* the caller thinks it's ok to sleep, but is it really? */ 3131 if (flags & SLEEP_OK) 3132 pause("t4slptst", 1); 3133 #endif 3134 3135 if (INTR_OK) 3136 pri = PCATCH; 3137 else 3138 pri = 0; 3139 3140 ADAPTER_LOCK(sc); 3141 for (;;) { 3142 3143 if (pi && IS_DOOMED(pi)) { 3144 rc = ENXIO; 3145 goto done; 3146 } 3147 3148 if (!IS_BUSY(sc)) { 3149 rc = 0; 3150 break; 3151 } 3152 3153 if (!(flags & SLEEP_OK)) { 3154 rc = EBUSY; 3155 goto done; 3156 } 3157 3158 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3159 rc = EINTR; 3160 goto done; 3161 } 3162 } 3163 3164 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3165 SET_BUSY(sc); 3166 #ifdef INVARIANTS 3167 sc->last_op = wmesg; 3168 sc->last_op_thr = curthread; 3169 sc->last_op_flags = flags; 3170 #endif 3171 3172 done: 3173 if (!(flags & HOLD_LOCK) || rc) 3174 ADAPTER_UNLOCK(sc); 3175 3176 return (rc); 3177 } 3178 3179 /* 3180 * {begin|end}_synchronized_op must be called from the same thread. 3181 */ 3182 void 3183 end_synchronized_op(struct adapter *sc, int flags) 3184 { 3185 3186 if (flags & LOCK_HELD) 3187 ADAPTER_LOCK_ASSERT_OWNED(sc); 3188 else 3189 ADAPTER_LOCK(sc); 3190 3191 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3192 CLR_BUSY(sc); 3193 wakeup(&sc->flags); 3194 ADAPTER_UNLOCK(sc); 3195 } 3196 3197 static int 3198 cxgbe_init_synchronized(struct port_info *pi) 3199 { 3200 struct adapter *sc = pi->adapter; 3201 struct ifnet *ifp = pi->ifp; 3202 int rc = 0, i; 3203 struct sge_txq *txq; 3204 3205 ASSERT_SYNCHRONIZED_OP(sc); 3206 3207 if (isset(&sc->open_device_map, pi->port_id)) { 3208 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3209 ("mismatch between open_device_map and if_drv_flags")); 3210 return (0); /* already running */ 3211 } 3212 3213 if (!(sc->flags & FULL_INIT_DONE) && 3214 ((rc = adapter_full_init(sc)) != 0)) 3215 return (rc); /* error message displayed already */ 3216 3217 if (!(pi->flags & PORT_INIT_DONE) && 3218 ((rc = port_full_init(pi)) != 0)) 3219 return (rc); /* error message displayed already */ 3220 3221 rc = update_mac_settings(ifp, XGMAC_ALL); 3222 if (rc) 3223 goto done; /* error message displayed already */ 3224 3225 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3226 if (rc != 0) { 3227 if_printf(ifp, "enable_vi failed: %d\n", rc); 3228 goto done; 3229 } 3230 3231 /* 3232 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3233 * if this changes. 3234 */ 3235 3236 for_each_txq(pi, i, txq) { 3237 TXQ_LOCK(txq); 3238 txq->eq.flags |= EQ_ENABLED; 3239 TXQ_UNLOCK(txq); 3240 } 3241 3242 /* 3243 * The first iq of the first port to come up is used for tracing. 3244 */ 3245 if (sc->traceq < 0) { 3246 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3247 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3248 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3249 V_QUEUENUMBER(sc->traceq)); 3250 pi->flags |= HAS_TRACEQ; 3251 } 3252 3253 /* all ok */ 3254 setbit(&sc->open_device_map, pi->port_id); 3255 PORT_LOCK(pi); 3256 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3257 PORT_UNLOCK(pi); 3258 3259 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3260 done: 3261 if (rc != 0) 3262 cxgbe_uninit_synchronized(pi); 3263 3264 return (rc); 3265 } 3266 3267 /* 3268 * Idempotent. 3269 */ 3270 static int 3271 cxgbe_uninit_synchronized(struct port_info *pi) 3272 { 3273 struct adapter *sc = pi->adapter; 3274 struct ifnet *ifp = pi->ifp; 3275 int rc, i; 3276 struct sge_txq *txq; 3277 3278 ASSERT_SYNCHRONIZED_OP(sc); 3279 3280 if (!(pi->flags & PORT_INIT_DONE)) { 3281 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3282 ("uninited port is running")); 3283 return (0); 3284 } 3285 3286 /* 3287 * Disable the VI so that all its data in either direction is discarded 3288 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3289 * tick) intact as the TP can deliver negative advice or data that it's 3290 * holding in its RAM (for an offloaded connection) even after the VI is 3291 * disabled. 3292 */ 3293 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3294 if (rc) { 3295 if_printf(ifp, "disable_vi failed: %d\n", rc); 3296 return (rc); 3297 } 3298 3299 for_each_txq(pi, i, txq) { 3300 TXQ_LOCK(txq); 3301 txq->eq.flags &= ~EQ_ENABLED; 3302 TXQ_UNLOCK(txq); 3303 } 3304 3305 clrbit(&sc->open_device_map, pi->port_id); 3306 PORT_LOCK(pi); 3307 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3308 PORT_UNLOCK(pi); 3309 3310 pi->link_cfg.link_ok = 0; 3311 pi->link_cfg.speed = 0; 3312 pi->linkdnrc = -1; 3313 t4_os_link_changed(sc, pi->port_id, 0, -1); 3314 3315 return (0); 3316 } 3317 3318 /* 3319 * It is ok for this function to fail midway and return right away. t4_detach 3320 * will walk the entire sc->irq list and clean up whatever is valid. 3321 */ 3322 static int 3323 setup_intr_handlers(struct adapter *sc) 3324 { 3325 int rc, rid, p, q; 3326 char s[8]; 3327 struct irq *irq; 3328 struct port_info *pi; 3329 struct sge_rxq *rxq; 3330 #ifdef TCP_OFFLOAD 3331 struct sge_ofld_rxq *ofld_rxq; 3332 #endif 3333 #ifdef DEV_NETMAP 3334 struct sge_nm_rxq *nm_rxq; 3335 #endif 3336 3337 /* 3338 * Setup interrupts. 3339 */ 3340 irq = &sc->irq[0]; 3341 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3342 if (sc->intr_count == 1) 3343 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3344 3345 /* Multiple interrupts. */ 3346 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3347 ("%s: too few intr.", __func__)); 3348 3349 /* The first one is always error intr */ 3350 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3351 if (rc != 0) 3352 return (rc); 3353 irq++; 3354 rid++; 3355 3356 /* The second one is always the firmware event queue */ 3357 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3358 if (rc != 0) 3359 return (rc); 3360 irq++; 3361 rid++; 3362 3363 for_each_port(sc, p) { 3364 pi = sc->port[p]; 3365 3366 if (pi->flags & INTR_RXQ) { 3367 for_each_rxq(pi, q, rxq) { 3368 snprintf(s, sizeof(s), "%d.%d", p, q); 3369 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3370 s); 3371 if (rc != 0) 3372 return (rc); 3373 irq++; 3374 rid++; 3375 } 3376 } 3377 #ifdef TCP_OFFLOAD 3378 if (pi->flags & INTR_OFLD_RXQ) { 3379 for_each_ofld_rxq(pi, q, ofld_rxq) { 3380 snprintf(s, sizeof(s), "%d,%d", p, q); 3381 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3382 ofld_rxq, s); 3383 if (rc != 0) 3384 return (rc); 3385 irq++; 3386 rid++; 3387 } 3388 } 3389 #endif 3390 #ifdef DEV_NETMAP 3391 if (pi->flags & INTR_NM_RXQ) { 3392 for_each_nm_rxq(pi, q, nm_rxq) { 3393 snprintf(s, sizeof(s), "%d-%d", p, q); 3394 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3395 nm_rxq, s); 3396 if (rc != 0) 3397 return (rc); 3398 irq++; 3399 rid++; 3400 } 3401 } 3402 #endif 3403 } 3404 MPASS(irq == &sc->irq[sc->intr_count]); 3405 3406 return (0); 3407 } 3408 3409 int 3410 adapter_full_init(struct adapter *sc) 3411 { 3412 int rc, i; 3413 3414 ASSERT_SYNCHRONIZED_OP(sc); 3415 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3416 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3417 ("%s: FULL_INIT_DONE already", __func__)); 3418 3419 /* 3420 * queues that belong to the adapter (not any particular port). 3421 */ 3422 rc = t4_setup_adapter_queues(sc); 3423 if (rc != 0) 3424 goto done; 3425 3426 for (i = 0; i < nitems(sc->tq); i++) { 3427 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3428 taskqueue_thread_enqueue, &sc->tq[i]); 3429 if (sc->tq[i] == NULL) { 3430 device_printf(sc->dev, 3431 "failed to allocate task queue %d\n", i); 3432 rc = ENOMEM; 3433 goto done; 3434 } 3435 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3436 device_get_nameunit(sc->dev), i); 3437 } 3438 3439 t4_intr_enable(sc); 3440 sc->flags |= FULL_INIT_DONE; 3441 done: 3442 if (rc != 0) 3443 adapter_full_uninit(sc); 3444 3445 return (rc); 3446 } 3447 3448 int 3449 adapter_full_uninit(struct adapter *sc) 3450 { 3451 int i; 3452 3453 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3454 3455 t4_teardown_adapter_queues(sc); 3456 3457 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3458 taskqueue_free(sc->tq[i]); 3459 sc->tq[i] = NULL; 3460 } 3461 3462 sc->flags &= ~FULL_INIT_DONE; 3463 3464 return (0); 3465 } 3466 3467 int 3468 port_full_init(struct port_info *pi) 3469 { 3470 struct adapter *sc = pi->adapter; 3471 struct ifnet *ifp = pi->ifp; 3472 uint16_t *rss; 3473 struct sge_rxq *rxq; 3474 int rc, i, j; 3475 3476 ASSERT_SYNCHRONIZED_OP(sc); 3477 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3478 ("%s: PORT_INIT_DONE already", __func__)); 3479 3480 sysctl_ctx_init(&pi->ctx); 3481 pi->flags |= PORT_SYSCTL_CTX; 3482 3483 /* 3484 * Allocate tx/rx/fl queues for this port. 3485 */ 3486 rc = t4_setup_port_queues(pi); 3487 if (rc != 0) 3488 goto done; /* error message displayed already */ 3489 3490 /* 3491 * Setup RSS for this port. Save a copy of the RSS table for later use. 3492 */ 3493 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3494 for (i = 0; i < pi->rss_size;) { 3495 for_each_rxq(pi, j, rxq) { 3496 rss[i++] = rxq->iq.abs_id; 3497 if (i == pi->rss_size) 3498 break; 3499 } 3500 } 3501 3502 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3503 pi->rss_size); 3504 if (rc != 0) { 3505 if_printf(ifp, "rss_config failed: %d\n", rc); 3506 goto done; 3507 } 3508 3509 pi->rss = rss; 3510 pi->flags |= PORT_INIT_DONE; 3511 done: 3512 if (rc != 0) 3513 port_full_uninit(pi); 3514 3515 return (rc); 3516 } 3517 3518 /* 3519 * Idempotent. 3520 */ 3521 int 3522 port_full_uninit(struct port_info *pi) 3523 { 3524 struct adapter *sc = pi->adapter; 3525 int i; 3526 struct sge_rxq *rxq; 3527 struct sge_txq *txq; 3528 #ifdef TCP_OFFLOAD 3529 struct sge_ofld_rxq *ofld_rxq; 3530 struct sge_wrq *ofld_txq; 3531 #endif 3532 3533 if (pi->flags & PORT_INIT_DONE) { 3534 3535 /* Need to quiesce queues. */ 3536 3537 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3538 3539 for_each_txq(pi, i, txq) { 3540 quiesce_txq(sc, txq); 3541 } 3542 3543 #ifdef TCP_OFFLOAD 3544 for_each_ofld_txq(pi, i, ofld_txq) { 3545 quiesce_wrq(sc, ofld_txq); 3546 } 3547 #endif 3548 3549 for_each_rxq(pi, i, rxq) { 3550 quiesce_iq(sc, &rxq->iq); 3551 quiesce_fl(sc, &rxq->fl); 3552 } 3553 3554 #ifdef TCP_OFFLOAD 3555 for_each_ofld_rxq(pi, i, ofld_rxq) { 3556 quiesce_iq(sc, &ofld_rxq->iq); 3557 quiesce_fl(sc, &ofld_rxq->fl); 3558 } 3559 #endif 3560 free(pi->rss, M_CXGBE); 3561 } 3562 3563 t4_teardown_port_queues(pi); 3564 pi->flags &= ~PORT_INIT_DONE; 3565 3566 return (0); 3567 } 3568 3569 static void 3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3571 { 3572 struct sge_eq *eq = &txq->eq; 3573 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3574 3575 (void) sc; /* unused */ 3576 3577 #ifdef INVARIANTS 3578 TXQ_LOCK(txq); 3579 MPASS((eq->flags & EQ_ENABLED) == 0); 3580 TXQ_UNLOCK(txq); 3581 #endif 3582 3583 /* Wait for the mp_ring to empty. */ 3584 while (!mp_ring_is_idle(txq->r)) { 3585 mp_ring_check_drainage(txq->r, 0); 3586 pause("rquiesce", 1); 3587 } 3588 3589 /* Then wait for the hardware to finish. */ 3590 while (spg->cidx != htobe16(eq->pidx)) 3591 pause("equiesce", 1); 3592 3593 /* Finally, wait for the driver to reclaim all descriptors. */ 3594 while (eq->cidx != eq->pidx) 3595 pause("dquiesce", 1); 3596 } 3597 3598 static void 3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3600 { 3601 3602 /* XXXTX */ 3603 } 3604 3605 static void 3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3607 { 3608 (void) sc; /* unused */ 3609 3610 /* Synchronize with the interrupt handler */ 3611 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3612 pause("iqfree", 1); 3613 } 3614 3615 static void 3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3617 { 3618 mtx_lock(&sc->sfl_lock); 3619 FL_LOCK(fl); 3620 fl->flags |= FL_DOOMED; 3621 FL_UNLOCK(fl); 3622 mtx_unlock(&sc->sfl_lock); 3623 3624 callout_drain(&sc->sfl_callout); 3625 KASSERT((fl->flags & FL_STARVING) == 0, 3626 ("%s: still starving", __func__)); 3627 } 3628 3629 static int 3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3631 driver_intr_t *handler, void *arg, char *name) 3632 { 3633 int rc; 3634 3635 irq->rid = rid; 3636 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3637 RF_SHAREABLE | RF_ACTIVE); 3638 if (irq->res == NULL) { 3639 device_printf(sc->dev, 3640 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3641 return (ENOMEM); 3642 } 3643 3644 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3645 NULL, handler, arg, &irq->tag); 3646 if (rc != 0) { 3647 device_printf(sc->dev, 3648 "failed to setup interrupt for rid %d, name %s: %d\n", 3649 rid, name, rc); 3650 } else if (name) 3651 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3652 3653 return (rc); 3654 } 3655 3656 static int 3657 t4_free_irq(struct adapter *sc, struct irq *irq) 3658 { 3659 if (irq->tag) 3660 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3661 if (irq->res) 3662 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3663 3664 bzero(irq, sizeof(*irq)); 3665 3666 return (0); 3667 } 3668 3669 static void 3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3671 unsigned int end) 3672 { 3673 uint32_t *p = (uint32_t *)(buf + start); 3674 3675 for ( ; start <= end; start += sizeof(uint32_t)) 3676 *p++ = t4_read_reg(sc, start); 3677 } 3678 3679 static void 3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3681 { 3682 int i, n; 3683 const unsigned int *reg_ranges; 3684 static const unsigned int t4_reg_ranges[] = { 3685 0x1008, 0x1108, 3686 0x1180, 0x11b4, 3687 0x11fc, 0x123c, 3688 0x1300, 0x173c, 3689 0x1800, 0x18fc, 3690 0x3000, 0x30d8, 3691 0x30e0, 0x5924, 3692 0x5960, 0x59d4, 3693 0x5a00, 0x5af8, 3694 0x6000, 0x6098, 3695 0x6100, 0x6150, 3696 0x6200, 0x6208, 3697 0x6240, 0x6248, 3698 0x6280, 0x6338, 3699 0x6370, 0x638c, 3700 0x6400, 0x643c, 3701 0x6500, 0x6524, 3702 0x6a00, 0x6a38, 3703 0x6a60, 0x6a78, 3704 0x6b00, 0x6b84, 3705 0x6bf0, 0x6c84, 3706 0x6cf0, 0x6d84, 3707 0x6df0, 0x6e84, 3708 0x6ef0, 0x6f84, 3709 0x6ff0, 0x7084, 3710 0x70f0, 0x7184, 3711 0x71f0, 0x7284, 3712 0x72f0, 0x7384, 3713 0x73f0, 0x7450, 3714 0x7500, 0x7530, 3715 0x7600, 0x761c, 3716 0x7680, 0x76cc, 3717 0x7700, 0x7798, 3718 0x77c0, 0x77fc, 3719 0x7900, 0x79fc, 3720 0x7b00, 0x7c38, 3721 0x7d00, 0x7efc, 3722 0x8dc0, 0x8e1c, 3723 0x8e30, 0x8e78, 3724 0x8ea0, 0x8f6c, 3725 0x8fc0, 0x9074, 3726 0x90fc, 0x90fc, 3727 0x9400, 0x9458, 3728 0x9600, 0x96bc, 3729 0x9800, 0x9808, 3730 0x9820, 0x983c, 3731 0x9850, 0x9864, 3732 0x9c00, 0x9c6c, 3733 0x9c80, 0x9cec, 3734 0x9d00, 0x9d6c, 3735 0x9d80, 0x9dec, 3736 0x9e00, 0x9e6c, 3737 0x9e80, 0x9eec, 3738 0x9f00, 0x9f6c, 3739 0x9f80, 0x9fec, 3740 0xd004, 0xd03c, 3741 0xdfc0, 0xdfe0, 3742 0xe000, 0xea7c, 3743 0xf000, 0x11110, 3744 0x11118, 0x11190, 3745 0x19040, 0x1906c, 3746 0x19078, 0x19080, 3747 0x1908c, 0x19124, 3748 0x19150, 0x191b0, 3749 0x191d0, 0x191e8, 3750 0x19238, 0x1924c, 3751 0x193f8, 0x19474, 3752 0x19490, 0x194f8, 3753 0x19800, 0x19f30, 3754 0x1a000, 0x1a06c, 3755 0x1a0b0, 0x1a120, 3756 0x1a128, 0x1a138, 3757 0x1a190, 0x1a1c4, 3758 0x1a1fc, 0x1a1fc, 3759 0x1e040, 0x1e04c, 3760 0x1e284, 0x1e28c, 3761 0x1e2c0, 0x1e2c0, 3762 0x1e2e0, 0x1e2e0, 3763 0x1e300, 0x1e384, 3764 0x1e3c0, 0x1e3c8, 3765 0x1e440, 0x1e44c, 3766 0x1e684, 0x1e68c, 3767 0x1e6c0, 0x1e6c0, 3768 0x1e6e0, 0x1e6e0, 3769 0x1e700, 0x1e784, 3770 0x1e7c0, 0x1e7c8, 3771 0x1e840, 0x1e84c, 3772 0x1ea84, 0x1ea8c, 3773 0x1eac0, 0x1eac0, 3774 0x1eae0, 0x1eae0, 3775 0x1eb00, 0x1eb84, 3776 0x1ebc0, 0x1ebc8, 3777 0x1ec40, 0x1ec4c, 3778 0x1ee84, 0x1ee8c, 3779 0x1eec0, 0x1eec0, 3780 0x1eee0, 0x1eee0, 3781 0x1ef00, 0x1ef84, 3782 0x1efc0, 0x1efc8, 3783 0x1f040, 0x1f04c, 3784 0x1f284, 0x1f28c, 3785 0x1f2c0, 0x1f2c0, 3786 0x1f2e0, 0x1f2e0, 3787 0x1f300, 0x1f384, 3788 0x1f3c0, 0x1f3c8, 3789 0x1f440, 0x1f44c, 3790 0x1f684, 0x1f68c, 3791 0x1f6c0, 0x1f6c0, 3792 0x1f6e0, 0x1f6e0, 3793 0x1f700, 0x1f784, 3794 0x1f7c0, 0x1f7c8, 3795 0x1f840, 0x1f84c, 3796 0x1fa84, 0x1fa8c, 3797 0x1fac0, 0x1fac0, 3798 0x1fae0, 0x1fae0, 3799 0x1fb00, 0x1fb84, 3800 0x1fbc0, 0x1fbc8, 3801 0x1fc40, 0x1fc4c, 3802 0x1fe84, 0x1fe8c, 3803 0x1fec0, 0x1fec0, 3804 0x1fee0, 0x1fee0, 3805 0x1ff00, 0x1ff84, 3806 0x1ffc0, 0x1ffc8, 3807 0x20000, 0x2002c, 3808 0x20100, 0x2013c, 3809 0x20190, 0x201c8, 3810 0x20200, 0x20318, 3811 0x20400, 0x20528, 3812 0x20540, 0x20614, 3813 0x21000, 0x21040, 3814 0x2104c, 0x21060, 3815 0x210c0, 0x210ec, 3816 0x21200, 0x21268, 3817 0x21270, 0x21284, 3818 0x212fc, 0x21388, 3819 0x21400, 0x21404, 3820 0x21500, 0x21518, 3821 0x2152c, 0x2153c, 3822 0x21550, 0x21554, 3823 0x21600, 0x21600, 3824 0x21608, 0x21628, 3825 0x21630, 0x2163c, 3826 0x21700, 0x2171c, 3827 0x21780, 0x2178c, 3828 0x21800, 0x21c38, 3829 0x21c80, 0x21d7c, 3830 0x21e00, 0x21e04, 3831 0x22000, 0x2202c, 3832 0x22100, 0x2213c, 3833 0x22190, 0x221c8, 3834 0x22200, 0x22318, 3835 0x22400, 0x22528, 3836 0x22540, 0x22614, 3837 0x23000, 0x23040, 3838 0x2304c, 0x23060, 3839 0x230c0, 0x230ec, 3840 0x23200, 0x23268, 3841 0x23270, 0x23284, 3842 0x232fc, 0x23388, 3843 0x23400, 0x23404, 3844 0x23500, 0x23518, 3845 0x2352c, 0x2353c, 3846 0x23550, 0x23554, 3847 0x23600, 0x23600, 3848 0x23608, 0x23628, 3849 0x23630, 0x2363c, 3850 0x23700, 0x2371c, 3851 0x23780, 0x2378c, 3852 0x23800, 0x23c38, 3853 0x23c80, 0x23d7c, 3854 0x23e00, 0x23e04, 3855 0x24000, 0x2402c, 3856 0x24100, 0x2413c, 3857 0x24190, 0x241c8, 3858 0x24200, 0x24318, 3859 0x24400, 0x24528, 3860 0x24540, 0x24614, 3861 0x25000, 0x25040, 3862 0x2504c, 0x25060, 3863 0x250c0, 0x250ec, 3864 0x25200, 0x25268, 3865 0x25270, 0x25284, 3866 0x252fc, 0x25388, 3867 0x25400, 0x25404, 3868 0x25500, 0x25518, 3869 0x2552c, 0x2553c, 3870 0x25550, 0x25554, 3871 0x25600, 0x25600, 3872 0x25608, 0x25628, 3873 0x25630, 0x2563c, 3874 0x25700, 0x2571c, 3875 0x25780, 0x2578c, 3876 0x25800, 0x25c38, 3877 0x25c80, 0x25d7c, 3878 0x25e00, 0x25e04, 3879 0x26000, 0x2602c, 3880 0x26100, 0x2613c, 3881 0x26190, 0x261c8, 3882 0x26200, 0x26318, 3883 0x26400, 0x26528, 3884 0x26540, 0x26614, 3885 0x27000, 0x27040, 3886 0x2704c, 0x27060, 3887 0x270c0, 0x270ec, 3888 0x27200, 0x27268, 3889 0x27270, 0x27284, 3890 0x272fc, 0x27388, 3891 0x27400, 0x27404, 3892 0x27500, 0x27518, 3893 0x2752c, 0x2753c, 3894 0x27550, 0x27554, 3895 0x27600, 0x27600, 3896 0x27608, 0x27628, 3897 0x27630, 0x2763c, 3898 0x27700, 0x2771c, 3899 0x27780, 0x2778c, 3900 0x27800, 0x27c38, 3901 0x27c80, 0x27d7c, 3902 0x27e00, 0x27e04 3903 }; 3904 static const unsigned int t5_reg_ranges[] = { 3905 0x1008, 0x1148, 3906 0x1180, 0x11b4, 3907 0x11fc, 0x123c, 3908 0x1280, 0x173c, 3909 0x1800, 0x18fc, 3910 0x3000, 0x3028, 3911 0x3060, 0x30d8, 3912 0x30e0, 0x30fc, 3913 0x3140, 0x357c, 3914 0x35a8, 0x35cc, 3915 0x35ec, 0x35ec, 3916 0x3600, 0x5624, 3917 0x56cc, 0x575c, 3918 0x580c, 0x5814, 3919 0x5890, 0x58bc, 3920 0x5940, 0x59dc, 3921 0x59fc, 0x5a18, 3922 0x5a60, 0x5a9c, 3923 0x5b94, 0x5bfc, 3924 0x6000, 0x6040, 3925 0x6058, 0x614c, 3926 0x7700, 0x7798, 3927 0x77c0, 0x78fc, 3928 0x7b00, 0x7c54, 3929 0x7d00, 0x7efc, 3930 0x8dc0, 0x8de0, 3931 0x8df8, 0x8e84, 3932 0x8ea0, 0x8f84, 3933 0x8fc0, 0x90f8, 3934 0x9400, 0x9470, 3935 0x9600, 0x96f4, 3936 0x9800, 0x9808, 3937 0x9820, 0x983c, 3938 0x9850, 0x9864, 3939 0x9c00, 0x9c6c, 3940 0x9c80, 0x9cec, 3941 0x9d00, 0x9d6c, 3942 0x9d80, 0x9dec, 3943 0x9e00, 0x9e6c, 3944 0x9e80, 0x9eec, 3945 0x9f00, 0x9f6c, 3946 0x9f80, 0xa020, 3947 0xd004, 0xd03c, 3948 0xdfc0, 0xdfe0, 3949 0xe000, 0x11088, 3950 0x1109c, 0x11110, 3951 0x11118, 0x1117c, 3952 0x11190, 0x11204, 3953 0x19040, 0x1906c, 3954 0x19078, 0x19080, 3955 0x1908c, 0x19124, 3956 0x19150, 0x191b0, 3957 0x191d0, 0x191e8, 3958 0x19238, 0x19290, 3959 0x193f8, 0x19474, 3960 0x19490, 0x194cc, 3961 0x194f0, 0x194f8, 3962 0x19c00, 0x19c60, 3963 0x19c94, 0x19e10, 3964 0x19e50, 0x19f34, 3965 0x19f40, 0x19f50, 3966 0x19f90, 0x19fe4, 3967 0x1a000, 0x1a06c, 3968 0x1a0b0, 0x1a120, 3969 0x1a128, 0x1a138, 3970 0x1a190, 0x1a1c4, 3971 0x1a1fc, 0x1a1fc, 3972 0x1e008, 0x1e00c, 3973 0x1e040, 0x1e04c, 3974 0x1e284, 0x1e290, 3975 0x1e2c0, 0x1e2c0, 3976 0x1e2e0, 0x1e2e0, 3977 0x1e300, 0x1e384, 3978 0x1e3c0, 0x1e3c8, 3979 0x1e408, 0x1e40c, 3980 0x1e440, 0x1e44c, 3981 0x1e684, 0x1e690, 3982 0x1e6c0, 0x1e6c0, 3983 0x1e6e0, 0x1e6e0, 3984 0x1e700, 0x1e784, 3985 0x1e7c0, 0x1e7c8, 3986 0x1e808, 0x1e80c, 3987 0x1e840, 0x1e84c, 3988 0x1ea84, 0x1ea90, 3989 0x1eac0, 0x1eac0, 3990 0x1eae0, 0x1eae0, 3991 0x1eb00, 0x1eb84, 3992 0x1ebc0, 0x1ebc8, 3993 0x1ec08, 0x1ec0c, 3994 0x1ec40, 0x1ec4c, 3995 0x1ee84, 0x1ee90, 3996 0x1eec0, 0x1eec0, 3997 0x1eee0, 0x1eee0, 3998 0x1ef00, 0x1ef84, 3999 0x1efc0, 0x1efc8, 4000 0x1f008, 0x1f00c, 4001 0x1f040, 0x1f04c, 4002 0x1f284, 0x1f290, 4003 0x1f2c0, 0x1f2c0, 4004 0x1f2e0, 0x1f2e0, 4005 0x1f300, 0x1f384, 4006 0x1f3c0, 0x1f3c8, 4007 0x1f408, 0x1f40c, 4008 0x1f440, 0x1f44c, 4009 0x1f684, 0x1f690, 4010 0x1f6c0, 0x1f6c0, 4011 0x1f6e0, 0x1f6e0, 4012 0x1f700, 0x1f784, 4013 0x1f7c0, 0x1f7c8, 4014 0x1f808, 0x1f80c, 4015 0x1f840, 0x1f84c, 4016 0x1fa84, 0x1fa90, 4017 0x1fac0, 0x1fac0, 4018 0x1fae0, 0x1fae0, 4019 0x1fb00, 0x1fb84, 4020 0x1fbc0, 0x1fbc8, 4021 0x1fc08, 0x1fc0c, 4022 0x1fc40, 0x1fc4c, 4023 0x1fe84, 0x1fe90, 4024 0x1fec0, 0x1fec0, 4025 0x1fee0, 0x1fee0, 4026 0x1ff00, 0x1ff84, 4027 0x1ffc0, 0x1ffc8, 4028 0x30000, 0x30030, 4029 0x30100, 0x30144, 4030 0x30190, 0x301d0, 4031 0x30200, 0x30318, 4032 0x30400, 0x3052c, 4033 0x30540, 0x3061c, 4034 0x30800, 0x30834, 4035 0x308c0, 0x30908, 4036 0x30910, 0x309ac, 4037 0x30a00, 0x30a2c, 4038 0x30a44, 0x30a50, 4039 0x30a74, 0x30c24, 4040 0x30d00, 0x30d00, 4041 0x30d08, 0x30d14, 4042 0x30d1c, 0x30d20, 4043 0x30d3c, 0x30d50, 4044 0x31200, 0x3120c, 4045 0x31220, 0x31220, 4046 0x31240, 0x31240, 4047 0x31600, 0x3160c, 4048 0x31a00, 0x31a1c, 4049 0x31e00, 0x31e20, 4050 0x31e38, 0x31e3c, 4051 0x31e80, 0x31e80, 4052 0x31e88, 0x31ea8, 4053 0x31eb0, 0x31eb4, 4054 0x31ec8, 0x31ed4, 4055 0x31fb8, 0x32004, 4056 0x32200, 0x32200, 4057 0x32208, 0x32240, 4058 0x32248, 0x32280, 4059 0x32288, 0x322c0, 4060 0x322c8, 0x322fc, 4061 0x32600, 0x32630, 4062 0x32a00, 0x32abc, 4063 0x32b00, 0x32b70, 4064 0x33000, 0x33048, 4065 0x33060, 0x3309c, 4066 0x330f0, 0x33148, 4067 0x33160, 0x3319c, 4068 0x331f0, 0x332e4, 4069 0x332f8, 0x333e4, 4070 0x333f8, 0x33448, 4071 0x33460, 0x3349c, 4072 0x334f0, 0x33548, 4073 0x33560, 0x3359c, 4074 0x335f0, 0x336e4, 4075 0x336f8, 0x337e4, 4076 0x337f8, 0x337fc, 4077 0x33814, 0x33814, 4078 0x3382c, 0x3382c, 4079 0x33880, 0x3388c, 4080 0x338e8, 0x338ec, 4081 0x33900, 0x33948, 4082 0x33960, 0x3399c, 4083 0x339f0, 0x33ae4, 4084 0x33af8, 0x33b10, 4085 0x33b28, 0x33b28, 4086 0x33b3c, 0x33b50, 4087 0x33bf0, 0x33c10, 4088 0x33c28, 0x33c28, 4089 0x33c3c, 0x33c50, 4090 0x33cf0, 0x33cfc, 4091 0x34000, 0x34030, 4092 0x34100, 0x34144, 4093 0x34190, 0x341d0, 4094 0x34200, 0x34318, 4095 0x34400, 0x3452c, 4096 0x34540, 0x3461c, 4097 0x34800, 0x34834, 4098 0x348c0, 0x34908, 4099 0x34910, 0x349ac, 4100 0x34a00, 0x34a2c, 4101 0x34a44, 0x34a50, 4102 0x34a74, 0x34c24, 4103 0x34d00, 0x34d00, 4104 0x34d08, 0x34d14, 4105 0x34d1c, 0x34d20, 4106 0x34d3c, 0x34d50, 4107 0x35200, 0x3520c, 4108 0x35220, 0x35220, 4109 0x35240, 0x35240, 4110 0x35600, 0x3560c, 4111 0x35a00, 0x35a1c, 4112 0x35e00, 0x35e20, 4113 0x35e38, 0x35e3c, 4114 0x35e80, 0x35e80, 4115 0x35e88, 0x35ea8, 4116 0x35eb0, 0x35eb4, 4117 0x35ec8, 0x35ed4, 4118 0x35fb8, 0x36004, 4119 0x36200, 0x36200, 4120 0x36208, 0x36240, 4121 0x36248, 0x36280, 4122 0x36288, 0x362c0, 4123 0x362c8, 0x362fc, 4124 0x36600, 0x36630, 4125 0x36a00, 0x36abc, 4126 0x36b00, 0x36b70, 4127 0x37000, 0x37048, 4128 0x37060, 0x3709c, 4129 0x370f0, 0x37148, 4130 0x37160, 0x3719c, 4131 0x371f0, 0x372e4, 4132 0x372f8, 0x373e4, 4133 0x373f8, 0x37448, 4134 0x37460, 0x3749c, 4135 0x374f0, 0x37548, 4136 0x37560, 0x3759c, 4137 0x375f0, 0x376e4, 4138 0x376f8, 0x377e4, 4139 0x377f8, 0x377fc, 4140 0x37814, 0x37814, 4141 0x3782c, 0x3782c, 4142 0x37880, 0x3788c, 4143 0x378e8, 0x378ec, 4144 0x37900, 0x37948, 4145 0x37960, 0x3799c, 4146 0x379f0, 0x37ae4, 4147 0x37af8, 0x37b10, 4148 0x37b28, 0x37b28, 4149 0x37b3c, 0x37b50, 4150 0x37bf0, 0x37c10, 4151 0x37c28, 0x37c28, 4152 0x37c3c, 0x37c50, 4153 0x37cf0, 0x37cfc, 4154 0x38000, 0x38030, 4155 0x38100, 0x38144, 4156 0x38190, 0x381d0, 4157 0x38200, 0x38318, 4158 0x38400, 0x3852c, 4159 0x38540, 0x3861c, 4160 0x38800, 0x38834, 4161 0x388c0, 0x38908, 4162 0x38910, 0x389ac, 4163 0x38a00, 0x38a2c, 4164 0x38a44, 0x38a50, 4165 0x38a74, 0x38c24, 4166 0x38d00, 0x38d00, 4167 0x38d08, 0x38d14, 4168 0x38d1c, 0x38d20, 4169 0x38d3c, 0x38d50, 4170 0x39200, 0x3920c, 4171 0x39220, 0x39220, 4172 0x39240, 0x39240, 4173 0x39600, 0x3960c, 4174 0x39a00, 0x39a1c, 4175 0x39e00, 0x39e20, 4176 0x39e38, 0x39e3c, 4177 0x39e80, 0x39e80, 4178 0x39e88, 0x39ea8, 4179 0x39eb0, 0x39eb4, 4180 0x39ec8, 0x39ed4, 4181 0x39fb8, 0x3a004, 4182 0x3a200, 0x3a200, 4183 0x3a208, 0x3a240, 4184 0x3a248, 0x3a280, 4185 0x3a288, 0x3a2c0, 4186 0x3a2c8, 0x3a2fc, 4187 0x3a600, 0x3a630, 4188 0x3aa00, 0x3aabc, 4189 0x3ab00, 0x3ab70, 4190 0x3b000, 0x3b048, 4191 0x3b060, 0x3b09c, 4192 0x3b0f0, 0x3b148, 4193 0x3b160, 0x3b19c, 4194 0x3b1f0, 0x3b2e4, 4195 0x3b2f8, 0x3b3e4, 4196 0x3b3f8, 0x3b448, 4197 0x3b460, 0x3b49c, 4198 0x3b4f0, 0x3b548, 4199 0x3b560, 0x3b59c, 4200 0x3b5f0, 0x3b6e4, 4201 0x3b6f8, 0x3b7e4, 4202 0x3b7f8, 0x3b7fc, 4203 0x3b814, 0x3b814, 4204 0x3b82c, 0x3b82c, 4205 0x3b880, 0x3b88c, 4206 0x3b8e8, 0x3b8ec, 4207 0x3b900, 0x3b948, 4208 0x3b960, 0x3b99c, 4209 0x3b9f0, 0x3bae4, 4210 0x3baf8, 0x3bb10, 4211 0x3bb28, 0x3bb28, 4212 0x3bb3c, 0x3bb50, 4213 0x3bbf0, 0x3bc10, 4214 0x3bc28, 0x3bc28, 4215 0x3bc3c, 0x3bc50, 4216 0x3bcf0, 0x3bcfc, 4217 0x3c000, 0x3c030, 4218 0x3c100, 0x3c144, 4219 0x3c190, 0x3c1d0, 4220 0x3c200, 0x3c318, 4221 0x3c400, 0x3c52c, 4222 0x3c540, 0x3c61c, 4223 0x3c800, 0x3c834, 4224 0x3c8c0, 0x3c908, 4225 0x3c910, 0x3c9ac, 4226 0x3ca00, 0x3ca2c, 4227 0x3ca44, 0x3ca50, 4228 0x3ca74, 0x3cc24, 4229 0x3cd00, 0x3cd00, 4230 0x3cd08, 0x3cd14, 4231 0x3cd1c, 0x3cd20, 4232 0x3cd3c, 0x3cd50, 4233 0x3d200, 0x3d20c, 4234 0x3d220, 0x3d220, 4235 0x3d240, 0x3d240, 4236 0x3d600, 0x3d60c, 4237 0x3da00, 0x3da1c, 4238 0x3de00, 0x3de20, 4239 0x3de38, 0x3de3c, 4240 0x3de80, 0x3de80, 4241 0x3de88, 0x3dea8, 4242 0x3deb0, 0x3deb4, 4243 0x3dec8, 0x3ded4, 4244 0x3dfb8, 0x3e004, 4245 0x3e200, 0x3e200, 4246 0x3e208, 0x3e240, 4247 0x3e248, 0x3e280, 4248 0x3e288, 0x3e2c0, 4249 0x3e2c8, 0x3e2fc, 4250 0x3e600, 0x3e630, 4251 0x3ea00, 0x3eabc, 4252 0x3eb00, 0x3eb70, 4253 0x3f000, 0x3f048, 4254 0x3f060, 0x3f09c, 4255 0x3f0f0, 0x3f148, 4256 0x3f160, 0x3f19c, 4257 0x3f1f0, 0x3f2e4, 4258 0x3f2f8, 0x3f3e4, 4259 0x3f3f8, 0x3f448, 4260 0x3f460, 0x3f49c, 4261 0x3f4f0, 0x3f548, 4262 0x3f560, 0x3f59c, 4263 0x3f5f0, 0x3f6e4, 4264 0x3f6f8, 0x3f7e4, 4265 0x3f7f8, 0x3f7fc, 4266 0x3f814, 0x3f814, 4267 0x3f82c, 0x3f82c, 4268 0x3f880, 0x3f88c, 4269 0x3f8e8, 0x3f8ec, 4270 0x3f900, 0x3f948, 4271 0x3f960, 0x3f99c, 4272 0x3f9f0, 0x3fae4, 4273 0x3faf8, 0x3fb10, 4274 0x3fb28, 0x3fb28, 4275 0x3fb3c, 0x3fb50, 4276 0x3fbf0, 0x3fc10, 4277 0x3fc28, 0x3fc28, 4278 0x3fc3c, 0x3fc50, 4279 0x3fcf0, 0x3fcfc, 4280 0x40000, 0x4000c, 4281 0x40040, 0x40068, 4282 0x4007c, 0x40144, 4283 0x40180, 0x4018c, 4284 0x40200, 0x40298, 4285 0x402ac, 0x4033c, 4286 0x403f8, 0x403fc, 4287 0x41304, 0x413c4, 4288 0x41400, 0x4141c, 4289 0x41480, 0x414d0, 4290 0x44000, 0x44078, 4291 0x440c0, 0x44278, 4292 0x442c0, 0x44478, 4293 0x444c0, 0x44678, 4294 0x446c0, 0x44878, 4295 0x448c0, 0x449fc, 4296 0x45000, 0x45068, 4297 0x45080, 0x45084, 4298 0x450a0, 0x450b0, 4299 0x45200, 0x45268, 4300 0x45280, 0x45284, 4301 0x452a0, 0x452b0, 4302 0x460c0, 0x460e4, 4303 0x47000, 0x4708c, 4304 0x47200, 0x47250, 4305 0x47400, 0x47420, 4306 0x47600, 0x47618, 4307 0x47800, 0x47814, 4308 0x48000, 0x4800c, 4309 0x48040, 0x48068, 4310 0x4807c, 0x48144, 4311 0x48180, 0x4818c, 4312 0x48200, 0x48298, 4313 0x482ac, 0x4833c, 4314 0x483f8, 0x483fc, 4315 0x49304, 0x493c4, 4316 0x49400, 0x4941c, 4317 0x49480, 0x494d0, 4318 0x4c000, 0x4c078, 4319 0x4c0c0, 0x4c278, 4320 0x4c2c0, 0x4c478, 4321 0x4c4c0, 0x4c678, 4322 0x4c6c0, 0x4c878, 4323 0x4c8c0, 0x4c9fc, 4324 0x4d000, 0x4d068, 4325 0x4d080, 0x4d084, 4326 0x4d0a0, 0x4d0b0, 4327 0x4d200, 0x4d268, 4328 0x4d280, 0x4d284, 4329 0x4d2a0, 0x4d2b0, 4330 0x4e0c0, 0x4e0e4, 4331 0x4f000, 0x4f08c, 4332 0x4f200, 0x4f250, 4333 0x4f400, 0x4f420, 4334 0x4f600, 0x4f618, 4335 0x4f800, 0x4f814, 4336 0x50000, 0x500cc, 4337 0x50400, 0x50400, 4338 0x50800, 0x508cc, 4339 0x50c00, 0x50c00, 4340 0x51000, 0x5101c, 4341 0x51300, 0x51308, 4342 }; 4343 4344 if (is_t4(sc)) { 4345 reg_ranges = &t4_reg_ranges[0]; 4346 n = nitems(t4_reg_ranges); 4347 } else { 4348 reg_ranges = &t5_reg_ranges[0]; 4349 n = nitems(t5_reg_ranges); 4350 } 4351 4352 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4353 for (i = 0; i < n; i += 2) 4354 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4355 } 4356 4357 static void 4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4359 { 4360 int i; 4361 u_int v, tnl_cong_drops; 4362 struct timeval tv; 4363 const struct timeval interval = {0, 250000}; /* 250ms */ 4364 4365 getmicrotime(&tv); 4366 timevalsub(&tv, &interval); 4367 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4368 return; 4369 4370 tnl_cong_drops = 0; 4371 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4372 for (i = 0; i < NCHAN; i++) { 4373 if (pi->rx_chan_map & (1 << i)) { 4374 mtx_lock(&sc->regwin_lock); 4375 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4376 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4377 mtx_unlock(&sc->regwin_lock); 4378 tnl_cong_drops += v; 4379 } 4380 } 4381 pi->tnl_cong_drops = tnl_cong_drops; 4382 getmicrotime(&pi->last_refreshed); 4383 } 4384 4385 static void 4386 cxgbe_tick(void *arg) 4387 { 4388 struct port_info *pi = arg; 4389 struct adapter *sc = pi->adapter; 4390 struct ifnet *ifp = pi->ifp; 4391 4392 PORT_LOCK(pi); 4393 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4394 PORT_UNLOCK(pi); 4395 return; /* without scheduling another callout */ 4396 } 4397 4398 cxgbe_refresh_stats(sc, pi); 4399 4400 callout_schedule(&pi->tick, hz); 4401 PORT_UNLOCK(pi); 4402 } 4403 4404 static void 4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4406 { 4407 struct ifnet *vlan; 4408 4409 if (arg != ifp || ifp->if_type != IFT_ETHER) 4410 return; 4411 4412 vlan = VLAN_DEVAT(ifp, vid); 4413 VLAN_SETCOOKIE(vlan, ifp); 4414 } 4415 4416 static int 4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4418 { 4419 4420 #ifdef INVARIANTS 4421 panic("%s: opcode 0x%02x on iq %p with payload %p", 4422 __func__, rss->opcode, iq, m); 4423 #else 4424 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4425 __func__, rss->opcode, iq, m); 4426 m_freem(m); 4427 #endif 4428 return (EDOOFUS); 4429 } 4430 4431 int 4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4433 { 4434 uintptr_t *loc, new; 4435 4436 if (opcode >= nitems(sc->cpl_handler)) 4437 return (EINVAL); 4438 4439 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4440 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4441 atomic_store_rel_ptr(loc, new); 4442 4443 return (0); 4444 } 4445 4446 static int 4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4448 { 4449 4450 #ifdef INVARIANTS 4451 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4452 #else 4453 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4454 __func__, iq, ctrl); 4455 #endif 4456 return (EDOOFUS); 4457 } 4458 4459 int 4460 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4461 { 4462 uintptr_t *loc, new; 4463 4464 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4465 loc = (uintptr_t *) &sc->an_handler; 4466 atomic_store_rel_ptr(loc, new); 4467 4468 return (0); 4469 } 4470 4471 static int 4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4473 { 4474 const struct cpl_fw6_msg *cpl = 4475 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4476 4477 #ifdef INVARIANTS 4478 panic("%s: fw_msg type %d", __func__, cpl->type); 4479 #else 4480 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4481 #endif 4482 return (EDOOFUS); 4483 } 4484 4485 int 4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4487 { 4488 uintptr_t *loc, new; 4489 4490 if (type >= nitems(sc->fw_msg_handler)) 4491 return (EINVAL); 4492 4493 /* 4494 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4495 * handler dispatch table. Reject any attempt to install a handler for 4496 * this subtype. 4497 */ 4498 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4499 return (EINVAL); 4500 4501 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4502 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4503 atomic_store_rel_ptr(loc, new); 4504 4505 return (0); 4506 } 4507 4508 static int 4509 t4_sysctls(struct adapter *sc) 4510 { 4511 struct sysctl_ctx_list *ctx; 4512 struct sysctl_oid *oid; 4513 struct sysctl_oid_list *children, *c0; 4514 static char *caps[] = { 4515 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4516 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4517 "\6HASHFILTER\7ETHOFLD", 4518 "\20\1TOE", /* caps[2] toecaps */ 4519 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4520 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4521 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4522 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4523 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4524 "\4PO_INITIAOR\5PO_TARGET" 4525 }; 4526 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4527 4528 ctx = device_get_sysctl_ctx(sc->dev); 4529 4530 /* 4531 * dev.t4nex.X. 4532 */ 4533 oid = device_get_sysctl_tree(sc->dev); 4534 c0 = children = SYSCTL_CHILDREN(oid); 4535 4536 sc->sc_do_rxcopy = 1; 4537 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4538 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4539 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4541 sc->params.nports, "# of ports"); 4542 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4544 NULL, chip_rev(sc), "chip hardware revision"); 4545 4546 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4547 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4548 4549 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4550 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4551 4552 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4553 sc->cfcsum, "config file checksum"); 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4556 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4557 sysctl_bitfield, "A", "available doorbells"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4560 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4561 sysctl_bitfield, "A", "available link capabilities"); 4562 4563 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4564 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4565 sysctl_bitfield, "A", "available NIC capabilities"); 4566 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4568 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4569 sysctl_bitfield, "A", "available TCP offload capabilities"); 4570 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4572 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4573 sysctl_bitfield, "A", "available RDMA capabilities"); 4574 4575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4576 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4577 sysctl_bitfield, "A", "available iSCSI capabilities"); 4578 4579 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4580 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4581 sysctl_bitfield, "A", "available FCoE capabilities"); 4582 4583 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4584 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4588 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4589 "interrupt holdoff timer values (us)"); 4590 4591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4592 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4593 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4594 "interrupt holdoff packet counter values"); 4595 4596 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4597 NULL, sc->tids.nftids, "number of filters"); 4598 4599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4600 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4601 "chip temperature (in Celsius)"); 4602 4603 t4_sge_sysctls(sc, ctx, children); 4604 4605 sc->lro_timeout = 100; 4606 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4607 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4608 4609 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4610 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4611 4612 #ifdef SBUF_DRAIN 4613 /* 4614 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4615 */ 4616 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4617 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4618 "logs and miscellaneous information"); 4619 children = SYSCTL_CHILDREN(oid); 4620 4621 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4622 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4623 sysctl_cctrl, "A", "congestion control"); 4624 4625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4626 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4627 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4628 4629 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4630 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4631 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4632 4633 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4634 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4635 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4636 4637 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4638 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4639 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4640 4641 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4642 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4643 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4644 4645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4646 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4647 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4651 sysctl_cim_la, "A", "CIM logic analyzer"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4655 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4659 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4660 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4663 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4664 4665 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4666 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4667 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4668 4669 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4670 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4671 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4672 4673 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4674 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4675 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4676 4677 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4678 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4679 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4680 4681 if (is_t5(sc)) { 4682 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4683 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4684 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4685 4686 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4687 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4688 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4689 } 4690 4691 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4692 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4693 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4694 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4696 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4697 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4698 4699 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4700 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4701 sysctl_cpl_stats, "A", "CPL statistics"); 4702 4703 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4704 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4705 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4706 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4708 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4709 sysctl_devlog, "A", "firmware's device log"); 4710 4711 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4712 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4713 sysctl_fcoe_stats, "A", "FCoE statistics"); 4714 4715 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4716 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4717 sysctl_hw_sched, "A", "hardware scheduler "); 4718 4719 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4720 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4721 sysctl_l2t, "A", "hardware L2 table"); 4722 4723 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4724 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4725 sysctl_lb_stats, "A", "loopback statistics"); 4726 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4728 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4729 sysctl_meminfo, "A", "memory regions"); 4730 4731 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4732 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4733 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4734 4735 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4736 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4737 sysctl_path_mtus, "A", "path MTUs"); 4738 4739 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4740 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4741 sysctl_pm_stats, "A", "PM statistics"); 4742 4743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4744 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4745 sysctl_rdma_stats, "A", "RDMA statistics"); 4746 4747 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4748 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4749 sysctl_tcp_stats, "A", "TCP statistics"); 4750 4751 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4752 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4753 sysctl_tids, "A", "TID information"); 4754 4755 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4756 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4757 sysctl_tp_err_stats, "A", "TP error statistics"); 4758 4759 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4760 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4761 sysctl_tp_la, "A", "TP logic analyzer"); 4762 4763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4764 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4765 sysctl_tx_rate, "A", "Tx rate"); 4766 4767 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4768 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4769 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4770 4771 if (is_t5(sc)) { 4772 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4773 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4774 sysctl_wcwr_stats, "A", "write combined work requests"); 4775 } 4776 #endif 4777 4778 #ifdef TCP_OFFLOAD 4779 if (is_offload(sc)) { 4780 /* 4781 * dev.t4nex.X.toe. 4782 */ 4783 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4784 NULL, "TOE parameters"); 4785 children = SYSCTL_CHILDREN(oid); 4786 4787 sc->tt.sndbuf = 256 * 1024; 4788 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4789 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4790 4791 sc->tt.ddp = 0; 4792 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4793 &sc->tt.ddp, 0, "DDP allowed"); 4794 4795 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4796 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4797 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4798 4799 sc->tt.ddp_thres = 4800 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4801 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4802 &sc->tt.ddp_thres, 0, "DDP threshold"); 4803 4804 sc->tt.rx_coalesce = 1; 4805 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4806 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4807 4808 sc->tt.tx_align = 1; 4809 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4810 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4811 } 4812 #endif 4813 4814 4815 return (0); 4816 } 4817 4818 static int 4819 cxgbe_sysctls(struct port_info *pi) 4820 { 4821 struct sysctl_ctx_list *ctx; 4822 struct sysctl_oid *oid; 4823 struct sysctl_oid_list *children; 4824 struct adapter *sc = pi->adapter; 4825 4826 ctx = device_get_sysctl_ctx(pi->dev); 4827 4828 /* 4829 * dev.cxgbe.X. 4830 */ 4831 oid = device_get_sysctl_tree(pi->dev); 4832 children = SYSCTL_CHILDREN(oid); 4833 4834 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4835 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4836 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4837 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4838 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4839 "PHY temperature (in Celsius)"); 4840 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4841 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4842 "PHY firmware version"); 4843 } 4844 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4845 &pi->nrxq, 0, "# of rx queues"); 4846 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4847 &pi->ntxq, 0, "# of tx queues"); 4848 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4849 &pi->first_rxq, 0, "index of first rx queue"); 4850 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4851 &pi->first_txq, 0, "index of first tx queue"); 4852 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4853 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4854 "Reserve queue 0 for non-flowid packets"); 4855 4856 #ifdef TCP_OFFLOAD 4857 if (is_offload(sc)) { 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4859 &pi->nofldrxq, 0, 4860 "# of rx queues for offloaded TCP connections"); 4861 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4862 &pi->nofldtxq, 0, 4863 "# of tx queues for offloaded TCP connections"); 4864 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4865 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4866 "index of first TOE rx queue"); 4867 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4868 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4869 "index of first TOE tx queue"); 4870 } 4871 #endif 4872 #ifdef DEV_NETMAP 4873 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4874 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4875 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4876 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4877 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4878 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4879 "index of first netmap rx queue"); 4880 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4881 CTLFLAG_RD, &pi->first_nm_txq, 0, 4882 "index of first netmap tx queue"); 4883 #endif 4884 4885 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4886 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4887 "holdoff timer index"); 4888 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4889 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4890 "holdoff packet counter index"); 4891 4892 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4893 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4894 "rx queue size"); 4895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4896 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4897 "tx queue size"); 4898 4899 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4900 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4901 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4902 4903 /* 4904 * dev.cxgbe.X.stats. 4905 */ 4906 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4907 NULL, "port statistics"); 4908 children = SYSCTL_CHILDREN(oid); 4909 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4910 &pi->tx_parse_error, 0, 4911 "# of tx packets with invalid length or # of segments"); 4912 4913 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4914 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4915 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4916 sysctl_handle_t4_reg64, "QU", desc) 4917 4918 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4923 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4924 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4925 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4926 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4931 "# of tx frames in this range", 4932 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4933 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4934 "# of tx frames in this range", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4937 "# of tx frames in this range", 4938 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4939 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4940 "# of tx frames in this range", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4943 "# of tx frames in this range", 4944 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4945 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4946 "# of tx frames in this range", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4949 "# of tx frames in this range", 4950 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4951 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4952 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4953 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4971 4972 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4975 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4976 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4979 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4980 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4981 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4982 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4983 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4984 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4987 "# of frames received with bad FCS", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4990 "# of frames received with length error", 4991 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4992 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4997 "# of rx frames in this range", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5000 "# of rx frames in this range", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5003 "# of rx frames in this range", 5004 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5005 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5006 "# of rx frames in this range", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5009 "# of rx frames in this range", 5010 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5011 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5012 "# of rx frames in this range", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5015 "# of rx frames in this range", 5016 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5017 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5018 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5019 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5020 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5021 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5022 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5023 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5024 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5025 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5026 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5027 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5028 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5029 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5030 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5031 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5032 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5033 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5034 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5035 5036 #undef SYSCTL_ADD_T4_REG64 5037 5038 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5039 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5040 &pi->stats.name, desc) 5041 5042 /* We get these from port_stats and they may be stale by upto 1s */ 5043 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5044 "# drops due to buffer-group 0 overflows"); 5045 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5046 "# drops due to buffer-group 1 overflows"); 5047 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5048 "# drops due to buffer-group 2 overflows"); 5049 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5050 "# drops due to buffer-group 3 overflows"); 5051 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5052 "# of buffer-group 0 truncated packets"); 5053 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5054 "# of buffer-group 1 truncated packets"); 5055 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5056 "# of buffer-group 2 truncated packets"); 5057 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5058 "# of buffer-group 3 truncated packets"); 5059 5060 #undef SYSCTL_ADD_T4_PORTSTAT 5061 5062 return (0); 5063 } 5064 5065 static int 5066 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5067 { 5068 int rc, *i, space = 0; 5069 struct sbuf sb; 5070 5071 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5072 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5073 if (space) 5074 sbuf_printf(&sb, " "); 5075 sbuf_printf(&sb, "%d", *i); 5076 space = 1; 5077 } 5078 rc = sbuf_finish(&sb); 5079 sbuf_delete(&sb); 5080 return (rc); 5081 } 5082 5083 static int 5084 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5085 { 5086 int rc; 5087 struct sbuf *sb; 5088 5089 rc = sysctl_wire_old_buffer(req, 0); 5090 if (rc != 0) 5091 return(rc); 5092 5093 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5094 if (sb == NULL) 5095 return (ENOMEM); 5096 5097 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5098 rc = sbuf_finish(sb); 5099 sbuf_delete(sb); 5100 5101 return (rc); 5102 } 5103 5104 static int 5105 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5106 { 5107 struct port_info *pi = arg1; 5108 int op = arg2; 5109 struct adapter *sc = pi->adapter; 5110 u_int v; 5111 int rc; 5112 5113 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5114 if (rc) 5115 return (rc); 5116 /* XXX: magic numbers */ 5117 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5118 &v); 5119 end_synchronized_op(sc, 0); 5120 if (rc) 5121 return (rc); 5122 if (op == 0) 5123 v /= 256; 5124 5125 rc = sysctl_handle_int(oidp, &v, 0, req); 5126 return (rc); 5127 } 5128 5129 static int 5130 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5131 { 5132 struct port_info *pi = arg1; 5133 int rc, val; 5134 5135 val = pi->rsrv_noflowq; 5136 rc = sysctl_handle_int(oidp, &val, 0, req); 5137 if (rc != 0 || req->newptr == NULL) 5138 return (rc); 5139 5140 if ((val >= 1) && (pi->ntxq > 1)) 5141 pi->rsrv_noflowq = 1; 5142 else 5143 pi->rsrv_noflowq = 0; 5144 5145 return (rc); 5146 } 5147 5148 static int 5149 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5150 { 5151 struct port_info *pi = arg1; 5152 struct adapter *sc = pi->adapter; 5153 int idx, rc, i; 5154 struct sge_rxq *rxq; 5155 #ifdef TCP_OFFLOAD 5156 struct sge_ofld_rxq *ofld_rxq; 5157 #endif 5158 uint8_t v; 5159 5160 idx = pi->tmr_idx; 5161 5162 rc = sysctl_handle_int(oidp, &idx, 0, req); 5163 if (rc != 0 || req->newptr == NULL) 5164 return (rc); 5165 5166 if (idx < 0 || idx >= SGE_NTIMERS) 5167 return (EINVAL); 5168 5169 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5170 "t4tmr"); 5171 if (rc) 5172 return (rc); 5173 5174 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5175 for_each_rxq(pi, i, rxq) { 5176 #ifdef atomic_store_rel_8 5177 atomic_store_rel_8(&rxq->iq.intr_params, v); 5178 #else 5179 rxq->iq.intr_params = v; 5180 #endif 5181 } 5182 #ifdef TCP_OFFLOAD 5183 for_each_ofld_rxq(pi, i, ofld_rxq) { 5184 #ifdef atomic_store_rel_8 5185 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5186 #else 5187 ofld_rxq->iq.intr_params = v; 5188 #endif 5189 } 5190 #endif 5191 pi->tmr_idx = idx; 5192 5193 end_synchronized_op(sc, LOCK_HELD); 5194 return (0); 5195 } 5196 5197 static int 5198 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5199 { 5200 struct port_info *pi = arg1; 5201 struct adapter *sc = pi->adapter; 5202 int idx, rc; 5203 5204 idx = pi->pktc_idx; 5205 5206 rc = sysctl_handle_int(oidp, &idx, 0, req); 5207 if (rc != 0 || req->newptr == NULL) 5208 return (rc); 5209 5210 if (idx < -1 || idx >= SGE_NCOUNTERS) 5211 return (EINVAL); 5212 5213 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5214 "t4pktc"); 5215 if (rc) 5216 return (rc); 5217 5218 if (pi->flags & PORT_INIT_DONE) 5219 rc = EBUSY; /* cannot be changed once the queues are created */ 5220 else 5221 pi->pktc_idx = idx; 5222 5223 end_synchronized_op(sc, LOCK_HELD); 5224 return (rc); 5225 } 5226 5227 static int 5228 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5229 { 5230 struct port_info *pi = arg1; 5231 struct adapter *sc = pi->adapter; 5232 int qsize, rc; 5233 5234 qsize = pi->qsize_rxq; 5235 5236 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5237 if (rc != 0 || req->newptr == NULL) 5238 return (rc); 5239 5240 if (qsize < 128 || (qsize & 7)) 5241 return (EINVAL); 5242 5243 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5244 "t4rxqs"); 5245 if (rc) 5246 return (rc); 5247 5248 if (pi->flags & PORT_INIT_DONE) 5249 rc = EBUSY; /* cannot be changed once the queues are created */ 5250 else 5251 pi->qsize_rxq = qsize; 5252 5253 end_synchronized_op(sc, LOCK_HELD); 5254 return (rc); 5255 } 5256 5257 static int 5258 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5259 { 5260 struct port_info *pi = arg1; 5261 struct adapter *sc = pi->adapter; 5262 int qsize, rc; 5263 5264 qsize = pi->qsize_txq; 5265 5266 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5267 if (rc != 0 || req->newptr == NULL) 5268 return (rc); 5269 5270 if (qsize < 128 || qsize > 65536) 5271 return (EINVAL); 5272 5273 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5274 "t4txqs"); 5275 if (rc) 5276 return (rc); 5277 5278 if (pi->flags & PORT_INIT_DONE) 5279 rc = EBUSY; /* cannot be changed once the queues are created */ 5280 else 5281 pi->qsize_txq = qsize; 5282 5283 end_synchronized_op(sc, LOCK_HELD); 5284 return (rc); 5285 } 5286 5287 static int 5288 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5289 { 5290 struct port_info *pi = arg1; 5291 struct adapter *sc = pi->adapter; 5292 struct link_config *lc = &pi->link_cfg; 5293 int rc; 5294 5295 if (req->newptr == NULL) { 5296 struct sbuf *sb; 5297 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5298 5299 rc = sysctl_wire_old_buffer(req, 0); 5300 if (rc != 0) 5301 return(rc); 5302 5303 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5304 if (sb == NULL) 5305 return (ENOMEM); 5306 5307 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5308 rc = sbuf_finish(sb); 5309 sbuf_delete(sb); 5310 } else { 5311 char s[2]; 5312 int n; 5313 5314 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5315 s[1] = 0; 5316 5317 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5318 if (rc != 0) 5319 return(rc); 5320 5321 if (s[1] != 0) 5322 return (EINVAL); 5323 if (s[0] < '0' || s[0] > '9') 5324 return (EINVAL); /* not a number */ 5325 n = s[0] - '0'; 5326 if (n & ~(PAUSE_TX | PAUSE_RX)) 5327 return (EINVAL); /* some other bit is set too */ 5328 5329 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5330 if (rc) 5331 return (rc); 5332 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5333 int link_ok = lc->link_ok; 5334 5335 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5336 lc->requested_fc |= n; 5337 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5338 lc->link_ok = link_ok; /* restore */ 5339 } 5340 end_synchronized_op(sc, 0); 5341 } 5342 5343 return (rc); 5344 } 5345 5346 static int 5347 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5348 { 5349 struct adapter *sc = arg1; 5350 int reg = arg2; 5351 uint64_t val; 5352 5353 val = t4_read_reg64(sc, reg); 5354 5355 return (sysctl_handle_64(oidp, &val, 0, req)); 5356 } 5357 5358 static int 5359 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5360 { 5361 struct adapter *sc = arg1; 5362 int rc, t; 5363 uint32_t param, val; 5364 5365 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5366 if (rc) 5367 return (rc); 5368 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5369 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5370 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5371 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5372 end_synchronized_op(sc, 0); 5373 if (rc) 5374 return (rc); 5375 5376 /* unknown is returned as 0 but we display -1 in that case */ 5377 t = val == 0 ? -1 : val; 5378 5379 rc = sysctl_handle_int(oidp, &t, 0, req); 5380 return (rc); 5381 } 5382 5383 #ifdef SBUF_DRAIN 5384 static int 5385 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5386 { 5387 struct adapter *sc = arg1; 5388 struct sbuf *sb; 5389 int rc, i; 5390 uint16_t incr[NMTUS][NCCTRL_WIN]; 5391 static const char *dec_fac[] = { 5392 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5393 "0.9375" 5394 }; 5395 5396 rc = sysctl_wire_old_buffer(req, 0); 5397 if (rc != 0) 5398 return (rc); 5399 5400 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5401 if (sb == NULL) 5402 return (ENOMEM); 5403 5404 t4_read_cong_tbl(sc, incr); 5405 5406 for (i = 0; i < NCCTRL_WIN; ++i) { 5407 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5408 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5409 incr[5][i], incr[6][i], incr[7][i]); 5410 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5411 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5412 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5413 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5414 } 5415 5416 rc = sbuf_finish(sb); 5417 sbuf_delete(sb); 5418 5419 return (rc); 5420 } 5421 5422 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5423 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5424 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5425 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5426 }; 5427 5428 static int 5429 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5430 { 5431 struct adapter *sc = arg1; 5432 struct sbuf *sb; 5433 int rc, i, n, qid = arg2; 5434 uint32_t *buf, *p; 5435 char *qtype; 5436 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5437 5438 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5439 ("%s: bad qid %d\n", __func__, qid)); 5440 5441 if (qid < CIM_NUM_IBQ) { 5442 /* inbound queue */ 5443 qtype = "IBQ"; 5444 n = 4 * CIM_IBQ_SIZE; 5445 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5446 rc = t4_read_cim_ibq(sc, qid, buf, n); 5447 } else { 5448 /* outbound queue */ 5449 qtype = "OBQ"; 5450 qid -= CIM_NUM_IBQ; 5451 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5452 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5453 rc = t4_read_cim_obq(sc, qid, buf, n); 5454 } 5455 5456 if (rc < 0) { 5457 rc = -rc; 5458 goto done; 5459 } 5460 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5461 5462 rc = sysctl_wire_old_buffer(req, 0); 5463 if (rc != 0) 5464 goto done; 5465 5466 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5467 if (sb == NULL) { 5468 rc = ENOMEM; 5469 goto done; 5470 } 5471 5472 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5473 for (i = 0, p = buf; i < n; i += 16, p += 4) 5474 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5475 p[2], p[3]); 5476 5477 rc = sbuf_finish(sb); 5478 sbuf_delete(sb); 5479 done: 5480 free(buf, M_CXGBE); 5481 return (rc); 5482 } 5483 5484 static int 5485 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5486 { 5487 struct adapter *sc = arg1; 5488 u_int cfg; 5489 struct sbuf *sb; 5490 uint32_t *buf, *p; 5491 int rc; 5492 5493 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5494 if (rc != 0) 5495 return (rc); 5496 5497 rc = sysctl_wire_old_buffer(req, 0); 5498 if (rc != 0) 5499 return (rc); 5500 5501 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5502 if (sb == NULL) 5503 return (ENOMEM); 5504 5505 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5506 M_ZERO | M_WAITOK); 5507 5508 rc = -t4_cim_read_la(sc, buf, NULL); 5509 if (rc != 0) 5510 goto done; 5511 5512 sbuf_printf(sb, "Status Data PC%s", 5513 cfg & F_UPDBGLACAPTPCONLY ? "" : 5514 " LS0Stat LS0Addr LS0Data"); 5515 5516 KASSERT((sc->params.cim_la_size & 7) == 0, 5517 ("%s: p will walk off the end of buf", __func__)); 5518 5519 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5520 if (cfg & F_UPDBGLACAPTPCONLY) { 5521 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5522 p[6], p[7]); 5523 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5524 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5525 p[4] & 0xff, p[5] >> 8); 5526 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5527 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5528 p[1] & 0xf, p[2] >> 4); 5529 } else { 5530 sbuf_printf(sb, 5531 "\n %02x %x%07x %x%07x %08x %08x " 5532 "%08x%08x%08x%08x", 5533 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5534 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5535 p[6], p[7]); 5536 } 5537 } 5538 5539 rc = sbuf_finish(sb); 5540 sbuf_delete(sb); 5541 done: 5542 free(buf, M_CXGBE); 5543 return (rc); 5544 } 5545 5546 static int 5547 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5548 { 5549 struct adapter *sc = arg1; 5550 u_int i; 5551 struct sbuf *sb; 5552 uint32_t *buf, *p; 5553 int rc; 5554 5555 rc = sysctl_wire_old_buffer(req, 0); 5556 if (rc != 0) 5557 return (rc); 5558 5559 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5560 if (sb == NULL) 5561 return (ENOMEM); 5562 5563 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5564 M_ZERO | M_WAITOK); 5565 5566 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5567 p = buf; 5568 5569 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5570 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5571 p[1], p[0]); 5572 } 5573 5574 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5575 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5576 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5577 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5578 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5579 (p[1] >> 2) | ((p[2] & 3) << 30), 5580 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5581 p[0] & 1); 5582 } 5583 5584 rc = sbuf_finish(sb); 5585 sbuf_delete(sb); 5586 free(buf, M_CXGBE); 5587 return (rc); 5588 } 5589 5590 static int 5591 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5592 { 5593 struct adapter *sc = arg1; 5594 u_int i; 5595 struct sbuf *sb; 5596 uint32_t *buf, *p; 5597 int rc; 5598 5599 rc = sysctl_wire_old_buffer(req, 0); 5600 if (rc != 0) 5601 return (rc); 5602 5603 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5604 if (sb == NULL) 5605 return (ENOMEM); 5606 5607 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5608 M_ZERO | M_WAITOK); 5609 5610 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5611 p = buf; 5612 5613 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5614 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5615 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5616 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5617 p[4], p[3], p[2], p[1], p[0]); 5618 } 5619 5620 sbuf_printf(sb, "\n\nCntl ID Data"); 5621 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5622 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5623 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5624 } 5625 5626 rc = sbuf_finish(sb); 5627 sbuf_delete(sb); 5628 free(buf, M_CXGBE); 5629 return (rc); 5630 } 5631 5632 static int 5633 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5634 { 5635 struct adapter *sc = arg1; 5636 struct sbuf *sb; 5637 int rc, i; 5638 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5639 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5640 uint16_t thres[CIM_NUM_IBQ]; 5641 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5642 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5643 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5644 5645 if (is_t4(sc)) { 5646 cim_num_obq = CIM_NUM_OBQ; 5647 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5648 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5649 } else { 5650 cim_num_obq = CIM_NUM_OBQ_T5; 5651 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5652 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5653 } 5654 nq = CIM_NUM_IBQ + cim_num_obq; 5655 5656 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5657 if (rc == 0) 5658 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5659 if (rc != 0) 5660 return (rc); 5661 5662 t4_read_cimq_cfg(sc, base, size, thres); 5663 5664 rc = sysctl_wire_old_buffer(req, 0); 5665 if (rc != 0) 5666 return (rc); 5667 5668 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5669 if (sb == NULL) 5670 return (ENOMEM); 5671 5672 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5673 5674 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5675 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5676 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5677 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5678 G_QUEREMFLITS(p[2]) * 16); 5679 for ( ; i < nq; i++, p += 4, wr += 2) 5680 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5681 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5682 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5683 G_QUEREMFLITS(p[2]) * 16); 5684 5685 rc = sbuf_finish(sb); 5686 sbuf_delete(sb); 5687 5688 return (rc); 5689 } 5690 5691 static int 5692 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5693 { 5694 struct adapter *sc = arg1; 5695 struct sbuf *sb; 5696 int rc; 5697 struct tp_cpl_stats stats; 5698 5699 rc = sysctl_wire_old_buffer(req, 0); 5700 if (rc != 0) 5701 return (rc); 5702 5703 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5704 if (sb == NULL) 5705 return (ENOMEM); 5706 5707 t4_tp_get_cpl_stats(sc, &stats); 5708 5709 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5710 "channel 3\n"); 5711 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5712 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5713 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5714 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5715 5716 rc = sbuf_finish(sb); 5717 sbuf_delete(sb); 5718 5719 return (rc); 5720 } 5721 5722 static int 5723 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5724 { 5725 struct adapter *sc = arg1; 5726 struct sbuf *sb; 5727 int rc; 5728 struct tp_usm_stats stats; 5729 5730 rc = sysctl_wire_old_buffer(req, 0); 5731 if (rc != 0) 5732 return(rc); 5733 5734 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5735 if (sb == NULL) 5736 return (ENOMEM); 5737 5738 t4_get_usm_stats(sc, &stats); 5739 5740 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5741 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5742 sbuf_printf(sb, "Drops: %u", stats.drops); 5743 5744 rc = sbuf_finish(sb); 5745 sbuf_delete(sb); 5746 5747 return (rc); 5748 } 5749 5750 const char *devlog_level_strings[] = { 5751 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5752 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5753 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5754 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5755 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5756 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5757 }; 5758 5759 const char *devlog_facility_strings[] = { 5760 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5761 [FW_DEVLOG_FACILITY_CF] = "CF", 5762 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5763 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5764 [FW_DEVLOG_FACILITY_RES] = "RES", 5765 [FW_DEVLOG_FACILITY_HW] = "HW", 5766 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5767 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5768 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5769 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5770 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5771 [FW_DEVLOG_FACILITY_VI] = "VI", 5772 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5773 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5774 [FW_DEVLOG_FACILITY_TM] = "TM", 5775 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5776 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5777 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5778 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5779 [FW_DEVLOG_FACILITY_RI] = "RI", 5780 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5781 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5782 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5783 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5784 }; 5785 5786 static int 5787 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5788 { 5789 struct adapter *sc = arg1; 5790 struct devlog_params *dparams = &sc->params.devlog; 5791 struct fw_devlog_e *buf, *e; 5792 int i, j, rc, nentries, first = 0, m; 5793 struct sbuf *sb; 5794 uint64_t ftstamp = UINT64_MAX; 5795 5796 if (dparams->start == 0) { 5797 dparams->memtype = FW_MEMTYPE_EDC0; 5798 dparams->start = 0x84000; 5799 dparams->size = 32768; 5800 } 5801 5802 nentries = dparams->size / sizeof(struct fw_devlog_e); 5803 5804 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5805 if (buf == NULL) 5806 return (ENOMEM); 5807 5808 m = fwmtype_to_hwmtype(dparams->memtype); 5809 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5810 if (rc != 0) 5811 goto done; 5812 5813 for (i = 0; i < nentries; i++) { 5814 e = &buf[i]; 5815 5816 if (e->timestamp == 0) 5817 break; /* end */ 5818 5819 e->timestamp = be64toh(e->timestamp); 5820 e->seqno = be32toh(e->seqno); 5821 for (j = 0; j < 8; j++) 5822 e->params[j] = be32toh(e->params[j]); 5823 5824 if (e->timestamp < ftstamp) { 5825 ftstamp = e->timestamp; 5826 first = i; 5827 } 5828 } 5829 5830 if (buf[first].timestamp == 0) 5831 goto done; /* nothing in the log */ 5832 5833 rc = sysctl_wire_old_buffer(req, 0); 5834 if (rc != 0) 5835 goto done; 5836 5837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5838 if (sb == NULL) { 5839 rc = ENOMEM; 5840 goto done; 5841 } 5842 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5843 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5844 5845 i = first; 5846 do { 5847 e = &buf[i]; 5848 if (e->timestamp == 0) 5849 break; /* end */ 5850 5851 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5852 e->seqno, e->timestamp, 5853 (e->level < nitems(devlog_level_strings) ? 5854 devlog_level_strings[e->level] : "UNKNOWN"), 5855 (e->facility < nitems(devlog_facility_strings) ? 5856 devlog_facility_strings[e->facility] : "UNKNOWN")); 5857 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5858 e->params[2], e->params[3], e->params[4], 5859 e->params[5], e->params[6], e->params[7]); 5860 5861 if (++i == nentries) 5862 i = 0; 5863 } while (i != first); 5864 5865 rc = sbuf_finish(sb); 5866 sbuf_delete(sb); 5867 done: 5868 free(buf, M_CXGBE); 5869 return (rc); 5870 } 5871 5872 static int 5873 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5874 { 5875 struct adapter *sc = arg1; 5876 struct sbuf *sb; 5877 int rc; 5878 struct tp_fcoe_stats stats[4]; 5879 5880 rc = sysctl_wire_old_buffer(req, 0); 5881 if (rc != 0) 5882 return (rc); 5883 5884 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5885 if (sb == NULL) 5886 return (ENOMEM); 5887 5888 t4_get_fcoe_stats(sc, 0, &stats[0]); 5889 t4_get_fcoe_stats(sc, 1, &stats[1]); 5890 t4_get_fcoe_stats(sc, 2, &stats[2]); 5891 t4_get_fcoe_stats(sc, 3, &stats[3]); 5892 5893 sbuf_printf(sb, " channel 0 channel 1 " 5894 "channel 2 channel 3\n"); 5895 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5896 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5897 stats[3].octetsDDP); 5898 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5899 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5900 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5901 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5902 stats[3].framesDrop); 5903 5904 rc = sbuf_finish(sb); 5905 sbuf_delete(sb); 5906 5907 return (rc); 5908 } 5909 5910 static int 5911 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5912 { 5913 struct adapter *sc = arg1; 5914 struct sbuf *sb; 5915 int rc, i; 5916 unsigned int map, kbps, ipg, mode; 5917 unsigned int pace_tab[NTX_SCHED]; 5918 5919 rc = sysctl_wire_old_buffer(req, 0); 5920 if (rc != 0) 5921 return (rc); 5922 5923 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5924 if (sb == NULL) 5925 return (ENOMEM); 5926 5927 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5928 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5929 t4_read_pace_tbl(sc, pace_tab); 5930 5931 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5932 "Class IPG (0.1 ns) Flow IPG (us)"); 5933 5934 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5935 t4_get_tx_sched(sc, i, &kbps, &ipg); 5936 sbuf_printf(sb, "\n %u %-5s %u ", i, 5937 (mode & (1 << i)) ? "flow" : "class", map & 3); 5938 if (kbps) 5939 sbuf_printf(sb, "%9u ", kbps); 5940 else 5941 sbuf_printf(sb, " disabled "); 5942 5943 if (ipg) 5944 sbuf_printf(sb, "%13u ", ipg); 5945 else 5946 sbuf_printf(sb, " disabled "); 5947 5948 if (pace_tab[i]) 5949 sbuf_printf(sb, "%10u", pace_tab[i]); 5950 else 5951 sbuf_printf(sb, " disabled"); 5952 } 5953 5954 rc = sbuf_finish(sb); 5955 sbuf_delete(sb); 5956 5957 return (rc); 5958 } 5959 5960 static int 5961 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5962 { 5963 struct adapter *sc = arg1; 5964 struct sbuf *sb; 5965 int rc, i, j; 5966 uint64_t *p0, *p1; 5967 struct lb_port_stats s[2]; 5968 static const char *stat_name[] = { 5969 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5970 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5971 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5972 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5973 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5974 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5975 "BG2FramesTrunc:", "BG3FramesTrunc:" 5976 }; 5977 5978 rc = sysctl_wire_old_buffer(req, 0); 5979 if (rc != 0) 5980 return (rc); 5981 5982 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5983 if (sb == NULL) 5984 return (ENOMEM); 5985 5986 memset(s, 0, sizeof(s)); 5987 5988 for (i = 0; i < 4; i += 2) { 5989 t4_get_lb_stats(sc, i, &s[0]); 5990 t4_get_lb_stats(sc, i + 1, &s[1]); 5991 5992 p0 = &s[0].octets; 5993 p1 = &s[1].octets; 5994 sbuf_printf(sb, "%s Loopback %u" 5995 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5996 5997 for (j = 0; j < nitems(stat_name); j++) 5998 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5999 *p0++, *p1++); 6000 } 6001 6002 rc = sbuf_finish(sb); 6003 sbuf_delete(sb); 6004 6005 return (rc); 6006 } 6007 6008 static int 6009 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6010 { 6011 int rc = 0; 6012 struct port_info *pi = arg1; 6013 struct sbuf *sb; 6014 static const char *linkdnreasons[] = { 6015 "non-specific", "remote fault", "autoneg failed", "reserved3", 6016 "PHY overheated", "unknown", "rx los", "reserved7" 6017 }; 6018 6019 rc = sysctl_wire_old_buffer(req, 0); 6020 if (rc != 0) 6021 return(rc); 6022 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6023 if (sb == NULL) 6024 return (ENOMEM); 6025 6026 if (pi->linkdnrc < 0) 6027 sbuf_printf(sb, "n/a"); 6028 else if (pi->linkdnrc < nitems(linkdnreasons)) 6029 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6030 else 6031 sbuf_printf(sb, "%d", pi->linkdnrc); 6032 6033 rc = sbuf_finish(sb); 6034 sbuf_delete(sb); 6035 6036 return (rc); 6037 } 6038 6039 struct mem_desc { 6040 unsigned int base; 6041 unsigned int limit; 6042 unsigned int idx; 6043 }; 6044 6045 static int 6046 mem_desc_cmp(const void *a, const void *b) 6047 { 6048 return ((const struct mem_desc *)a)->base - 6049 ((const struct mem_desc *)b)->base; 6050 } 6051 6052 static void 6053 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6054 unsigned int to) 6055 { 6056 unsigned int size; 6057 6058 size = to - from + 1; 6059 if (size == 0) 6060 return; 6061 6062 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6063 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6064 } 6065 6066 static int 6067 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6068 { 6069 struct adapter *sc = arg1; 6070 struct sbuf *sb; 6071 int rc, i, n; 6072 uint32_t lo, hi, used, alloc; 6073 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6074 static const char *region[] = { 6075 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6076 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6077 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6078 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6079 "RQUDP region:", "PBL region:", "TXPBL region:", 6080 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6081 "On-chip queues:" 6082 }; 6083 struct mem_desc avail[4]; 6084 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6085 struct mem_desc *md = mem; 6086 6087 rc = sysctl_wire_old_buffer(req, 0); 6088 if (rc != 0) 6089 return (rc); 6090 6091 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6092 if (sb == NULL) 6093 return (ENOMEM); 6094 6095 for (i = 0; i < nitems(mem); i++) { 6096 mem[i].limit = 0; 6097 mem[i].idx = i; 6098 } 6099 6100 /* Find and sort the populated memory ranges */ 6101 i = 0; 6102 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6103 if (lo & F_EDRAM0_ENABLE) { 6104 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6105 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6106 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6107 avail[i].idx = 0; 6108 i++; 6109 } 6110 if (lo & F_EDRAM1_ENABLE) { 6111 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6112 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6113 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6114 avail[i].idx = 1; 6115 i++; 6116 } 6117 if (lo & F_EXT_MEM_ENABLE) { 6118 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6119 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6120 avail[i].limit = avail[i].base + 6121 (G_EXT_MEM_SIZE(hi) << 20); 6122 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6123 i++; 6124 } 6125 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6126 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6127 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6128 avail[i].limit = avail[i].base + 6129 (G_EXT_MEM1_SIZE(hi) << 20); 6130 avail[i].idx = 4; 6131 i++; 6132 } 6133 if (!i) /* no memory available */ 6134 return 0; 6135 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6136 6137 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6138 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6139 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6140 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6141 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6142 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6143 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6144 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6145 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6146 6147 /* the next few have explicit upper bounds */ 6148 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6149 md->limit = md->base - 1 + 6150 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6151 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6152 md++; 6153 6154 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6155 md->limit = md->base - 1 + 6156 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6157 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6158 md++; 6159 6160 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6161 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6162 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6163 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6164 } else { 6165 md->base = 0; 6166 md->idx = nitems(region); /* hide it */ 6167 } 6168 md++; 6169 6170 #define ulp_region(reg) \ 6171 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6172 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6173 6174 ulp_region(RX_ISCSI); 6175 ulp_region(RX_TDDP); 6176 ulp_region(TX_TPT); 6177 ulp_region(RX_STAG); 6178 ulp_region(RX_RQ); 6179 ulp_region(RX_RQUDP); 6180 ulp_region(RX_PBL); 6181 ulp_region(TX_PBL); 6182 #undef ulp_region 6183 6184 md->base = 0; 6185 md->idx = nitems(region); 6186 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6187 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6188 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6189 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6190 } 6191 md++; 6192 6193 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6194 md->limit = md->base + sc->tids.ntids - 1; 6195 md++; 6196 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6197 md->limit = md->base + sc->tids.ntids - 1; 6198 md++; 6199 6200 md->base = sc->vres.ocq.start; 6201 if (sc->vres.ocq.size) 6202 md->limit = md->base + sc->vres.ocq.size - 1; 6203 else 6204 md->idx = nitems(region); /* hide it */ 6205 md++; 6206 6207 /* add any address-space holes, there can be up to 3 */ 6208 for (n = 0; n < i - 1; n++) 6209 if (avail[n].limit < avail[n + 1].base) 6210 (md++)->base = avail[n].limit; 6211 if (avail[n].limit) 6212 (md++)->base = avail[n].limit; 6213 6214 n = md - mem; 6215 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6216 6217 for (lo = 0; lo < i; lo++) 6218 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6219 avail[lo].limit - 1); 6220 6221 sbuf_printf(sb, "\n"); 6222 for (i = 0; i < n; i++) { 6223 if (mem[i].idx >= nitems(region)) 6224 continue; /* skip holes */ 6225 if (!mem[i].limit) 6226 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6227 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6228 mem[i].limit); 6229 } 6230 6231 sbuf_printf(sb, "\n"); 6232 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6233 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6234 mem_region_show(sb, "uP RAM:", lo, hi); 6235 6236 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6237 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6238 mem_region_show(sb, "uP Extmem2:", lo, hi); 6239 6240 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6241 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6242 G_PMRXMAXPAGE(lo), 6243 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6244 (lo & F_PMRXNUMCHN) ? 2 : 1); 6245 6246 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6247 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6248 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6249 G_PMTXMAXPAGE(lo), 6250 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6251 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6252 sbuf_printf(sb, "%u p-structs\n", 6253 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6254 6255 for (i = 0; i < 4; i++) { 6256 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6257 if (is_t4(sc)) { 6258 used = G_USED(lo); 6259 alloc = G_ALLOC(lo); 6260 } else { 6261 used = G_T5_USED(lo); 6262 alloc = G_T5_ALLOC(lo); 6263 } 6264 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6265 i, used, alloc); 6266 } 6267 for (i = 0; i < 4; i++) { 6268 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6269 if (is_t4(sc)) { 6270 used = G_USED(lo); 6271 alloc = G_ALLOC(lo); 6272 } else { 6273 used = G_T5_USED(lo); 6274 alloc = G_T5_ALLOC(lo); 6275 } 6276 sbuf_printf(sb, 6277 "\nLoopback %d using %u pages out of %u allocated", 6278 i, used, alloc); 6279 } 6280 6281 rc = sbuf_finish(sb); 6282 sbuf_delete(sb); 6283 6284 return (rc); 6285 } 6286 6287 static inline void 6288 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6289 { 6290 *mask = x | y; 6291 y = htobe64(y); 6292 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6293 } 6294 6295 static int 6296 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6297 { 6298 struct adapter *sc = arg1; 6299 struct sbuf *sb; 6300 int rc, i, n; 6301 6302 rc = sysctl_wire_old_buffer(req, 0); 6303 if (rc != 0) 6304 return (rc); 6305 6306 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6307 if (sb == NULL) 6308 return (ENOMEM); 6309 6310 sbuf_printf(sb, 6311 "Idx Ethernet address Mask Vld Ports PF" 6312 " VF Replication P0 P1 P2 P3 ML"); 6313 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6314 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6315 for (i = 0; i < n; i++) { 6316 uint64_t tcamx, tcamy, mask; 6317 uint32_t cls_lo, cls_hi; 6318 uint8_t addr[ETHER_ADDR_LEN]; 6319 6320 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6321 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6322 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6323 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6324 6325 if (tcamx & tcamy) 6326 continue; 6327 6328 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6329 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6330 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6331 addr[3], addr[4], addr[5], (uintmax_t)mask, 6332 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6333 G_PORTMAP(cls_hi), G_PF(cls_lo), 6334 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6335 6336 if (cls_lo & F_REPLICATE) { 6337 struct fw_ldst_cmd ldst_cmd; 6338 6339 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6340 ldst_cmd.op_to_addrspace = 6341 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6342 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6343 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6344 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6345 ldst_cmd.u.mps.rplc.fid_idx = 6346 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6347 V_FW_LDST_CMD_IDX(i)); 6348 6349 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6350 "t4mps"); 6351 if (rc) 6352 break; 6353 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6354 sizeof(ldst_cmd), &ldst_cmd); 6355 end_synchronized_op(sc, 0); 6356 6357 if (rc != 0) { 6358 sbuf_printf(sb, 6359 " ------------ error %3u ------------", rc); 6360 rc = 0; 6361 } else { 6362 sbuf_printf(sb, " %08x %08x %08x %08x", 6363 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6364 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6365 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6366 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6367 } 6368 } else 6369 sbuf_printf(sb, "%36s", ""); 6370 6371 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6372 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6373 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6374 } 6375 6376 if (rc) 6377 (void) sbuf_finish(sb); 6378 else 6379 rc = sbuf_finish(sb); 6380 sbuf_delete(sb); 6381 6382 return (rc); 6383 } 6384 6385 static int 6386 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6387 { 6388 struct adapter *sc = arg1; 6389 struct sbuf *sb; 6390 int rc; 6391 uint16_t mtus[NMTUS]; 6392 6393 rc = sysctl_wire_old_buffer(req, 0); 6394 if (rc != 0) 6395 return (rc); 6396 6397 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6398 if (sb == NULL) 6399 return (ENOMEM); 6400 6401 t4_read_mtu_tbl(sc, mtus, NULL); 6402 6403 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6404 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6405 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6406 mtus[14], mtus[15]); 6407 6408 rc = sbuf_finish(sb); 6409 sbuf_delete(sb); 6410 6411 return (rc); 6412 } 6413 6414 static int 6415 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6416 { 6417 struct adapter *sc = arg1; 6418 struct sbuf *sb; 6419 int rc, i; 6420 uint32_t cnt[PM_NSTATS]; 6421 uint64_t cyc[PM_NSTATS]; 6422 static const char *rx_stats[] = { 6423 "Read:", "Write bypass:", "Write mem:", "Flush:" 6424 }; 6425 static const char *tx_stats[] = { 6426 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6427 }; 6428 6429 rc = sysctl_wire_old_buffer(req, 0); 6430 if (rc != 0) 6431 return (rc); 6432 6433 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6434 if (sb == NULL) 6435 return (ENOMEM); 6436 6437 t4_pmtx_get_stats(sc, cnt, cyc); 6438 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6439 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6440 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6441 cyc[i]); 6442 6443 t4_pmrx_get_stats(sc, cnt, cyc); 6444 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6445 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6446 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6447 cyc[i]); 6448 6449 rc = sbuf_finish(sb); 6450 sbuf_delete(sb); 6451 6452 return (rc); 6453 } 6454 6455 static int 6456 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6457 { 6458 struct adapter *sc = arg1; 6459 struct sbuf *sb; 6460 int rc; 6461 struct tp_rdma_stats stats; 6462 6463 rc = sysctl_wire_old_buffer(req, 0); 6464 if (rc != 0) 6465 return (rc); 6466 6467 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6468 if (sb == NULL) 6469 return (ENOMEM); 6470 6471 t4_tp_get_rdma_stats(sc, &stats); 6472 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6473 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6474 6475 rc = sbuf_finish(sb); 6476 sbuf_delete(sb); 6477 6478 return (rc); 6479 } 6480 6481 static int 6482 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6483 { 6484 struct adapter *sc = arg1; 6485 struct sbuf *sb; 6486 int rc; 6487 struct tp_tcp_stats v4, v6; 6488 6489 rc = sysctl_wire_old_buffer(req, 0); 6490 if (rc != 0) 6491 return (rc); 6492 6493 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6494 if (sb == NULL) 6495 return (ENOMEM); 6496 6497 t4_tp_get_tcp_stats(sc, &v4, &v6); 6498 sbuf_printf(sb, 6499 " IP IPv6\n"); 6500 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6501 v4.tcpOutRsts, v6.tcpOutRsts); 6502 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6503 v4.tcpInSegs, v6.tcpInSegs); 6504 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6505 v4.tcpOutSegs, v6.tcpOutSegs); 6506 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6507 v4.tcpRetransSegs, v6.tcpRetransSegs); 6508 6509 rc = sbuf_finish(sb); 6510 sbuf_delete(sb); 6511 6512 return (rc); 6513 } 6514 6515 static int 6516 sysctl_tids(SYSCTL_HANDLER_ARGS) 6517 { 6518 struct adapter *sc = arg1; 6519 struct sbuf *sb; 6520 int rc; 6521 struct tid_info *t = &sc->tids; 6522 6523 rc = sysctl_wire_old_buffer(req, 0); 6524 if (rc != 0) 6525 return (rc); 6526 6527 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6528 if (sb == NULL) 6529 return (ENOMEM); 6530 6531 if (t->natids) { 6532 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6533 t->atids_in_use); 6534 } 6535 6536 if (t->ntids) { 6537 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6538 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6539 6540 if (b) { 6541 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6542 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6543 t->ntids - 1); 6544 } else { 6545 sbuf_printf(sb, "TID range: %u-%u", 6546 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6547 t->ntids - 1); 6548 } 6549 } else 6550 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6551 sbuf_printf(sb, ", in use: %u\n", 6552 atomic_load_acq_int(&t->tids_in_use)); 6553 } 6554 6555 if (t->nstids) { 6556 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6557 t->stid_base + t->nstids - 1, t->stids_in_use); 6558 } 6559 6560 if (t->nftids) { 6561 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6562 t->ftid_base + t->nftids - 1); 6563 } 6564 6565 if (t->netids) { 6566 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6567 t->etid_base + t->netids - 1); 6568 } 6569 6570 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6571 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6572 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6573 6574 rc = sbuf_finish(sb); 6575 sbuf_delete(sb); 6576 6577 return (rc); 6578 } 6579 6580 static int 6581 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6582 { 6583 struct adapter *sc = arg1; 6584 struct sbuf *sb; 6585 int rc; 6586 struct tp_err_stats stats; 6587 6588 rc = sysctl_wire_old_buffer(req, 0); 6589 if (rc != 0) 6590 return (rc); 6591 6592 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6593 if (sb == NULL) 6594 return (ENOMEM); 6595 6596 t4_tp_get_err_stats(sc, &stats); 6597 6598 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6599 "channel 3\n"); 6600 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6601 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6602 stats.macInErrs[3]); 6603 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6604 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6605 stats.hdrInErrs[3]); 6606 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6607 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6608 stats.tcpInErrs[3]); 6609 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6610 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6611 stats.tcp6InErrs[3]); 6612 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6613 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6614 stats.tnlCongDrops[3]); 6615 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6616 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6617 stats.tnlTxDrops[3]); 6618 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6619 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6620 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6621 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6622 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6623 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6624 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6625 stats.ofldNoNeigh, stats.ofldCongDefer); 6626 6627 rc = sbuf_finish(sb); 6628 sbuf_delete(sb); 6629 6630 return (rc); 6631 } 6632 6633 struct field_desc { 6634 const char *name; 6635 u_int start; 6636 u_int width; 6637 }; 6638 6639 static void 6640 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6641 { 6642 char buf[32]; 6643 int line_size = 0; 6644 6645 while (f->name) { 6646 uint64_t mask = (1ULL << f->width) - 1; 6647 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6648 ((uintmax_t)v >> f->start) & mask); 6649 6650 if (line_size + len >= 79) { 6651 line_size = 8; 6652 sbuf_printf(sb, "\n "); 6653 } 6654 sbuf_printf(sb, "%s ", buf); 6655 line_size += len + 1; 6656 f++; 6657 } 6658 sbuf_printf(sb, "\n"); 6659 } 6660 6661 static struct field_desc tp_la0[] = { 6662 { "RcfOpCodeOut", 60, 4 }, 6663 { "State", 56, 4 }, 6664 { "WcfState", 52, 4 }, 6665 { "RcfOpcSrcOut", 50, 2 }, 6666 { "CRxError", 49, 1 }, 6667 { "ERxError", 48, 1 }, 6668 { "SanityFailed", 47, 1 }, 6669 { "SpuriousMsg", 46, 1 }, 6670 { "FlushInputMsg", 45, 1 }, 6671 { "FlushInputCpl", 44, 1 }, 6672 { "RssUpBit", 43, 1 }, 6673 { "RssFilterHit", 42, 1 }, 6674 { "Tid", 32, 10 }, 6675 { "InitTcb", 31, 1 }, 6676 { "LineNumber", 24, 7 }, 6677 { "Emsg", 23, 1 }, 6678 { "EdataOut", 22, 1 }, 6679 { "Cmsg", 21, 1 }, 6680 { "CdataOut", 20, 1 }, 6681 { "EreadPdu", 19, 1 }, 6682 { "CreadPdu", 18, 1 }, 6683 { "TunnelPkt", 17, 1 }, 6684 { "RcfPeerFin", 16, 1 }, 6685 { "RcfReasonOut", 12, 4 }, 6686 { "TxCchannel", 10, 2 }, 6687 { "RcfTxChannel", 8, 2 }, 6688 { "RxEchannel", 6, 2 }, 6689 { "RcfRxChannel", 5, 1 }, 6690 { "RcfDataOutSrdy", 4, 1 }, 6691 { "RxDvld", 3, 1 }, 6692 { "RxOoDvld", 2, 1 }, 6693 { "RxCongestion", 1, 1 }, 6694 { "TxCongestion", 0, 1 }, 6695 { NULL } 6696 }; 6697 6698 static struct field_desc tp_la1[] = { 6699 { "CplCmdIn", 56, 8 }, 6700 { "CplCmdOut", 48, 8 }, 6701 { "ESynOut", 47, 1 }, 6702 { "EAckOut", 46, 1 }, 6703 { "EFinOut", 45, 1 }, 6704 { "ERstOut", 44, 1 }, 6705 { "SynIn", 43, 1 }, 6706 { "AckIn", 42, 1 }, 6707 { "FinIn", 41, 1 }, 6708 { "RstIn", 40, 1 }, 6709 { "DataIn", 39, 1 }, 6710 { "DataInVld", 38, 1 }, 6711 { "PadIn", 37, 1 }, 6712 { "RxBufEmpty", 36, 1 }, 6713 { "RxDdp", 35, 1 }, 6714 { "RxFbCongestion", 34, 1 }, 6715 { "TxFbCongestion", 33, 1 }, 6716 { "TxPktSumSrdy", 32, 1 }, 6717 { "RcfUlpType", 28, 4 }, 6718 { "Eread", 27, 1 }, 6719 { "Ebypass", 26, 1 }, 6720 { "Esave", 25, 1 }, 6721 { "Static0", 24, 1 }, 6722 { "Cread", 23, 1 }, 6723 { "Cbypass", 22, 1 }, 6724 { "Csave", 21, 1 }, 6725 { "CPktOut", 20, 1 }, 6726 { "RxPagePoolFull", 18, 2 }, 6727 { "RxLpbkPkt", 17, 1 }, 6728 { "TxLpbkPkt", 16, 1 }, 6729 { "RxVfValid", 15, 1 }, 6730 { "SynLearned", 14, 1 }, 6731 { "SetDelEntry", 13, 1 }, 6732 { "SetInvEntry", 12, 1 }, 6733 { "CpcmdDvld", 11, 1 }, 6734 { "CpcmdSave", 10, 1 }, 6735 { "RxPstructsFull", 8, 2 }, 6736 { "EpcmdDvld", 7, 1 }, 6737 { "EpcmdFlush", 6, 1 }, 6738 { "EpcmdTrimPrefix", 5, 1 }, 6739 { "EpcmdTrimPostfix", 4, 1 }, 6740 { "ERssIp4Pkt", 3, 1 }, 6741 { "ERssIp6Pkt", 2, 1 }, 6742 { "ERssTcpUdpPkt", 1, 1 }, 6743 { "ERssFceFipPkt", 0, 1 }, 6744 { NULL } 6745 }; 6746 6747 static struct field_desc tp_la2[] = { 6748 { "CplCmdIn", 56, 8 }, 6749 { "MpsVfVld", 55, 1 }, 6750 { "MpsPf", 52, 3 }, 6751 { "MpsVf", 44, 8 }, 6752 { "SynIn", 43, 1 }, 6753 { "AckIn", 42, 1 }, 6754 { "FinIn", 41, 1 }, 6755 { "RstIn", 40, 1 }, 6756 { "DataIn", 39, 1 }, 6757 { "DataInVld", 38, 1 }, 6758 { "PadIn", 37, 1 }, 6759 { "RxBufEmpty", 36, 1 }, 6760 { "RxDdp", 35, 1 }, 6761 { "RxFbCongestion", 34, 1 }, 6762 { "TxFbCongestion", 33, 1 }, 6763 { "TxPktSumSrdy", 32, 1 }, 6764 { "RcfUlpType", 28, 4 }, 6765 { "Eread", 27, 1 }, 6766 { "Ebypass", 26, 1 }, 6767 { "Esave", 25, 1 }, 6768 { "Static0", 24, 1 }, 6769 { "Cread", 23, 1 }, 6770 { "Cbypass", 22, 1 }, 6771 { "Csave", 21, 1 }, 6772 { "CPktOut", 20, 1 }, 6773 { "RxPagePoolFull", 18, 2 }, 6774 { "RxLpbkPkt", 17, 1 }, 6775 { "TxLpbkPkt", 16, 1 }, 6776 { "RxVfValid", 15, 1 }, 6777 { "SynLearned", 14, 1 }, 6778 { "SetDelEntry", 13, 1 }, 6779 { "SetInvEntry", 12, 1 }, 6780 { "CpcmdDvld", 11, 1 }, 6781 { "CpcmdSave", 10, 1 }, 6782 { "RxPstructsFull", 8, 2 }, 6783 { "EpcmdDvld", 7, 1 }, 6784 { "EpcmdFlush", 6, 1 }, 6785 { "EpcmdTrimPrefix", 5, 1 }, 6786 { "EpcmdTrimPostfix", 4, 1 }, 6787 { "ERssIp4Pkt", 3, 1 }, 6788 { "ERssIp6Pkt", 2, 1 }, 6789 { "ERssTcpUdpPkt", 1, 1 }, 6790 { "ERssFceFipPkt", 0, 1 }, 6791 { NULL } 6792 }; 6793 6794 static void 6795 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6796 { 6797 6798 field_desc_show(sb, *p, tp_la0); 6799 } 6800 6801 static void 6802 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6803 { 6804 6805 if (idx) 6806 sbuf_printf(sb, "\n"); 6807 field_desc_show(sb, p[0], tp_la0); 6808 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6809 field_desc_show(sb, p[1], tp_la0); 6810 } 6811 6812 static void 6813 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6814 { 6815 6816 if (idx) 6817 sbuf_printf(sb, "\n"); 6818 field_desc_show(sb, p[0], tp_la0); 6819 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6820 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6821 } 6822 6823 static int 6824 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6825 { 6826 struct adapter *sc = arg1; 6827 struct sbuf *sb; 6828 uint64_t *buf, *p; 6829 int rc; 6830 u_int i, inc; 6831 void (*show_func)(struct sbuf *, uint64_t *, int); 6832 6833 rc = sysctl_wire_old_buffer(req, 0); 6834 if (rc != 0) 6835 return (rc); 6836 6837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6838 if (sb == NULL) 6839 return (ENOMEM); 6840 6841 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6842 6843 t4_tp_read_la(sc, buf, NULL); 6844 p = buf; 6845 6846 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6847 case 2: 6848 inc = 2; 6849 show_func = tp_la_show2; 6850 break; 6851 case 3: 6852 inc = 2; 6853 show_func = tp_la_show3; 6854 break; 6855 default: 6856 inc = 1; 6857 show_func = tp_la_show; 6858 } 6859 6860 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6861 (*show_func)(sb, p, i); 6862 6863 rc = sbuf_finish(sb); 6864 sbuf_delete(sb); 6865 free(buf, M_CXGBE); 6866 return (rc); 6867 } 6868 6869 static int 6870 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6871 { 6872 struct adapter *sc = arg1; 6873 struct sbuf *sb; 6874 int rc; 6875 u64 nrate[NCHAN], orate[NCHAN]; 6876 6877 rc = sysctl_wire_old_buffer(req, 0); 6878 if (rc != 0) 6879 return (rc); 6880 6881 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6882 if (sb == NULL) 6883 return (ENOMEM); 6884 6885 t4_get_chan_txrate(sc, nrate, orate); 6886 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6887 "channel 3\n"); 6888 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6889 nrate[0], nrate[1], nrate[2], nrate[3]); 6890 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6891 orate[0], orate[1], orate[2], orate[3]); 6892 6893 rc = sbuf_finish(sb); 6894 sbuf_delete(sb); 6895 6896 return (rc); 6897 } 6898 6899 static int 6900 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6901 { 6902 struct adapter *sc = arg1; 6903 struct sbuf *sb; 6904 uint32_t *buf, *p; 6905 int rc, i; 6906 6907 rc = sysctl_wire_old_buffer(req, 0); 6908 if (rc != 0) 6909 return (rc); 6910 6911 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6912 if (sb == NULL) 6913 return (ENOMEM); 6914 6915 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6916 M_ZERO | M_WAITOK); 6917 6918 t4_ulprx_read_la(sc, buf); 6919 p = buf; 6920 6921 sbuf_printf(sb, " Pcmd Type Message" 6922 " Data"); 6923 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6924 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6925 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6926 } 6927 6928 rc = sbuf_finish(sb); 6929 sbuf_delete(sb); 6930 free(buf, M_CXGBE); 6931 return (rc); 6932 } 6933 6934 static int 6935 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6936 { 6937 struct adapter *sc = arg1; 6938 struct sbuf *sb; 6939 int rc, v; 6940 6941 rc = sysctl_wire_old_buffer(req, 0); 6942 if (rc != 0) 6943 return (rc); 6944 6945 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6946 if (sb == NULL) 6947 return (ENOMEM); 6948 6949 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6950 if (G_STATSOURCE_T5(v) == 7) { 6951 if (G_STATMODE(v) == 0) { 6952 sbuf_printf(sb, "total %d, incomplete %d", 6953 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6954 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6955 } else if (G_STATMODE(v) == 1) { 6956 sbuf_printf(sb, "total %d, data overflow %d", 6957 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6958 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6959 } 6960 } 6961 rc = sbuf_finish(sb); 6962 sbuf_delete(sb); 6963 6964 return (rc); 6965 } 6966 #endif 6967 6968 static uint32_t 6969 fconf_to_mode(uint32_t fconf) 6970 { 6971 uint32_t mode; 6972 6973 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6974 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6975 6976 if (fconf & F_FRAGMENTATION) 6977 mode |= T4_FILTER_IP_FRAGMENT; 6978 6979 if (fconf & F_MPSHITTYPE) 6980 mode |= T4_FILTER_MPS_HIT_TYPE; 6981 6982 if (fconf & F_MACMATCH) 6983 mode |= T4_FILTER_MAC_IDX; 6984 6985 if (fconf & F_ETHERTYPE) 6986 mode |= T4_FILTER_ETH_TYPE; 6987 6988 if (fconf & F_PROTOCOL) 6989 mode |= T4_FILTER_IP_PROTO; 6990 6991 if (fconf & F_TOS) 6992 mode |= T4_FILTER_IP_TOS; 6993 6994 if (fconf & F_VLAN) 6995 mode |= T4_FILTER_VLAN; 6996 6997 if (fconf & F_VNIC_ID) 6998 mode |= T4_FILTER_VNIC; 6999 7000 if (fconf & F_PORT) 7001 mode |= T4_FILTER_PORT; 7002 7003 if (fconf & F_FCOE) 7004 mode |= T4_FILTER_FCoE; 7005 7006 return (mode); 7007 } 7008 7009 static uint32_t 7010 mode_to_fconf(uint32_t mode) 7011 { 7012 uint32_t fconf = 0; 7013 7014 if (mode & T4_FILTER_IP_FRAGMENT) 7015 fconf |= F_FRAGMENTATION; 7016 7017 if (mode & T4_FILTER_MPS_HIT_TYPE) 7018 fconf |= F_MPSHITTYPE; 7019 7020 if (mode & T4_FILTER_MAC_IDX) 7021 fconf |= F_MACMATCH; 7022 7023 if (mode & T4_FILTER_ETH_TYPE) 7024 fconf |= F_ETHERTYPE; 7025 7026 if (mode & T4_FILTER_IP_PROTO) 7027 fconf |= F_PROTOCOL; 7028 7029 if (mode & T4_FILTER_IP_TOS) 7030 fconf |= F_TOS; 7031 7032 if (mode & T4_FILTER_VLAN) 7033 fconf |= F_VLAN; 7034 7035 if (mode & T4_FILTER_VNIC) 7036 fconf |= F_VNIC_ID; 7037 7038 if (mode & T4_FILTER_PORT) 7039 fconf |= F_PORT; 7040 7041 if (mode & T4_FILTER_FCoE) 7042 fconf |= F_FCOE; 7043 7044 return (fconf); 7045 } 7046 7047 static uint32_t 7048 fspec_to_fconf(struct t4_filter_specification *fs) 7049 { 7050 uint32_t fconf = 0; 7051 7052 if (fs->val.frag || fs->mask.frag) 7053 fconf |= F_FRAGMENTATION; 7054 7055 if (fs->val.matchtype || fs->mask.matchtype) 7056 fconf |= F_MPSHITTYPE; 7057 7058 if (fs->val.macidx || fs->mask.macidx) 7059 fconf |= F_MACMATCH; 7060 7061 if (fs->val.ethtype || fs->mask.ethtype) 7062 fconf |= F_ETHERTYPE; 7063 7064 if (fs->val.proto || fs->mask.proto) 7065 fconf |= F_PROTOCOL; 7066 7067 if (fs->val.tos || fs->mask.tos) 7068 fconf |= F_TOS; 7069 7070 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7071 fconf |= F_VLAN; 7072 7073 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7074 fconf |= F_VNIC_ID; 7075 7076 if (fs->val.iport || fs->mask.iport) 7077 fconf |= F_PORT; 7078 7079 if (fs->val.fcoe || fs->mask.fcoe) 7080 fconf |= F_FCOE; 7081 7082 return (fconf); 7083 } 7084 7085 static int 7086 get_filter_mode(struct adapter *sc, uint32_t *mode) 7087 { 7088 int rc; 7089 uint32_t fconf; 7090 7091 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7092 "t4getfm"); 7093 if (rc) 7094 return (rc); 7095 7096 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7097 A_TP_VLAN_PRI_MAP); 7098 7099 if (sc->params.tp.vlan_pri_map != fconf) { 7100 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7101 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7102 fconf); 7103 } 7104 7105 *mode = fconf_to_mode(fconf); 7106 7107 end_synchronized_op(sc, LOCK_HELD); 7108 return (0); 7109 } 7110 7111 static int 7112 set_filter_mode(struct adapter *sc, uint32_t mode) 7113 { 7114 uint32_t fconf; 7115 int rc; 7116 7117 fconf = mode_to_fconf(mode); 7118 7119 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7120 "t4setfm"); 7121 if (rc) 7122 return (rc); 7123 7124 if (sc->tids.ftids_in_use > 0) { 7125 rc = EBUSY; 7126 goto done; 7127 } 7128 7129 #ifdef TCP_OFFLOAD 7130 if (uld_active(sc, ULD_TOM)) { 7131 rc = EBUSY; 7132 goto done; 7133 } 7134 #endif 7135 7136 rc = -t4_set_filter_mode(sc, fconf); 7137 done: 7138 end_synchronized_op(sc, LOCK_HELD); 7139 return (rc); 7140 } 7141 7142 static inline uint64_t 7143 get_filter_hits(struct adapter *sc, uint32_t fid) 7144 { 7145 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7146 uint64_t hits; 7147 7148 memwin_info(sc, 0, &mw_base, NULL); 7149 off = position_memwin(sc, 0, 7150 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7151 if (is_t4(sc)) { 7152 hits = t4_read_reg64(sc, mw_base + off + 16); 7153 hits = be64toh(hits); 7154 } else { 7155 hits = t4_read_reg(sc, mw_base + off + 24); 7156 hits = be32toh(hits); 7157 } 7158 7159 return (hits); 7160 } 7161 7162 static int 7163 get_filter(struct adapter *sc, struct t4_filter *t) 7164 { 7165 int i, rc, nfilters = sc->tids.nftids; 7166 struct filter_entry *f; 7167 7168 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7169 "t4getf"); 7170 if (rc) 7171 return (rc); 7172 7173 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7174 t->idx >= nfilters) { 7175 t->idx = 0xffffffff; 7176 goto done; 7177 } 7178 7179 f = &sc->tids.ftid_tab[t->idx]; 7180 for (i = t->idx; i < nfilters; i++, f++) { 7181 if (f->valid) { 7182 t->idx = i; 7183 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7184 t->smtidx = f->smtidx; 7185 if (f->fs.hitcnts) 7186 t->hits = get_filter_hits(sc, t->idx); 7187 else 7188 t->hits = UINT64_MAX; 7189 t->fs = f->fs; 7190 7191 goto done; 7192 } 7193 } 7194 7195 t->idx = 0xffffffff; 7196 done: 7197 end_synchronized_op(sc, LOCK_HELD); 7198 return (0); 7199 } 7200 7201 static int 7202 set_filter(struct adapter *sc, struct t4_filter *t) 7203 { 7204 unsigned int nfilters, nports; 7205 struct filter_entry *f; 7206 int i, rc; 7207 7208 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7209 if (rc) 7210 return (rc); 7211 7212 nfilters = sc->tids.nftids; 7213 nports = sc->params.nports; 7214 7215 if (nfilters == 0) { 7216 rc = ENOTSUP; 7217 goto done; 7218 } 7219 7220 if (!(sc->flags & FULL_INIT_DONE)) { 7221 rc = EAGAIN; 7222 goto done; 7223 } 7224 7225 if (t->idx >= nfilters) { 7226 rc = EINVAL; 7227 goto done; 7228 } 7229 7230 /* Validate against the global filter mode */ 7231 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7232 sc->params.tp.vlan_pri_map) { 7233 rc = E2BIG; 7234 goto done; 7235 } 7236 7237 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7238 rc = EINVAL; 7239 goto done; 7240 } 7241 7242 if (t->fs.val.iport >= nports) { 7243 rc = EINVAL; 7244 goto done; 7245 } 7246 7247 /* Can't specify an iq if not steering to it */ 7248 if (!t->fs.dirsteer && t->fs.iq) { 7249 rc = EINVAL; 7250 goto done; 7251 } 7252 7253 /* IPv6 filter idx must be 4 aligned */ 7254 if (t->fs.type == 1 && 7255 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7256 rc = EINVAL; 7257 goto done; 7258 } 7259 7260 if (sc->tids.ftid_tab == NULL) { 7261 KASSERT(sc->tids.ftids_in_use == 0, 7262 ("%s: no memory allocated but filters_in_use > 0", 7263 __func__)); 7264 7265 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7266 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7267 if (sc->tids.ftid_tab == NULL) { 7268 rc = ENOMEM; 7269 goto done; 7270 } 7271 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7272 } 7273 7274 for (i = 0; i < 4; i++) { 7275 f = &sc->tids.ftid_tab[t->idx + i]; 7276 7277 if (f->pending || f->valid) { 7278 rc = EBUSY; 7279 goto done; 7280 } 7281 if (f->locked) { 7282 rc = EPERM; 7283 goto done; 7284 } 7285 7286 if (t->fs.type == 0) 7287 break; 7288 } 7289 7290 f = &sc->tids.ftid_tab[t->idx]; 7291 f->fs = t->fs; 7292 7293 rc = set_filter_wr(sc, t->idx); 7294 done: 7295 end_synchronized_op(sc, 0); 7296 7297 if (rc == 0) { 7298 mtx_lock(&sc->tids.ftid_lock); 7299 for (;;) { 7300 if (f->pending == 0) { 7301 rc = f->valid ? 0 : EIO; 7302 break; 7303 } 7304 7305 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7306 PCATCH, "t4setfw", 0)) { 7307 rc = EINPROGRESS; 7308 break; 7309 } 7310 } 7311 mtx_unlock(&sc->tids.ftid_lock); 7312 } 7313 return (rc); 7314 } 7315 7316 static int 7317 del_filter(struct adapter *sc, struct t4_filter *t) 7318 { 7319 unsigned int nfilters; 7320 struct filter_entry *f; 7321 int rc; 7322 7323 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7324 if (rc) 7325 return (rc); 7326 7327 nfilters = sc->tids.nftids; 7328 7329 if (nfilters == 0) { 7330 rc = ENOTSUP; 7331 goto done; 7332 } 7333 7334 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7335 t->idx >= nfilters) { 7336 rc = EINVAL; 7337 goto done; 7338 } 7339 7340 if (!(sc->flags & FULL_INIT_DONE)) { 7341 rc = EAGAIN; 7342 goto done; 7343 } 7344 7345 f = &sc->tids.ftid_tab[t->idx]; 7346 7347 if (f->pending) { 7348 rc = EBUSY; 7349 goto done; 7350 } 7351 if (f->locked) { 7352 rc = EPERM; 7353 goto done; 7354 } 7355 7356 if (f->valid) { 7357 t->fs = f->fs; /* extra info for the caller */ 7358 rc = del_filter_wr(sc, t->idx); 7359 } 7360 7361 done: 7362 end_synchronized_op(sc, 0); 7363 7364 if (rc == 0) { 7365 mtx_lock(&sc->tids.ftid_lock); 7366 for (;;) { 7367 if (f->pending == 0) { 7368 rc = f->valid ? EIO : 0; 7369 break; 7370 } 7371 7372 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7373 PCATCH, "t4delfw", 0)) { 7374 rc = EINPROGRESS; 7375 break; 7376 } 7377 } 7378 mtx_unlock(&sc->tids.ftid_lock); 7379 } 7380 7381 return (rc); 7382 } 7383 7384 static void 7385 clear_filter(struct filter_entry *f) 7386 { 7387 if (f->l2t) 7388 t4_l2t_release(f->l2t); 7389 7390 bzero(f, sizeof (*f)); 7391 } 7392 7393 static int 7394 set_filter_wr(struct adapter *sc, int fidx) 7395 { 7396 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7397 struct fw_filter_wr *fwr; 7398 unsigned int ftid; 7399 struct wrq_cookie cookie; 7400 7401 ASSERT_SYNCHRONIZED_OP(sc); 7402 7403 if (f->fs.newdmac || f->fs.newvlan) { 7404 /* This filter needs an L2T entry; allocate one. */ 7405 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7406 if (f->l2t == NULL) 7407 return (EAGAIN); 7408 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7409 f->fs.dmac)) { 7410 t4_l2t_release(f->l2t); 7411 f->l2t = NULL; 7412 return (ENOMEM); 7413 } 7414 } 7415 7416 ftid = sc->tids.ftid_base + fidx; 7417 7418 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7419 if (fwr == NULL) 7420 return (ENOMEM); 7421 bzero(fwr, sizeof(*fwr)); 7422 7423 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7424 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7425 fwr->tid_to_iq = 7426 htobe32(V_FW_FILTER_WR_TID(ftid) | 7427 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7428 V_FW_FILTER_WR_NOREPLY(0) | 7429 V_FW_FILTER_WR_IQ(f->fs.iq)); 7430 fwr->del_filter_to_l2tix = 7431 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7432 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7433 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7434 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7435 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7436 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7437 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7438 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7439 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7440 f->fs.newvlan == VLAN_REWRITE) | 7441 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7442 f->fs.newvlan == VLAN_REWRITE) | 7443 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7444 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7445 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7446 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7447 fwr->ethtype = htobe16(f->fs.val.ethtype); 7448 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7449 fwr->frag_to_ovlan_vldm = 7450 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7451 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7452 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7453 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7454 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7455 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7456 fwr->smac_sel = 0; 7457 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7458 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7459 fwr->maci_to_matchtypem = 7460 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7461 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7462 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7463 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7464 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7465 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7466 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7467 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7468 fwr->ptcl = f->fs.val.proto; 7469 fwr->ptclm = f->fs.mask.proto; 7470 fwr->ttyp = f->fs.val.tos; 7471 fwr->ttypm = f->fs.mask.tos; 7472 fwr->ivlan = htobe16(f->fs.val.vlan); 7473 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7474 fwr->ovlan = htobe16(f->fs.val.vnic); 7475 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7476 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7477 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7478 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7479 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7480 fwr->lp = htobe16(f->fs.val.dport); 7481 fwr->lpm = htobe16(f->fs.mask.dport); 7482 fwr->fp = htobe16(f->fs.val.sport); 7483 fwr->fpm = htobe16(f->fs.mask.sport); 7484 if (f->fs.newsmac) 7485 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7486 7487 f->pending = 1; 7488 sc->tids.ftids_in_use++; 7489 7490 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7491 return (0); 7492 } 7493 7494 static int 7495 del_filter_wr(struct adapter *sc, int fidx) 7496 { 7497 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7498 struct fw_filter_wr *fwr; 7499 unsigned int ftid; 7500 struct wrq_cookie cookie; 7501 7502 ftid = sc->tids.ftid_base + fidx; 7503 7504 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7505 if (fwr == NULL) 7506 return (ENOMEM); 7507 bzero(fwr, sizeof (*fwr)); 7508 7509 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7510 7511 f->pending = 1; 7512 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7513 return (0); 7514 } 7515 7516 int 7517 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7518 { 7519 struct adapter *sc = iq->adapter; 7520 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7521 unsigned int idx = GET_TID(rpl); 7522 unsigned int rc; 7523 struct filter_entry *f; 7524 7525 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7526 rss->opcode)); 7527 7528 if (is_ftid(sc, idx)) { 7529 7530 idx -= sc->tids.ftid_base; 7531 f = &sc->tids.ftid_tab[idx]; 7532 rc = G_COOKIE(rpl->cookie); 7533 7534 mtx_lock(&sc->tids.ftid_lock); 7535 if (rc == FW_FILTER_WR_FLT_ADDED) { 7536 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7537 __func__, idx)); 7538 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7539 f->pending = 0; /* asynchronous setup completed */ 7540 f->valid = 1; 7541 } else { 7542 if (rc != FW_FILTER_WR_FLT_DELETED) { 7543 /* Add or delete failed, display an error */ 7544 log(LOG_ERR, 7545 "filter %u setup failed with error %u\n", 7546 idx, rc); 7547 } 7548 7549 clear_filter(f); 7550 sc->tids.ftids_in_use--; 7551 } 7552 wakeup(&sc->tids.ftid_tab); 7553 mtx_unlock(&sc->tids.ftid_lock); 7554 } 7555 7556 return (0); 7557 } 7558 7559 static int 7560 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7561 { 7562 int rc; 7563 7564 if (cntxt->cid > M_CTXTQID) 7565 return (EINVAL); 7566 7567 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7568 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7569 return (EINVAL); 7570 7571 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7572 if (rc) 7573 return (rc); 7574 7575 if (sc->flags & FW_OK) { 7576 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7577 &cntxt->data[0]); 7578 if (rc == 0) 7579 goto done; 7580 } 7581 7582 /* 7583 * Read via firmware failed or wasn't even attempted. Read directly via 7584 * the backdoor. 7585 */ 7586 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7587 done: 7588 end_synchronized_op(sc, 0); 7589 return (rc); 7590 } 7591 7592 static int 7593 load_fw(struct adapter *sc, struct t4_data *fw) 7594 { 7595 int rc; 7596 uint8_t *fw_data; 7597 7598 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7599 if (rc) 7600 return (rc); 7601 7602 if (sc->flags & FULL_INIT_DONE) { 7603 rc = EBUSY; 7604 goto done; 7605 } 7606 7607 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7608 if (fw_data == NULL) { 7609 rc = ENOMEM; 7610 goto done; 7611 } 7612 7613 rc = copyin(fw->data, fw_data, fw->len); 7614 if (rc == 0) 7615 rc = -t4_load_fw(sc, fw_data, fw->len); 7616 7617 free(fw_data, M_CXGBE); 7618 done: 7619 end_synchronized_op(sc, 0); 7620 return (rc); 7621 } 7622 7623 static int 7624 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7625 { 7626 uint32_t addr, off, remaining, i, n; 7627 uint32_t *buf, *b; 7628 uint32_t mw_base, mw_aperture; 7629 int rc; 7630 uint8_t *dst; 7631 7632 rc = validate_mem_range(sc, mr->addr, mr->len); 7633 if (rc != 0) 7634 return (rc); 7635 7636 memwin_info(sc, win, &mw_base, &mw_aperture); 7637 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7638 addr = mr->addr; 7639 remaining = mr->len; 7640 dst = (void *)mr->data; 7641 7642 while (remaining) { 7643 off = position_memwin(sc, win, addr); 7644 7645 /* number of bytes that we'll copy in the inner loop */ 7646 n = min(remaining, mw_aperture - off); 7647 for (i = 0; i < n; i += 4) 7648 *b++ = t4_read_reg(sc, mw_base + off + i); 7649 7650 rc = copyout(buf, dst, n); 7651 if (rc != 0) 7652 break; 7653 7654 b = buf; 7655 dst += n; 7656 remaining -= n; 7657 addr += n; 7658 } 7659 7660 free(buf, M_CXGBE); 7661 return (rc); 7662 } 7663 7664 static int 7665 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7666 { 7667 int rc; 7668 7669 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7670 return (EINVAL); 7671 7672 if (i2cd->len > sizeof(i2cd->data)) 7673 return (EFBIG); 7674 7675 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7676 if (rc) 7677 return (rc); 7678 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7679 i2cd->offset, i2cd->len, &i2cd->data[0]); 7680 end_synchronized_op(sc, 0); 7681 7682 return (rc); 7683 } 7684 7685 static int 7686 in_range(int val, int lo, int hi) 7687 { 7688 7689 return (val < 0 || (val <= hi && val >= lo)); 7690 } 7691 7692 static int 7693 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7694 { 7695 int fw_subcmd, fw_type, rc; 7696 7697 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7698 if (rc) 7699 return (rc); 7700 7701 if (!(sc->flags & FULL_INIT_DONE)) { 7702 rc = EAGAIN; 7703 goto done; 7704 } 7705 7706 /* 7707 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7708 * sub-command and type are in common locations.) 7709 */ 7710 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7711 fw_subcmd = FW_SCHED_SC_CONFIG; 7712 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7713 fw_subcmd = FW_SCHED_SC_PARAMS; 7714 else { 7715 rc = EINVAL; 7716 goto done; 7717 } 7718 if (p->type == SCHED_CLASS_TYPE_PACKET) 7719 fw_type = FW_SCHED_TYPE_PKTSCHED; 7720 else { 7721 rc = EINVAL; 7722 goto done; 7723 } 7724 7725 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7726 /* Vet our parameters ..*/ 7727 if (p->u.config.minmax < 0) { 7728 rc = EINVAL; 7729 goto done; 7730 } 7731 7732 /* And pass the request to the firmware ...*/ 7733 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7734 goto done; 7735 } 7736 7737 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7738 int fw_level; 7739 int fw_mode; 7740 int fw_rateunit; 7741 int fw_ratemode; 7742 7743 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7744 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7745 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7746 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7747 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7748 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7749 else { 7750 rc = EINVAL; 7751 goto done; 7752 } 7753 7754 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7755 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7756 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7757 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7758 else { 7759 rc = EINVAL; 7760 goto done; 7761 } 7762 7763 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7764 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7765 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7766 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7767 else { 7768 rc = EINVAL; 7769 goto done; 7770 } 7771 7772 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7773 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7774 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7775 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7776 else { 7777 rc = EINVAL; 7778 goto done; 7779 } 7780 7781 /* Vet our parameters ... */ 7782 if (!in_range(p->u.params.channel, 0, 3) || 7783 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7784 !in_range(p->u.params.minrate, 0, 10000000) || 7785 !in_range(p->u.params.maxrate, 0, 10000000) || 7786 !in_range(p->u.params.weight, 0, 100)) { 7787 rc = ERANGE; 7788 goto done; 7789 } 7790 7791 /* 7792 * Translate any unset parameters into the firmware's 7793 * nomenclature and/or fail the call if the parameters 7794 * are required ... 7795 */ 7796 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7797 p->u.params.channel < 0 || p->u.params.cl < 0) { 7798 rc = EINVAL; 7799 goto done; 7800 } 7801 if (p->u.params.minrate < 0) 7802 p->u.params.minrate = 0; 7803 if (p->u.params.maxrate < 0) { 7804 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7805 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7806 rc = EINVAL; 7807 goto done; 7808 } else 7809 p->u.params.maxrate = 0; 7810 } 7811 if (p->u.params.weight < 0) { 7812 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7813 rc = EINVAL; 7814 goto done; 7815 } else 7816 p->u.params.weight = 0; 7817 } 7818 if (p->u.params.pktsize < 0) { 7819 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7820 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7821 rc = EINVAL; 7822 goto done; 7823 } else 7824 p->u.params.pktsize = 0; 7825 } 7826 7827 /* See what the firmware thinks of the request ... */ 7828 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7829 fw_rateunit, fw_ratemode, p->u.params.channel, 7830 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7831 p->u.params.weight, p->u.params.pktsize, 1); 7832 goto done; 7833 } 7834 7835 rc = EINVAL; 7836 done: 7837 end_synchronized_op(sc, 0); 7838 return (rc); 7839 } 7840 7841 static int 7842 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7843 { 7844 struct port_info *pi = NULL; 7845 struct sge_txq *txq; 7846 uint32_t fw_mnem, fw_queue, fw_class; 7847 int i, rc; 7848 7849 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7850 if (rc) 7851 return (rc); 7852 7853 if (!(sc->flags & FULL_INIT_DONE)) { 7854 rc = EAGAIN; 7855 goto done; 7856 } 7857 7858 if (p->port >= sc->params.nports) { 7859 rc = EINVAL; 7860 goto done; 7861 } 7862 7863 pi = sc->port[p->port]; 7864 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7865 rc = EINVAL; 7866 goto done; 7867 } 7868 7869 /* 7870 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7871 * Scheduling Class in this case). 7872 */ 7873 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7874 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7875 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7876 7877 /* 7878 * If op.queue is non-negative, then we're only changing the scheduling 7879 * on a single specified TX queue. 7880 */ 7881 if (p->queue >= 0) { 7882 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7883 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7884 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7885 &fw_class); 7886 goto done; 7887 } 7888 7889 /* 7890 * Change the scheduling on all the TX queues for the 7891 * interface. 7892 */ 7893 for_each_txq(pi, i, txq) { 7894 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7895 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7896 &fw_class); 7897 if (rc) 7898 goto done; 7899 } 7900 7901 rc = 0; 7902 done: 7903 end_synchronized_op(sc, 0); 7904 return (rc); 7905 } 7906 7907 int 7908 t4_os_find_pci_capability(struct adapter *sc, int cap) 7909 { 7910 int i; 7911 7912 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7913 } 7914 7915 int 7916 t4_os_pci_save_state(struct adapter *sc) 7917 { 7918 device_t dev; 7919 struct pci_devinfo *dinfo; 7920 7921 dev = sc->dev; 7922 dinfo = device_get_ivars(dev); 7923 7924 pci_cfg_save(dev, dinfo, 0); 7925 return (0); 7926 } 7927 7928 int 7929 t4_os_pci_restore_state(struct adapter *sc) 7930 { 7931 device_t dev; 7932 struct pci_devinfo *dinfo; 7933 7934 dev = sc->dev; 7935 dinfo = device_get_ivars(dev); 7936 7937 pci_cfg_restore(dev, dinfo); 7938 return (0); 7939 } 7940 7941 void 7942 t4_os_portmod_changed(const struct adapter *sc, int idx) 7943 { 7944 struct port_info *pi = sc->port[idx]; 7945 static const char *mod_str[] = { 7946 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7947 }; 7948 7949 build_medialist(pi, &pi->media); 7950 #ifdef DEV_NETMAP 7951 build_medialist(pi, &pi->nm_media); 7952 #endif 7953 7954 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7955 if_printf(pi->ifp, "transceiver unplugged.\n"); 7956 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7957 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7958 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7959 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7960 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7961 if_printf(pi->ifp, "%s transceiver inserted.\n", 7962 mod_str[pi->mod_type]); 7963 } else { 7964 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7965 pi->mod_type); 7966 } 7967 } 7968 7969 void 7970 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7971 { 7972 struct port_info *pi = sc->port[idx]; 7973 struct ifnet *ifp = pi->ifp; 7974 7975 if (link_stat) { 7976 pi->linkdnrc = -1; 7977 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7978 if_link_state_change(ifp, LINK_STATE_UP); 7979 } else { 7980 if (reason >= 0) 7981 pi->linkdnrc = reason; 7982 if_link_state_change(ifp, LINK_STATE_DOWN); 7983 } 7984 } 7985 7986 void 7987 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7988 { 7989 struct adapter *sc; 7990 7991 sx_slock(&t4_list_lock); 7992 SLIST_FOREACH(sc, &t4_list, link) { 7993 /* 7994 * func should not make any assumptions about what state sc is 7995 * in - the only guarantee is that sc->sc_lock is a valid lock. 7996 */ 7997 func(sc, arg); 7998 } 7999 sx_sunlock(&t4_list_lock); 8000 } 8001 8002 static int 8003 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8004 { 8005 return (0); 8006 } 8007 8008 static int 8009 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8010 { 8011 return (0); 8012 } 8013 8014 static int 8015 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8016 struct thread *td) 8017 { 8018 int rc; 8019 struct adapter *sc = dev->si_drv1; 8020 8021 rc = priv_check(td, PRIV_DRIVER); 8022 if (rc != 0) 8023 return (rc); 8024 8025 switch (cmd) { 8026 case CHELSIO_T4_GETREG: { 8027 struct t4_reg *edata = (struct t4_reg *)data; 8028 8029 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8030 return (EFAULT); 8031 8032 if (edata->size == 4) 8033 edata->val = t4_read_reg(sc, edata->addr); 8034 else if (edata->size == 8) 8035 edata->val = t4_read_reg64(sc, edata->addr); 8036 else 8037 return (EINVAL); 8038 8039 break; 8040 } 8041 case CHELSIO_T4_SETREG: { 8042 struct t4_reg *edata = (struct t4_reg *)data; 8043 8044 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8045 return (EFAULT); 8046 8047 if (edata->size == 4) { 8048 if (edata->val & 0xffffffff00000000) 8049 return (EINVAL); 8050 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8051 } else if (edata->size == 8) 8052 t4_write_reg64(sc, edata->addr, edata->val); 8053 else 8054 return (EINVAL); 8055 break; 8056 } 8057 case CHELSIO_T4_REGDUMP: { 8058 struct t4_regdump *regs = (struct t4_regdump *)data; 8059 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8060 uint8_t *buf; 8061 8062 if (regs->len < reglen) { 8063 regs->len = reglen; /* hint to the caller */ 8064 return (ENOBUFS); 8065 } 8066 8067 regs->len = reglen; 8068 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8069 t4_get_regs(sc, regs, buf); 8070 rc = copyout(buf, regs->data, reglen); 8071 free(buf, M_CXGBE); 8072 break; 8073 } 8074 case CHELSIO_T4_GET_FILTER_MODE: 8075 rc = get_filter_mode(sc, (uint32_t *)data); 8076 break; 8077 case CHELSIO_T4_SET_FILTER_MODE: 8078 rc = set_filter_mode(sc, *(uint32_t *)data); 8079 break; 8080 case CHELSIO_T4_GET_FILTER: 8081 rc = get_filter(sc, (struct t4_filter *)data); 8082 break; 8083 case CHELSIO_T4_SET_FILTER: 8084 rc = set_filter(sc, (struct t4_filter *)data); 8085 break; 8086 case CHELSIO_T4_DEL_FILTER: 8087 rc = del_filter(sc, (struct t4_filter *)data); 8088 break; 8089 case CHELSIO_T4_GET_SGE_CONTEXT: 8090 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8091 break; 8092 case CHELSIO_T4_LOAD_FW: 8093 rc = load_fw(sc, (struct t4_data *)data); 8094 break; 8095 case CHELSIO_T4_GET_MEM: 8096 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8097 break; 8098 case CHELSIO_T4_GET_I2C: 8099 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8100 break; 8101 case CHELSIO_T4_CLEAR_STATS: { 8102 int i; 8103 u_int port_id = *(uint32_t *)data; 8104 struct port_info *pi; 8105 8106 if (port_id >= sc->params.nports) 8107 return (EINVAL); 8108 pi = sc->port[port_id]; 8109 8110 /* MAC stats */ 8111 t4_clr_port_stats(sc, pi->tx_chan); 8112 pi->tx_parse_error = 0; 8113 8114 if (pi->flags & PORT_INIT_DONE) { 8115 struct sge_rxq *rxq; 8116 struct sge_txq *txq; 8117 struct sge_wrq *wrq; 8118 8119 for_each_rxq(pi, i, rxq) { 8120 #if defined(INET) || defined(INET6) 8121 rxq->lro.lro_queued = 0; 8122 rxq->lro.lro_flushed = 0; 8123 #endif 8124 rxq->rxcsum = 0; 8125 rxq->vlan_extraction = 0; 8126 } 8127 8128 for_each_txq(pi, i, txq) { 8129 txq->txcsum = 0; 8130 txq->tso_wrs = 0; 8131 txq->vlan_insertion = 0; 8132 txq->imm_wrs = 0; 8133 txq->sgl_wrs = 0; 8134 txq->txpkt_wrs = 0; 8135 txq->txpkts0_wrs = 0; 8136 txq->txpkts1_wrs = 0; 8137 txq->txpkts0_pkts = 0; 8138 txq->txpkts1_pkts = 0; 8139 mp_ring_reset_stats(txq->r); 8140 } 8141 8142 #ifdef TCP_OFFLOAD 8143 /* nothing to clear for each ofld_rxq */ 8144 8145 for_each_ofld_txq(pi, i, wrq) { 8146 wrq->tx_wrs_direct = 0; 8147 wrq->tx_wrs_copied = 0; 8148 } 8149 #endif 8150 wrq = &sc->sge.ctrlq[pi->port_id]; 8151 wrq->tx_wrs_direct = 0; 8152 wrq->tx_wrs_copied = 0; 8153 } 8154 break; 8155 } 8156 case CHELSIO_T4_SCHED_CLASS: 8157 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8158 break; 8159 case CHELSIO_T4_SCHED_QUEUE: 8160 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8161 break; 8162 case CHELSIO_T4_GET_TRACER: 8163 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8164 break; 8165 case CHELSIO_T4_SET_TRACER: 8166 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8167 break; 8168 default: 8169 rc = EINVAL; 8170 } 8171 8172 return (rc); 8173 } 8174 8175 #ifdef TCP_OFFLOAD 8176 void 8177 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8178 const unsigned int *pgsz_order) 8179 { 8180 struct port_info *pi = ifp->if_softc; 8181 struct adapter *sc = pi->adapter; 8182 8183 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8184 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8185 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8186 V_HPZ3(pgsz_order[3])); 8187 } 8188 8189 static int 8190 toe_capability(struct port_info *pi, int enable) 8191 { 8192 int rc; 8193 struct adapter *sc = pi->adapter; 8194 8195 ASSERT_SYNCHRONIZED_OP(sc); 8196 8197 if (!is_offload(sc)) 8198 return (ENODEV); 8199 8200 if (enable) { 8201 /* 8202 * We need the port's queues around so that we're able to send 8203 * and receive CPLs to/from the TOE even if the ifnet for this 8204 * port has never been UP'd administratively. 8205 */ 8206 if (!(pi->flags & PORT_INIT_DONE)) { 8207 rc = cxgbe_init_synchronized(pi); 8208 if (rc) 8209 return (rc); 8210 } 8211 8212 if (isset(&sc->offload_map, pi->port_id)) 8213 return (0); 8214 8215 if (!uld_active(sc, ULD_TOM)) { 8216 rc = t4_activate_uld(sc, ULD_TOM); 8217 if (rc == EAGAIN) { 8218 log(LOG_WARNING, 8219 "You must kldload t4_tom.ko before trying " 8220 "to enable TOE on a cxgbe interface.\n"); 8221 } 8222 if (rc != 0) 8223 return (rc); 8224 KASSERT(sc->tom_softc != NULL, 8225 ("%s: TOM activated but softc NULL", __func__)); 8226 KASSERT(uld_active(sc, ULD_TOM), 8227 ("%s: TOM activated but flag not set", __func__)); 8228 } 8229 8230 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8231 if (!uld_active(sc, ULD_IWARP)) 8232 (void) t4_activate_uld(sc, ULD_IWARP); 8233 if (!uld_active(sc, ULD_ISCSI)) 8234 (void) t4_activate_uld(sc, ULD_ISCSI); 8235 8236 setbit(&sc->offload_map, pi->port_id); 8237 } else { 8238 if (!isset(&sc->offload_map, pi->port_id)) 8239 return (0); 8240 8241 KASSERT(uld_active(sc, ULD_TOM), 8242 ("%s: TOM never initialized?", __func__)); 8243 clrbit(&sc->offload_map, pi->port_id); 8244 } 8245 8246 return (0); 8247 } 8248 8249 /* 8250 * Add an upper layer driver to the global list. 8251 */ 8252 int 8253 t4_register_uld(struct uld_info *ui) 8254 { 8255 int rc = 0; 8256 struct uld_info *u; 8257 8258 sx_xlock(&t4_uld_list_lock); 8259 SLIST_FOREACH(u, &t4_uld_list, link) { 8260 if (u->uld_id == ui->uld_id) { 8261 rc = EEXIST; 8262 goto done; 8263 } 8264 } 8265 8266 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8267 ui->refcount = 0; 8268 done: 8269 sx_xunlock(&t4_uld_list_lock); 8270 return (rc); 8271 } 8272 8273 int 8274 t4_unregister_uld(struct uld_info *ui) 8275 { 8276 int rc = EINVAL; 8277 struct uld_info *u; 8278 8279 sx_xlock(&t4_uld_list_lock); 8280 8281 SLIST_FOREACH(u, &t4_uld_list, link) { 8282 if (u == ui) { 8283 if (ui->refcount > 0) { 8284 rc = EBUSY; 8285 goto done; 8286 } 8287 8288 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8289 rc = 0; 8290 goto done; 8291 } 8292 } 8293 done: 8294 sx_xunlock(&t4_uld_list_lock); 8295 return (rc); 8296 } 8297 8298 int 8299 t4_activate_uld(struct adapter *sc, int id) 8300 { 8301 int rc; 8302 struct uld_info *ui; 8303 8304 ASSERT_SYNCHRONIZED_OP(sc); 8305 8306 if (id < 0 || id > ULD_MAX) 8307 return (EINVAL); 8308 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8309 8310 sx_slock(&t4_uld_list_lock); 8311 8312 SLIST_FOREACH(ui, &t4_uld_list, link) { 8313 if (ui->uld_id == id) { 8314 if (!(sc->flags & FULL_INIT_DONE)) { 8315 rc = adapter_full_init(sc); 8316 if (rc != 0) 8317 break; 8318 } 8319 8320 rc = ui->activate(sc); 8321 if (rc == 0) { 8322 setbit(&sc->active_ulds, id); 8323 ui->refcount++; 8324 } 8325 break; 8326 } 8327 } 8328 8329 sx_sunlock(&t4_uld_list_lock); 8330 8331 return (rc); 8332 } 8333 8334 int 8335 t4_deactivate_uld(struct adapter *sc, int id) 8336 { 8337 int rc; 8338 struct uld_info *ui; 8339 8340 ASSERT_SYNCHRONIZED_OP(sc); 8341 8342 if (id < 0 || id > ULD_MAX) 8343 return (EINVAL); 8344 rc = ENXIO; 8345 8346 sx_slock(&t4_uld_list_lock); 8347 8348 SLIST_FOREACH(ui, &t4_uld_list, link) { 8349 if (ui->uld_id == id) { 8350 rc = ui->deactivate(sc); 8351 if (rc == 0) { 8352 clrbit(&sc->active_ulds, id); 8353 ui->refcount--; 8354 } 8355 break; 8356 } 8357 } 8358 8359 sx_sunlock(&t4_uld_list_lock); 8360 8361 return (rc); 8362 } 8363 8364 int 8365 uld_active(struct adapter *sc, int uld_id) 8366 { 8367 8368 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8369 8370 return (isset(&sc->active_ulds, uld_id)); 8371 } 8372 #endif 8373 8374 /* 8375 * Come up with reasonable defaults for some of the tunables, provided they're 8376 * not set by the user (in which case we'll use the values as is). 8377 */ 8378 static void 8379 tweak_tunables(void) 8380 { 8381 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8382 8383 if (t4_ntxq10g < 1) 8384 t4_ntxq10g = min(nc, NTXQ_10G); 8385 8386 if (t4_ntxq1g < 1) 8387 t4_ntxq1g = min(nc, NTXQ_1G); 8388 8389 if (t4_nrxq10g < 1) 8390 t4_nrxq10g = min(nc, NRXQ_10G); 8391 8392 if (t4_nrxq1g < 1) 8393 t4_nrxq1g = min(nc, NRXQ_1G); 8394 8395 #ifdef TCP_OFFLOAD 8396 if (t4_nofldtxq10g < 1) 8397 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8398 8399 if (t4_nofldtxq1g < 1) 8400 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8401 8402 if (t4_nofldrxq10g < 1) 8403 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8404 8405 if (t4_nofldrxq1g < 1) 8406 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8407 8408 if (t4_toecaps_allowed == -1) 8409 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8410 #else 8411 if (t4_toecaps_allowed == -1) 8412 t4_toecaps_allowed = 0; 8413 #endif 8414 8415 #ifdef DEV_NETMAP 8416 if (t4_nnmtxq10g < 1) 8417 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8418 8419 if (t4_nnmtxq1g < 1) 8420 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8421 8422 if (t4_nnmrxq10g < 1) 8423 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8424 8425 if (t4_nnmrxq1g < 1) 8426 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8427 #endif 8428 8429 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8430 t4_tmr_idx_10g = TMR_IDX_10G; 8431 8432 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8433 t4_pktc_idx_10g = PKTC_IDX_10G; 8434 8435 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8436 t4_tmr_idx_1g = TMR_IDX_1G; 8437 8438 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8439 t4_pktc_idx_1g = PKTC_IDX_1G; 8440 8441 if (t4_qsize_txq < 128) 8442 t4_qsize_txq = 128; 8443 8444 if (t4_qsize_rxq < 128) 8445 t4_qsize_rxq = 128; 8446 while (t4_qsize_rxq & 7) 8447 t4_qsize_rxq++; 8448 8449 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8450 } 8451 8452 static struct sx mlu; /* mod load unload */ 8453 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8454 8455 static int 8456 mod_event(module_t mod, int cmd, void *arg) 8457 { 8458 int rc = 0; 8459 static int loaded = 0; 8460 8461 switch (cmd) { 8462 case MOD_LOAD: 8463 sx_xlock(&mlu); 8464 if (loaded++ == 0) { 8465 t4_sge_modload(); 8466 sx_init(&t4_list_lock, "T4/T5 adapters"); 8467 SLIST_INIT(&t4_list); 8468 #ifdef TCP_OFFLOAD 8469 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8470 SLIST_INIT(&t4_uld_list); 8471 #endif 8472 t4_tracer_modload(); 8473 tweak_tunables(); 8474 } 8475 sx_xunlock(&mlu); 8476 break; 8477 8478 case MOD_UNLOAD: 8479 sx_xlock(&mlu); 8480 if (--loaded == 0) { 8481 int tries; 8482 8483 sx_slock(&t4_list_lock); 8484 if (!SLIST_EMPTY(&t4_list)) { 8485 rc = EBUSY; 8486 sx_sunlock(&t4_list_lock); 8487 goto done_unload; 8488 } 8489 #ifdef TCP_OFFLOAD 8490 sx_slock(&t4_uld_list_lock); 8491 if (!SLIST_EMPTY(&t4_uld_list)) { 8492 rc = EBUSY; 8493 sx_sunlock(&t4_uld_list_lock); 8494 sx_sunlock(&t4_list_lock); 8495 goto done_unload; 8496 } 8497 #endif 8498 tries = 0; 8499 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8500 uprintf("%ju clusters with custom free routine " 8501 "still is use.\n", t4_sge_extfree_refs()); 8502 pause("t4unload", 2 * hz); 8503 } 8504 #ifdef TCP_OFFLOAD 8505 sx_sunlock(&t4_uld_list_lock); 8506 #endif 8507 sx_sunlock(&t4_list_lock); 8508 8509 if (t4_sge_extfree_refs() == 0) { 8510 t4_tracer_modunload(); 8511 #ifdef TCP_OFFLOAD 8512 sx_destroy(&t4_uld_list_lock); 8513 #endif 8514 sx_destroy(&t4_list_lock); 8515 t4_sge_modunload(); 8516 loaded = 0; 8517 } else { 8518 rc = EBUSY; 8519 loaded++; /* undo earlier decrement */ 8520 } 8521 } 8522 done_unload: 8523 sx_xunlock(&mlu); 8524 break; 8525 } 8526 8527 return (rc); 8528 } 8529 8530 static devclass_t t4_devclass, t5_devclass; 8531 static devclass_t cxgbe_devclass, cxl_devclass; 8532 8533 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8534 MODULE_VERSION(t4nex, 1); 8535 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8536 #ifdef DEV_NETMAP 8537 MODULE_DEPEND(t4nex, netmap, 1, 1, 1); 8538 #endif /* DEV_NETMAP */ 8539 8540 8541 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8542 MODULE_VERSION(t5nex, 1); 8543 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8544 #ifdef DEV_NETMAP 8545 MODULE_DEPEND(t5nex, netmap, 1, 1, 1); 8546 #endif /* DEV_NETMAP */ 8547 8548 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8549 MODULE_VERSION(cxgbe, 1); 8550 8551 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8552 MODULE_VERSION(cxl, 1); 8553