1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 70 /* T4 bus driver interface */ 71 static int t4_probe(device_t); 72 static int t4_attach(device_t); 73 static int t4_detach(device_t); 74 static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80 }; 81 static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85 }; 86 87 88 /* T4 port (cxgbe) interface */ 89 static int cxgbe_probe(device_t); 90 static int cxgbe_attach(device_t); 91 static int cxgbe_detach(device_t); 92 static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97 }; 98 static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102 }; 103 104 static d_ioctl_t t4_ioctl; 105 static d_open_t t4_open; 106 static d_close_t t4_close; 107 108 static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115 }; 116 117 /* T5 bus driver interface */ 118 static int t5_probe(device_t); 119 static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125 }; 126 static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130 }; 131 132 133 /* T5 port (cxl) interface */ 134 static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138 }; 139 140 static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147 }; 148 149 /* ifnet + media interface */ 150 static void cxgbe_init(void *); 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153 static void cxgbe_qflush(struct ifnet *); 154 static int cxgbe_media_change(struct ifnet *); 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159 /* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163 static struct mtx t4_list_lock; 164 static SLIST_HEAD(, adapter) t4_list; 165 #ifdef TCP_OFFLOAD 166 static struct mtx t4_uld_list_lock; 167 static SLIST_HEAD(, uld_info) t4_uld_list; 168 #endif 169 170 /* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181 /* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184 #define NTXQ_10G 16 185 static int t4_ntxq10g = -1; 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188 #define NRXQ_10G 8 189 static int t4_nrxq10g = -1; 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192 #define NTXQ_1G 4 193 static int t4_ntxq1g = -1; 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196 #define NRXQ_1G 2 197 static int t4_nrxq1g = -1; 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200 #ifdef TCP_OFFLOAD 201 #define NOFLDTXQ_10G 8 202 static int t4_nofldtxq10g = -1; 203 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 204 205 #define NOFLDRXQ_10G 2 206 static int t4_nofldrxq10g = -1; 207 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 208 209 #define NOFLDTXQ_1G 2 210 static int t4_nofldtxq1g = -1; 211 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 212 213 #define NOFLDRXQ_1G 1 214 static int t4_nofldrxq1g = -1; 215 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 216 #endif 217 218 /* 219 * Holdoff parameters for 10G and 1G ports. 220 */ 221 #define TMR_IDX_10G 1 222 static int t4_tmr_idx_10g = TMR_IDX_10G; 223 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 224 225 #define PKTC_IDX_10G (-1) 226 static int t4_pktc_idx_10g = PKTC_IDX_10G; 227 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 228 229 #define TMR_IDX_1G 1 230 static int t4_tmr_idx_1g = TMR_IDX_1G; 231 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 232 233 #define PKTC_IDX_1G (-1) 234 static int t4_pktc_idx_1g = PKTC_IDX_1G; 235 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 236 237 /* 238 * Size (# of entries) of each tx and rx queue. 239 */ 240 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 241 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 242 243 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 244 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 245 246 /* 247 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 248 */ 249 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 250 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 251 252 /* 253 * Configuration file. 254 */ 255 #define DEFAULT_CF "default" 256 #define FLASH_CF "flash" 257 #define UWIRE_CF "uwire" 258 static char t4_cfg_file[32] = DEFAULT_CF; 259 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 260 261 /* 262 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 263 * encouraged respectively). 264 */ 265 static unsigned int t4_fw_install = 1; 266 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 267 268 /* 269 * ASIC features that will be used. Disable the ones you don't want so that the 270 * chip resources aren't wasted on features that will not be used. 271 */ 272 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 273 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 274 275 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 276 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 277 278 static int t4_toecaps_allowed = -1; 279 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 280 281 static int t4_rdmacaps_allowed = 0; 282 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 283 284 static int t4_iscsicaps_allowed = 0; 285 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 286 287 static int t4_fcoecaps_allowed = 0; 288 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 289 290 static int t5_write_combine = 0; 291 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 292 293 struct intrs_and_queues { 294 int intr_type; /* INTx, MSI, or MSI-X */ 295 int nirq; /* Number of vectors */ 296 int intr_flags; 297 int ntxq10g; /* # of NIC txq's for each 10G port */ 298 int nrxq10g; /* # of NIC rxq's for each 10G port */ 299 int ntxq1g; /* # of NIC txq's for each 1G port */ 300 int nrxq1g; /* # of NIC rxq's for each 1G port */ 301 #ifdef TCP_OFFLOAD 302 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 303 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 304 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 305 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 306 #endif 307 }; 308 309 struct filter_entry { 310 uint32_t valid:1; /* filter allocated and valid */ 311 uint32_t locked:1; /* filter is administratively locked */ 312 uint32_t pending:1; /* filter action is pending firmware reply */ 313 uint32_t smtidx:8; /* Source MAC Table index for smac */ 314 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 315 316 struct t4_filter_specification fs; 317 }; 318 319 enum { 320 XGMAC_MTU = (1 << 0), 321 XGMAC_PROMISC = (1 << 1), 322 XGMAC_ALLMULTI = (1 << 2), 323 XGMAC_VLANEX = (1 << 3), 324 XGMAC_UCADDR = (1 << 4), 325 XGMAC_MCADDRS = (1 << 5), 326 327 XGMAC_ALL = 0xffff 328 }; 329 330 static int map_bars_0_and_4(struct adapter *); 331 static int map_bar_2(struct adapter *); 332 static void setup_memwin(struct adapter *); 333 static int validate_mem_range(struct adapter *, uint32_t, int); 334 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 335 uint32_t *); 336 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 337 static uint32_t position_memwin(struct adapter *, int, uint32_t); 338 static int cfg_itype_and_nqueues(struct adapter *, int, int, 339 struct intrs_and_queues *); 340 static int prep_firmware(struct adapter *); 341 static int partition_resources(struct adapter *, const struct firmware *, 342 const char *); 343 static int get_params__pre_init(struct adapter *); 344 static int get_params__post_init(struct adapter *); 345 static int set_params__post_init(struct adapter *); 346 static void t4_set_desc(struct adapter *); 347 static void build_medialist(struct port_info *); 348 static int update_mac_settings(struct port_info *, int); 349 static int cxgbe_init_synchronized(struct port_info *); 350 static int cxgbe_uninit_synchronized(struct port_info *); 351 static int setup_intr_handlers(struct adapter *); 352 static int adapter_full_init(struct adapter *); 353 static int adapter_full_uninit(struct adapter *); 354 static int port_full_init(struct port_info *); 355 static int port_full_uninit(struct port_info *); 356 static void quiesce_eq(struct adapter *, struct sge_eq *); 357 static void quiesce_iq(struct adapter *, struct sge_iq *); 358 static void quiesce_fl(struct adapter *, struct sge_fl *); 359 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 360 driver_intr_t *, void *, char *); 361 static int t4_free_irq(struct adapter *, struct irq *); 362 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 363 unsigned int); 364 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 365 static void cxgbe_tick(void *); 366 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 367 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 368 struct mbuf *); 369 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 370 static int fw_msg_not_handled(struct adapter *, const __be64 *); 371 static int t4_sysctls(struct adapter *); 372 static int cxgbe_sysctls(struct port_info *); 373 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 374 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 375 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 376 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 377 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 378 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 379 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 380 #ifdef SBUF_DRAIN 381 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 382 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 383 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 384 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 385 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 386 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 387 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 388 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 389 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 390 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 391 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 392 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 393 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 394 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 395 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 396 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 397 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 398 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 399 static int sysctl_wrwc_stats(SYSCTL_HANDLER_ARGS); 400 #endif 401 static inline void txq_start(struct ifnet *, struct sge_txq *); 402 static uint32_t fconf_to_mode(uint32_t); 403 static uint32_t mode_to_fconf(uint32_t); 404 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 405 static int get_filter_mode(struct adapter *, uint32_t *); 406 static int set_filter_mode(struct adapter *, uint32_t); 407 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 408 static int get_filter(struct adapter *, struct t4_filter *); 409 static int set_filter(struct adapter *, struct t4_filter *); 410 static int del_filter(struct adapter *, struct t4_filter *); 411 static void clear_filter(struct filter_entry *); 412 static int set_filter_wr(struct adapter *, int); 413 static int del_filter_wr(struct adapter *, int); 414 static int get_sge_context(struct adapter *, struct t4_sge_context *); 415 static int load_fw(struct adapter *, struct t4_data *); 416 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 417 static int read_i2c(struct adapter *, struct t4_i2c_data *); 418 #ifdef TCP_OFFLOAD 419 static int toe_capability(struct port_info *, int); 420 #endif 421 static int t4_mod_event(module_t, int, void *); 422 423 struct { 424 uint16_t device; 425 char *desc; 426 } t4_pciids[] = { 427 {0xa000, "Chelsio Terminator 4 FPGA"}, 428 {0x4400, "Chelsio T440-dbg"}, 429 {0x4401, "Chelsio T420-CR"}, 430 {0x4402, "Chelsio T422-CR"}, 431 {0x4403, "Chelsio T440-CR"}, 432 {0x4404, "Chelsio T420-BCH"}, 433 {0x4405, "Chelsio T440-BCH"}, 434 {0x4406, "Chelsio T440-CH"}, 435 {0x4407, "Chelsio T420-SO"}, 436 {0x4408, "Chelsio T420-CX"}, 437 {0x4409, "Chelsio T420-BT"}, 438 {0x440a, "Chelsio T404-BT"}, 439 {0x440e, "Chelsio T440-LP-CR"}, 440 }, t5_pciids[] = { 441 {0xb000, "Chelsio Terminator 5 FPGA"}, 442 {0x5400, "Chelsio T580-dbg"}, 443 }; 444 445 #ifdef TCP_OFFLOAD 446 /* 447 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 448 * exactly the same for both rxq and ofld_rxq. 449 */ 450 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 451 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 452 #endif 453 454 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 455 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 456 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 457 458 static int 459 t4_probe(device_t dev) 460 { 461 int i; 462 uint16_t v = pci_get_vendor(dev); 463 uint16_t d = pci_get_device(dev); 464 uint8_t f = pci_get_function(dev); 465 466 if (v != PCI_VENDOR_ID_CHELSIO) 467 return (ENXIO); 468 469 /* Attach only to PF0 of the FPGA */ 470 if (d == 0xa000 && f != 0) 471 return (ENXIO); 472 473 for (i = 0; i < nitems(t4_pciids); i++) { 474 if (d == t4_pciids[i].device) { 475 device_set_desc(dev, t4_pciids[i].desc); 476 return (BUS_PROBE_DEFAULT); 477 } 478 } 479 480 return (ENXIO); 481 } 482 483 static int 484 t5_probe(device_t dev) 485 { 486 int i; 487 uint16_t v = pci_get_vendor(dev); 488 uint16_t d = pci_get_device(dev); 489 uint8_t f = pci_get_function(dev); 490 491 if (v != PCI_VENDOR_ID_CHELSIO) 492 return (ENXIO); 493 494 /* Attach only to PF0 of the FPGA */ 495 if (d == 0xb000 && f != 0) 496 return (ENXIO); 497 498 for (i = 0; i < nitems(t5_pciids); i++) { 499 if (d == t5_pciids[i].device) { 500 device_set_desc(dev, t5_pciids[i].desc); 501 return (BUS_PROBE_DEFAULT); 502 } 503 } 504 505 return (ENXIO); 506 } 507 508 static int 509 t4_attach(device_t dev) 510 { 511 struct adapter *sc; 512 int rc = 0, i, n10g, n1g, rqidx, tqidx; 513 struct intrs_and_queues iaq; 514 struct sge *s; 515 #ifdef TCP_OFFLOAD 516 int ofld_rqidx, ofld_tqidx; 517 #endif 518 519 sc = device_get_softc(dev); 520 sc->dev = dev; 521 522 pci_enable_busmaster(dev); 523 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 524 uint32_t v; 525 526 pci_set_max_read_req(dev, 4096); 527 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 528 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 529 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 530 } 531 532 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 533 device_get_nameunit(dev)); 534 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 535 mtx_lock(&t4_list_lock); 536 SLIST_INSERT_HEAD(&t4_list, sc, link); 537 mtx_unlock(&t4_list_lock); 538 539 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 540 TAILQ_INIT(&sc->sfl); 541 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 542 543 rc = map_bars_0_and_4(sc); 544 if (rc != 0) 545 goto done; /* error message displayed already */ 546 547 /* 548 * This is the real PF# to which we're attaching. Works from within PCI 549 * passthrough environments too, where pci_get_function() could return a 550 * different PF# depending on the passthrough configuration. We need to 551 * use the real PF# in all our communication with the firmware. 552 */ 553 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 554 sc->mbox = sc->pf; 555 556 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 557 sc->an_handler = an_not_handled; 558 for (i = 0; i < nitems(sc->cpl_handler); i++) 559 sc->cpl_handler[i] = cpl_not_handled; 560 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 561 sc->fw_msg_handler[i] = fw_msg_not_handled; 562 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 563 t4_init_sge_cpl_handlers(sc); 564 565 /* Prepare the adapter for operation */ 566 rc = -t4_prep_adapter(sc); 567 if (rc != 0) { 568 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 569 goto done; 570 } 571 572 /* 573 * Do this really early, with the memory windows set up even before the 574 * character device. The userland tool's register i/o and mem read 575 * will work even in "recovery mode". 576 */ 577 setup_memwin(sc); 578 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 579 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 580 device_get_nameunit(dev)); 581 if (sc->cdev == NULL) 582 device_printf(dev, "failed to create nexus char device.\n"); 583 else 584 sc->cdev->si_drv1 = sc; 585 586 /* Go no further if recovery mode has been requested. */ 587 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 588 device_printf(dev, "recovery mode.\n"); 589 goto done; 590 } 591 592 /* Prepare the firmware for operation */ 593 rc = prep_firmware(sc); 594 if (rc != 0) 595 goto done; /* error message displayed already */ 596 597 rc = get_params__post_init(sc); 598 if (rc != 0) 599 goto done; /* error message displayed already */ 600 601 rc = set_params__post_init(sc); 602 if (rc != 0) 603 goto done; /* error message displayed already */ 604 605 rc = map_bar_2(sc); 606 if (rc != 0) 607 goto done; /* error message displayed already */ 608 609 for (i = 0; i < NCHAN; i++) 610 sc->params.tp.tx_modq[i] = i; 611 612 rc = t4_create_dma_tag(sc); 613 if (rc != 0) 614 goto done; /* error message displayed already */ 615 616 /* 617 * First pass over all the ports - allocate VIs and initialize some 618 * basic parameters like mac address, port type, etc. We also figure 619 * out whether a port is 10G or 1G and use that information when 620 * calculating how many interrupts to attempt to allocate. 621 */ 622 n10g = n1g = 0; 623 for_each_port(sc, i) { 624 struct port_info *pi; 625 626 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 627 sc->port[i] = pi; 628 629 /* These must be set before t4_port_init */ 630 pi->adapter = sc; 631 pi->port_id = i; 632 633 /* Allocate the vi and initialize parameters like mac addr */ 634 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 635 if (rc != 0) { 636 device_printf(dev, "unable to initialize port %d: %d\n", 637 i, rc); 638 free(pi, M_CXGBE); 639 sc->port[i] = NULL; 640 goto done; 641 } 642 643 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 644 device_get_nameunit(dev), i); 645 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 646 647 if (is_10G_port(pi)) { 648 n10g++; 649 pi->tmr_idx = t4_tmr_idx_10g; 650 pi->pktc_idx = t4_pktc_idx_10g; 651 } else { 652 n1g++; 653 pi->tmr_idx = t4_tmr_idx_1g; 654 pi->pktc_idx = t4_pktc_idx_1g; 655 } 656 657 pi->xact_addr_filt = -1; 658 659 pi->qsize_rxq = t4_qsize_rxq; 660 pi->qsize_txq = t4_qsize_txq; 661 662 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 663 if (pi->dev == NULL) { 664 device_printf(dev, 665 "failed to add device for port %d.\n", i); 666 rc = ENXIO; 667 goto done; 668 } 669 device_set_softc(pi->dev, pi); 670 } 671 672 /* 673 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 674 */ 675 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 676 if (rc != 0) 677 goto done; /* error message displayed already */ 678 679 sc->intr_type = iaq.intr_type; 680 sc->intr_count = iaq.nirq; 681 sc->flags |= iaq.intr_flags; 682 683 s = &sc->sge; 684 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 685 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 686 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 687 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 688 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 689 690 #ifdef TCP_OFFLOAD 691 if (is_offload(sc)) { 692 693 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 694 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 695 s->neq += s->nofldtxq + s->nofldrxq; 696 s->niq += s->nofldrxq; 697 698 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 699 M_CXGBE, M_ZERO | M_WAITOK); 700 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 701 M_CXGBE, M_ZERO | M_WAITOK); 702 } 703 #endif 704 705 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 706 M_ZERO | M_WAITOK); 707 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 708 M_ZERO | M_WAITOK); 709 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 710 M_ZERO | M_WAITOK); 711 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 712 M_ZERO | M_WAITOK); 713 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 714 M_ZERO | M_WAITOK); 715 716 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 717 M_ZERO | M_WAITOK); 718 719 t4_init_l2t(sc, M_WAITOK); 720 721 /* 722 * Second pass over the ports. This time we know the number of rx and 723 * tx queues that each port should get. 724 */ 725 rqidx = tqidx = 0; 726 #ifdef TCP_OFFLOAD 727 ofld_rqidx = ofld_tqidx = 0; 728 #endif 729 for_each_port(sc, i) { 730 struct port_info *pi = sc->port[i]; 731 732 if (pi == NULL) 733 continue; 734 735 pi->first_rxq = rqidx; 736 pi->first_txq = tqidx; 737 if (is_10G_port(pi)) { 738 pi->nrxq = iaq.nrxq10g; 739 pi->ntxq = iaq.ntxq10g; 740 } else { 741 pi->nrxq = iaq.nrxq1g; 742 pi->ntxq = iaq.ntxq1g; 743 } 744 745 rqidx += pi->nrxq; 746 tqidx += pi->ntxq; 747 748 #ifdef TCP_OFFLOAD 749 if (is_offload(sc)) { 750 pi->first_ofld_rxq = ofld_rqidx; 751 pi->first_ofld_txq = ofld_tqidx; 752 if (is_10G_port(pi)) { 753 pi->nofldrxq = iaq.nofldrxq10g; 754 pi->nofldtxq = iaq.nofldtxq10g; 755 } else { 756 pi->nofldrxq = iaq.nofldrxq1g; 757 pi->nofldtxq = iaq.nofldtxq1g; 758 } 759 ofld_rqidx += pi->nofldrxq; 760 ofld_tqidx += pi->nofldtxq; 761 } 762 #endif 763 } 764 765 rc = setup_intr_handlers(sc); 766 if (rc != 0) { 767 device_printf(dev, 768 "failed to setup interrupt handlers: %d\n", rc); 769 goto done; 770 } 771 772 rc = bus_generic_attach(dev); 773 if (rc != 0) { 774 device_printf(dev, 775 "failed to attach all child ports: %d\n", rc); 776 goto done; 777 } 778 779 device_printf(dev, 780 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 781 sc->params.pci.width, sc->params.nports, sc->intr_count, 782 sc->intr_type == INTR_MSIX ? "MSI-X" : 783 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 784 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 785 786 t4_set_desc(sc); 787 788 done: 789 if (rc != 0 && sc->cdev) { 790 /* cdev was created and so cxgbetool works; recover that way. */ 791 device_printf(dev, 792 "error during attach, adapter is now in recovery mode.\n"); 793 rc = 0; 794 } 795 796 if (rc != 0) 797 t4_detach(dev); 798 else 799 t4_sysctls(sc); 800 801 return (rc); 802 } 803 804 /* 805 * Idempotent 806 */ 807 static int 808 t4_detach(device_t dev) 809 { 810 struct adapter *sc; 811 struct port_info *pi; 812 int i, rc; 813 814 sc = device_get_softc(dev); 815 816 if (sc->flags & FULL_INIT_DONE) 817 t4_intr_disable(sc); 818 819 if (sc->cdev) { 820 destroy_dev(sc->cdev); 821 sc->cdev = NULL; 822 } 823 824 rc = bus_generic_detach(dev); 825 if (rc) { 826 device_printf(dev, 827 "failed to detach child devices: %d\n", rc); 828 return (rc); 829 } 830 831 for (i = 0; i < sc->intr_count; i++) 832 t4_free_irq(sc, &sc->irq[i]); 833 834 for (i = 0; i < MAX_NPORTS; i++) { 835 pi = sc->port[i]; 836 if (pi) { 837 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 838 if (pi->dev) 839 device_delete_child(dev, pi->dev); 840 841 mtx_destroy(&pi->pi_lock); 842 free(pi, M_CXGBE); 843 } 844 } 845 846 if (sc->flags & FULL_INIT_DONE) 847 adapter_full_uninit(sc); 848 849 if (sc->flags & FW_OK) 850 t4_fw_bye(sc, sc->mbox); 851 852 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 853 pci_release_msi(dev); 854 855 if (sc->regs_res) 856 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 857 sc->regs_res); 858 859 if (sc->udbs_res) 860 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 861 sc->udbs_res); 862 863 if (sc->msix_res) 864 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 865 sc->msix_res); 866 867 if (sc->l2t) 868 t4_free_l2t(sc->l2t); 869 870 #ifdef TCP_OFFLOAD 871 free(sc->sge.ofld_rxq, M_CXGBE); 872 free(sc->sge.ofld_txq, M_CXGBE); 873 #endif 874 free(sc->irq, M_CXGBE); 875 free(sc->sge.rxq, M_CXGBE); 876 free(sc->sge.txq, M_CXGBE); 877 free(sc->sge.ctrlq, M_CXGBE); 878 free(sc->sge.iqmap, M_CXGBE); 879 free(sc->sge.eqmap, M_CXGBE); 880 free(sc->tids.ftid_tab, M_CXGBE); 881 t4_destroy_dma_tag(sc); 882 if (mtx_initialized(&sc->sc_lock)) { 883 mtx_lock(&t4_list_lock); 884 SLIST_REMOVE(&t4_list, sc, adapter, link); 885 mtx_unlock(&t4_list_lock); 886 mtx_destroy(&sc->sc_lock); 887 } 888 889 if (mtx_initialized(&sc->tids.ftid_lock)) 890 mtx_destroy(&sc->tids.ftid_lock); 891 if (mtx_initialized(&sc->sfl_lock)) 892 mtx_destroy(&sc->sfl_lock); 893 894 bzero(sc, sizeof(*sc)); 895 896 return (0); 897 } 898 899 900 static int 901 cxgbe_probe(device_t dev) 902 { 903 char buf[128]; 904 struct port_info *pi = device_get_softc(dev); 905 906 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 907 device_set_desc_copy(dev, buf); 908 909 return (BUS_PROBE_DEFAULT); 910 } 911 912 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 913 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 914 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 915 #define T4_CAP_ENABLE (T4_CAP) 916 917 static int 918 cxgbe_attach(device_t dev) 919 { 920 struct port_info *pi = device_get_softc(dev); 921 struct ifnet *ifp; 922 923 /* Allocate an ifnet and set it up */ 924 ifp = if_alloc(IFT_ETHER); 925 if (ifp == NULL) { 926 device_printf(dev, "Cannot allocate ifnet\n"); 927 return (ENOMEM); 928 } 929 pi->ifp = ifp; 930 ifp->if_softc = pi; 931 932 callout_init(&pi->tick, CALLOUT_MPSAFE); 933 934 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 935 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 936 937 ifp->if_init = cxgbe_init; 938 ifp->if_ioctl = cxgbe_ioctl; 939 ifp->if_transmit = cxgbe_transmit; 940 ifp->if_qflush = cxgbe_qflush; 941 942 ifp->if_capabilities = T4_CAP; 943 #ifdef TCP_OFFLOAD 944 if (is_offload(pi->adapter)) 945 ifp->if_capabilities |= IFCAP_TOE; 946 #endif 947 ifp->if_capenable = T4_CAP_ENABLE; 948 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 949 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 950 951 /* Initialize ifmedia for this port */ 952 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 953 cxgbe_media_status); 954 build_medialist(pi); 955 956 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 957 EVENTHANDLER_PRI_ANY); 958 959 ether_ifattach(ifp, pi->hw_addr); 960 961 #ifdef TCP_OFFLOAD 962 if (is_offload(pi->adapter)) { 963 device_printf(dev, 964 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 965 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 966 } else 967 #endif 968 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 969 970 cxgbe_sysctls(pi); 971 972 return (0); 973 } 974 975 static int 976 cxgbe_detach(device_t dev) 977 { 978 struct port_info *pi = device_get_softc(dev); 979 struct adapter *sc = pi->adapter; 980 struct ifnet *ifp = pi->ifp; 981 982 /* Tell if_ioctl and if_init that the port is going away */ 983 ADAPTER_LOCK(sc); 984 SET_DOOMED(pi); 985 wakeup(&sc->flags); 986 while (IS_BUSY(sc)) 987 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 988 SET_BUSY(sc); 989 #ifdef INVARIANTS 990 sc->last_op = "t4detach"; 991 sc->last_op_thr = curthread; 992 #endif 993 ADAPTER_UNLOCK(sc); 994 995 if (pi->vlan_c) 996 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 997 998 PORT_LOCK(pi); 999 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1000 callout_stop(&pi->tick); 1001 PORT_UNLOCK(pi); 1002 callout_drain(&pi->tick); 1003 1004 /* Let detach proceed even if these fail. */ 1005 cxgbe_uninit_synchronized(pi); 1006 port_full_uninit(pi); 1007 1008 ifmedia_removeall(&pi->media); 1009 ether_ifdetach(pi->ifp); 1010 if_free(pi->ifp); 1011 1012 ADAPTER_LOCK(sc); 1013 CLR_BUSY(sc); 1014 wakeup(&sc->flags); 1015 ADAPTER_UNLOCK(sc); 1016 1017 return (0); 1018 } 1019 1020 static void 1021 cxgbe_init(void *arg) 1022 { 1023 struct port_info *pi = arg; 1024 struct adapter *sc = pi->adapter; 1025 1026 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1027 return; 1028 cxgbe_init_synchronized(pi); 1029 end_synchronized_op(sc, 0); 1030 } 1031 1032 static int 1033 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1034 { 1035 int rc = 0, mtu, flags; 1036 struct port_info *pi = ifp->if_softc; 1037 struct adapter *sc = pi->adapter; 1038 struct ifreq *ifr = (struct ifreq *)data; 1039 uint32_t mask; 1040 1041 switch (cmd) { 1042 case SIOCSIFMTU: 1043 mtu = ifr->ifr_mtu; 1044 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1045 return (EINVAL); 1046 1047 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1048 if (rc) 1049 return (rc); 1050 ifp->if_mtu = mtu; 1051 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1052 t4_update_fl_bufsize(ifp); 1053 rc = update_mac_settings(pi, XGMAC_MTU); 1054 } 1055 end_synchronized_op(sc, 0); 1056 break; 1057 1058 case SIOCSIFFLAGS: 1059 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1060 if (rc) 1061 return (rc); 1062 1063 if (ifp->if_flags & IFF_UP) { 1064 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1065 flags = pi->if_flags; 1066 if ((ifp->if_flags ^ flags) & 1067 (IFF_PROMISC | IFF_ALLMULTI)) { 1068 rc = update_mac_settings(pi, 1069 XGMAC_PROMISC | XGMAC_ALLMULTI); 1070 } 1071 } else 1072 rc = cxgbe_init_synchronized(pi); 1073 pi->if_flags = ifp->if_flags; 1074 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1075 rc = cxgbe_uninit_synchronized(pi); 1076 end_synchronized_op(sc, 0); 1077 break; 1078 1079 case SIOCADDMULTI: 1080 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1081 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1082 if (rc) 1083 return (rc); 1084 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1085 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1086 end_synchronized_op(sc, LOCK_HELD); 1087 break; 1088 1089 case SIOCSIFCAP: 1090 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1091 if (rc) 1092 return (rc); 1093 1094 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1095 if (mask & IFCAP_TXCSUM) { 1096 ifp->if_capenable ^= IFCAP_TXCSUM; 1097 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1098 1099 if (IFCAP_TSO4 & ifp->if_capenable && 1100 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1101 ifp->if_capenable &= ~IFCAP_TSO4; 1102 if_printf(ifp, 1103 "tso4 disabled due to -txcsum.\n"); 1104 } 1105 } 1106 if (mask & IFCAP_TXCSUM_IPV6) { 1107 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1108 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1109 1110 if (IFCAP_TSO6 & ifp->if_capenable && 1111 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1112 ifp->if_capenable &= ~IFCAP_TSO6; 1113 if_printf(ifp, 1114 "tso6 disabled due to -txcsum6.\n"); 1115 } 1116 } 1117 if (mask & IFCAP_RXCSUM) 1118 ifp->if_capenable ^= IFCAP_RXCSUM; 1119 if (mask & IFCAP_RXCSUM_IPV6) 1120 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1121 1122 /* 1123 * Note that we leave CSUM_TSO alone (it is always set). The 1124 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1125 * sending a TSO request our way, so it's sufficient to toggle 1126 * IFCAP_TSOx only. 1127 */ 1128 if (mask & IFCAP_TSO4) { 1129 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1130 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1131 if_printf(ifp, "enable txcsum first.\n"); 1132 rc = EAGAIN; 1133 goto fail; 1134 } 1135 ifp->if_capenable ^= IFCAP_TSO4; 1136 } 1137 if (mask & IFCAP_TSO6) { 1138 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1139 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1140 if_printf(ifp, "enable txcsum6 first.\n"); 1141 rc = EAGAIN; 1142 goto fail; 1143 } 1144 ifp->if_capenable ^= IFCAP_TSO6; 1145 } 1146 if (mask & IFCAP_LRO) { 1147 #if defined(INET) || defined(INET6) 1148 int i; 1149 struct sge_rxq *rxq; 1150 1151 ifp->if_capenable ^= IFCAP_LRO; 1152 for_each_rxq(pi, i, rxq) { 1153 if (ifp->if_capenable & IFCAP_LRO) 1154 rxq->iq.flags |= IQ_LRO_ENABLED; 1155 else 1156 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1157 } 1158 #endif 1159 } 1160 #ifdef TCP_OFFLOAD 1161 if (mask & IFCAP_TOE) { 1162 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1163 1164 rc = toe_capability(pi, enable); 1165 if (rc != 0) 1166 goto fail; 1167 1168 ifp->if_capenable ^= mask; 1169 } 1170 #endif 1171 if (mask & IFCAP_VLAN_HWTAGGING) { 1172 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1173 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1174 rc = update_mac_settings(pi, XGMAC_VLANEX); 1175 } 1176 if (mask & IFCAP_VLAN_MTU) { 1177 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1178 1179 /* Need to find out how to disable auto-mtu-inflation */ 1180 } 1181 if (mask & IFCAP_VLAN_HWTSO) 1182 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1183 if (mask & IFCAP_VLAN_HWCSUM) 1184 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1185 1186 #ifdef VLAN_CAPABILITIES 1187 VLAN_CAPABILITIES(ifp); 1188 #endif 1189 fail: 1190 end_synchronized_op(sc, 0); 1191 break; 1192 1193 case SIOCSIFMEDIA: 1194 case SIOCGIFMEDIA: 1195 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1196 break; 1197 1198 default: 1199 rc = ether_ioctl(ifp, cmd, data); 1200 } 1201 1202 return (rc); 1203 } 1204 1205 static int 1206 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1207 { 1208 struct port_info *pi = ifp->if_softc; 1209 struct adapter *sc = pi->adapter; 1210 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1211 struct buf_ring *br; 1212 int rc; 1213 1214 M_ASSERTPKTHDR(m); 1215 1216 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1217 m_freem(m); 1218 return (ENETDOWN); 1219 } 1220 1221 if (m->m_flags & M_FLOWID) 1222 txq += (m->m_pkthdr.flowid % pi->ntxq); 1223 br = txq->br; 1224 1225 if (TXQ_TRYLOCK(txq) == 0) { 1226 struct sge_eq *eq = &txq->eq; 1227 1228 /* 1229 * It is possible that t4_eth_tx finishes up and releases the 1230 * lock between the TRYLOCK above and the drbr_enqueue here. We 1231 * need to make sure that this mbuf doesn't just sit there in 1232 * the drbr. 1233 */ 1234 1235 rc = drbr_enqueue(ifp, br, m); 1236 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1237 !(eq->flags & EQ_DOOMED)) 1238 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1239 return (rc); 1240 } 1241 1242 /* 1243 * txq->m is the mbuf that is held up due to a temporary shortage of 1244 * resources and it should be put on the wire first. Then what's in 1245 * drbr and finally the mbuf that was just passed in to us. 1246 * 1247 * Return code should indicate the fate of the mbuf that was passed in 1248 * this time. 1249 */ 1250 1251 TXQ_LOCK_ASSERT_OWNED(txq); 1252 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1253 1254 /* Queued for transmission. */ 1255 1256 rc = drbr_enqueue(ifp, br, m); 1257 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1258 (void) t4_eth_tx(ifp, txq, m); 1259 TXQ_UNLOCK(txq); 1260 return (rc); 1261 } 1262 1263 /* Direct transmission. */ 1264 rc = t4_eth_tx(ifp, txq, m); 1265 if (rc != 0 && txq->m) 1266 rc = 0; /* held, will be transmitted soon (hopefully) */ 1267 1268 TXQ_UNLOCK(txq); 1269 return (rc); 1270 } 1271 1272 static void 1273 cxgbe_qflush(struct ifnet *ifp) 1274 { 1275 struct port_info *pi = ifp->if_softc; 1276 struct sge_txq *txq; 1277 int i; 1278 struct mbuf *m; 1279 1280 /* queues do not exist if !PORT_INIT_DONE. */ 1281 if (pi->flags & PORT_INIT_DONE) { 1282 for_each_txq(pi, i, txq) { 1283 TXQ_LOCK(txq); 1284 m_freem(txq->m); 1285 txq->m = NULL; 1286 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1287 m_freem(m); 1288 TXQ_UNLOCK(txq); 1289 } 1290 } 1291 if_qflush(ifp); 1292 } 1293 1294 static int 1295 cxgbe_media_change(struct ifnet *ifp) 1296 { 1297 struct port_info *pi = ifp->if_softc; 1298 1299 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1300 1301 return (EOPNOTSUPP); 1302 } 1303 1304 static void 1305 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1306 { 1307 struct port_info *pi = ifp->if_softc; 1308 struct ifmedia_entry *cur = pi->media.ifm_cur; 1309 int speed = pi->link_cfg.speed; 1310 int data = (pi->port_type << 8) | pi->mod_type; 1311 1312 if (cur->ifm_data != data) { 1313 build_medialist(pi); 1314 cur = pi->media.ifm_cur; 1315 } 1316 1317 ifmr->ifm_status = IFM_AVALID; 1318 if (!pi->link_cfg.link_ok) 1319 return; 1320 1321 ifmr->ifm_status |= IFM_ACTIVE; 1322 1323 /* active and current will differ iff current media is autoselect. */ 1324 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1325 return; 1326 1327 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1328 if (speed == SPEED_10000) 1329 ifmr->ifm_active |= IFM_10G_T; 1330 else if (speed == SPEED_1000) 1331 ifmr->ifm_active |= IFM_1000_T; 1332 else if (speed == SPEED_100) 1333 ifmr->ifm_active |= IFM_100_TX; 1334 else if (speed == SPEED_10) 1335 ifmr->ifm_active |= IFM_10_T; 1336 else 1337 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1338 speed)); 1339 } 1340 1341 void 1342 t4_fatal_err(struct adapter *sc) 1343 { 1344 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1345 t4_intr_disable(sc); 1346 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1347 device_get_nameunit(sc->dev)); 1348 } 1349 1350 static int 1351 map_bars_0_and_4(struct adapter *sc) 1352 { 1353 sc->regs_rid = PCIR_BAR(0); 1354 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1355 &sc->regs_rid, RF_ACTIVE); 1356 if (sc->regs_res == NULL) { 1357 device_printf(sc->dev, "cannot map registers.\n"); 1358 return (ENXIO); 1359 } 1360 sc->bt = rman_get_bustag(sc->regs_res); 1361 sc->bh = rman_get_bushandle(sc->regs_res); 1362 sc->mmio_len = rman_get_size(sc->regs_res); 1363 setbit(&sc->doorbells, DOORBELL_KDB); 1364 1365 sc->msix_rid = PCIR_BAR(4); 1366 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1367 &sc->msix_rid, RF_ACTIVE); 1368 if (sc->msix_res == NULL) { 1369 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1370 return (ENXIO); 1371 } 1372 1373 return (0); 1374 } 1375 1376 static int 1377 map_bar_2(struct adapter *sc) 1378 { 1379 1380 /* 1381 * T4: only iWARP driver uses the userspace doorbells. There is no need 1382 * to map it if RDMA is disabled. 1383 */ 1384 if (is_t4(sc) && sc->rdmacaps == 0) 1385 return (0); 1386 1387 sc->udbs_rid = PCIR_BAR(2); 1388 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1389 &sc->udbs_rid, RF_ACTIVE); 1390 if (sc->udbs_res == NULL) { 1391 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1392 return (ENXIO); 1393 } 1394 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1395 1396 if (is_t5(sc)) { 1397 setbit(&sc->doorbells, DOORBELL_UDB); 1398 #if defined(__i386__) || defined(__amd64__) 1399 if (t5_write_combine) { 1400 int rc; 1401 1402 /* 1403 * Enable write combining on BAR2. This is the 1404 * userspace doorbell BAR and is split into 128B 1405 * (UDBS_SEG_SIZE) doorbell regions, each associated 1406 * with an egress queue. The first 64B has the doorbell 1407 * and the second 64B can be used to submit a tx work 1408 * request with an implicit doorbell. 1409 */ 1410 1411 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1412 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1413 if (rc == 0) { 1414 clrbit(&sc->doorbells, DOORBELL_UDB); 1415 setbit(&sc->doorbells, DOORBELL_WRWC); 1416 setbit(&sc->doorbells, DOORBELL_UDBWC); 1417 } else { 1418 device_printf(sc->dev, 1419 "couldn't enable write combining: %d\n", 1420 rc); 1421 } 1422 1423 t4_write_reg(sc, A_SGE_STAT_CFG, 1424 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1425 } 1426 #endif 1427 } 1428 1429 return (0); 1430 } 1431 1432 static const struct memwin t4_memwin[] = { 1433 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1434 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1435 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1436 }; 1437 1438 static const struct memwin t5_memwin[] = { 1439 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1440 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1441 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1442 }; 1443 1444 static void 1445 setup_memwin(struct adapter *sc) 1446 { 1447 const struct memwin *mw; 1448 int i, n; 1449 uint32_t bar0; 1450 1451 if (is_t4(sc)) { 1452 /* 1453 * Read low 32b of bar0 indirectly via the hardware backdoor 1454 * mechanism. Works from within PCI passthrough environments 1455 * too, where rman_get_start() can return a different value. We 1456 * need to program the T4 memory window decoders with the actual 1457 * addresses that will be coming across the PCIe link. 1458 */ 1459 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1460 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1461 1462 mw = &t4_memwin[0]; 1463 n = nitems(t4_memwin); 1464 } else { 1465 /* T5 uses the relative offset inside the PCIe BAR */ 1466 bar0 = 0; 1467 1468 mw = &t5_memwin[0]; 1469 n = nitems(t5_memwin); 1470 } 1471 1472 for (i = 0; i < n; i++, mw++) { 1473 t4_write_reg(sc, 1474 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1475 (mw->base + bar0) | V_BIR(0) | 1476 V_WINDOW(ilog2(mw->aperture) - 10)); 1477 } 1478 1479 /* flush */ 1480 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1481 } 1482 1483 /* 1484 * Verify that the memory range specified by the addr/len pair is valid and lies 1485 * entirely within a single region (EDCx or MCx). 1486 */ 1487 static int 1488 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1489 { 1490 uint32_t em, addr_len, maddr, mlen; 1491 1492 /* Memory can only be accessed in naturally aligned 4 byte units */ 1493 if (addr & 3 || len & 3 || len == 0) 1494 return (EINVAL); 1495 1496 /* Enabled memories */ 1497 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1498 if (em & F_EDRAM0_ENABLE) { 1499 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1500 maddr = G_EDRAM0_BASE(addr_len) << 20; 1501 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1502 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1503 addr + len <= maddr + mlen) 1504 return (0); 1505 } 1506 if (em & F_EDRAM1_ENABLE) { 1507 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1508 maddr = G_EDRAM1_BASE(addr_len) << 20; 1509 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1510 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1511 addr + len <= maddr + mlen) 1512 return (0); 1513 } 1514 if (em & F_EXT_MEM_ENABLE) { 1515 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1516 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1517 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1518 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1519 addr + len <= maddr + mlen) 1520 return (0); 1521 } 1522 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1523 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1524 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1525 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1526 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1527 addr + len <= maddr + mlen) 1528 return (0); 1529 } 1530 1531 return (EFAULT); 1532 } 1533 1534 /* 1535 * Verify that the memory range specified by the memtype/offset/len pair is 1536 * valid and lies entirely within the memtype specified. The global address of 1537 * the start of the range is returned in addr. 1538 */ 1539 static int 1540 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1541 uint32_t *addr) 1542 { 1543 uint32_t em, addr_len, maddr, mlen; 1544 1545 /* Memory can only be accessed in naturally aligned 4 byte units */ 1546 if (off & 3 || len & 3 || len == 0) 1547 return (EINVAL); 1548 1549 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1550 switch (mtype) { 1551 case MEM_EDC0: 1552 if (!(em & F_EDRAM0_ENABLE)) 1553 return (EINVAL); 1554 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1555 maddr = G_EDRAM0_BASE(addr_len) << 20; 1556 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1557 break; 1558 case MEM_EDC1: 1559 if (!(em & F_EDRAM1_ENABLE)) 1560 return (EINVAL); 1561 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1562 maddr = G_EDRAM1_BASE(addr_len) << 20; 1563 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1564 break; 1565 case MEM_MC: 1566 if (!(em & F_EXT_MEM_ENABLE)) 1567 return (EINVAL); 1568 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1569 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1570 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1571 break; 1572 case MEM_MC1: 1573 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1574 return (EINVAL); 1575 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1576 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1577 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1578 break; 1579 default: 1580 return (EINVAL); 1581 } 1582 1583 if (mlen > 0 && off < mlen && off + len <= mlen) { 1584 *addr = maddr + off; /* global address */ 1585 return (0); 1586 } 1587 1588 return (EFAULT); 1589 } 1590 1591 static void 1592 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1593 { 1594 const struct memwin *mw; 1595 1596 if (is_t4(sc)) { 1597 KASSERT(win >= 0 && win < nitems(t4_memwin), 1598 ("%s: incorrect memwin# (%d)", __func__, win)); 1599 mw = &t4_memwin[win]; 1600 } else { 1601 KASSERT(win >= 0 && win < nitems(t5_memwin), 1602 ("%s: incorrect memwin# (%d)", __func__, win)); 1603 mw = &t5_memwin[win]; 1604 } 1605 1606 if (base != NULL) 1607 *base = mw->base; 1608 if (aperture != NULL) 1609 *aperture = mw->aperture; 1610 } 1611 1612 /* 1613 * Positions the memory window such that it can be used to access the specified 1614 * address in the chip's address space. The return value is the offset of addr 1615 * from the start of the window. 1616 */ 1617 static uint32_t 1618 position_memwin(struct adapter *sc, int n, uint32_t addr) 1619 { 1620 uint32_t start, pf; 1621 uint32_t reg; 1622 1623 KASSERT(n >= 0 && n <= 3, 1624 ("%s: invalid window %d.", __func__, n)); 1625 KASSERT((addr & 3) == 0, 1626 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1627 1628 if (is_t4(sc)) { 1629 pf = 0; 1630 start = addr & ~0xf; /* start must be 16B aligned */ 1631 } else { 1632 pf = V_PFNUM(sc->pf); 1633 start = addr & ~0x7f; /* start must be 128B aligned */ 1634 } 1635 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1636 1637 t4_write_reg(sc, reg, start | pf); 1638 t4_read_reg(sc, reg); 1639 1640 return (addr - start); 1641 } 1642 1643 static int 1644 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1645 struct intrs_and_queues *iaq) 1646 { 1647 int rc, itype, navail, nrxq10g, nrxq1g, n; 1648 int nofldrxq10g = 0, nofldrxq1g = 0; 1649 1650 bzero(iaq, sizeof(*iaq)); 1651 1652 iaq->ntxq10g = t4_ntxq10g; 1653 iaq->ntxq1g = t4_ntxq1g; 1654 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1655 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1656 #ifdef TCP_OFFLOAD 1657 if (is_offload(sc)) { 1658 iaq->nofldtxq10g = t4_nofldtxq10g; 1659 iaq->nofldtxq1g = t4_nofldtxq1g; 1660 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1661 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1662 } 1663 #endif 1664 1665 for (itype = INTR_MSIX; itype; itype >>= 1) { 1666 1667 if ((itype & t4_intr_types) == 0) 1668 continue; /* not allowed */ 1669 1670 if (itype == INTR_MSIX) 1671 navail = pci_msix_count(sc->dev); 1672 else if (itype == INTR_MSI) 1673 navail = pci_msi_count(sc->dev); 1674 else 1675 navail = 1; 1676 restart: 1677 if (navail == 0) 1678 continue; 1679 1680 iaq->intr_type = itype; 1681 iaq->intr_flags = 0; 1682 1683 /* 1684 * Best option: an interrupt vector for errors, one for the 1685 * firmware event queue, and one each for each rxq (NIC as well 1686 * as offload). 1687 */ 1688 iaq->nirq = T4_EXTRA_INTR; 1689 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1690 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1691 if (iaq->nirq <= navail && 1692 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1693 iaq->intr_flags |= INTR_DIRECT; 1694 goto allocate; 1695 } 1696 1697 /* 1698 * Second best option: an interrupt vector for errors, one for 1699 * the firmware event queue, and one each for either NIC or 1700 * offload rxq's. 1701 */ 1702 iaq->nirq = T4_EXTRA_INTR; 1703 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1704 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1705 if (iaq->nirq <= navail && 1706 (itype != INTR_MSI || powerof2(iaq->nirq))) 1707 goto allocate; 1708 1709 /* 1710 * Next best option: an interrupt vector for errors, one for the 1711 * firmware event queue, and at least one per port. At this 1712 * point we know we'll have to downsize nrxq or nofldrxq to fit 1713 * what's available to us. 1714 */ 1715 iaq->nirq = T4_EXTRA_INTR; 1716 iaq->nirq += n10g + n1g; 1717 if (iaq->nirq <= navail) { 1718 int leftover = navail - iaq->nirq; 1719 1720 if (n10g > 0) { 1721 int target = max(nrxq10g, nofldrxq10g); 1722 1723 n = 1; 1724 while (n < target && leftover >= n10g) { 1725 leftover -= n10g; 1726 iaq->nirq += n10g; 1727 n++; 1728 } 1729 iaq->nrxq10g = min(n, nrxq10g); 1730 #ifdef TCP_OFFLOAD 1731 if (is_offload(sc)) 1732 iaq->nofldrxq10g = min(n, nofldrxq10g); 1733 #endif 1734 } 1735 1736 if (n1g > 0) { 1737 int target = max(nrxq1g, nofldrxq1g); 1738 1739 n = 1; 1740 while (n < target && leftover >= n1g) { 1741 leftover -= n1g; 1742 iaq->nirq += n1g; 1743 n++; 1744 } 1745 iaq->nrxq1g = min(n, nrxq1g); 1746 #ifdef TCP_OFFLOAD 1747 if (is_offload(sc)) 1748 iaq->nofldrxq1g = min(n, nofldrxq1g); 1749 #endif 1750 } 1751 1752 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1753 goto allocate; 1754 } 1755 1756 /* 1757 * Least desirable option: one interrupt vector for everything. 1758 */ 1759 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1760 #ifdef TCP_OFFLOAD 1761 if (is_offload(sc)) 1762 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1763 #endif 1764 1765 allocate: 1766 navail = iaq->nirq; 1767 rc = 0; 1768 if (itype == INTR_MSIX) 1769 rc = pci_alloc_msix(sc->dev, &navail); 1770 else if (itype == INTR_MSI) 1771 rc = pci_alloc_msi(sc->dev, &navail); 1772 1773 if (rc == 0) { 1774 if (navail == iaq->nirq) 1775 return (0); 1776 1777 /* 1778 * Didn't get the number requested. Use whatever number 1779 * the kernel is willing to allocate (it's in navail). 1780 */ 1781 device_printf(sc->dev, "fewer vectors than requested, " 1782 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1783 itype, iaq->nirq, navail); 1784 pci_release_msi(sc->dev); 1785 goto restart; 1786 } 1787 1788 device_printf(sc->dev, 1789 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1790 itype, rc, iaq->nirq, navail); 1791 } 1792 1793 device_printf(sc->dev, 1794 "failed to find a usable interrupt type. " 1795 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1796 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1797 1798 return (ENXIO); 1799 } 1800 1801 #define FW_VERSION(chip) ( \ 1802 V_FW_HDR_FW_VER_MAJOR(FW_VERSION_MAJOR_##chip) | \ 1803 V_FW_HDR_FW_VER_MINOR(FW_VERSION_MINOR_##chip) | \ 1804 V_FW_HDR_FW_VER_MICRO(FW_VERSION_MICRO_##chip) | \ 1805 V_FW_HDR_FW_VER_BUILD(FW_VERSION_BUILD_##chip)) 1806 #define FW_INTFVER(chip, intf) (FW_HDR_INTFVER_##intf) 1807 1808 struct fw_info { 1809 uint8_t chip; 1810 char *kld_name; 1811 char *fw_mod_name; 1812 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1813 } fw_info[] = { 1814 { 1815 .chip = CHELSIO_T4, 1816 .kld_name = "t4fw_cfg", 1817 .fw_mod_name = "t4fw", 1818 .fw_hdr = { 1819 .chip = FW_HDR_CHIP_T4, 1820 .fw_ver = htobe32_const(FW_VERSION(T4)), 1821 .intfver_nic = FW_INTFVER(T4, NIC), 1822 .intfver_vnic = FW_INTFVER(T4, VNIC), 1823 .intfver_ofld = FW_INTFVER(T4, OFLD), 1824 .intfver_ri = FW_INTFVER(T4, RI), 1825 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1826 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1827 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1828 .intfver_fcoe = FW_INTFVER(T4, FCOE), 1829 }, 1830 }, { 1831 .chip = CHELSIO_T5, 1832 .kld_name = "t5fw_cfg", 1833 .fw_mod_name = "t5fw", 1834 .fw_hdr = { 1835 .chip = FW_HDR_CHIP_T5, 1836 .fw_ver = htobe32_const(FW_VERSION(T5)), 1837 .intfver_nic = FW_INTFVER(T5, NIC), 1838 .intfver_vnic = FW_INTFVER(T5, VNIC), 1839 .intfver_ofld = FW_INTFVER(T5, OFLD), 1840 .intfver_ri = FW_INTFVER(T5, RI), 1841 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1842 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1843 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1844 .intfver_fcoe = FW_INTFVER(T5, FCOE), 1845 }, 1846 } 1847 }; 1848 1849 static struct fw_info * 1850 find_fw_info(int chip) 1851 { 1852 int i; 1853 1854 for (i = 0; i < nitems(fw_info); i++) { 1855 if (fw_info[i].chip == chip) 1856 return (&fw_info[i]); 1857 } 1858 return (NULL); 1859 } 1860 1861 /* 1862 * Is the given firmware API compatible with the one the driver was compiled 1863 * with? 1864 */ 1865 static int 1866 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1867 { 1868 1869 /* short circuit if it's the exact same firmware version */ 1870 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1871 return (1); 1872 1873 /* 1874 * XXX: Is this too conservative? Perhaps I should limit this to the 1875 * features that are supported in the driver. 1876 */ 1877 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1878 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1879 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1880 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1881 return (1); 1882 #undef SAME_INTF 1883 1884 return (0); 1885 } 1886 1887 /* 1888 * Establish contact with the firmware and determine if we are the master driver 1889 * or not, and whether we are responsible for chip initialization. 1890 */ 1891 static int 1892 prep_firmware(struct adapter *sc) 1893 { 1894 const struct firmware *fw = NULL, *default_cfg; 1895 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 1896 enum dev_state state; 1897 struct fw_info *fw_info; 1898 struct fw_hdr *card_fw; /* fw on the card */ 1899 const struct fw_hdr *kld_fw; /* fw in the KLD */ 1900 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 1901 against */ 1902 1903 /* Contact firmware. */ 1904 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1905 if (rc < 0 || state == DEV_STATE_ERR) { 1906 rc = -rc; 1907 device_printf(sc->dev, 1908 "failed to connect to the firmware: %d, %d.\n", rc, state); 1909 return (rc); 1910 } 1911 pf = rc; 1912 if (pf == sc->mbox) 1913 sc->flags |= MASTER_PF; 1914 else if (state == DEV_STATE_UNINIT) { 1915 /* 1916 * We didn't get to be the master so we definitely won't be 1917 * configuring the chip. It's a bug if someone else hasn't 1918 * configured it already. 1919 */ 1920 device_printf(sc->dev, "couldn't be master(%d), " 1921 "device not already initialized either(%d).\n", rc, state); 1922 return (EDOOFUS); 1923 } 1924 1925 /* This is the firmware whose headers the driver was compiled against */ 1926 fw_info = find_fw_info(chip_id(sc)); 1927 if (fw_info == NULL) { 1928 device_printf(sc->dev, 1929 "unable to look up firmware information for chip %d.\n", 1930 chip_id(sc)); 1931 return (EINVAL); 1932 } 1933 drv_fw = &fw_info->fw_hdr; 1934 1935 /* 1936 * The firmware KLD contains many modules. The KLD name is also the 1937 * name of the module that contains the default config file. 1938 */ 1939 default_cfg = firmware_get(fw_info->kld_name); 1940 1941 /* Read the header of the firmware on the card */ 1942 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 1943 rc = -t4_read_flash(sc, FLASH_FW_START, 1944 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 1945 if (rc == 0) 1946 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 1947 else { 1948 device_printf(sc->dev, 1949 "Unable to read card's firmware header: %d\n", rc); 1950 card_fw_usable = 0; 1951 } 1952 1953 /* This is the firmware in the KLD */ 1954 fw = firmware_get(fw_info->fw_mod_name); 1955 if (fw != NULL) { 1956 kld_fw = (const void *)fw->data; 1957 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 1958 } else { 1959 kld_fw = NULL; 1960 kld_fw_usable = 0; 1961 } 1962 1963 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 1964 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver || 1965 t4_fw_install == 0)) { 1966 /* 1967 * Common case: the firmware on the card is an exact match and 1968 * the KLD is an exact match too, or the KLD is 1969 * absent/incompatible, or we're prohibited from using it. Note 1970 * that t4_fw_install = 2 is ignored here -- use cxgbetool 1971 * loadfw if you want to reinstall the same firmware as the one 1972 * on the card. 1973 */ 1974 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 1975 (!card_fw_usable || 1976 be32toh(kld_fw->fw_ver) > be32toh(card_fw->fw_ver) || 1977 (t4_fw_install == 2 && kld_fw->fw_ver != card_fw->fw_ver))) { 1978 uint32_t v = ntohl(kld_fw->fw_ver); 1979 1980 device_printf(sc->dev, 1981 "installing firmware %d.%d.%d.%d on card.\n", 1982 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1983 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1984 1985 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1986 if (rc != 0) { 1987 device_printf(sc->dev, 1988 "failed to install firmware: %d\n", rc); 1989 goto done; 1990 } 1991 1992 /* Installed successfully, update the cached header too. */ 1993 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 1994 card_fw_usable = 1; 1995 need_fw_reset = 0; /* already reset as part of load_fw */ 1996 } 1997 1998 if (!card_fw_usable) { 1999 uint32_t d, c, k; 2000 2001 d = ntohl(drv_fw->fw_ver); 2002 c = ntohl(card_fw->fw_ver); 2003 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2004 2005 device_printf(sc->dev, "Cannot find a usable firmware: " 2006 "fw_install %d, chip state %d, " 2007 "driver compiled with %d.%d.%d.%d, " 2008 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2009 t4_fw_install, state, 2010 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2011 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2012 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2013 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2014 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2015 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2016 rc = EINVAL; 2017 goto done; 2018 } 2019 2020 /* We're using whatever's on the card and it's known to be good. */ 2021 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2022 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2023 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2024 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2025 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2026 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2027 2028 /* Reset device */ 2029 if (need_fw_reset && 2030 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2031 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2032 if (rc != ETIMEDOUT && rc != EIO) 2033 t4_fw_bye(sc, sc->mbox); 2034 goto done; 2035 } 2036 sc->flags |= FW_OK; 2037 2038 rc = get_params__pre_init(sc); 2039 if (rc != 0) 2040 goto done; /* error message displayed already */ 2041 2042 /* Partition adapter resources as specified in the config file. */ 2043 if (state == DEV_STATE_UNINIT) { 2044 2045 KASSERT(sc->flags & MASTER_PF, 2046 ("%s: trying to change chip settings when not master.", 2047 __func__)); 2048 2049 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2050 if (rc != 0) 2051 goto done; /* error message displayed already */ 2052 2053 t4_tweak_chip_settings(sc); 2054 2055 /* get basic stuff going */ 2056 rc = -t4_fw_initialize(sc, sc->mbox); 2057 if (rc != 0) { 2058 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2059 goto done; 2060 } 2061 } else { 2062 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2063 sc->cfcsum = 0; 2064 } 2065 2066 done: 2067 free(card_fw, M_CXGBE); 2068 if (fw != NULL) 2069 firmware_put(fw, FIRMWARE_UNLOAD); 2070 if (default_cfg != NULL) 2071 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2072 2073 return (rc); 2074 } 2075 2076 #define FW_PARAM_DEV(param) \ 2077 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2078 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2079 #define FW_PARAM_PFVF(param) \ 2080 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2081 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2082 2083 /* 2084 * Partition chip resources for use between various PFs, VFs, etc. 2085 */ 2086 static int 2087 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2088 const char *name_prefix) 2089 { 2090 const struct firmware *cfg = NULL; 2091 int rc = 0; 2092 struct fw_caps_config_cmd caps; 2093 uint32_t mtype, moff, finicsum, cfcsum; 2094 2095 /* 2096 * Figure out what configuration file to use. Pick the default config 2097 * file for the card if the user hasn't specified one explicitly. 2098 */ 2099 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2100 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2101 /* Card specific overrides go here. */ 2102 if (pci_get_device(sc->dev) == 0x440a) 2103 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2104 } 2105 2106 /* 2107 * We need to load another module if the profile is anything except 2108 * "default" or "flash". 2109 */ 2110 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2111 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2112 char s[32]; 2113 2114 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2115 cfg = firmware_get(s); 2116 if (cfg == NULL) { 2117 device_printf(sc->dev, "unable to load module \"%s\" " 2118 "for configuration profile \"%s\", ", 2119 s, sc->cfg_file); 2120 if (default_cfg != NULL) { 2121 device_printf(sc->dev, "will use the default " 2122 "config file instead.\n"); 2123 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2124 "%s", DEFAULT_CF); 2125 } else { 2126 device_printf(sc->dev, "will use the config " 2127 "file on the card's flash instead.\n"); 2128 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2129 "%s", FLASH_CF); 2130 } 2131 } 2132 } 2133 2134 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2135 default_cfg == NULL) { 2136 device_printf(sc->dev, 2137 "default config file not available, will use the config " 2138 "file on the card's flash instead.\n"); 2139 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2140 } 2141 2142 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2143 u_int cflen, i, n; 2144 const uint32_t *cfdata; 2145 uint32_t param, val, addr, off, mw_base, mw_aperture; 2146 2147 KASSERT(cfg != NULL || default_cfg != NULL, 2148 ("%s: no config to upload", __func__)); 2149 2150 /* 2151 * Ask the firmware where it wants us to upload the config file. 2152 */ 2153 param = FW_PARAM_DEV(CF); 2154 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2155 if (rc != 0) { 2156 /* No support for config file? Shouldn't happen. */ 2157 device_printf(sc->dev, 2158 "failed to query config file location: %d.\n", rc); 2159 goto done; 2160 } 2161 mtype = G_FW_PARAMS_PARAM_Y(val); 2162 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2163 2164 /* 2165 * XXX: sheer laziness. We deliberately added 4 bytes of 2166 * useless stuffing/comments at the end of the config file so 2167 * it's ok to simply throw away the last remaining bytes when 2168 * the config file is not an exact multiple of 4. This also 2169 * helps with the validate_mt_off_len check. 2170 */ 2171 if (cfg != NULL) { 2172 cflen = cfg->datasize & ~3; 2173 cfdata = cfg->data; 2174 } else { 2175 cflen = default_cfg->datasize & ~3; 2176 cfdata = default_cfg->data; 2177 } 2178 2179 if (cflen > FLASH_CFG_MAX_SIZE) { 2180 device_printf(sc->dev, 2181 "config file too long (%d, max allowed is %d). " 2182 "Will try to use the config on the card, if any.\n", 2183 cflen, FLASH_CFG_MAX_SIZE); 2184 goto use_config_on_flash; 2185 } 2186 2187 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2188 if (rc != 0) { 2189 device_printf(sc->dev, 2190 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2191 "Will try to use the config on the card, if any.\n", 2192 __func__, mtype, moff, cflen, rc); 2193 goto use_config_on_flash; 2194 } 2195 2196 memwin_info(sc, 2, &mw_base, &mw_aperture); 2197 while (cflen) { 2198 off = position_memwin(sc, 2, addr); 2199 n = min(cflen, mw_aperture - off); 2200 for (i = 0; i < n; i += 4) 2201 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2202 cflen -= n; 2203 addr += n; 2204 } 2205 } else { 2206 use_config_on_flash: 2207 mtype = FW_MEMTYPE_CF_FLASH; 2208 moff = t4_flash_cfg_addr(sc); 2209 } 2210 2211 bzero(&caps, sizeof(caps)); 2212 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2213 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2214 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2215 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2216 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2217 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2218 if (rc != 0) { 2219 device_printf(sc->dev, 2220 "failed to pre-process config file: %d (mtype %d).\n", rc, 2221 mtype); 2222 goto done; 2223 } 2224 2225 finicsum = be32toh(caps.finicsum); 2226 cfcsum = be32toh(caps.cfcsum); 2227 if (finicsum != cfcsum) { 2228 device_printf(sc->dev, 2229 "WARNING: config file checksum mismatch: %08x %08x\n", 2230 finicsum, cfcsum); 2231 } 2232 sc->cfcsum = cfcsum; 2233 2234 #define LIMIT_CAPS(x) do { \ 2235 caps.x &= htobe16(t4_##x##_allowed); \ 2236 sc->x = htobe16(caps.x); \ 2237 } while (0) 2238 2239 /* 2240 * Let the firmware know what features will (not) be used so it can tune 2241 * things accordingly. 2242 */ 2243 LIMIT_CAPS(linkcaps); 2244 LIMIT_CAPS(niccaps); 2245 LIMIT_CAPS(toecaps); 2246 LIMIT_CAPS(rdmacaps); 2247 LIMIT_CAPS(iscsicaps); 2248 LIMIT_CAPS(fcoecaps); 2249 #undef LIMIT_CAPS 2250 2251 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2252 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2253 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2254 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2255 if (rc != 0) { 2256 device_printf(sc->dev, 2257 "failed to process config file: %d.\n", rc); 2258 } 2259 done: 2260 if (cfg != NULL) 2261 firmware_put(cfg, FIRMWARE_UNLOAD); 2262 return (rc); 2263 } 2264 2265 /* 2266 * Retrieve parameters that are needed (or nice to have) very early. 2267 */ 2268 static int 2269 get_params__pre_init(struct adapter *sc) 2270 { 2271 int rc; 2272 uint32_t param[2], val[2]; 2273 struct fw_devlog_cmd cmd; 2274 struct devlog_params *dlog = &sc->params.devlog; 2275 2276 param[0] = FW_PARAM_DEV(PORTVEC); 2277 param[1] = FW_PARAM_DEV(CCLK); 2278 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2279 if (rc != 0) { 2280 device_printf(sc->dev, 2281 "failed to query parameters (pre_init): %d.\n", rc); 2282 return (rc); 2283 } 2284 2285 sc->params.portvec = val[0]; 2286 sc->params.nports = bitcount32(val[0]); 2287 sc->params.vpd.cclk = val[1]; 2288 2289 /* Read device log parameters. */ 2290 bzero(&cmd, sizeof(cmd)); 2291 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2292 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2293 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2294 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2295 if (rc != 0) { 2296 device_printf(sc->dev, 2297 "failed to get devlog parameters: %d.\n", rc); 2298 bzero(dlog, sizeof (*dlog)); 2299 rc = 0; /* devlog isn't critical for device operation */ 2300 } else { 2301 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2302 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2303 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2304 dlog->size = be32toh(cmd.memsize_devlog); 2305 } 2306 2307 return (rc); 2308 } 2309 2310 /* 2311 * Retrieve various parameters that are of interest to the driver. The device 2312 * has been initialized by the firmware at this point. 2313 */ 2314 static int 2315 get_params__post_init(struct adapter *sc) 2316 { 2317 int rc; 2318 uint32_t param[7], val[7]; 2319 struct fw_caps_config_cmd caps; 2320 2321 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2322 param[1] = FW_PARAM_PFVF(EQ_START); 2323 param[2] = FW_PARAM_PFVF(FILTER_START); 2324 param[3] = FW_PARAM_PFVF(FILTER_END); 2325 param[4] = FW_PARAM_PFVF(L2T_START); 2326 param[5] = FW_PARAM_PFVF(L2T_END); 2327 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2328 if (rc != 0) { 2329 device_printf(sc->dev, 2330 "failed to query parameters (post_init): %d.\n", rc); 2331 return (rc); 2332 } 2333 2334 sc->sge.iq_start = val[0]; 2335 sc->sge.eq_start = val[1]; 2336 sc->tids.ftid_base = val[2]; 2337 sc->tids.nftids = val[3] - val[2] + 1; 2338 sc->vres.l2t.start = val[4]; 2339 sc->vres.l2t.size = val[5] - val[4] + 1; 2340 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2341 ("%s: L2 table size (%u) larger than expected (%u)", 2342 __func__, sc->vres.l2t.size, L2T_SIZE)); 2343 2344 /* get capabilites */ 2345 bzero(&caps, sizeof(caps)); 2346 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2347 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2348 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2349 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2350 if (rc != 0) { 2351 device_printf(sc->dev, 2352 "failed to get card capabilities: %d.\n", rc); 2353 return (rc); 2354 } 2355 2356 if (caps.toecaps) { 2357 /* query offload-related parameters */ 2358 param[0] = FW_PARAM_DEV(NTID); 2359 param[1] = FW_PARAM_PFVF(SERVER_START); 2360 param[2] = FW_PARAM_PFVF(SERVER_END); 2361 param[3] = FW_PARAM_PFVF(TDDP_START); 2362 param[4] = FW_PARAM_PFVF(TDDP_END); 2363 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2364 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2365 if (rc != 0) { 2366 device_printf(sc->dev, 2367 "failed to query TOE parameters: %d.\n", rc); 2368 return (rc); 2369 } 2370 sc->tids.ntids = val[0]; 2371 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2372 sc->tids.stid_base = val[1]; 2373 sc->tids.nstids = val[2] - val[1] + 1; 2374 sc->vres.ddp.start = val[3]; 2375 sc->vres.ddp.size = val[4] - val[3] + 1; 2376 sc->params.ofldq_wr_cred = val[5]; 2377 sc->params.offload = 1; 2378 } 2379 if (caps.rdmacaps) { 2380 param[0] = FW_PARAM_PFVF(STAG_START); 2381 param[1] = FW_PARAM_PFVF(STAG_END); 2382 param[2] = FW_PARAM_PFVF(RQ_START); 2383 param[3] = FW_PARAM_PFVF(RQ_END); 2384 param[4] = FW_PARAM_PFVF(PBL_START); 2385 param[5] = FW_PARAM_PFVF(PBL_END); 2386 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2387 if (rc != 0) { 2388 device_printf(sc->dev, 2389 "failed to query RDMA parameters(1): %d.\n", rc); 2390 return (rc); 2391 } 2392 sc->vres.stag.start = val[0]; 2393 sc->vres.stag.size = val[1] - val[0] + 1; 2394 sc->vres.rq.start = val[2]; 2395 sc->vres.rq.size = val[3] - val[2] + 1; 2396 sc->vres.pbl.start = val[4]; 2397 sc->vres.pbl.size = val[5] - val[4] + 1; 2398 2399 param[0] = FW_PARAM_PFVF(SQRQ_START); 2400 param[1] = FW_PARAM_PFVF(SQRQ_END); 2401 param[2] = FW_PARAM_PFVF(CQ_START); 2402 param[3] = FW_PARAM_PFVF(CQ_END); 2403 param[4] = FW_PARAM_PFVF(OCQ_START); 2404 param[5] = FW_PARAM_PFVF(OCQ_END); 2405 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 2406 if (rc != 0) { 2407 device_printf(sc->dev, 2408 "failed to query RDMA parameters(2): %d.\n", rc); 2409 return (rc); 2410 } 2411 sc->vres.qp.start = val[0]; 2412 sc->vres.qp.size = val[1] - val[0] + 1; 2413 sc->vres.cq.start = val[2]; 2414 sc->vres.cq.size = val[3] - val[2] + 1; 2415 sc->vres.ocq.start = val[4]; 2416 sc->vres.ocq.size = val[5] - val[4] + 1; 2417 } 2418 if (caps.iscsicaps) { 2419 param[0] = FW_PARAM_PFVF(ISCSI_START); 2420 param[1] = FW_PARAM_PFVF(ISCSI_END); 2421 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2422 if (rc != 0) { 2423 device_printf(sc->dev, 2424 "failed to query iSCSI parameters: %d.\n", rc); 2425 return (rc); 2426 } 2427 sc->vres.iscsi.start = val[0]; 2428 sc->vres.iscsi.size = val[1] - val[0] + 1; 2429 } 2430 2431 /* 2432 * We've got the params we wanted to query via the firmware. Now grab 2433 * some others directly from the chip. 2434 */ 2435 rc = t4_read_chip_settings(sc); 2436 2437 return (rc); 2438 } 2439 2440 static int 2441 set_params__post_init(struct adapter *sc) 2442 { 2443 uint32_t param, val; 2444 int rc; 2445 2446 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2447 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2448 if (rc == 0) { 2449 /* ask for encapsulated CPLs */ 2450 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2451 val = 1; 2452 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2453 if (rc != 0) { 2454 device_printf(sc->dev, 2455 "failed to set parameter (post_init): %d.\n", rc); 2456 return (rc); 2457 } 2458 } else if (rc != FW_EINVAL) { 2459 device_printf(sc->dev, 2460 "failed to check for encapsulated CPLs: %d.\n", rc); 2461 } else 2462 rc = 0; /* the firmware doesn't support the param, no worries */ 2463 2464 return (rc); 2465 } 2466 2467 #undef FW_PARAM_PFVF 2468 #undef FW_PARAM_DEV 2469 2470 static void 2471 t4_set_desc(struct adapter *sc) 2472 { 2473 char buf[128]; 2474 struct adapter_params *p = &sc->params; 2475 2476 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2477 p->vpd.id, is_offload(sc) ? "R" : "", chip_rev(sc), p->vpd.sn, 2478 p->vpd.ec); 2479 2480 device_set_desc_copy(sc->dev, buf); 2481 } 2482 2483 static void 2484 build_medialist(struct port_info *pi) 2485 { 2486 struct ifmedia *media = &pi->media; 2487 int data, m; 2488 2489 PORT_LOCK(pi); 2490 2491 ifmedia_removeall(media); 2492 2493 m = IFM_ETHER | IFM_FDX; 2494 data = (pi->port_type << 8) | pi->mod_type; 2495 2496 switch(pi->port_type) { 2497 case FW_PORT_TYPE_BT_XFI: 2498 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2499 break; 2500 2501 case FW_PORT_TYPE_BT_XAUI: 2502 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2503 /* fall through */ 2504 2505 case FW_PORT_TYPE_BT_SGMII: 2506 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2507 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2508 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2509 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2510 break; 2511 2512 case FW_PORT_TYPE_CX4: 2513 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2514 ifmedia_set(media, m | IFM_10G_CX4); 2515 break; 2516 2517 case FW_PORT_TYPE_SFP: 2518 case FW_PORT_TYPE_FIBER_XFI: 2519 case FW_PORT_TYPE_FIBER_XAUI: 2520 switch (pi->mod_type) { 2521 2522 case FW_PORT_MOD_TYPE_LR: 2523 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2524 ifmedia_set(media, m | IFM_10G_LR); 2525 break; 2526 2527 case FW_PORT_MOD_TYPE_SR: 2528 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2529 ifmedia_set(media, m | IFM_10G_SR); 2530 break; 2531 2532 case FW_PORT_MOD_TYPE_LRM: 2533 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2534 ifmedia_set(media, m | IFM_10G_LRM); 2535 break; 2536 2537 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2538 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2539 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2540 ifmedia_set(media, m | IFM_10G_TWINAX); 2541 break; 2542 2543 case FW_PORT_MOD_TYPE_NONE: 2544 m &= ~IFM_FDX; 2545 ifmedia_add(media, m | IFM_NONE, data, NULL); 2546 ifmedia_set(media, m | IFM_NONE); 2547 break; 2548 2549 case FW_PORT_MOD_TYPE_NA: 2550 case FW_PORT_MOD_TYPE_ER: 2551 default: 2552 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2553 ifmedia_set(media, m | IFM_UNKNOWN); 2554 break; 2555 } 2556 break; 2557 2558 case FW_PORT_TYPE_KX4: 2559 case FW_PORT_TYPE_KX: 2560 case FW_PORT_TYPE_KR: 2561 default: 2562 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2563 ifmedia_set(media, m | IFM_UNKNOWN); 2564 break; 2565 } 2566 2567 PORT_UNLOCK(pi); 2568 } 2569 2570 #define FW_MAC_EXACT_CHUNK 7 2571 2572 /* 2573 * Program the port's XGMAC based on parameters in ifnet. The caller also 2574 * indicates which parameters should be programmed (the rest are left alone). 2575 */ 2576 static int 2577 update_mac_settings(struct port_info *pi, int flags) 2578 { 2579 int rc; 2580 struct ifnet *ifp = pi->ifp; 2581 struct adapter *sc = pi->adapter; 2582 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2583 2584 ASSERT_SYNCHRONIZED_OP(sc); 2585 KASSERT(flags, ("%s: not told what to update.", __func__)); 2586 2587 if (flags & XGMAC_MTU) 2588 mtu = ifp->if_mtu; 2589 2590 if (flags & XGMAC_PROMISC) 2591 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2592 2593 if (flags & XGMAC_ALLMULTI) 2594 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2595 2596 if (flags & XGMAC_VLANEX) 2597 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2598 2599 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2600 vlanex, false); 2601 if (rc) { 2602 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2603 return (rc); 2604 } 2605 2606 if (flags & XGMAC_UCADDR) { 2607 uint8_t ucaddr[ETHER_ADDR_LEN]; 2608 2609 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2610 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2611 ucaddr, true, true); 2612 if (rc < 0) { 2613 rc = -rc; 2614 if_printf(ifp, "change_mac failed: %d\n", rc); 2615 return (rc); 2616 } else { 2617 pi->xact_addr_filt = rc; 2618 rc = 0; 2619 } 2620 } 2621 2622 if (flags & XGMAC_MCADDRS) { 2623 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2624 int del = 1; 2625 uint64_t hash = 0; 2626 struct ifmultiaddr *ifma; 2627 int i = 0, j; 2628 2629 if_maddr_rlock(ifp); 2630 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2631 if (ifma->ifma_addr->sa_family != AF_LINK) 2632 continue; 2633 mcaddr[i++] = 2634 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2635 2636 if (i == FW_MAC_EXACT_CHUNK) { 2637 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2638 del, i, mcaddr, NULL, &hash, 0); 2639 if (rc < 0) { 2640 rc = -rc; 2641 for (j = 0; j < i; j++) { 2642 if_printf(ifp, 2643 "failed to add mc address" 2644 " %02x:%02x:%02x:" 2645 "%02x:%02x:%02x rc=%d\n", 2646 mcaddr[j][0], mcaddr[j][1], 2647 mcaddr[j][2], mcaddr[j][3], 2648 mcaddr[j][4], mcaddr[j][5], 2649 rc); 2650 } 2651 goto mcfail; 2652 } 2653 del = 0; 2654 i = 0; 2655 } 2656 } 2657 if (i > 0) { 2658 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2659 del, i, mcaddr, NULL, &hash, 0); 2660 if (rc < 0) { 2661 rc = -rc; 2662 for (j = 0; j < i; j++) { 2663 if_printf(ifp, 2664 "failed to add mc address" 2665 " %02x:%02x:%02x:" 2666 "%02x:%02x:%02x rc=%d\n", 2667 mcaddr[j][0], mcaddr[j][1], 2668 mcaddr[j][2], mcaddr[j][3], 2669 mcaddr[j][4], mcaddr[j][5], 2670 rc); 2671 } 2672 goto mcfail; 2673 } 2674 } 2675 2676 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2677 if (rc != 0) 2678 if_printf(ifp, "failed to set mc address hash: %d", rc); 2679 mcfail: 2680 if_maddr_runlock(ifp); 2681 } 2682 2683 return (rc); 2684 } 2685 2686 int 2687 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2688 char *wmesg) 2689 { 2690 int rc, pri; 2691 2692 #ifdef WITNESS 2693 /* the caller thinks it's ok to sleep, but is it really? */ 2694 if (flags & SLEEP_OK) 2695 pause("t4slptst", 1); 2696 #endif 2697 2698 if (INTR_OK) 2699 pri = PCATCH; 2700 else 2701 pri = 0; 2702 2703 ADAPTER_LOCK(sc); 2704 for (;;) { 2705 2706 if (pi && IS_DOOMED(pi)) { 2707 rc = ENXIO; 2708 goto done; 2709 } 2710 2711 if (!IS_BUSY(sc)) { 2712 rc = 0; 2713 break; 2714 } 2715 2716 if (!(flags & SLEEP_OK)) { 2717 rc = EBUSY; 2718 goto done; 2719 } 2720 2721 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2722 rc = EINTR; 2723 goto done; 2724 } 2725 } 2726 2727 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2728 SET_BUSY(sc); 2729 #ifdef INVARIANTS 2730 sc->last_op = wmesg; 2731 sc->last_op_thr = curthread; 2732 #endif 2733 2734 done: 2735 if (!(flags & HOLD_LOCK) || rc) 2736 ADAPTER_UNLOCK(sc); 2737 2738 return (rc); 2739 } 2740 2741 void 2742 end_synchronized_op(struct adapter *sc, int flags) 2743 { 2744 2745 if (flags & LOCK_HELD) 2746 ADAPTER_LOCK_ASSERT_OWNED(sc); 2747 else 2748 ADAPTER_LOCK(sc); 2749 2750 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2751 CLR_BUSY(sc); 2752 wakeup(&sc->flags); 2753 ADAPTER_UNLOCK(sc); 2754 } 2755 2756 static int 2757 cxgbe_init_synchronized(struct port_info *pi) 2758 { 2759 struct adapter *sc = pi->adapter; 2760 struct ifnet *ifp = pi->ifp; 2761 int rc = 0; 2762 2763 ASSERT_SYNCHRONIZED_OP(sc); 2764 2765 if (isset(&sc->open_device_map, pi->port_id)) { 2766 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2767 ("mismatch between open_device_map and if_drv_flags")); 2768 return (0); /* already running */ 2769 } 2770 2771 if (!(sc->flags & FULL_INIT_DONE) && 2772 ((rc = adapter_full_init(sc)) != 0)) 2773 return (rc); /* error message displayed already */ 2774 2775 if (!(pi->flags & PORT_INIT_DONE) && 2776 ((rc = port_full_init(pi)) != 0)) 2777 return (rc); /* error message displayed already */ 2778 2779 rc = update_mac_settings(pi, XGMAC_ALL); 2780 if (rc) 2781 goto done; /* error message displayed already */ 2782 2783 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2784 if (rc != 0) { 2785 if_printf(ifp, "start_link failed: %d\n", rc); 2786 goto done; 2787 } 2788 2789 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2790 if (rc != 0) { 2791 if_printf(ifp, "enable_vi failed: %d\n", rc); 2792 goto done; 2793 } 2794 2795 /* all ok */ 2796 setbit(&sc->open_device_map, pi->port_id); 2797 PORT_LOCK(pi); 2798 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2799 PORT_UNLOCK(pi); 2800 2801 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2802 done: 2803 if (rc != 0) 2804 cxgbe_uninit_synchronized(pi); 2805 2806 return (rc); 2807 } 2808 2809 /* 2810 * Idempotent. 2811 */ 2812 static int 2813 cxgbe_uninit_synchronized(struct port_info *pi) 2814 { 2815 struct adapter *sc = pi->adapter; 2816 struct ifnet *ifp = pi->ifp; 2817 int rc; 2818 2819 ASSERT_SYNCHRONIZED_OP(sc); 2820 2821 /* 2822 * Disable the VI so that all its data in either direction is discarded 2823 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2824 * tick) intact as the TP can deliver negative advice or data that it's 2825 * holding in its RAM (for an offloaded connection) even after the VI is 2826 * disabled. 2827 */ 2828 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2829 if (rc) { 2830 if_printf(ifp, "disable_vi failed: %d\n", rc); 2831 return (rc); 2832 } 2833 2834 clrbit(&sc->open_device_map, pi->port_id); 2835 PORT_LOCK(pi); 2836 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2837 PORT_UNLOCK(pi); 2838 2839 pi->link_cfg.link_ok = 0; 2840 pi->link_cfg.speed = 0; 2841 t4_os_link_changed(sc, pi->port_id, 0); 2842 2843 return (0); 2844 } 2845 2846 /* 2847 * It is ok for this function to fail midway and return right away. t4_detach 2848 * will walk the entire sc->irq list and clean up whatever is valid. 2849 */ 2850 static int 2851 setup_intr_handlers(struct adapter *sc) 2852 { 2853 int rc, rid, p, q; 2854 char s[8]; 2855 struct irq *irq; 2856 struct port_info *pi; 2857 struct sge_rxq *rxq; 2858 #ifdef TCP_OFFLOAD 2859 struct sge_ofld_rxq *ofld_rxq; 2860 #endif 2861 2862 /* 2863 * Setup interrupts. 2864 */ 2865 irq = &sc->irq[0]; 2866 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2867 if (sc->intr_count == 1) { 2868 KASSERT(!(sc->flags & INTR_DIRECT), 2869 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2870 2871 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2872 if (rc != 0) 2873 return (rc); 2874 } else { 2875 /* Multiple interrupts. */ 2876 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2877 ("%s: too few intr.", __func__)); 2878 2879 /* The first one is always error intr */ 2880 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2881 if (rc != 0) 2882 return (rc); 2883 irq++; 2884 rid++; 2885 2886 /* The second one is always the firmware event queue */ 2887 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2888 "evt"); 2889 if (rc != 0) 2890 return (rc); 2891 irq++; 2892 rid++; 2893 2894 /* 2895 * Note that if INTR_DIRECT is not set then either the NIC rx 2896 * queues or (exclusive or) the TOE rx queueus will be taking 2897 * direct interrupts. 2898 * 2899 * There is no need to check for is_offload(sc) as nofldrxq 2900 * will be 0 if offload is disabled. 2901 */ 2902 for_each_port(sc, p) { 2903 pi = sc->port[p]; 2904 2905 #ifdef TCP_OFFLOAD 2906 /* 2907 * Skip over the NIC queues if they aren't taking direct 2908 * interrupts. 2909 */ 2910 if (!(sc->flags & INTR_DIRECT) && 2911 pi->nofldrxq > pi->nrxq) 2912 goto ofld_queues; 2913 #endif 2914 rxq = &sc->sge.rxq[pi->first_rxq]; 2915 for (q = 0; q < pi->nrxq; q++, rxq++) { 2916 snprintf(s, sizeof(s), "%d.%d", p, q); 2917 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2918 s); 2919 if (rc != 0) 2920 return (rc); 2921 irq++; 2922 rid++; 2923 } 2924 2925 #ifdef TCP_OFFLOAD 2926 /* 2927 * Skip over the offload queues if they aren't taking 2928 * direct interrupts. 2929 */ 2930 if (!(sc->flags & INTR_DIRECT)) 2931 continue; 2932 ofld_queues: 2933 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2934 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2935 snprintf(s, sizeof(s), "%d,%d", p, q); 2936 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2937 ofld_rxq, s); 2938 if (rc != 0) 2939 return (rc); 2940 irq++; 2941 rid++; 2942 } 2943 #endif 2944 } 2945 } 2946 2947 return (0); 2948 } 2949 2950 static int 2951 adapter_full_init(struct adapter *sc) 2952 { 2953 int rc, i; 2954 2955 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2956 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2957 ("%s: FULL_INIT_DONE already", __func__)); 2958 2959 /* 2960 * queues that belong to the adapter (not any particular port). 2961 */ 2962 rc = t4_setup_adapter_queues(sc); 2963 if (rc != 0) 2964 goto done; 2965 2966 for (i = 0; i < nitems(sc->tq); i++) { 2967 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2968 taskqueue_thread_enqueue, &sc->tq[i]); 2969 if (sc->tq[i] == NULL) { 2970 device_printf(sc->dev, 2971 "failed to allocate task queue %d\n", i); 2972 rc = ENOMEM; 2973 goto done; 2974 } 2975 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2976 device_get_nameunit(sc->dev), i); 2977 } 2978 2979 t4_intr_enable(sc); 2980 sc->flags |= FULL_INIT_DONE; 2981 done: 2982 if (rc != 0) 2983 adapter_full_uninit(sc); 2984 2985 return (rc); 2986 } 2987 2988 static int 2989 adapter_full_uninit(struct adapter *sc) 2990 { 2991 int i; 2992 2993 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2994 2995 t4_teardown_adapter_queues(sc); 2996 2997 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 2998 taskqueue_free(sc->tq[i]); 2999 sc->tq[i] = NULL; 3000 } 3001 3002 sc->flags &= ~FULL_INIT_DONE; 3003 3004 return (0); 3005 } 3006 3007 static int 3008 port_full_init(struct port_info *pi) 3009 { 3010 struct adapter *sc = pi->adapter; 3011 struct ifnet *ifp = pi->ifp; 3012 uint16_t *rss; 3013 struct sge_rxq *rxq; 3014 int rc, i; 3015 3016 ASSERT_SYNCHRONIZED_OP(sc); 3017 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3018 ("%s: PORT_INIT_DONE already", __func__)); 3019 3020 sysctl_ctx_init(&pi->ctx); 3021 pi->flags |= PORT_SYSCTL_CTX; 3022 3023 /* 3024 * Allocate tx/rx/fl queues for this port. 3025 */ 3026 rc = t4_setup_port_queues(pi); 3027 if (rc != 0) 3028 goto done; /* error message displayed already */ 3029 3030 /* 3031 * Setup RSS for this port. 3032 */ 3033 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 3034 M_ZERO | M_WAITOK); 3035 for_each_rxq(pi, i, rxq) { 3036 rss[i] = rxq->iq.abs_id; 3037 } 3038 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 3039 pi->rss_size, rss, pi->nrxq); 3040 free(rss, M_CXGBE); 3041 if (rc != 0) { 3042 if_printf(ifp, "rss_config failed: %d\n", rc); 3043 goto done; 3044 } 3045 3046 pi->flags |= PORT_INIT_DONE; 3047 done: 3048 if (rc != 0) 3049 port_full_uninit(pi); 3050 3051 return (rc); 3052 } 3053 3054 /* 3055 * Idempotent. 3056 */ 3057 static int 3058 port_full_uninit(struct port_info *pi) 3059 { 3060 struct adapter *sc = pi->adapter; 3061 int i; 3062 struct sge_rxq *rxq; 3063 struct sge_txq *txq; 3064 #ifdef TCP_OFFLOAD 3065 struct sge_ofld_rxq *ofld_rxq; 3066 struct sge_wrq *ofld_txq; 3067 #endif 3068 3069 if (pi->flags & PORT_INIT_DONE) { 3070 3071 /* Need to quiesce queues. XXX: ctrl queues? */ 3072 3073 for_each_txq(pi, i, txq) { 3074 quiesce_eq(sc, &txq->eq); 3075 } 3076 3077 #ifdef TCP_OFFLOAD 3078 for_each_ofld_txq(pi, i, ofld_txq) { 3079 quiesce_eq(sc, &ofld_txq->eq); 3080 } 3081 #endif 3082 3083 for_each_rxq(pi, i, rxq) { 3084 quiesce_iq(sc, &rxq->iq); 3085 quiesce_fl(sc, &rxq->fl); 3086 } 3087 3088 #ifdef TCP_OFFLOAD 3089 for_each_ofld_rxq(pi, i, ofld_rxq) { 3090 quiesce_iq(sc, &ofld_rxq->iq); 3091 quiesce_fl(sc, &ofld_rxq->fl); 3092 } 3093 #endif 3094 } 3095 3096 t4_teardown_port_queues(pi); 3097 pi->flags &= ~PORT_INIT_DONE; 3098 3099 return (0); 3100 } 3101 3102 static void 3103 quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3104 { 3105 EQ_LOCK(eq); 3106 eq->flags |= EQ_DOOMED; 3107 3108 /* 3109 * Wait for the response to a credit flush if one's 3110 * pending. 3111 */ 3112 while (eq->flags & EQ_CRFLUSHED) 3113 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3114 EQ_UNLOCK(eq); 3115 3116 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3117 pause("callout", 10); /* Still iffy */ 3118 3119 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3120 } 3121 3122 static void 3123 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3124 { 3125 (void) sc; /* unused */ 3126 3127 /* Synchronize with the interrupt handler */ 3128 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3129 pause("iqfree", 1); 3130 } 3131 3132 static void 3133 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3134 { 3135 mtx_lock(&sc->sfl_lock); 3136 FL_LOCK(fl); 3137 fl->flags |= FL_DOOMED; 3138 FL_UNLOCK(fl); 3139 mtx_unlock(&sc->sfl_lock); 3140 3141 callout_drain(&sc->sfl_callout); 3142 KASSERT((fl->flags & FL_STARVING) == 0, 3143 ("%s: still starving", __func__)); 3144 } 3145 3146 static int 3147 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3148 driver_intr_t *handler, void *arg, char *name) 3149 { 3150 int rc; 3151 3152 irq->rid = rid; 3153 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3154 RF_SHAREABLE | RF_ACTIVE); 3155 if (irq->res == NULL) { 3156 device_printf(sc->dev, 3157 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3158 return (ENOMEM); 3159 } 3160 3161 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3162 NULL, handler, arg, &irq->tag); 3163 if (rc != 0) { 3164 device_printf(sc->dev, 3165 "failed to setup interrupt for rid %d, name %s: %d\n", 3166 rid, name, rc); 3167 } else if (name) 3168 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3169 3170 return (rc); 3171 } 3172 3173 static int 3174 t4_free_irq(struct adapter *sc, struct irq *irq) 3175 { 3176 if (irq->tag) 3177 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3178 if (irq->res) 3179 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3180 3181 bzero(irq, sizeof(*irq)); 3182 3183 return (0); 3184 } 3185 3186 static void 3187 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3188 unsigned int end) 3189 { 3190 uint32_t *p = (uint32_t *)(buf + start); 3191 3192 for ( ; start <= end; start += sizeof(uint32_t)) 3193 *p++ = t4_read_reg(sc, start); 3194 } 3195 3196 static void 3197 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3198 { 3199 int i, n; 3200 const unsigned int *reg_ranges; 3201 static const unsigned int t4_reg_ranges[] = { 3202 0x1008, 0x1108, 3203 0x1180, 0x11b4, 3204 0x11fc, 0x123c, 3205 0x1300, 0x173c, 3206 0x1800, 0x18fc, 3207 0x3000, 0x30d8, 3208 0x30e0, 0x5924, 3209 0x5960, 0x59d4, 3210 0x5a00, 0x5af8, 3211 0x6000, 0x6098, 3212 0x6100, 0x6150, 3213 0x6200, 0x6208, 3214 0x6240, 0x6248, 3215 0x6280, 0x6338, 3216 0x6370, 0x638c, 3217 0x6400, 0x643c, 3218 0x6500, 0x6524, 3219 0x6a00, 0x6a38, 3220 0x6a60, 0x6a78, 3221 0x6b00, 0x6b84, 3222 0x6bf0, 0x6c84, 3223 0x6cf0, 0x6d84, 3224 0x6df0, 0x6e84, 3225 0x6ef0, 0x6f84, 3226 0x6ff0, 0x7084, 3227 0x70f0, 0x7184, 3228 0x71f0, 0x7284, 3229 0x72f0, 0x7384, 3230 0x73f0, 0x7450, 3231 0x7500, 0x7530, 3232 0x7600, 0x761c, 3233 0x7680, 0x76cc, 3234 0x7700, 0x7798, 3235 0x77c0, 0x77fc, 3236 0x7900, 0x79fc, 3237 0x7b00, 0x7c38, 3238 0x7d00, 0x7efc, 3239 0x8dc0, 0x8e1c, 3240 0x8e30, 0x8e78, 3241 0x8ea0, 0x8f6c, 3242 0x8fc0, 0x9074, 3243 0x90fc, 0x90fc, 3244 0x9400, 0x9458, 3245 0x9600, 0x96bc, 3246 0x9800, 0x9808, 3247 0x9820, 0x983c, 3248 0x9850, 0x9864, 3249 0x9c00, 0x9c6c, 3250 0x9c80, 0x9cec, 3251 0x9d00, 0x9d6c, 3252 0x9d80, 0x9dec, 3253 0x9e00, 0x9e6c, 3254 0x9e80, 0x9eec, 3255 0x9f00, 0x9f6c, 3256 0x9f80, 0x9fec, 3257 0xd004, 0xd03c, 3258 0xdfc0, 0xdfe0, 3259 0xe000, 0xea7c, 3260 0xf000, 0x11190, 3261 0x19040, 0x1906c, 3262 0x19078, 0x19080, 3263 0x1908c, 0x19124, 3264 0x19150, 0x191b0, 3265 0x191d0, 0x191e8, 3266 0x19238, 0x1924c, 3267 0x193f8, 0x19474, 3268 0x19490, 0x194f8, 3269 0x19800, 0x19f30, 3270 0x1a000, 0x1a06c, 3271 0x1a0b0, 0x1a120, 3272 0x1a128, 0x1a138, 3273 0x1a190, 0x1a1c4, 3274 0x1a1fc, 0x1a1fc, 3275 0x1e040, 0x1e04c, 3276 0x1e284, 0x1e28c, 3277 0x1e2c0, 0x1e2c0, 3278 0x1e2e0, 0x1e2e0, 3279 0x1e300, 0x1e384, 3280 0x1e3c0, 0x1e3c8, 3281 0x1e440, 0x1e44c, 3282 0x1e684, 0x1e68c, 3283 0x1e6c0, 0x1e6c0, 3284 0x1e6e0, 0x1e6e0, 3285 0x1e700, 0x1e784, 3286 0x1e7c0, 0x1e7c8, 3287 0x1e840, 0x1e84c, 3288 0x1ea84, 0x1ea8c, 3289 0x1eac0, 0x1eac0, 3290 0x1eae0, 0x1eae0, 3291 0x1eb00, 0x1eb84, 3292 0x1ebc0, 0x1ebc8, 3293 0x1ec40, 0x1ec4c, 3294 0x1ee84, 0x1ee8c, 3295 0x1eec0, 0x1eec0, 3296 0x1eee0, 0x1eee0, 3297 0x1ef00, 0x1ef84, 3298 0x1efc0, 0x1efc8, 3299 0x1f040, 0x1f04c, 3300 0x1f284, 0x1f28c, 3301 0x1f2c0, 0x1f2c0, 3302 0x1f2e0, 0x1f2e0, 3303 0x1f300, 0x1f384, 3304 0x1f3c0, 0x1f3c8, 3305 0x1f440, 0x1f44c, 3306 0x1f684, 0x1f68c, 3307 0x1f6c0, 0x1f6c0, 3308 0x1f6e0, 0x1f6e0, 3309 0x1f700, 0x1f784, 3310 0x1f7c0, 0x1f7c8, 3311 0x1f840, 0x1f84c, 3312 0x1fa84, 0x1fa8c, 3313 0x1fac0, 0x1fac0, 3314 0x1fae0, 0x1fae0, 3315 0x1fb00, 0x1fb84, 3316 0x1fbc0, 0x1fbc8, 3317 0x1fc40, 0x1fc4c, 3318 0x1fe84, 0x1fe8c, 3319 0x1fec0, 0x1fec0, 3320 0x1fee0, 0x1fee0, 3321 0x1ff00, 0x1ff84, 3322 0x1ffc0, 0x1ffc8, 3323 0x20000, 0x2002c, 3324 0x20100, 0x2013c, 3325 0x20190, 0x201c8, 3326 0x20200, 0x20318, 3327 0x20400, 0x20528, 3328 0x20540, 0x20614, 3329 0x21000, 0x21040, 3330 0x2104c, 0x21060, 3331 0x210c0, 0x210ec, 3332 0x21200, 0x21268, 3333 0x21270, 0x21284, 3334 0x212fc, 0x21388, 3335 0x21400, 0x21404, 3336 0x21500, 0x21518, 3337 0x2152c, 0x2153c, 3338 0x21550, 0x21554, 3339 0x21600, 0x21600, 3340 0x21608, 0x21628, 3341 0x21630, 0x2163c, 3342 0x21700, 0x2171c, 3343 0x21780, 0x2178c, 3344 0x21800, 0x21c38, 3345 0x21c80, 0x21d7c, 3346 0x21e00, 0x21e04, 3347 0x22000, 0x2202c, 3348 0x22100, 0x2213c, 3349 0x22190, 0x221c8, 3350 0x22200, 0x22318, 3351 0x22400, 0x22528, 3352 0x22540, 0x22614, 3353 0x23000, 0x23040, 3354 0x2304c, 0x23060, 3355 0x230c0, 0x230ec, 3356 0x23200, 0x23268, 3357 0x23270, 0x23284, 3358 0x232fc, 0x23388, 3359 0x23400, 0x23404, 3360 0x23500, 0x23518, 3361 0x2352c, 0x2353c, 3362 0x23550, 0x23554, 3363 0x23600, 0x23600, 3364 0x23608, 0x23628, 3365 0x23630, 0x2363c, 3366 0x23700, 0x2371c, 3367 0x23780, 0x2378c, 3368 0x23800, 0x23c38, 3369 0x23c80, 0x23d7c, 3370 0x23e00, 0x23e04, 3371 0x24000, 0x2402c, 3372 0x24100, 0x2413c, 3373 0x24190, 0x241c8, 3374 0x24200, 0x24318, 3375 0x24400, 0x24528, 3376 0x24540, 0x24614, 3377 0x25000, 0x25040, 3378 0x2504c, 0x25060, 3379 0x250c0, 0x250ec, 3380 0x25200, 0x25268, 3381 0x25270, 0x25284, 3382 0x252fc, 0x25388, 3383 0x25400, 0x25404, 3384 0x25500, 0x25518, 3385 0x2552c, 0x2553c, 3386 0x25550, 0x25554, 3387 0x25600, 0x25600, 3388 0x25608, 0x25628, 3389 0x25630, 0x2563c, 3390 0x25700, 0x2571c, 3391 0x25780, 0x2578c, 3392 0x25800, 0x25c38, 3393 0x25c80, 0x25d7c, 3394 0x25e00, 0x25e04, 3395 0x26000, 0x2602c, 3396 0x26100, 0x2613c, 3397 0x26190, 0x261c8, 3398 0x26200, 0x26318, 3399 0x26400, 0x26528, 3400 0x26540, 0x26614, 3401 0x27000, 0x27040, 3402 0x2704c, 0x27060, 3403 0x270c0, 0x270ec, 3404 0x27200, 0x27268, 3405 0x27270, 0x27284, 3406 0x272fc, 0x27388, 3407 0x27400, 0x27404, 3408 0x27500, 0x27518, 3409 0x2752c, 0x2753c, 3410 0x27550, 0x27554, 3411 0x27600, 0x27600, 3412 0x27608, 0x27628, 3413 0x27630, 0x2763c, 3414 0x27700, 0x2771c, 3415 0x27780, 0x2778c, 3416 0x27800, 0x27c38, 3417 0x27c80, 0x27d7c, 3418 0x27e00, 0x27e04 3419 }; 3420 static const unsigned int t5_reg_ranges[] = { 3421 0x1008, 0x1148, 3422 0x1180, 0x11b4, 3423 0x11fc, 0x123c, 3424 0x1280, 0x173c, 3425 0x1800, 0x18fc, 3426 0x3000, 0x3028, 3427 0x3060, 0x30d8, 3428 0x30e0, 0x30fc, 3429 0x3140, 0x357c, 3430 0x35a8, 0x35cc, 3431 0x35ec, 0x35ec, 3432 0x3600, 0x5624, 3433 0x56cc, 0x575c, 3434 0x580c, 0x5814, 3435 0x5890, 0x58bc, 3436 0x5940, 0x59dc, 3437 0x59fc, 0x5a18, 3438 0x5a60, 0x5a9c, 3439 0x5b94, 0x5bfc, 3440 0x6000, 0x6040, 3441 0x6058, 0x614c, 3442 0x7700, 0x7798, 3443 0x77c0, 0x78fc, 3444 0x7b00, 0x7c54, 3445 0x7d00, 0x7efc, 3446 0x8dc0, 0x8de0, 3447 0x8df8, 0x8e84, 3448 0x8ea0, 0x8f84, 3449 0x8fc0, 0x90f8, 3450 0x9400, 0x9470, 3451 0x9600, 0x96f4, 3452 0x9800, 0x9808, 3453 0x9820, 0x983c, 3454 0x9850, 0x9864, 3455 0x9c00, 0x9c6c, 3456 0x9c80, 0x9cec, 3457 0x9d00, 0x9d6c, 3458 0x9d80, 0x9dec, 3459 0x9e00, 0x9e6c, 3460 0x9e80, 0x9eec, 3461 0x9f00, 0x9f6c, 3462 0x9f80, 0xa020, 3463 0xd004, 0xd03c, 3464 0xdfc0, 0xdfe0, 3465 0xe000, 0x11088, 3466 0x1109c, 0x1117c, 3467 0x11190, 0x11204, 3468 0x19040, 0x1906c, 3469 0x19078, 0x19080, 3470 0x1908c, 0x19124, 3471 0x19150, 0x191b0, 3472 0x191d0, 0x191e8, 3473 0x19238, 0x19290, 3474 0x193f8, 0x19474, 3475 0x19490, 0x194cc, 3476 0x194f0, 0x194f8, 3477 0x19c00, 0x19c60, 3478 0x19c94, 0x19e10, 3479 0x19e50, 0x19f34, 3480 0x19f40, 0x19f50, 3481 0x19f90, 0x19fe4, 3482 0x1a000, 0x1a06c, 3483 0x1a0b0, 0x1a120, 3484 0x1a128, 0x1a138, 3485 0x1a190, 0x1a1c4, 3486 0x1a1fc, 0x1a1fc, 3487 0x1e008, 0x1e00c, 3488 0x1e040, 0x1e04c, 3489 0x1e284, 0x1e290, 3490 0x1e2c0, 0x1e2c0, 3491 0x1e2e0, 0x1e2e0, 3492 0x1e300, 0x1e384, 3493 0x1e3c0, 0x1e3c8, 3494 0x1e408, 0x1e40c, 3495 0x1e440, 0x1e44c, 3496 0x1e684, 0x1e690, 3497 0x1e6c0, 0x1e6c0, 3498 0x1e6e0, 0x1e6e0, 3499 0x1e700, 0x1e784, 3500 0x1e7c0, 0x1e7c8, 3501 0x1e808, 0x1e80c, 3502 0x1e840, 0x1e84c, 3503 0x1ea84, 0x1ea90, 3504 0x1eac0, 0x1eac0, 3505 0x1eae0, 0x1eae0, 3506 0x1eb00, 0x1eb84, 3507 0x1ebc0, 0x1ebc8, 3508 0x1ec08, 0x1ec0c, 3509 0x1ec40, 0x1ec4c, 3510 0x1ee84, 0x1ee90, 3511 0x1eec0, 0x1eec0, 3512 0x1eee0, 0x1eee0, 3513 0x1ef00, 0x1ef84, 3514 0x1efc0, 0x1efc8, 3515 0x1f008, 0x1f00c, 3516 0x1f040, 0x1f04c, 3517 0x1f284, 0x1f290, 3518 0x1f2c0, 0x1f2c0, 3519 0x1f2e0, 0x1f2e0, 3520 0x1f300, 0x1f384, 3521 0x1f3c0, 0x1f3c8, 3522 0x1f408, 0x1f40c, 3523 0x1f440, 0x1f44c, 3524 0x1f684, 0x1f690, 3525 0x1f6c0, 0x1f6c0, 3526 0x1f6e0, 0x1f6e0, 3527 0x1f700, 0x1f784, 3528 0x1f7c0, 0x1f7c8, 3529 0x1f808, 0x1f80c, 3530 0x1f840, 0x1f84c, 3531 0x1fa84, 0x1fa90, 3532 0x1fac0, 0x1fac0, 3533 0x1fae0, 0x1fae0, 3534 0x1fb00, 0x1fb84, 3535 0x1fbc0, 0x1fbc8, 3536 0x1fc08, 0x1fc0c, 3537 0x1fc40, 0x1fc4c, 3538 0x1fe84, 0x1fe90, 3539 0x1fec0, 0x1fec0, 3540 0x1fee0, 0x1fee0, 3541 0x1ff00, 0x1ff84, 3542 0x1ffc0, 0x1ffc8, 3543 0x30000, 0x30040, 3544 0x30100, 0x30144, 3545 0x30190, 0x301d0, 3546 0x30200, 0x30318, 3547 0x30400, 0x3052c, 3548 0x30540, 0x3061c, 3549 0x30800, 0x30834, 3550 0x308c0, 0x30908, 3551 0x30910, 0x309ac, 3552 0x30a00, 0x30a04, 3553 0x30a0c, 0x30a2c, 3554 0x30a44, 0x30a50, 3555 0x30a74, 0x30c24, 3556 0x30d08, 0x30d14, 3557 0x30d1c, 0x30d20, 3558 0x30d3c, 0x30d50, 3559 0x31200, 0x3120c, 3560 0x31220, 0x31220, 3561 0x31240, 0x31240, 3562 0x31600, 0x31600, 3563 0x31608, 0x3160c, 3564 0x31a00, 0x31a1c, 3565 0x31e04, 0x31e20, 3566 0x31e38, 0x31e3c, 3567 0x31e80, 0x31e80, 3568 0x31e88, 0x31ea8, 3569 0x31eb0, 0x31eb4, 3570 0x31ec8, 0x31ed4, 3571 0x31fb8, 0x32004, 3572 0x32208, 0x3223c, 3573 0x32248, 0x3227c, 3574 0x32288, 0x322bc, 3575 0x322c8, 0x322fc, 3576 0x32600, 0x32630, 3577 0x32a00, 0x32abc, 3578 0x32b00, 0x32b70, 3579 0x33000, 0x33048, 3580 0x33060, 0x3309c, 3581 0x330f0, 0x33148, 3582 0x33160, 0x3319c, 3583 0x331f0, 0x332e4, 3584 0x332f8, 0x333e4, 3585 0x333f8, 0x33448, 3586 0x33460, 0x3349c, 3587 0x334f0, 0x33548, 3588 0x33560, 0x3359c, 3589 0x335f0, 0x336e4, 3590 0x336f8, 0x337e4, 3591 0x337f8, 0x337fc, 3592 0x33814, 0x33814, 3593 0x3382c, 0x3382c, 3594 0x33880, 0x3388c, 3595 0x338e8, 0x338ec, 3596 0x33900, 0x33948, 3597 0x33960, 0x3399c, 3598 0x339f0, 0x33ae4, 3599 0x33af8, 0x33b10, 3600 0x33b28, 0x33b28, 3601 0x33b3c, 0x33b50, 3602 0x33bf0, 0x33c10, 3603 0x33c28, 0x33c28, 3604 0x33c3c, 0x33c50, 3605 0x33cf0, 0x33cfc, 3606 0x34000, 0x34040, 3607 0x34100, 0x34144, 3608 0x34190, 0x341d0, 3609 0x34200, 0x34318, 3610 0x34400, 0x3452c, 3611 0x34540, 0x3461c, 3612 0x34800, 0x34834, 3613 0x348c0, 0x34908, 3614 0x34910, 0x349ac, 3615 0x34a00, 0x34a04, 3616 0x34a0c, 0x34a2c, 3617 0x34a44, 0x34a50, 3618 0x34a74, 0x34c24, 3619 0x34d08, 0x34d14, 3620 0x34d1c, 0x34d20, 3621 0x34d3c, 0x34d50, 3622 0x35200, 0x3520c, 3623 0x35220, 0x35220, 3624 0x35240, 0x35240, 3625 0x35600, 0x35600, 3626 0x35608, 0x3560c, 3627 0x35a00, 0x35a1c, 3628 0x35e04, 0x35e20, 3629 0x35e38, 0x35e3c, 3630 0x35e80, 0x35e80, 3631 0x35e88, 0x35ea8, 3632 0x35eb0, 0x35eb4, 3633 0x35ec8, 0x35ed4, 3634 0x35fb8, 0x36004, 3635 0x36208, 0x3623c, 3636 0x36248, 0x3627c, 3637 0x36288, 0x362bc, 3638 0x362c8, 0x362fc, 3639 0x36600, 0x36630, 3640 0x36a00, 0x36abc, 3641 0x36b00, 0x36b70, 3642 0x37000, 0x37048, 3643 0x37060, 0x3709c, 3644 0x370f0, 0x37148, 3645 0x37160, 0x3719c, 3646 0x371f0, 0x372e4, 3647 0x372f8, 0x373e4, 3648 0x373f8, 0x37448, 3649 0x37460, 0x3749c, 3650 0x374f0, 0x37548, 3651 0x37560, 0x3759c, 3652 0x375f0, 0x376e4, 3653 0x376f8, 0x377e4, 3654 0x377f8, 0x377fc, 3655 0x37814, 0x37814, 3656 0x3782c, 0x3782c, 3657 0x37880, 0x3788c, 3658 0x378e8, 0x378ec, 3659 0x37900, 0x37948, 3660 0x37960, 0x3799c, 3661 0x379f0, 0x37ae4, 3662 0x37af8, 0x37b10, 3663 0x37b28, 0x37b28, 3664 0x37b3c, 0x37b50, 3665 0x37bf0, 0x37c10, 3666 0x37c28, 0x37c28, 3667 0x37c3c, 0x37c50, 3668 0x37cf0, 0x37cfc, 3669 0x38000, 0x38040, 3670 0x38100, 0x38144, 3671 0x38190, 0x381d0, 3672 0x38200, 0x38318, 3673 0x38400, 0x3852c, 3674 0x38540, 0x3861c, 3675 0x38800, 0x38834, 3676 0x388c0, 0x38908, 3677 0x38910, 0x389ac, 3678 0x38a00, 0x38a04, 3679 0x38a0c, 0x38a2c, 3680 0x38a44, 0x38a50, 3681 0x38a74, 0x38c24, 3682 0x38d08, 0x38d14, 3683 0x38d1c, 0x38d20, 3684 0x38d3c, 0x38d50, 3685 0x39200, 0x3920c, 3686 0x39220, 0x39220, 3687 0x39240, 0x39240, 3688 0x39600, 0x39600, 3689 0x39608, 0x3960c, 3690 0x39a00, 0x39a1c, 3691 0x39e04, 0x39e20, 3692 0x39e38, 0x39e3c, 3693 0x39e80, 0x39e80, 3694 0x39e88, 0x39ea8, 3695 0x39eb0, 0x39eb4, 3696 0x39ec8, 0x39ed4, 3697 0x39fb8, 0x3a004, 3698 0x3a208, 0x3a23c, 3699 0x3a248, 0x3a27c, 3700 0x3a288, 0x3a2bc, 3701 0x3a2c8, 0x3a2fc, 3702 0x3a600, 0x3a630, 3703 0x3aa00, 0x3aabc, 3704 0x3ab00, 0x3ab70, 3705 0x3b000, 0x3b048, 3706 0x3b060, 0x3b09c, 3707 0x3b0f0, 0x3b148, 3708 0x3b160, 0x3b19c, 3709 0x3b1f0, 0x3b2e4, 3710 0x3b2f8, 0x3b3e4, 3711 0x3b3f8, 0x3b448, 3712 0x3b460, 0x3b49c, 3713 0x3b4f0, 0x3b548, 3714 0x3b560, 0x3b59c, 3715 0x3b5f0, 0x3b6e4, 3716 0x3b6f8, 0x3b7e4, 3717 0x3b7f8, 0x3b7fc, 3718 0x3b814, 0x3b814, 3719 0x3b82c, 0x3b82c, 3720 0x3b880, 0x3b88c, 3721 0x3b8e8, 0x3b8ec, 3722 0x3b900, 0x3b948, 3723 0x3b960, 0x3b99c, 3724 0x3b9f0, 0x3bae4, 3725 0x3baf8, 0x3bb10, 3726 0x3bb28, 0x3bb28, 3727 0x3bb3c, 0x3bb50, 3728 0x3bbf0, 0x3bc10, 3729 0x3bc28, 0x3bc28, 3730 0x3bc3c, 0x3bc50, 3731 0x3bcf0, 0x3bcfc, 3732 0x3c000, 0x3c040, 3733 0x3c100, 0x3c144, 3734 0x3c190, 0x3c1d0, 3735 0x3c200, 0x3c318, 3736 0x3c400, 0x3c52c, 3737 0x3c540, 0x3c61c, 3738 0x3c800, 0x3c834, 3739 0x3c8c0, 0x3c908, 3740 0x3c910, 0x3c9ac, 3741 0x3ca00, 0x3ca04, 3742 0x3ca0c, 0x3ca2c, 3743 0x3ca44, 0x3ca50, 3744 0x3ca74, 0x3cc24, 3745 0x3cd08, 0x3cd14, 3746 0x3cd1c, 0x3cd20, 3747 0x3cd3c, 0x3cd50, 3748 0x3d200, 0x3d20c, 3749 0x3d220, 0x3d220, 3750 0x3d240, 0x3d240, 3751 0x3d600, 0x3d600, 3752 0x3d608, 0x3d60c, 3753 0x3da00, 0x3da1c, 3754 0x3de04, 0x3de20, 3755 0x3de38, 0x3de3c, 3756 0x3de80, 0x3de80, 3757 0x3de88, 0x3dea8, 3758 0x3deb0, 0x3deb4, 3759 0x3dec8, 0x3ded4, 3760 0x3dfb8, 0x3e004, 3761 0x3e208, 0x3e23c, 3762 0x3e248, 0x3e27c, 3763 0x3e288, 0x3e2bc, 3764 0x3e2c8, 0x3e2fc, 3765 0x3e600, 0x3e630, 3766 0x3ea00, 0x3eabc, 3767 0x3eb00, 0x3eb70, 3768 0x3f000, 0x3f048, 3769 0x3f060, 0x3f09c, 3770 0x3f0f0, 0x3f148, 3771 0x3f160, 0x3f19c, 3772 0x3f1f0, 0x3f2e4, 3773 0x3f2f8, 0x3f3e4, 3774 0x3f3f8, 0x3f448, 3775 0x3f460, 0x3f49c, 3776 0x3f4f0, 0x3f548, 3777 0x3f560, 0x3f59c, 3778 0x3f5f0, 0x3f6e4, 3779 0x3f6f8, 0x3f7e4, 3780 0x3f7f8, 0x3f7fc, 3781 0x3f814, 0x3f814, 3782 0x3f82c, 0x3f82c, 3783 0x3f880, 0x3f88c, 3784 0x3f8e8, 0x3f8ec, 3785 0x3f900, 0x3f948, 3786 0x3f960, 0x3f99c, 3787 0x3f9f0, 0x3fae4, 3788 0x3faf8, 0x3fb10, 3789 0x3fb28, 0x3fb28, 3790 0x3fb3c, 0x3fb50, 3791 0x3fbf0, 0x3fc10, 3792 0x3fc28, 0x3fc28, 3793 0x3fc3c, 0x3fc50, 3794 0x3fcf0, 0x3fcfc, 3795 0x40000, 0x4000c, 3796 0x40040, 0x40068, 3797 0x4007c, 0x40144, 3798 0x40180, 0x4018c, 3799 0x40200, 0x40298, 3800 0x402ac, 0x4033c, 3801 0x403f8, 0x403fc, 3802 0x41300, 0x413c4, 3803 0x41400, 0x4141c, 3804 0x41480, 0x414d0, 3805 0x44000, 0x44078, 3806 0x440c0, 0x44278, 3807 0x442c0, 0x44478, 3808 0x444c0, 0x44678, 3809 0x446c0, 0x44878, 3810 0x448c0, 0x449fc, 3811 0x45000, 0x45068, 3812 0x45080, 0x45084, 3813 0x450a0, 0x450b0, 3814 0x45200, 0x45268, 3815 0x45280, 0x45284, 3816 0x452a0, 0x452b0, 3817 0x460c0, 0x460e4, 3818 0x47000, 0x4708c, 3819 0x47200, 0x47250, 3820 0x47400, 0x47420, 3821 0x47600, 0x47618, 3822 0x47800, 0x47814, 3823 0x48000, 0x4800c, 3824 0x48040, 0x48068, 3825 0x4807c, 0x48144, 3826 0x48180, 0x4818c, 3827 0x48200, 0x48298, 3828 0x482ac, 0x4833c, 3829 0x483f8, 0x483fc, 3830 0x49300, 0x493c4, 3831 0x49400, 0x4941c, 3832 0x49480, 0x494d0, 3833 0x4c000, 0x4c078, 3834 0x4c0c0, 0x4c278, 3835 0x4c2c0, 0x4c478, 3836 0x4c4c0, 0x4c678, 3837 0x4c6c0, 0x4c878, 3838 0x4c8c0, 0x4c9fc, 3839 0x4d000, 0x4d068, 3840 0x4d080, 0x4d084, 3841 0x4d0a0, 0x4d0b0, 3842 0x4d200, 0x4d268, 3843 0x4d280, 0x4d284, 3844 0x4d2a0, 0x4d2b0, 3845 0x4e0c0, 0x4e0e4, 3846 0x4f000, 0x4f08c, 3847 0x4f200, 0x4f250, 3848 0x4f400, 0x4f420, 3849 0x4f600, 0x4f618, 3850 0x4f800, 0x4f814, 3851 0x50000, 0x500cc, 3852 0x50400, 0x50400, 3853 0x50800, 0x508cc, 3854 0x50c00, 0x50c00, 3855 0x51000, 0x5101c, 3856 0x51300, 0x51308, 3857 }; 3858 3859 if (is_t4(sc)) { 3860 reg_ranges = &t4_reg_ranges[0]; 3861 n = nitems(t4_reg_ranges); 3862 } else { 3863 reg_ranges = &t5_reg_ranges[0]; 3864 n = nitems(t5_reg_ranges); 3865 } 3866 3867 regs->version = chip_id(sc) | chip_rev(sc) << 10; 3868 for (i = 0; i < n; i += 2) 3869 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 3870 } 3871 3872 static void 3873 cxgbe_tick(void *arg) 3874 { 3875 struct port_info *pi = arg; 3876 struct ifnet *ifp = pi->ifp; 3877 struct sge_txq *txq; 3878 int i, drops; 3879 struct port_stats *s = &pi->stats; 3880 3881 PORT_LOCK(pi); 3882 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3883 PORT_UNLOCK(pi); 3884 return; /* without scheduling another callout */ 3885 } 3886 3887 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 3888 3889 ifp->if_opackets = s->tx_frames - s->tx_pause; 3890 ifp->if_ipackets = s->rx_frames - s->rx_pause; 3891 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 3892 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 3893 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 3894 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 3895 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 3896 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 3897 s->rx_trunc3; 3898 3899 drops = s->tx_drop; 3900 for_each_txq(pi, i, txq) 3901 drops += txq->br->br_drops; 3902 ifp->if_snd.ifq_drops = drops; 3903 3904 ifp->if_oerrors = s->tx_error_frames; 3905 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 3906 s->rx_fcs_err + s->rx_len_err; 3907 3908 callout_schedule(&pi->tick, hz); 3909 PORT_UNLOCK(pi); 3910 } 3911 3912 static void 3913 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 3914 { 3915 struct ifnet *vlan; 3916 3917 if (arg != ifp || ifp->if_type != IFT_ETHER) 3918 return; 3919 3920 vlan = VLAN_DEVAT(ifp, vid); 3921 VLAN_SETCOOKIE(vlan, ifp); 3922 } 3923 3924 static int 3925 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3926 { 3927 3928 #ifdef INVARIANTS 3929 panic("%s: opcode 0x%02x on iq %p with payload %p", 3930 __func__, rss->opcode, iq, m); 3931 #else 3932 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3933 __func__, rss->opcode, iq, m); 3934 m_freem(m); 3935 #endif 3936 return (EDOOFUS); 3937 } 3938 3939 int 3940 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3941 { 3942 uintptr_t *loc, new; 3943 3944 if (opcode >= nitems(sc->cpl_handler)) 3945 return (EINVAL); 3946 3947 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3948 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3949 atomic_store_rel_ptr(loc, new); 3950 3951 return (0); 3952 } 3953 3954 static int 3955 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3956 { 3957 3958 #ifdef INVARIANTS 3959 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3960 #else 3961 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 3962 __func__, iq, ctrl); 3963 #endif 3964 return (EDOOFUS); 3965 } 3966 3967 int 3968 t4_register_an_handler(struct adapter *sc, an_handler_t h) 3969 { 3970 uintptr_t *loc, new; 3971 3972 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 3973 loc = (uintptr_t *) &sc->an_handler; 3974 atomic_store_rel_ptr(loc, new); 3975 3976 return (0); 3977 } 3978 3979 static int 3980 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 3981 { 3982 const struct cpl_fw6_msg *cpl = 3983 __containerof(rpl, struct cpl_fw6_msg, data[0]); 3984 3985 #ifdef INVARIANTS 3986 panic("%s: fw_msg type %d", __func__, cpl->type); 3987 #else 3988 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 3989 #endif 3990 return (EDOOFUS); 3991 } 3992 3993 int 3994 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 3995 { 3996 uintptr_t *loc, new; 3997 3998 if (type >= nitems(sc->fw_msg_handler)) 3999 return (EINVAL); 4000 4001 /* 4002 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4003 * handler dispatch table. Reject any attempt to install a handler for 4004 * this subtype. 4005 */ 4006 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4007 return (EINVAL); 4008 4009 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4010 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4011 atomic_store_rel_ptr(loc, new); 4012 4013 return (0); 4014 } 4015 4016 static int 4017 t4_sysctls(struct adapter *sc) 4018 { 4019 struct sysctl_ctx_list *ctx; 4020 struct sysctl_oid *oid; 4021 struct sysctl_oid_list *children, *c0; 4022 static char *caps[] = { 4023 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4024 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 4025 "\20\1TOE", /* caps[2] toecaps */ 4026 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4027 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4028 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4029 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4030 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4031 }; 4032 static char *doorbells = {"\20\1UDB\2WRWC\3UDBWC\4KDB"}; 4033 4034 ctx = device_get_sysctl_ctx(sc->dev); 4035 4036 /* 4037 * dev.t4nex.X. 4038 */ 4039 oid = device_get_sysctl_tree(sc->dev); 4040 c0 = children = SYSCTL_CHILDREN(oid); 4041 4042 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4043 sc->params.nports, "# of ports"); 4044 4045 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4046 NULL, chip_rev(sc), "chip hardware revision"); 4047 4048 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4049 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4050 4051 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4052 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4053 4054 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4055 sc->cfcsum, "config file checksum"); 4056 4057 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4058 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4059 sysctl_bitfield, "A", "available doorbells"); 4060 4061 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4062 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4063 sysctl_bitfield, "A", "available link capabilities"); 4064 4065 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4066 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4067 sysctl_bitfield, "A", "available NIC capabilities"); 4068 4069 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4070 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4071 sysctl_bitfield, "A", "available TCP offload capabilities"); 4072 4073 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4074 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4075 sysctl_bitfield, "A", "available RDMA capabilities"); 4076 4077 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4078 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4079 sysctl_bitfield, "A", "available iSCSI capabilities"); 4080 4081 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4082 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4083 sysctl_bitfield, "A", "available FCoE capabilities"); 4084 4085 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4086 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4087 4088 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4089 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4090 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4091 "interrupt holdoff timer values (us)"); 4092 4093 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4094 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4095 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4096 "interrupt holdoff packet counter values"); 4097 4098 #ifdef SBUF_DRAIN 4099 /* 4100 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4101 */ 4102 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4103 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4104 "logs and miscellaneous information"); 4105 children = SYSCTL_CHILDREN(oid); 4106 4107 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4108 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4109 sysctl_cctrl, "A", "congestion control"); 4110 4111 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4112 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4113 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4114 4115 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4116 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4117 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4118 4119 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4120 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4121 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4122 4123 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4124 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4125 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4126 4127 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4128 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4129 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4130 4131 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4132 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4133 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4134 4135 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4136 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4137 sysctl_cim_la, "A", "CIM logic analyzer"); 4138 4139 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4140 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4141 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4142 4143 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4144 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4145 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4146 4147 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4148 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4149 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4150 4151 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4152 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4153 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4154 4155 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4156 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4157 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4158 4159 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4160 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4161 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4162 4163 if (is_t5(sc)) { 4164 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4165 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4166 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4167 4168 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4169 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4170 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4171 } 4172 4173 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4174 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4175 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4176 4177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4178 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4179 sysctl_cpl_stats, "A", "CPL statistics"); 4180 4181 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4182 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4183 sysctl_ddp_stats, "A", "DDP statistics"); 4184 4185 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4186 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4187 sysctl_devlog, "A", "firmware's device log"); 4188 4189 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4190 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4191 sysctl_fcoe_stats, "A", "FCoE statistics"); 4192 4193 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4194 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4195 sysctl_hw_sched, "A", "hardware scheduler "); 4196 4197 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4198 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4199 sysctl_l2t, "A", "hardware L2 table"); 4200 4201 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4202 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4203 sysctl_lb_stats, "A", "loopback statistics"); 4204 4205 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4206 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4207 sysctl_meminfo, "A", "memory regions"); 4208 4209 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4210 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4211 sysctl_path_mtus, "A", "path MTUs"); 4212 4213 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4214 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4215 sysctl_pm_stats, "A", "PM statistics"); 4216 4217 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4218 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4219 sysctl_rdma_stats, "A", "RDMA statistics"); 4220 4221 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4222 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4223 sysctl_tcp_stats, "A", "TCP statistics"); 4224 4225 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4226 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4227 sysctl_tids, "A", "TID information"); 4228 4229 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4230 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4231 sysctl_tp_err_stats, "A", "TP error statistics"); 4232 4233 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4234 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4235 sysctl_tx_rate, "A", "Tx rate"); 4236 4237 if (is_t5(sc)) { 4238 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wrwc_stats", 4239 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4240 sysctl_wrwc_stats, "A", "work request (WC) statistics"); 4241 } 4242 #endif 4243 4244 #ifdef TCP_OFFLOAD 4245 if (is_offload(sc)) { 4246 /* 4247 * dev.t4nex.X.toe. 4248 */ 4249 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4250 NULL, "TOE parameters"); 4251 children = SYSCTL_CHILDREN(oid); 4252 4253 sc->tt.sndbuf = 256 * 1024; 4254 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4255 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4256 4257 sc->tt.ddp = 0; 4258 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4259 &sc->tt.ddp, 0, "DDP allowed"); 4260 4261 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4262 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4263 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4264 4265 sc->tt.ddp_thres = 4266 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4267 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4268 &sc->tt.ddp_thres, 0, "DDP threshold"); 4269 } 4270 #endif 4271 4272 4273 return (0); 4274 } 4275 4276 static int 4277 cxgbe_sysctls(struct port_info *pi) 4278 { 4279 struct sysctl_ctx_list *ctx; 4280 struct sysctl_oid *oid; 4281 struct sysctl_oid_list *children; 4282 4283 ctx = device_get_sysctl_ctx(pi->dev); 4284 4285 /* 4286 * dev.cxgbe.X. 4287 */ 4288 oid = device_get_sysctl_tree(pi->dev); 4289 children = SYSCTL_CHILDREN(oid); 4290 4291 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4292 &pi->nrxq, 0, "# of rx queues"); 4293 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4294 &pi->ntxq, 0, "# of tx queues"); 4295 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4296 &pi->first_rxq, 0, "index of first rx queue"); 4297 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4298 &pi->first_txq, 0, "index of first tx queue"); 4299 4300 #ifdef TCP_OFFLOAD 4301 if (is_offload(pi->adapter)) { 4302 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4303 &pi->nofldrxq, 0, 4304 "# of rx queues for offloaded TCP connections"); 4305 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4306 &pi->nofldtxq, 0, 4307 "# of tx queues for offloaded TCP connections"); 4308 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4309 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4310 "index of first TOE rx queue"); 4311 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4312 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4313 "index of first TOE tx queue"); 4314 } 4315 #endif 4316 4317 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4318 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4319 "holdoff timer index"); 4320 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4321 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4322 "holdoff packet counter index"); 4323 4324 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4325 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4326 "rx queue size"); 4327 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4328 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4329 "tx queue size"); 4330 4331 /* 4332 * dev.cxgbe.X.stats. 4333 */ 4334 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4335 NULL, "port statistics"); 4336 children = SYSCTL_CHILDREN(oid); 4337 4338 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4339 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4340 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 4341 sysctl_handle_t4_reg64, "QU", desc) 4342 4343 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4344 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4345 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4346 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4347 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4348 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4349 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4350 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4351 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4352 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4353 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4354 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4355 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4356 "# of tx frames in this range", 4357 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4358 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4359 "# of tx frames in this range", 4360 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4361 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4362 "# of tx frames in this range", 4363 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4364 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4365 "# of tx frames in this range", 4366 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4367 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4368 "# of tx frames in this range", 4369 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4370 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4371 "# of tx frames in this range", 4372 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4373 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4374 "# of tx frames in this range", 4375 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4376 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4377 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4378 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4379 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4380 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4381 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4382 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4383 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4384 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4385 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4386 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4387 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4388 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4389 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4390 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4391 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4392 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4393 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4394 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4395 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4396 4397 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4398 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4399 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4400 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4401 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4402 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4403 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4404 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4405 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4406 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4407 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4408 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4409 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4410 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4411 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4412 "# of frames received with bad FCS", 4413 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4414 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4415 "# of frames received with length error", 4416 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4417 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4418 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4419 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4420 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4421 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4422 "# of rx frames in this range", 4423 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4424 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4425 "# of rx frames in this range", 4426 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4427 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4428 "# of rx frames in this range", 4429 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4430 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4431 "# of rx frames in this range", 4432 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4433 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4434 "# of rx frames in this range", 4435 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4436 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4437 "# of rx frames in this range", 4438 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4439 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4440 "# of rx frames in this range", 4441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4442 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4444 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4445 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4446 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4447 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4448 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4449 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4450 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4451 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4452 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4453 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4454 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4455 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4456 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4457 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4458 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4459 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4460 4461 #undef SYSCTL_ADD_T4_REG64 4462 4463 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4464 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4465 &pi->stats.name, desc) 4466 4467 /* We get these from port_stats and they may be stale by upto 1s */ 4468 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4469 "# drops due to buffer-group 0 overflows"); 4470 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4471 "# drops due to buffer-group 1 overflows"); 4472 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4473 "# drops due to buffer-group 2 overflows"); 4474 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4475 "# drops due to buffer-group 3 overflows"); 4476 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4477 "# of buffer-group 0 truncated packets"); 4478 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4479 "# of buffer-group 1 truncated packets"); 4480 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4481 "# of buffer-group 2 truncated packets"); 4482 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4483 "# of buffer-group 3 truncated packets"); 4484 4485 #undef SYSCTL_ADD_T4_PORTSTAT 4486 4487 return (0); 4488 } 4489 4490 static int 4491 sysctl_int_array(SYSCTL_HANDLER_ARGS) 4492 { 4493 int rc, *i; 4494 struct sbuf sb; 4495 4496 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4497 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4498 sbuf_printf(&sb, "%d ", *i); 4499 sbuf_trim(&sb); 4500 sbuf_finish(&sb); 4501 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4502 sbuf_delete(&sb); 4503 return (rc); 4504 } 4505 4506 static int 4507 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4508 { 4509 int rc; 4510 struct sbuf *sb; 4511 4512 rc = sysctl_wire_old_buffer(req, 0); 4513 if (rc != 0) 4514 return(rc); 4515 4516 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4517 if (sb == NULL) 4518 return (ENOMEM); 4519 4520 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4521 rc = sbuf_finish(sb); 4522 sbuf_delete(sb); 4523 4524 return (rc); 4525 } 4526 4527 static int 4528 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4529 { 4530 struct port_info *pi = arg1; 4531 struct adapter *sc = pi->adapter; 4532 int idx, rc, i; 4533 struct sge_rxq *rxq; 4534 uint8_t v; 4535 4536 idx = pi->tmr_idx; 4537 4538 rc = sysctl_handle_int(oidp, &idx, 0, req); 4539 if (rc != 0 || req->newptr == NULL) 4540 return (rc); 4541 4542 if (idx < 0 || idx >= SGE_NTIMERS) 4543 return (EINVAL); 4544 4545 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4546 "t4tmr"); 4547 if (rc) 4548 return (rc); 4549 4550 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4551 for_each_rxq(pi, i, rxq) { 4552 #ifdef atomic_store_rel_8 4553 atomic_store_rel_8(&rxq->iq.intr_params, v); 4554 #else 4555 rxq->iq.intr_params = v; 4556 #endif 4557 } 4558 pi->tmr_idx = idx; 4559 4560 end_synchronized_op(sc, LOCK_HELD); 4561 return (0); 4562 } 4563 4564 static int 4565 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4566 { 4567 struct port_info *pi = arg1; 4568 struct adapter *sc = pi->adapter; 4569 int idx, rc; 4570 4571 idx = pi->pktc_idx; 4572 4573 rc = sysctl_handle_int(oidp, &idx, 0, req); 4574 if (rc != 0 || req->newptr == NULL) 4575 return (rc); 4576 4577 if (idx < -1 || idx >= SGE_NCOUNTERS) 4578 return (EINVAL); 4579 4580 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4581 "t4pktc"); 4582 if (rc) 4583 return (rc); 4584 4585 if (pi->flags & PORT_INIT_DONE) 4586 rc = EBUSY; /* cannot be changed once the queues are created */ 4587 else 4588 pi->pktc_idx = idx; 4589 4590 end_synchronized_op(sc, LOCK_HELD); 4591 return (rc); 4592 } 4593 4594 static int 4595 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4596 { 4597 struct port_info *pi = arg1; 4598 struct adapter *sc = pi->adapter; 4599 int qsize, rc; 4600 4601 qsize = pi->qsize_rxq; 4602 4603 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4604 if (rc != 0 || req->newptr == NULL) 4605 return (rc); 4606 4607 if (qsize < 128 || (qsize & 7)) 4608 return (EINVAL); 4609 4610 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4611 "t4rxqs"); 4612 if (rc) 4613 return (rc); 4614 4615 if (pi->flags & PORT_INIT_DONE) 4616 rc = EBUSY; /* cannot be changed once the queues are created */ 4617 else 4618 pi->qsize_rxq = qsize; 4619 4620 end_synchronized_op(sc, LOCK_HELD); 4621 return (rc); 4622 } 4623 4624 static int 4625 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4626 { 4627 struct port_info *pi = arg1; 4628 struct adapter *sc = pi->adapter; 4629 int qsize, rc; 4630 4631 qsize = pi->qsize_txq; 4632 4633 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4634 if (rc != 0 || req->newptr == NULL) 4635 return (rc); 4636 4637 /* bufring size must be powerof2 */ 4638 if (qsize < 128 || !powerof2(qsize)) 4639 return (EINVAL); 4640 4641 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4642 "t4txqs"); 4643 if (rc) 4644 return (rc); 4645 4646 if (pi->flags & PORT_INIT_DONE) 4647 rc = EBUSY; /* cannot be changed once the queues are created */ 4648 else 4649 pi->qsize_txq = qsize; 4650 4651 end_synchronized_op(sc, LOCK_HELD); 4652 return (rc); 4653 } 4654 4655 static int 4656 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4657 { 4658 struct adapter *sc = arg1; 4659 int reg = arg2; 4660 uint64_t val; 4661 4662 val = t4_read_reg64(sc, reg); 4663 4664 return (sysctl_handle_64(oidp, &val, 0, req)); 4665 } 4666 4667 #ifdef SBUF_DRAIN 4668 static int 4669 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 4670 { 4671 struct adapter *sc = arg1; 4672 struct sbuf *sb; 4673 int rc, i; 4674 uint16_t incr[NMTUS][NCCTRL_WIN]; 4675 static const char *dec_fac[] = { 4676 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4677 "0.9375" 4678 }; 4679 4680 rc = sysctl_wire_old_buffer(req, 0); 4681 if (rc != 0) 4682 return (rc); 4683 4684 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4685 if (sb == NULL) 4686 return (ENOMEM); 4687 4688 t4_read_cong_tbl(sc, incr); 4689 4690 for (i = 0; i < NCCTRL_WIN; ++i) { 4691 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 4692 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 4693 incr[5][i], incr[6][i], incr[7][i]); 4694 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 4695 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 4696 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 4697 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 4698 } 4699 4700 rc = sbuf_finish(sb); 4701 sbuf_delete(sb); 4702 4703 return (rc); 4704 } 4705 4706 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 4707 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 4708 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 4709 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 4710 }; 4711 4712 static int 4713 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 4714 { 4715 struct adapter *sc = arg1; 4716 struct sbuf *sb; 4717 int rc, i, n, qid = arg2; 4718 uint32_t *buf, *p; 4719 char *qtype; 4720 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 4721 4722 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 4723 ("%s: bad qid %d\n", __func__, qid)); 4724 4725 if (qid < CIM_NUM_IBQ) { 4726 /* inbound queue */ 4727 qtype = "IBQ"; 4728 n = 4 * CIM_IBQ_SIZE; 4729 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4730 rc = t4_read_cim_ibq(sc, qid, buf, n); 4731 } else { 4732 /* outbound queue */ 4733 qtype = "OBQ"; 4734 qid -= CIM_NUM_IBQ; 4735 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 4736 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4737 rc = t4_read_cim_obq(sc, qid, buf, n); 4738 } 4739 4740 if (rc < 0) { 4741 rc = -rc; 4742 goto done; 4743 } 4744 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 4745 4746 rc = sysctl_wire_old_buffer(req, 0); 4747 if (rc != 0) 4748 goto done; 4749 4750 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4751 if (sb == NULL) { 4752 rc = ENOMEM; 4753 goto done; 4754 } 4755 4756 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 4757 for (i = 0, p = buf; i < n; i += 16, p += 4) 4758 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 4759 p[2], p[3]); 4760 4761 rc = sbuf_finish(sb); 4762 sbuf_delete(sb); 4763 done: 4764 free(buf, M_CXGBE); 4765 return (rc); 4766 } 4767 4768 static int 4769 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 4770 { 4771 struct adapter *sc = arg1; 4772 u_int cfg; 4773 struct sbuf *sb; 4774 uint32_t *buf, *p; 4775 int rc; 4776 4777 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 4778 if (rc != 0) 4779 return (rc); 4780 4781 rc = sysctl_wire_old_buffer(req, 0); 4782 if (rc != 0) 4783 return (rc); 4784 4785 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4786 if (sb == NULL) 4787 return (ENOMEM); 4788 4789 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 4790 M_ZERO | M_WAITOK); 4791 4792 rc = -t4_cim_read_la(sc, buf, NULL); 4793 if (rc != 0) 4794 goto done; 4795 4796 sbuf_printf(sb, "Status Data PC%s", 4797 cfg & F_UPDBGLACAPTPCONLY ? "" : 4798 " LS0Stat LS0Addr LS0Data"); 4799 4800 KASSERT((sc->params.cim_la_size & 7) == 0, 4801 ("%s: p will walk off the end of buf", __func__)); 4802 4803 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 4804 if (cfg & F_UPDBGLACAPTPCONLY) { 4805 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 4806 p[6], p[7]); 4807 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 4808 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 4809 p[4] & 0xff, p[5] >> 8); 4810 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 4811 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4812 p[1] & 0xf, p[2] >> 4); 4813 } else { 4814 sbuf_printf(sb, 4815 "\n %02x %x%07x %x%07x %08x %08x " 4816 "%08x%08x%08x%08x", 4817 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4818 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 4819 p[6], p[7]); 4820 } 4821 } 4822 4823 rc = sbuf_finish(sb); 4824 sbuf_delete(sb); 4825 done: 4826 free(buf, M_CXGBE); 4827 return (rc); 4828 } 4829 4830 static int 4831 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 4832 { 4833 struct adapter *sc = arg1; 4834 struct sbuf *sb; 4835 int rc, i; 4836 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4837 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4838 uint16_t thres[CIM_NUM_IBQ]; 4839 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 4840 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 4841 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 4842 4843 if (is_t4(sc)) { 4844 cim_num_obq = CIM_NUM_OBQ; 4845 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 4846 obq_rdaddr = A_UP_OBQ_0_REALADDR; 4847 } else { 4848 cim_num_obq = CIM_NUM_OBQ_T5; 4849 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 4850 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 4851 } 4852 nq = CIM_NUM_IBQ + cim_num_obq; 4853 4854 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 4855 if (rc == 0) 4856 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 4857 if (rc != 0) 4858 return (rc); 4859 4860 t4_read_cimq_cfg(sc, base, size, thres); 4861 4862 rc = sysctl_wire_old_buffer(req, 0); 4863 if (rc != 0) 4864 return (rc); 4865 4866 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4867 if (sb == NULL) 4868 return (ENOMEM); 4869 4870 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 4871 4872 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 4873 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 4874 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 4875 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4876 G_QUEREMFLITS(p[2]) * 16); 4877 for ( ; i < nq; i++, p += 4, wr += 2) 4878 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 4879 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 4880 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4881 G_QUEREMFLITS(p[2]) * 16); 4882 4883 rc = sbuf_finish(sb); 4884 sbuf_delete(sb); 4885 4886 return (rc); 4887 } 4888 4889 static int 4890 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 4891 { 4892 struct adapter *sc = arg1; 4893 struct sbuf *sb; 4894 int rc; 4895 struct tp_cpl_stats stats; 4896 4897 rc = sysctl_wire_old_buffer(req, 0); 4898 if (rc != 0) 4899 return (rc); 4900 4901 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4902 if (sb == NULL) 4903 return (ENOMEM); 4904 4905 t4_tp_get_cpl_stats(sc, &stats); 4906 4907 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4908 "channel 3\n"); 4909 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 4910 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 4911 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 4912 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 4913 4914 rc = sbuf_finish(sb); 4915 sbuf_delete(sb); 4916 4917 return (rc); 4918 } 4919 4920 static int 4921 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 4922 { 4923 struct adapter *sc = arg1; 4924 struct sbuf *sb; 4925 int rc; 4926 struct tp_usm_stats stats; 4927 4928 rc = sysctl_wire_old_buffer(req, 0); 4929 if (rc != 0) 4930 return(rc); 4931 4932 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4933 if (sb == NULL) 4934 return (ENOMEM); 4935 4936 t4_get_usm_stats(sc, &stats); 4937 4938 sbuf_printf(sb, "Frames: %u\n", stats.frames); 4939 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 4940 sbuf_printf(sb, "Drops: %u", stats.drops); 4941 4942 rc = sbuf_finish(sb); 4943 sbuf_delete(sb); 4944 4945 return (rc); 4946 } 4947 4948 const char *devlog_level_strings[] = { 4949 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 4950 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 4951 [FW_DEVLOG_LEVEL_ERR] = "ERR", 4952 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 4953 [FW_DEVLOG_LEVEL_INFO] = "INFO", 4954 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 4955 }; 4956 4957 const char *devlog_facility_strings[] = { 4958 [FW_DEVLOG_FACILITY_CORE] = "CORE", 4959 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 4960 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 4961 [FW_DEVLOG_FACILITY_RES] = "RES", 4962 [FW_DEVLOG_FACILITY_HW] = "HW", 4963 [FW_DEVLOG_FACILITY_FLR] = "FLR", 4964 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 4965 [FW_DEVLOG_FACILITY_PHY] = "PHY", 4966 [FW_DEVLOG_FACILITY_MAC] = "MAC", 4967 [FW_DEVLOG_FACILITY_PORT] = "PORT", 4968 [FW_DEVLOG_FACILITY_VI] = "VI", 4969 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 4970 [FW_DEVLOG_FACILITY_ACL] = "ACL", 4971 [FW_DEVLOG_FACILITY_TM] = "TM", 4972 [FW_DEVLOG_FACILITY_QFC] = "QFC", 4973 [FW_DEVLOG_FACILITY_DCB] = "DCB", 4974 [FW_DEVLOG_FACILITY_ETH] = "ETH", 4975 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 4976 [FW_DEVLOG_FACILITY_RI] = "RI", 4977 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 4978 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 4979 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 4980 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 4981 }; 4982 4983 static int 4984 sysctl_devlog(SYSCTL_HANDLER_ARGS) 4985 { 4986 struct adapter *sc = arg1; 4987 struct devlog_params *dparams = &sc->params.devlog; 4988 struct fw_devlog_e *buf, *e; 4989 int i, j, rc, nentries, first = 0; 4990 struct sbuf *sb; 4991 uint64_t ftstamp = UINT64_MAX; 4992 4993 if (dparams->start == 0) { 4994 dparams->memtype = 0; 4995 dparams->start = 0x84000; 4996 dparams->size = 32768; 4997 } 4998 4999 nentries = dparams->size / sizeof(struct fw_devlog_e); 5000 5001 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5002 if (buf == NULL) 5003 return (ENOMEM); 5004 5005 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 5006 (void *)buf); 5007 if (rc != 0) 5008 goto done; 5009 5010 for (i = 0; i < nentries; i++) { 5011 e = &buf[i]; 5012 5013 if (e->timestamp == 0) 5014 break; /* end */ 5015 5016 e->timestamp = be64toh(e->timestamp); 5017 e->seqno = be32toh(e->seqno); 5018 for (j = 0; j < 8; j++) 5019 e->params[j] = be32toh(e->params[j]); 5020 5021 if (e->timestamp < ftstamp) { 5022 ftstamp = e->timestamp; 5023 first = i; 5024 } 5025 } 5026 5027 if (buf[first].timestamp == 0) 5028 goto done; /* nothing in the log */ 5029 5030 rc = sysctl_wire_old_buffer(req, 0); 5031 if (rc != 0) 5032 goto done; 5033 5034 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5035 if (sb == NULL) { 5036 rc = ENOMEM; 5037 goto done; 5038 } 5039 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5040 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5041 5042 i = first; 5043 do { 5044 e = &buf[i]; 5045 if (e->timestamp == 0) 5046 break; /* end */ 5047 5048 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5049 e->seqno, e->timestamp, 5050 (e->level < nitems(devlog_level_strings) ? 5051 devlog_level_strings[e->level] : "UNKNOWN"), 5052 (e->facility < nitems(devlog_facility_strings) ? 5053 devlog_facility_strings[e->facility] : "UNKNOWN")); 5054 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5055 e->params[2], e->params[3], e->params[4], 5056 e->params[5], e->params[6], e->params[7]); 5057 5058 if (++i == nentries) 5059 i = 0; 5060 } while (i != first); 5061 5062 rc = sbuf_finish(sb); 5063 sbuf_delete(sb); 5064 done: 5065 free(buf, M_CXGBE); 5066 return (rc); 5067 } 5068 5069 static int 5070 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5071 { 5072 struct adapter *sc = arg1; 5073 struct sbuf *sb; 5074 int rc; 5075 struct tp_fcoe_stats stats[4]; 5076 5077 rc = sysctl_wire_old_buffer(req, 0); 5078 if (rc != 0) 5079 return (rc); 5080 5081 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5082 if (sb == NULL) 5083 return (ENOMEM); 5084 5085 t4_get_fcoe_stats(sc, 0, &stats[0]); 5086 t4_get_fcoe_stats(sc, 1, &stats[1]); 5087 t4_get_fcoe_stats(sc, 2, &stats[2]); 5088 t4_get_fcoe_stats(sc, 3, &stats[3]); 5089 5090 sbuf_printf(sb, " channel 0 channel 1 " 5091 "channel 2 channel 3\n"); 5092 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5093 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5094 stats[3].octetsDDP); 5095 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5096 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5097 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5098 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5099 stats[3].framesDrop); 5100 5101 rc = sbuf_finish(sb); 5102 sbuf_delete(sb); 5103 5104 return (rc); 5105 } 5106 5107 static int 5108 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5109 { 5110 struct adapter *sc = arg1; 5111 struct sbuf *sb; 5112 int rc, i; 5113 unsigned int map, kbps, ipg, mode; 5114 unsigned int pace_tab[NTX_SCHED]; 5115 5116 rc = sysctl_wire_old_buffer(req, 0); 5117 if (rc != 0) 5118 return (rc); 5119 5120 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5121 if (sb == NULL) 5122 return (ENOMEM); 5123 5124 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5125 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5126 t4_read_pace_tbl(sc, pace_tab); 5127 5128 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5129 "Class IPG (0.1 ns) Flow IPG (us)"); 5130 5131 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5132 t4_get_tx_sched(sc, i, &kbps, &ipg); 5133 sbuf_printf(sb, "\n %u %-5s %u ", i, 5134 (mode & (1 << i)) ? "flow" : "class", map & 3); 5135 if (kbps) 5136 sbuf_printf(sb, "%9u ", kbps); 5137 else 5138 sbuf_printf(sb, " disabled "); 5139 5140 if (ipg) 5141 sbuf_printf(sb, "%13u ", ipg); 5142 else 5143 sbuf_printf(sb, " disabled "); 5144 5145 if (pace_tab[i]) 5146 sbuf_printf(sb, "%10u", pace_tab[i]); 5147 else 5148 sbuf_printf(sb, " disabled"); 5149 } 5150 5151 rc = sbuf_finish(sb); 5152 sbuf_delete(sb); 5153 5154 return (rc); 5155 } 5156 5157 static int 5158 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5159 { 5160 struct adapter *sc = arg1; 5161 struct sbuf *sb; 5162 int rc, i, j; 5163 uint64_t *p0, *p1; 5164 struct lb_port_stats s[2]; 5165 static const char *stat_name[] = { 5166 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5167 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5168 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5169 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5170 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5171 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5172 "BG2FramesTrunc:", "BG3FramesTrunc:" 5173 }; 5174 5175 rc = sysctl_wire_old_buffer(req, 0); 5176 if (rc != 0) 5177 return (rc); 5178 5179 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5180 if (sb == NULL) 5181 return (ENOMEM); 5182 5183 memset(s, 0, sizeof(s)); 5184 5185 for (i = 0; i < 4; i += 2) { 5186 t4_get_lb_stats(sc, i, &s[0]); 5187 t4_get_lb_stats(sc, i + 1, &s[1]); 5188 5189 p0 = &s[0].octets; 5190 p1 = &s[1].octets; 5191 sbuf_printf(sb, "%s Loopback %u" 5192 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5193 5194 for (j = 0; j < nitems(stat_name); j++) 5195 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5196 *p0++, *p1++); 5197 } 5198 5199 rc = sbuf_finish(sb); 5200 sbuf_delete(sb); 5201 5202 return (rc); 5203 } 5204 5205 struct mem_desc { 5206 unsigned int base; 5207 unsigned int limit; 5208 unsigned int idx; 5209 }; 5210 5211 static int 5212 mem_desc_cmp(const void *a, const void *b) 5213 { 5214 return ((const struct mem_desc *)a)->base - 5215 ((const struct mem_desc *)b)->base; 5216 } 5217 5218 static void 5219 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5220 unsigned int to) 5221 { 5222 unsigned int size; 5223 5224 size = to - from + 1; 5225 if (size == 0) 5226 return; 5227 5228 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5229 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5230 } 5231 5232 static int 5233 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5234 { 5235 struct adapter *sc = arg1; 5236 struct sbuf *sb; 5237 int rc, i, n; 5238 uint32_t lo, hi, used, alloc; 5239 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5240 static const char *region[] = { 5241 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5242 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5243 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5244 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5245 "RQUDP region:", "PBL region:", "TXPBL region:", 5246 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5247 "On-chip queues:" 5248 }; 5249 struct mem_desc avail[4]; 5250 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5251 struct mem_desc *md = mem; 5252 5253 rc = sysctl_wire_old_buffer(req, 0); 5254 if (rc != 0) 5255 return (rc); 5256 5257 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5258 if (sb == NULL) 5259 return (ENOMEM); 5260 5261 for (i = 0; i < nitems(mem); i++) { 5262 mem[i].limit = 0; 5263 mem[i].idx = i; 5264 } 5265 5266 /* Find and sort the populated memory ranges */ 5267 i = 0; 5268 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5269 if (lo & F_EDRAM0_ENABLE) { 5270 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5271 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5272 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5273 avail[i].idx = 0; 5274 i++; 5275 } 5276 if (lo & F_EDRAM1_ENABLE) { 5277 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5278 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5279 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5280 avail[i].idx = 1; 5281 i++; 5282 } 5283 if (lo & F_EXT_MEM_ENABLE) { 5284 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5285 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5286 avail[i].limit = avail[i].base + 5287 (G_EXT_MEM_SIZE(hi) << 20); 5288 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5289 i++; 5290 } 5291 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5292 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5293 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5294 avail[i].limit = avail[i].base + 5295 (G_EXT_MEM1_SIZE(hi) << 20); 5296 avail[i].idx = 4; 5297 i++; 5298 } 5299 if (!i) /* no memory available */ 5300 return 0; 5301 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5302 5303 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5304 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5305 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5306 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5307 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5308 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5309 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5310 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5311 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5312 5313 /* the next few have explicit upper bounds */ 5314 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5315 md->limit = md->base - 1 + 5316 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5317 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5318 md++; 5319 5320 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5321 md->limit = md->base - 1 + 5322 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5323 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5324 md++; 5325 5326 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5327 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5328 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5329 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5330 } else { 5331 md->base = 0; 5332 md->idx = nitems(region); /* hide it */ 5333 } 5334 md++; 5335 5336 #define ulp_region(reg) \ 5337 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5338 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5339 5340 ulp_region(RX_ISCSI); 5341 ulp_region(RX_TDDP); 5342 ulp_region(TX_TPT); 5343 ulp_region(RX_STAG); 5344 ulp_region(RX_RQ); 5345 ulp_region(RX_RQUDP); 5346 ulp_region(RX_PBL); 5347 ulp_region(TX_PBL); 5348 #undef ulp_region 5349 5350 md->base = 0; 5351 md->idx = nitems(region); 5352 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5353 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5354 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5355 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5356 } 5357 md++; 5358 5359 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5360 md->limit = md->base + sc->tids.ntids - 1; 5361 md++; 5362 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5363 md->limit = md->base + sc->tids.ntids - 1; 5364 md++; 5365 5366 md->base = sc->vres.ocq.start; 5367 if (sc->vres.ocq.size) 5368 md->limit = md->base + sc->vres.ocq.size - 1; 5369 else 5370 md->idx = nitems(region); /* hide it */ 5371 md++; 5372 5373 /* add any address-space holes, there can be up to 3 */ 5374 for (n = 0; n < i - 1; n++) 5375 if (avail[n].limit < avail[n + 1].base) 5376 (md++)->base = avail[n].limit; 5377 if (avail[n].limit) 5378 (md++)->base = avail[n].limit; 5379 5380 n = md - mem; 5381 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5382 5383 for (lo = 0; lo < i; lo++) 5384 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5385 avail[lo].limit - 1); 5386 5387 sbuf_printf(sb, "\n"); 5388 for (i = 0; i < n; i++) { 5389 if (mem[i].idx >= nitems(region)) 5390 continue; /* skip holes */ 5391 if (!mem[i].limit) 5392 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5393 mem_region_show(sb, region[mem[i].idx], mem[i].base, 5394 mem[i].limit); 5395 } 5396 5397 sbuf_printf(sb, "\n"); 5398 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5399 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5400 mem_region_show(sb, "uP RAM:", lo, hi); 5401 5402 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5403 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5404 mem_region_show(sb, "uP Extmem2:", lo, hi); 5405 5406 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5407 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5408 G_PMRXMAXPAGE(lo), 5409 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5410 (lo & F_PMRXNUMCHN) ? 2 : 1); 5411 5412 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5413 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5414 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5415 G_PMTXMAXPAGE(lo), 5416 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5417 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5418 sbuf_printf(sb, "%u p-structs\n", 5419 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5420 5421 for (i = 0; i < 4; i++) { 5422 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5423 if (is_t4(sc)) { 5424 used = G_USED(lo); 5425 alloc = G_ALLOC(lo); 5426 } else { 5427 used = G_T5_USED(lo); 5428 alloc = G_T5_ALLOC(lo); 5429 } 5430 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5431 i, used, alloc); 5432 } 5433 for (i = 0; i < 4; i++) { 5434 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5435 if (is_t4(sc)) { 5436 used = G_USED(lo); 5437 alloc = G_ALLOC(lo); 5438 } else { 5439 used = G_T5_USED(lo); 5440 alloc = G_T5_ALLOC(lo); 5441 } 5442 sbuf_printf(sb, 5443 "\nLoopback %d using %u pages out of %u allocated", 5444 i, used, alloc); 5445 } 5446 5447 rc = sbuf_finish(sb); 5448 sbuf_delete(sb); 5449 5450 return (rc); 5451 } 5452 5453 static int 5454 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5455 { 5456 struct adapter *sc = arg1; 5457 struct sbuf *sb; 5458 int rc; 5459 uint16_t mtus[NMTUS]; 5460 5461 rc = sysctl_wire_old_buffer(req, 0); 5462 if (rc != 0) 5463 return (rc); 5464 5465 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5466 if (sb == NULL) 5467 return (ENOMEM); 5468 5469 t4_read_mtu_tbl(sc, mtus, NULL); 5470 5471 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 5472 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 5473 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 5474 mtus[14], mtus[15]); 5475 5476 rc = sbuf_finish(sb); 5477 sbuf_delete(sb); 5478 5479 return (rc); 5480 } 5481 5482 static int 5483 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 5484 { 5485 struct adapter *sc = arg1; 5486 struct sbuf *sb; 5487 int rc, i; 5488 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 5489 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 5490 static const char *pm_stats[] = { 5491 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 5492 }; 5493 5494 rc = sysctl_wire_old_buffer(req, 0); 5495 if (rc != 0) 5496 return (rc); 5497 5498 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5499 if (sb == NULL) 5500 return (ENOMEM); 5501 5502 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 5503 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 5504 5505 sbuf_printf(sb, " Tx count Tx cycles " 5506 "Rx count Rx cycles"); 5507 for (i = 0; i < PM_NSTATS; i++) 5508 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 5509 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 5510 5511 rc = sbuf_finish(sb); 5512 sbuf_delete(sb); 5513 5514 return (rc); 5515 } 5516 5517 static int 5518 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 5519 { 5520 struct adapter *sc = arg1; 5521 struct sbuf *sb; 5522 int rc; 5523 struct tp_rdma_stats stats; 5524 5525 rc = sysctl_wire_old_buffer(req, 0); 5526 if (rc != 0) 5527 return (rc); 5528 5529 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5530 if (sb == NULL) 5531 return (ENOMEM); 5532 5533 t4_tp_get_rdma_stats(sc, &stats); 5534 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 5535 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 5536 5537 rc = sbuf_finish(sb); 5538 sbuf_delete(sb); 5539 5540 return (rc); 5541 } 5542 5543 static int 5544 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 5545 { 5546 struct adapter *sc = arg1; 5547 struct sbuf *sb; 5548 int rc; 5549 struct tp_tcp_stats v4, v6; 5550 5551 rc = sysctl_wire_old_buffer(req, 0); 5552 if (rc != 0) 5553 return (rc); 5554 5555 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5556 if (sb == NULL) 5557 return (ENOMEM); 5558 5559 t4_tp_get_tcp_stats(sc, &v4, &v6); 5560 sbuf_printf(sb, 5561 " IP IPv6\n"); 5562 sbuf_printf(sb, "OutRsts: %20u %20u\n", 5563 v4.tcpOutRsts, v6.tcpOutRsts); 5564 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 5565 v4.tcpInSegs, v6.tcpInSegs); 5566 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 5567 v4.tcpOutSegs, v6.tcpOutSegs); 5568 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 5569 v4.tcpRetransSegs, v6.tcpRetransSegs); 5570 5571 rc = sbuf_finish(sb); 5572 sbuf_delete(sb); 5573 5574 return (rc); 5575 } 5576 5577 static int 5578 sysctl_tids(SYSCTL_HANDLER_ARGS) 5579 { 5580 struct adapter *sc = arg1; 5581 struct sbuf *sb; 5582 int rc; 5583 struct tid_info *t = &sc->tids; 5584 5585 rc = sysctl_wire_old_buffer(req, 0); 5586 if (rc != 0) 5587 return (rc); 5588 5589 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5590 if (sb == NULL) 5591 return (ENOMEM); 5592 5593 if (t->natids) { 5594 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 5595 t->atids_in_use); 5596 } 5597 5598 if (t->ntids) { 5599 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5600 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 5601 5602 if (b) { 5603 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 5604 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5605 t->ntids - 1); 5606 } else { 5607 sbuf_printf(sb, "TID range: %u-%u", 5608 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5609 t->ntids - 1); 5610 } 5611 } else 5612 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 5613 sbuf_printf(sb, ", in use: %u\n", 5614 atomic_load_acq_int(&t->tids_in_use)); 5615 } 5616 5617 if (t->nstids) { 5618 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 5619 t->stid_base + t->nstids - 1, t->stids_in_use); 5620 } 5621 5622 if (t->nftids) { 5623 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 5624 t->ftid_base + t->nftids - 1); 5625 } 5626 5627 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 5628 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 5629 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 5630 5631 rc = sbuf_finish(sb); 5632 sbuf_delete(sb); 5633 5634 return (rc); 5635 } 5636 5637 static int 5638 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 5639 { 5640 struct adapter *sc = arg1; 5641 struct sbuf *sb; 5642 int rc; 5643 struct tp_err_stats stats; 5644 5645 rc = sysctl_wire_old_buffer(req, 0); 5646 if (rc != 0) 5647 return (rc); 5648 5649 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5650 if (sb == NULL) 5651 return (ENOMEM); 5652 5653 t4_tp_get_err_stats(sc, &stats); 5654 5655 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5656 "channel 3\n"); 5657 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 5658 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 5659 stats.macInErrs[3]); 5660 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 5661 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 5662 stats.hdrInErrs[3]); 5663 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 5664 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 5665 stats.tcpInErrs[3]); 5666 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 5667 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 5668 stats.tcp6InErrs[3]); 5669 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 5670 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 5671 stats.tnlCongDrops[3]); 5672 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 5673 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 5674 stats.tnlTxDrops[3]); 5675 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 5676 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 5677 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 5678 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 5679 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 5680 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 5681 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 5682 stats.ofldNoNeigh, stats.ofldCongDefer); 5683 5684 rc = sbuf_finish(sb); 5685 sbuf_delete(sb); 5686 5687 return (rc); 5688 } 5689 5690 static int 5691 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 5692 { 5693 struct adapter *sc = arg1; 5694 struct sbuf *sb; 5695 int rc; 5696 u64 nrate[NCHAN], orate[NCHAN]; 5697 5698 rc = sysctl_wire_old_buffer(req, 0); 5699 if (rc != 0) 5700 return (rc); 5701 5702 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5703 if (sb == NULL) 5704 return (ENOMEM); 5705 5706 t4_get_chan_txrate(sc, nrate, orate); 5707 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5708 "channel 3\n"); 5709 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 5710 nrate[0], nrate[1], nrate[2], nrate[3]); 5711 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 5712 orate[0], orate[1], orate[2], orate[3]); 5713 5714 rc = sbuf_finish(sb); 5715 sbuf_delete(sb); 5716 5717 return (rc); 5718 } 5719 5720 static int 5721 sysctl_wrwc_stats(SYSCTL_HANDLER_ARGS) 5722 { 5723 struct adapter *sc = arg1; 5724 struct sbuf *sb; 5725 int rc, v; 5726 5727 rc = sysctl_wire_old_buffer(req, 0); 5728 if (rc != 0) 5729 return (rc); 5730 5731 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5732 if (sb == NULL) 5733 return (ENOMEM); 5734 5735 v = t4_read_reg(sc, A_SGE_STAT_CFG); 5736 if (G_STATSOURCE_T5(v) == 7) { 5737 if (G_STATMODE(v) == 0) { 5738 sbuf_printf(sb, "\ntotal %d, incomplete %d", 5739 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5740 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5741 } else if (G_STATMODE(v) == 1) { 5742 sbuf_printf(sb, "\ntotal %d, data overflow %d", 5743 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5744 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5745 } 5746 } 5747 rc = sbuf_finish(sb); 5748 sbuf_delete(sb); 5749 5750 return (rc); 5751 } 5752 #endif 5753 5754 static inline void 5755 txq_start(struct ifnet *ifp, struct sge_txq *txq) 5756 { 5757 struct buf_ring *br; 5758 struct mbuf *m; 5759 5760 TXQ_LOCK_ASSERT_OWNED(txq); 5761 5762 br = txq->br; 5763 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 5764 if (m) 5765 t4_eth_tx(ifp, txq, m); 5766 } 5767 5768 void 5769 t4_tx_callout(void *arg) 5770 { 5771 struct sge_eq *eq = arg; 5772 struct adapter *sc; 5773 5774 if (EQ_TRYLOCK(eq) == 0) 5775 goto reschedule; 5776 5777 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 5778 EQ_UNLOCK(eq); 5779 reschedule: 5780 if (__predict_true(!(eq->flags && EQ_DOOMED))) 5781 callout_schedule(&eq->tx_callout, 1); 5782 return; 5783 } 5784 5785 EQ_LOCK_ASSERT_OWNED(eq); 5786 5787 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 5788 5789 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5790 struct sge_txq *txq = arg; 5791 struct port_info *pi = txq->ifp->if_softc; 5792 5793 sc = pi->adapter; 5794 } else { 5795 struct sge_wrq *wrq = arg; 5796 5797 sc = wrq->adapter; 5798 } 5799 5800 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 5801 } 5802 5803 EQ_UNLOCK(eq); 5804 } 5805 5806 void 5807 t4_tx_task(void *arg, int count) 5808 { 5809 struct sge_eq *eq = arg; 5810 5811 EQ_LOCK(eq); 5812 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5813 struct sge_txq *txq = arg; 5814 txq_start(txq->ifp, txq); 5815 } else { 5816 struct sge_wrq *wrq = arg; 5817 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 5818 } 5819 EQ_UNLOCK(eq); 5820 } 5821 5822 static uint32_t 5823 fconf_to_mode(uint32_t fconf) 5824 { 5825 uint32_t mode; 5826 5827 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 5828 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 5829 5830 if (fconf & F_FRAGMENTATION) 5831 mode |= T4_FILTER_IP_FRAGMENT; 5832 5833 if (fconf & F_MPSHITTYPE) 5834 mode |= T4_FILTER_MPS_HIT_TYPE; 5835 5836 if (fconf & F_MACMATCH) 5837 mode |= T4_FILTER_MAC_IDX; 5838 5839 if (fconf & F_ETHERTYPE) 5840 mode |= T4_FILTER_ETH_TYPE; 5841 5842 if (fconf & F_PROTOCOL) 5843 mode |= T4_FILTER_IP_PROTO; 5844 5845 if (fconf & F_TOS) 5846 mode |= T4_FILTER_IP_TOS; 5847 5848 if (fconf & F_VLAN) 5849 mode |= T4_FILTER_VLAN; 5850 5851 if (fconf & F_VNIC_ID) 5852 mode |= T4_FILTER_VNIC; 5853 5854 if (fconf & F_PORT) 5855 mode |= T4_FILTER_PORT; 5856 5857 if (fconf & F_FCOE) 5858 mode |= T4_FILTER_FCoE; 5859 5860 return (mode); 5861 } 5862 5863 static uint32_t 5864 mode_to_fconf(uint32_t mode) 5865 { 5866 uint32_t fconf = 0; 5867 5868 if (mode & T4_FILTER_IP_FRAGMENT) 5869 fconf |= F_FRAGMENTATION; 5870 5871 if (mode & T4_FILTER_MPS_HIT_TYPE) 5872 fconf |= F_MPSHITTYPE; 5873 5874 if (mode & T4_FILTER_MAC_IDX) 5875 fconf |= F_MACMATCH; 5876 5877 if (mode & T4_FILTER_ETH_TYPE) 5878 fconf |= F_ETHERTYPE; 5879 5880 if (mode & T4_FILTER_IP_PROTO) 5881 fconf |= F_PROTOCOL; 5882 5883 if (mode & T4_FILTER_IP_TOS) 5884 fconf |= F_TOS; 5885 5886 if (mode & T4_FILTER_VLAN) 5887 fconf |= F_VLAN; 5888 5889 if (mode & T4_FILTER_VNIC) 5890 fconf |= F_VNIC_ID; 5891 5892 if (mode & T4_FILTER_PORT) 5893 fconf |= F_PORT; 5894 5895 if (mode & T4_FILTER_FCoE) 5896 fconf |= F_FCOE; 5897 5898 return (fconf); 5899 } 5900 5901 static uint32_t 5902 fspec_to_fconf(struct t4_filter_specification *fs) 5903 { 5904 uint32_t fconf = 0; 5905 5906 if (fs->val.frag || fs->mask.frag) 5907 fconf |= F_FRAGMENTATION; 5908 5909 if (fs->val.matchtype || fs->mask.matchtype) 5910 fconf |= F_MPSHITTYPE; 5911 5912 if (fs->val.macidx || fs->mask.macidx) 5913 fconf |= F_MACMATCH; 5914 5915 if (fs->val.ethtype || fs->mask.ethtype) 5916 fconf |= F_ETHERTYPE; 5917 5918 if (fs->val.proto || fs->mask.proto) 5919 fconf |= F_PROTOCOL; 5920 5921 if (fs->val.tos || fs->mask.tos) 5922 fconf |= F_TOS; 5923 5924 if (fs->val.vlan_vld || fs->mask.vlan_vld) 5925 fconf |= F_VLAN; 5926 5927 if (fs->val.vnic_vld || fs->mask.vnic_vld) 5928 fconf |= F_VNIC_ID; 5929 5930 if (fs->val.iport || fs->mask.iport) 5931 fconf |= F_PORT; 5932 5933 if (fs->val.fcoe || fs->mask.fcoe) 5934 fconf |= F_FCOE; 5935 5936 return (fconf); 5937 } 5938 5939 static int 5940 get_filter_mode(struct adapter *sc, uint32_t *mode) 5941 { 5942 int rc; 5943 uint32_t fconf; 5944 5945 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 5946 "t4getfm"); 5947 if (rc) 5948 return (rc); 5949 5950 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 5951 A_TP_VLAN_PRI_MAP); 5952 5953 if (sc->filter_mode != fconf) { 5954 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 5955 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 5956 sc->filter_mode = fconf; 5957 } 5958 5959 *mode = fconf_to_mode(sc->filter_mode); 5960 5961 end_synchronized_op(sc, LOCK_HELD); 5962 return (0); 5963 } 5964 5965 static int 5966 set_filter_mode(struct adapter *sc, uint32_t mode) 5967 { 5968 uint32_t fconf; 5969 int rc; 5970 5971 fconf = mode_to_fconf(mode); 5972 5973 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 5974 "t4setfm"); 5975 if (rc) 5976 return (rc); 5977 5978 if (sc->tids.ftids_in_use > 0) { 5979 rc = EBUSY; 5980 goto done; 5981 } 5982 5983 #ifdef TCP_OFFLOAD 5984 if (sc->offload_map) { 5985 rc = EBUSY; 5986 goto done; 5987 } 5988 #endif 5989 5990 #ifdef notyet 5991 rc = -t4_set_filter_mode(sc, fconf); 5992 if (rc == 0) 5993 sc->filter_mode = fconf; 5994 #else 5995 rc = ENOTSUP; 5996 #endif 5997 5998 done: 5999 end_synchronized_op(sc, LOCK_HELD); 6000 return (rc); 6001 } 6002 6003 static inline uint64_t 6004 get_filter_hits(struct adapter *sc, uint32_t fid) 6005 { 6006 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6007 uint64_t hits; 6008 6009 memwin_info(sc, 0, &mw_base, NULL); 6010 off = position_memwin(sc, 0, 6011 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6012 hits = t4_read_reg64(sc, mw_base + off + 16); 6013 6014 return (be64toh(hits)); 6015 } 6016 6017 static int 6018 get_filter(struct adapter *sc, struct t4_filter *t) 6019 { 6020 int i, rc, nfilters = sc->tids.nftids; 6021 struct filter_entry *f; 6022 6023 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6024 "t4getf"); 6025 if (rc) 6026 return (rc); 6027 6028 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6029 t->idx >= nfilters) { 6030 t->idx = 0xffffffff; 6031 goto done; 6032 } 6033 6034 f = &sc->tids.ftid_tab[t->idx]; 6035 for (i = t->idx; i < nfilters; i++, f++) { 6036 if (f->valid) { 6037 t->idx = i; 6038 t->l2tidx = f->l2t ? f->l2t->idx : 0; 6039 t->smtidx = f->smtidx; 6040 if (f->fs.hitcnts) 6041 t->hits = get_filter_hits(sc, t->idx); 6042 else 6043 t->hits = UINT64_MAX; 6044 t->fs = f->fs; 6045 6046 goto done; 6047 } 6048 } 6049 6050 t->idx = 0xffffffff; 6051 done: 6052 end_synchronized_op(sc, LOCK_HELD); 6053 return (0); 6054 } 6055 6056 static int 6057 set_filter(struct adapter *sc, struct t4_filter *t) 6058 { 6059 unsigned int nfilters, nports; 6060 struct filter_entry *f; 6061 int i, rc; 6062 6063 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6064 if (rc) 6065 return (rc); 6066 6067 nfilters = sc->tids.nftids; 6068 nports = sc->params.nports; 6069 6070 if (nfilters == 0) { 6071 rc = ENOTSUP; 6072 goto done; 6073 } 6074 6075 if (!(sc->flags & FULL_INIT_DONE)) { 6076 rc = EAGAIN; 6077 goto done; 6078 } 6079 6080 if (t->idx >= nfilters) { 6081 rc = EINVAL; 6082 goto done; 6083 } 6084 6085 /* Validate against the global filter mode */ 6086 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) { 6087 rc = E2BIG; 6088 goto done; 6089 } 6090 6091 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6092 rc = EINVAL; 6093 goto done; 6094 } 6095 6096 if (t->fs.val.iport >= nports) { 6097 rc = EINVAL; 6098 goto done; 6099 } 6100 6101 /* Can't specify an iq if not steering to it */ 6102 if (!t->fs.dirsteer && t->fs.iq) { 6103 rc = EINVAL; 6104 goto done; 6105 } 6106 6107 /* IPv6 filter idx must be 4 aligned */ 6108 if (t->fs.type == 1 && 6109 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6110 rc = EINVAL; 6111 goto done; 6112 } 6113 6114 if (sc->tids.ftid_tab == NULL) { 6115 KASSERT(sc->tids.ftids_in_use == 0, 6116 ("%s: no memory allocated but filters_in_use > 0", 6117 __func__)); 6118 6119 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6120 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6121 if (sc->tids.ftid_tab == NULL) { 6122 rc = ENOMEM; 6123 goto done; 6124 } 6125 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6126 } 6127 6128 for (i = 0; i < 4; i++) { 6129 f = &sc->tids.ftid_tab[t->idx + i]; 6130 6131 if (f->pending || f->valid) { 6132 rc = EBUSY; 6133 goto done; 6134 } 6135 if (f->locked) { 6136 rc = EPERM; 6137 goto done; 6138 } 6139 6140 if (t->fs.type == 0) 6141 break; 6142 } 6143 6144 f = &sc->tids.ftid_tab[t->idx]; 6145 f->fs = t->fs; 6146 6147 rc = set_filter_wr(sc, t->idx); 6148 done: 6149 end_synchronized_op(sc, 0); 6150 6151 if (rc == 0) { 6152 mtx_lock(&sc->tids.ftid_lock); 6153 for (;;) { 6154 if (f->pending == 0) { 6155 rc = f->valid ? 0 : EIO; 6156 break; 6157 } 6158 6159 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6160 PCATCH, "t4setfw", 0)) { 6161 rc = EINPROGRESS; 6162 break; 6163 } 6164 } 6165 mtx_unlock(&sc->tids.ftid_lock); 6166 } 6167 return (rc); 6168 } 6169 6170 static int 6171 del_filter(struct adapter *sc, struct t4_filter *t) 6172 { 6173 unsigned int nfilters; 6174 struct filter_entry *f; 6175 int rc; 6176 6177 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 6178 if (rc) 6179 return (rc); 6180 6181 nfilters = sc->tids.nftids; 6182 6183 if (nfilters == 0) { 6184 rc = ENOTSUP; 6185 goto done; 6186 } 6187 6188 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 6189 t->idx >= nfilters) { 6190 rc = EINVAL; 6191 goto done; 6192 } 6193 6194 if (!(sc->flags & FULL_INIT_DONE)) { 6195 rc = EAGAIN; 6196 goto done; 6197 } 6198 6199 f = &sc->tids.ftid_tab[t->idx]; 6200 6201 if (f->pending) { 6202 rc = EBUSY; 6203 goto done; 6204 } 6205 if (f->locked) { 6206 rc = EPERM; 6207 goto done; 6208 } 6209 6210 if (f->valid) { 6211 t->fs = f->fs; /* extra info for the caller */ 6212 rc = del_filter_wr(sc, t->idx); 6213 } 6214 6215 done: 6216 end_synchronized_op(sc, 0); 6217 6218 if (rc == 0) { 6219 mtx_lock(&sc->tids.ftid_lock); 6220 for (;;) { 6221 if (f->pending == 0) { 6222 rc = f->valid ? EIO : 0; 6223 break; 6224 } 6225 6226 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6227 PCATCH, "t4delfw", 0)) { 6228 rc = EINPROGRESS; 6229 break; 6230 } 6231 } 6232 mtx_unlock(&sc->tids.ftid_lock); 6233 } 6234 6235 return (rc); 6236 } 6237 6238 static void 6239 clear_filter(struct filter_entry *f) 6240 { 6241 if (f->l2t) 6242 t4_l2t_release(f->l2t); 6243 6244 bzero(f, sizeof (*f)); 6245 } 6246 6247 static int 6248 set_filter_wr(struct adapter *sc, int fidx) 6249 { 6250 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6251 struct wrqe *wr; 6252 struct fw_filter_wr *fwr; 6253 unsigned int ftid; 6254 6255 ASSERT_SYNCHRONIZED_OP(sc); 6256 6257 if (f->fs.newdmac || f->fs.newvlan) { 6258 /* This filter needs an L2T entry; allocate one. */ 6259 f->l2t = t4_l2t_alloc_switching(sc->l2t); 6260 if (f->l2t == NULL) 6261 return (EAGAIN); 6262 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 6263 f->fs.dmac)) { 6264 t4_l2t_release(f->l2t); 6265 f->l2t = NULL; 6266 return (ENOMEM); 6267 } 6268 } 6269 6270 ftid = sc->tids.ftid_base + fidx; 6271 6272 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6273 if (wr == NULL) 6274 return (ENOMEM); 6275 6276 fwr = wrtod(wr); 6277 bzero(fwr, sizeof (*fwr)); 6278 6279 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 6280 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 6281 fwr->tid_to_iq = 6282 htobe32(V_FW_FILTER_WR_TID(ftid) | 6283 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 6284 V_FW_FILTER_WR_NOREPLY(0) | 6285 V_FW_FILTER_WR_IQ(f->fs.iq)); 6286 fwr->del_filter_to_l2tix = 6287 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 6288 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 6289 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 6290 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 6291 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 6292 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 6293 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 6294 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 6295 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 6296 f->fs.newvlan == VLAN_REWRITE) | 6297 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 6298 f->fs.newvlan == VLAN_REWRITE) | 6299 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 6300 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 6301 V_FW_FILTER_WR_PRIO(f->fs.prio) | 6302 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 6303 fwr->ethtype = htobe16(f->fs.val.ethtype); 6304 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 6305 fwr->frag_to_ovlan_vldm = 6306 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 6307 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 6308 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 6309 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 6310 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 6311 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 6312 fwr->smac_sel = 0; 6313 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 6314 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 6315 fwr->maci_to_matchtypem = 6316 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 6317 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 6318 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 6319 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 6320 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 6321 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 6322 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 6323 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 6324 fwr->ptcl = f->fs.val.proto; 6325 fwr->ptclm = f->fs.mask.proto; 6326 fwr->ttyp = f->fs.val.tos; 6327 fwr->ttypm = f->fs.mask.tos; 6328 fwr->ivlan = htobe16(f->fs.val.vlan); 6329 fwr->ivlanm = htobe16(f->fs.mask.vlan); 6330 fwr->ovlan = htobe16(f->fs.val.vnic); 6331 fwr->ovlanm = htobe16(f->fs.mask.vnic); 6332 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 6333 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 6334 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 6335 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 6336 fwr->lp = htobe16(f->fs.val.dport); 6337 fwr->lpm = htobe16(f->fs.mask.dport); 6338 fwr->fp = htobe16(f->fs.val.sport); 6339 fwr->fpm = htobe16(f->fs.mask.sport); 6340 if (f->fs.newsmac) 6341 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 6342 6343 f->pending = 1; 6344 sc->tids.ftids_in_use++; 6345 6346 t4_wrq_tx(sc, wr); 6347 return (0); 6348 } 6349 6350 static int 6351 del_filter_wr(struct adapter *sc, int fidx) 6352 { 6353 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6354 struct wrqe *wr; 6355 struct fw_filter_wr *fwr; 6356 unsigned int ftid; 6357 6358 ftid = sc->tids.ftid_base + fidx; 6359 6360 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6361 if (wr == NULL) 6362 return (ENOMEM); 6363 fwr = wrtod(wr); 6364 bzero(fwr, sizeof (*fwr)); 6365 6366 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 6367 6368 f->pending = 1; 6369 t4_wrq_tx(sc, wr); 6370 return (0); 6371 } 6372 6373 int 6374 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 6375 { 6376 struct adapter *sc = iq->adapter; 6377 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 6378 unsigned int idx = GET_TID(rpl); 6379 6380 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6381 rss->opcode)); 6382 6383 if (idx >= sc->tids.ftid_base && 6384 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 6385 unsigned int rc = G_COOKIE(rpl->cookie); 6386 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 6387 6388 mtx_lock(&sc->tids.ftid_lock); 6389 if (rc == FW_FILTER_WR_FLT_ADDED) { 6390 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 6391 __func__, idx)); 6392 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 6393 f->pending = 0; /* asynchronous setup completed */ 6394 f->valid = 1; 6395 } else { 6396 if (rc != FW_FILTER_WR_FLT_DELETED) { 6397 /* Add or delete failed, display an error */ 6398 log(LOG_ERR, 6399 "filter %u setup failed with error %u\n", 6400 idx, rc); 6401 } 6402 6403 clear_filter(f); 6404 sc->tids.ftids_in_use--; 6405 } 6406 wakeup(&sc->tids.ftid_tab); 6407 mtx_unlock(&sc->tids.ftid_lock); 6408 } 6409 6410 return (0); 6411 } 6412 6413 static int 6414 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 6415 { 6416 int rc; 6417 6418 if (cntxt->cid > M_CTXTQID) 6419 return (EINVAL); 6420 6421 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 6422 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 6423 return (EINVAL); 6424 6425 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 6426 if (rc) 6427 return (rc); 6428 6429 if (sc->flags & FW_OK) { 6430 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 6431 &cntxt->data[0]); 6432 if (rc == 0) 6433 goto done; 6434 } 6435 6436 /* 6437 * Read via firmware failed or wasn't even attempted. Read directly via 6438 * the backdoor. 6439 */ 6440 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 6441 done: 6442 end_synchronized_op(sc, 0); 6443 return (rc); 6444 } 6445 6446 static int 6447 load_fw(struct adapter *sc, struct t4_data *fw) 6448 { 6449 int rc; 6450 uint8_t *fw_data; 6451 6452 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 6453 if (rc) 6454 return (rc); 6455 6456 if (sc->flags & FULL_INIT_DONE) { 6457 rc = EBUSY; 6458 goto done; 6459 } 6460 6461 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 6462 if (fw_data == NULL) { 6463 rc = ENOMEM; 6464 goto done; 6465 } 6466 6467 rc = copyin(fw->data, fw_data, fw->len); 6468 if (rc == 0) 6469 rc = -t4_load_fw(sc, fw_data, fw->len); 6470 6471 free(fw_data, M_CXGBE); 6472 done: 6473 end_synchronized_op(sc, 0); 6474 return (rc); 6475 } 6476 6477 static int 6478 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 6479 { 6480 uint32_t addr, off, remaining, i, n; 6481 uint32_t *buf, *b; 6482 uint32_t mw_base, mw_aperture; 6483 int rc; 6484 uint8_t *dst; 6485 6486 rc = validate_mem_range(sc, mr->addr, mr->len); 6487 if (rc != 0) 6488 return (rc); 6489 6490 memwin_info(sc, win, &mw_base, &mw_aperture); 6491 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 6492 addr = mr->addr; 6493 remaining = mr->len; 6494 dst = (void *)mr->data; 6495 6496 while (remaining) { 6497 off = position_memwin(sc, win, addr); 6498 6499 /* number of bytes that we'll copy in the inner loop */ 6500 n = min(remaining, mw_aperture - off); 6501 for (i = 0; i < n; i += 4) 6502 *b++ = t4_read_reg(sc, mw_base + off + i); 6503 6504 rc = copyout(buf, dst, n); 6505 if (rc != 0) 6506 break; 6507 6508 b = buf; 6509 dst += n; 6510 remaining -= n; 6511 addr += n; 6512 } 6513 6514 free(buf, M_CXGBE); 6515 return (rc); 6516 } 6517 6518 static int 6519 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 6520 { 6521 int rc; 6522 6523 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 6524 return (EINVAL); 6525 6526 if (i2cd->len > 1) { 6527 /* XXX: need fw support for longer reads in one go */ 6528 return (ENOTSUP); 6529 } 6530 6531 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 6532 if (rc) 6533 return (rc); 6534 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 6535 i2cd->offset, &i2cd->data[0]); 6536 end_synchronized_op(sc, 0); 6537 6538 return (rc); 6539 } 6540 6541 int 6542 t4_os_find_pci_capability(struct adapter *sc, int cap) 6543 { 6544 int i; 6545 6546 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 6547 } 6548 6549 int 6550 t4_os_pci_save_state(struct adapter *sc) 6551 { 6552 device_t dev; 6553 struct pci_devinfo *dinfo; 6554 6555 dev = sc->dev; 6556 dinfo = device_get_ivars(dev); 6557 6558 pci_cfg_save(dev, dinfo, 0); 6559 return (0); 6560 } 6561 6562 int 6563 t4_os_pci_restore_state(struct adapter *sc) 6564 { 6565 device_t dev; 6566 struct pci_devinfo *dinfo; 6567 6568 dev = sc->dev; 6569 dinfo = device_get_ivars(dev); 6570 6571 pci_cfg_restore(dev, dinfo); 6572 return (0); 6573 } 6574 6575 void 6576 t4_os_portmod_changed(const struct adapter *sc, int idx) 6577 { 6578 struct port_info *pi = sc->port[idx]; 6579 static const char *mod_str[] = { 6580 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 6581 }; 6582 6583 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 6584 if_printf(pi->ifp, "transceiver unplugged.\n"); 6585 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 6586 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 6587 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 6588 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 6589 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 6590 if_printf(pi->ifp, "%s transceiver inserted.\n", 6591 mod_str[pi->mod_type]); 6592 } else { 6593 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 6594 pi->mod_type); 6595 } 6596 } 6597 6598 void 6599 t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 6600 { 6601 struct port_info *pi = sc->port[idx]; 6602 struct ifnet *ifp = pi->ifp; 6603 6604 if (link_stat) { 6605 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 6606 if_link_state_change(ifp, LINK_STATE_UP); 6607 } else 6608 if_link_state_change(ifp, LINK_STATE_DOWN); 6609 } 6610 6611 void 6612 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 6613 { 6614 struct adapter *sc; 6615 6616 mtx_lock(&t4_list_lock); 6617 SLIST_FOREACH(sc, &t4_list, link) { 6618 /* 6619 * func should not make any assumptions about what state sc is 6620 * in - the only guarantee is that sc->sc_lock is a valid lock. 6621 */ 6622 func(sc, arg); 6623 } 6624 mtx_unlock(&t4_list_lock); 6625 } 6626 6627 static int 6628 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 6629 { 6630 return (0); 6631 } 6632 6633 static int 6634 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 6635 { 6636 return (0); 6637 } 6638 6639 static int 6640 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 6641 struct thread *td) 6642 { 6643 int rc; 6644 struct adapter *sc = dev->si_drv1; 6645 6646 rc = priv_check(td, PRIV_DRIVER); 6647 if (rc != 0) 6648 return (rc); 6649 6650 switch (cmd) { 6651 case CHELSIO_T4_GETREG: { 6652 struct t4_reg *edata = (struct t4_reg *)data; 6653 6654 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6655 return (EFAULT); 6656 6657 if (edata->size == 4) 6658 edata->val = t4_read_reg(sc, edata->addr); 6659 else if (edata->size == 8) 6660 edata->val = t4_read_reg64(sc, edata->addr); 6661 else 6662 return (EINVAL); 6663 6664 break; 6665 } 6666 case CHELSIO_T4_SETREG: { 6667 struct t4_reg *edata = (struct t4_reg *)data; 6668 6669 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6670 return (EFAULT); 6671 6672 if (edata->size == 4) { 6673 if (edata->val & 0xffffffff00000000) 6674 return (EINVAL); 6675 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 6676 } else if (edata->size == 8) 6677 t4_write_reg64(sc, edata->addr, edata->val); 6678 else 6679 return (EINVAL); 6680 break; 6681 } 6682 case CHELSIO_T4_REGDUMP: { 6683 struct t4_regdump *regs = (struct t4_regdump *)data; 6684 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 6685 uint8_t *buf; 6686 6687 if (regs->len < reglen) { 6688 regs->len = reglen; /* hint to the caller */ 6689 return (ENOBUFS); 6690 } 6691 6692 regs->len = reglen; 6693 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 6694 t4_get_regs(sc, regs, buf); 6695 rc = copyout(buf, regs->data, reglen); 6696 free(buf, M_CXGBE); 6697 break; 6698 } 6699 case CHELSIO_T4_GET_FILTER_MODE: 6700 rc = get_filter_mode(sc, (uint32_t *)data); 6701 break; 6702 case CHELSIO_T4_SET_FILTER_MODE: 6703 rc = set_filter_mode(sc, *(uint32_t *)data); 6704 break; 6705 case CHELSIO_T4_GET_FILTER: 6706 rc = get_filter(sc, (struct t4_filter *)data); 6707 break; 6708 case CHELSIO_T4_SET_FILTER: 6709 rc = set_filter(sc, (struct t4_filter *)data); 6710 break; 6711 case CHELSIO_T4_DEL_FILTER: 6712 rc = del_filter(sc, (struct t4_filter *)data); 6713 break; 6714 case CHELSIO_T4_GET_SGE_CONTEXT: 6715 rc = get_sge_context(sc, (struct t4_sge_context *)data); 6716 break; 6717 case CHELSIO_T4_LOAD_FW: 6718 rc = load_fw(sc, (struct t4_data *)data); 6719 break; 6720 case CHELSIO_T4_GET_MEM: 6721 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 6722 break; 6723 case CHELSIO_T4_GET_I2C: 6724 rc = read_i2c(sc, (struct t4_i2c_data *)data); 6725 break; 6726 case CHELSIO_T4_CLEAR_STATS: { 6727 int i; 6728 u_int port_id = *(uint32_t *)data; 6729 struct port_info *pi; 6730 6731 if (port_id >= sc->params.nports) 6732 return (EINVAL); 6733 6734 /* MAC stats */ 6735 t4_clr_port_stats(sc, port_id); 6736 6737 pi = sc->port[port_id]; 6738 if (pi->flags & PORT_INIT_DONE) { 6739 struct sge_rxq *rxq; 6740 struct sge_txq *txq; 6741 struct sge_wrq *wrq; 6742 6743 for_each_rxq(pi, i, rxq) { 6744 #if defined(INET) || defined(INET6) 6745 rxq->lro.lro_queued = 0; 6746 rxq->lro.lro_flushed = 0; 6747 #endif 6748 rxq->rxcsum = 0; 6749 rxq->vlan_extraction = 0; 6750 } 6751 6752 for_each_txq(pi, i, txq) { 6753 txq->txcsum = 0; 6754 txq->tso_wrs = 0; 6755 txq->vlan_insertion = 0; 6756 txq->imm_wrs = 0; 6757 txq->sgl_wrs = 0; 6758 txq->txpkt_wrs = 0; 6759 txq->txpkts_wrs = 0; 6760 txq->txpkts_pkts = 0; 6761 txq->br->br_drops = 0; 6762 txq->no_dmamap = 0; 6763 txq->no_desc = 0; 6764 } 6765 6766 #ifdef TCP_OFFLOAD 6767 /* nothing to clear for each ofld_rxq */ 6768 6769 for_each_ofld_txq(pi, i, wrq) { 6770 wrq->tx_wrs = 0; 6771 wrq->no_desc = 0; 6772 } 6773 #endif 6774 wrq = &sc->sge.ctrlq[pi->port_id]; 6775 wrq->tx_wrs = 0; 6776 wrq->no_desc = 0; 6777 } 6778 break; 6779 } 6780 default: 6781 rc = EINVAL; 6782 } 6783 6784 return (rc); 6785 } 6786 6787 #ifdef TCP_OFFLOAD 6788 static int 6789 toe_capability(struct port_info *pi, int enable) 6790 { 6791 int rc; 6792 struct adapter *sc = pi->adapter; 6793 6794 ASSERT_SYNCHRONIZED_OP(sc); 6795 6796 if (!is_offload(sc)) 6797 return (ENODEV); 6798 6799 if (enable) { 6800 if (!(sc->flags & FULL_INIT_DONE)) { 6801 rc = cxgbe_init_synchronized(pi); 6802 if (rc) 6803 return (rc); 6804 } 6805 6806 if (isset(&sc->offload_map, pi->port_id)) 6807 return (0); 6808 6809 if (!(sc->flags & TOM_INIT_DONE)) { 6810 rc = t4_activate_uld(sc, ULD_TOM); 6811 if (rc == EAGAIN) { 6812 log(LOG_WARNING, 6813 "You must kldload t4_tom.ko before trying " 6814 "to enable TOE on a cxgbe interface.\n"); 6815 } 6816 if (rc != 0) 6817 return (rc); 6818 KASSERT(sc->tom_softc != NULL, 6819 ("%s: TOM activated but softc NULL", __func__)); 6820 KASSERT(sc->flags & TOM_INIT_DONE, 6821 ("%s: TOM activated but flag not set", __func__)); 6822 } 6823 6824 setbit(&sc->offload_map, pi->port_id); 6825 } else { 6826 if (!isset(&sc->offload_map, pi->port_id)) 6827 return (0); 6828 6829 KASSERT(sc->flags & TOM_INIT_DONE, 6830 ("%s: TOM never initialized?", __func__)); 6831 clrbit(&sc->offload_map, pi->port_id); 6832 } 6833 6834 return (0); 6835 } 6836 6837 /* 6838 * Add an upper layer driver to the global list. 6839 */ 6840 int 6841 t4_register_uld(struct uld_info *ui) 6842 { 6843 int rc = 0; 6844 struct uld_info *u; 6845 6846 mtx_lock(&t4_uld_list_lock); 6847 SLIST_FOREACH(u, &t4_uld_list, link) { 6848 if (u->uld_id == ui->uld_id) { 6849 rc = EEXIST; 6850 goto done; 6851 } 6852 } 6853 6854 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 6855 ui->refcount = 0; 6856 done: 6857 mtx_unlock(&t4_uld_list_lock); 6858 return (rc); 6859 } 6860 6861 int 6862 t4_unregister_uld(struct uld_info *ui) 6863 { 6864 int rc = EINVAL; 6865 struct uld_info *u; 6866 6867 mtx_lock(&t4_uld_list_lock); 6868 6869 SLIST_FOREACH(u, &t4_uld_list, link) { 6870 if (u == ui) { 6871 if (ui->refcount > 0) { 6872 rc = EBUSY; 6873 goto done; 6874 } 6875 6876 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 6877 rc = 0; 6878 goto done; 6879 } 6880 } 6881 done: 6882 mtx_unlock(&t4_uld_list_lock); 6883 return (rc); 6884 } 6885 6886 int 6887 t4_activate_uld(struct adapter *sc, int id) 6888 { 6889 int rc = EAGAIN; 6890 struct uld_info *ui; 6891 6892 ASSERT_SYNCHRONIZED_OP(sc); 6893 6894 mtx_lock(&t4_uld_list_lock); 6895 6896 SLIST_FOREACH(ui, &t4_uld_list, link) { 6897 if (ui->uld_id == id) { 6898 rc = ui->activate(sc); 6899 if (rc == 0) 6900 ui->refcount++; 6901 goto done; 6902 } 6903 } 6904 done: 6905 mtx_unlock(&t4_uld_list_lock); 6906 6907 return (rc); 6908 } 6909 6910 int 6911 t4_deactivate_uld(struct adapter *sc, int id) 6912 { 6913 int rc = EINVAL; 6914 struct uld_info *ui; 6915 6916 ASSERT_SYNCHRONIZED_OP(sc); 6917 6918 mtx_lock(&t4_uld_list_lock); 6919 6920 SLIST_FOREACH(ui, &t4_uld_list, link) { 6921 if (ui->uld_id == id) { 6922 rc = ui->deactivate(sc); 6923 if (rc == 0) 6924 ui->refcount--; 6925 goto done; 6926 } 6927 } 6928 done: 6929 mtx_unlock(&t4_uld_list_lock); 6930 6931 return (rc); 6932 } 6933 #endif 6934 6935 /* 6936 * Come up with reasonable defaults for some of the tunables, provided they're 6937 * not set by the user (in which case we'll use the values as is). 6938 */ 6939 static void 6940 tweak_tunables(void) 6941 { 6942 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 6943 6944 if (t4_ntxq10g < 1) 6945 t4_ntxq10g = min(nc, NTXQ_10G); 6946 6947 if (t4_ntxq1g < 1) 6948 t4_ntxq1g = min(nc, NTXQ_1G); 6949 6950 if (t4_nrxq10g < 1) 6951 t4_nrxq10g = min(nc, NRXQ_10G); 6952 6953 if (t4_nrxq1g < 1) 6954 t4_nrxq1g = min(nc, NRXQ_1G); 6955 6956 #ifdef TCP_OFFLOAD 6957 if (t4_nofldtxq10g < 1) 6958 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 6959 6960 if (t4_nofldtxq1g < 1) 6961 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 6962 6963 if (t4_nofldrxq10g < 1) 6964 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 6965 6966 if (t4_nofldrxq1g < 1) 6967 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 6968 6969 if (t4_toecaps_allowed == -1) 6970 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 6971 #else 6972 if (t4_toecaps_allowed == -1) 6973 t4_toecaps_allowed = 0; 6974 #endif 6975 6976 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 6977 t4_tmr_idx_10g = TMR_IDX_10G; 6978 6979 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 6980 t4_pktc_idx_10g = PKTC_IDX_10G; 6981 6982 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 6983 t4_tmr_idx_1g = TMR_IDX_1G; 6984 6985 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 6986 t4_pktc_idx_1g = PKTC_IDX_1G; 6987 6988 if (t4_qsize_txq < 128) 6989 t4_qsize_txq = 128; 6990 6991 if (t4_qsize_rxq < 128) 6992 t4_qsize_rxq = 128; 6993 while (t4_qsize_rxq & 7) 6994 t4_qsize_rxq++; 6995 6996 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 6997 } 6998 6999 static int 7000 t4_mod_event(module_t mod, int cmd, void *arg) 7001 { 7002 int rc = 0; 7003 7004 switch (cmd) { 7005 case MOD_LOAD: 7006 t4_sge_modload(); 7007 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 7008 SLIST_INIT(&t4_list); 7009 #ifdef TCP_OFFLOAD 7010 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 7011 SLIST_INIT(&t4_uld_list); 7012 #endif 7013 tweak_tunables(); 7014 break; 7015 7016 case MOD_UNLOAD: 7017 #ifdef TCP_OFFLOAD 7018 mtx_lock(&t4_uld_list_lock); 7019 if (!SLIST_EMPTY(&t4_uld_list)) { 7020 rc = EBUSY; 7021 mtx_unlock(&t4_uld_list_lock); 7022 break; 7023 } 7024 mtx_unlock(&t4_uld_list_lock); 7025 mtx_destroy(&t4_uld_list_lock); 7026 #endif 7027 mtx_lock(&t4_list_lock); 7028 if (!SLIST_EMPTY(&t4_list)) { 7029 rc = EBUSY; 7030 mtx_unlock(&t4_list_lock); 7031 break; 7032 } 7033 mtx_unlock(&t4_list_lock); 7034 mtx_destroy(&t4_list_lock); 7035 break; 7036 } 7037 7038 return (rc); 7039 } 7040 7041 static devclass_t t4_devclass, t5_devclass; 7042 static devclass_t cxgbe_devclass, cxl_devclass; 7043 7044 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 7045 MODULE_VERSION(t4nex, 1); 7046 7047 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, 0, 0); 7048 MODULE_VERSION(t5nex, 1); 7049 7050 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 7051 MODULE_VERSION(cxgbe, 1); 7052 7053 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 7054 MODULE_VERSION(cxl, 1); 7055