1 /*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/priv.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/taskqueue.h> 43 #include <sys/pciio.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pci_private.h> 47 #include <sys/firmware.h> 48 #include <sys/sbuf.h> 49 #include <sys/smp.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <net/ethernet.h> 54 #include <net/if.h> 55 #include <net/if_types.h> 56 #include <net/if_dl.h> 57 #include <net/if_vlan_var.h> 58 #if defined(__i386__) || defined(__amd64__) 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #endif 62 63 #include "common/common.h" 64 #include "common/t4_msg.h" 65 #include "common/t4_regs.h" 66 #include "common/t4_regs_values.h" 67 #include "t4_ioctl.h" 68 #include "t4_l2t.h" 69 #include "t4_mp_ring.h" 70 71 /* T4 bus driver interface */ 72 static int t4_probe(device_t); 73 static int t4_attach(device_t); 74 static int t4_detach(device_t); 75 static device_method_t t4_methods[] = { 76 DEVMETHOD(device_probe, t4_probe), 77 DEVMETHOD(device_attach, t4_attach), 78 DEVMETHOD(device_detach, t4_detach), 79 80 DEVMETHOD_END 81 }; 82 static driver_t t4_driver = { 83 "t4nex", 84 t4_methods, 85 sizeof(struct adapter) 86 }; 87 88 89 /* T4 port (cxgbe) interface */ 90 static int cxgbe_probe(device_t); 91 static int cxgbe_attach(device_t); 92 static int cxgbe_detach(device_t); 93 static device_method_t cxgbe_methods[] = { 94 DEVMETHOD(device_probe, cxgbe_probe), 95 DEVMETHOD(device_attach, cxgbe_attach), 96 DEVMETHOD(device_detach, cxgbe_detach), 97 { 0, 0 } 98 }; 99 static driver_t cxgbe_driver = { 100 "cxgbe", 101 cxgbe_methods, 102 sizeof(struct port_info) 103 }; 104 105 static d_ioctl_t t4_ioctl; 106 static d_open_t t4_open; 107 static d_close_t t4_close; 108 109 static struct cdevsw t4_cdevsw = { 110 .d_version = D_VERSION, 111 .d_flags = 0, 112 .d_open = t4_open, 113 .d_close = t4_close, 114 .d_ioctl = t4_ioctl, 115 .d_name = "t4nex", 116 }; 117 118 /* T5 bus driver interface */ 119 static int t5_probe(device_t); 120 static device_method_t t5_methods[] = { 121 DEVMETHOD(device_probe, t5_probe), 122 DEVMETHOD(device_attach, t4_attach), 123 DEVMETHOD(device_detach, t4_detach), 124 125 DEVMETHOD_END 126 }; 127 static driver_t t5_driver = { 128 "t5nex", 129 t5_methods, 130 sizeof(struct adapter) 131 }; 132 133 134 /* T5 port (cxl) interface */ 135 static driver_t cxl_driver = { 136 "cxl", 137 cxgbe_methods, 138 sizeof(struct port_info) 139 }; 140 141 static struct cdevsw t5_cdevsw = { 142 .d_version = D_VERSION, 143 .d_flags = 0, 144 .d_open = t4_open, 145 .d_close = t4_close, 146 .d_ioctl = t4_ioctl, 147 .d_name = "t5nex", 148 }; 149 150 /* ifnet + media interface */ 151 static void cxgbe_init(void *); 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *); 154 static void cxgbe_qflush(struct ifnet *); 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); 156 static int cxgbe_media_change(struct ifnet *); 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 158 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 160 161 /* 162 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 163 * then ADAPTER_LOCK, then t4_uld_list_lock. 164 */ 165 static struct sx t4_list_lock; 166 SLIST_HEAD(, adapter) t4_list; 167 #ifdef TCP_OFFLOAD 168 static struct sx t4_uld_list_lock; 169 SLIST_HEAD(, uld_info) t4_uld_list; 170 #endif 171 172 /* 173 * Tunables. See tweak_tunables() too. 174 * 175 * Each tunable is set to a default value here if it's known at compile-time. 176 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 177 * provide a reasonable default when the driver is loaded. 178 * 179 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 180 * T5 are under hw.cxl. 181 */ 182 183 /* 184 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 185 */ 186 #define NTXQ_10G 16 187 static int t4_ntxq10g = -1; 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 189 190 #define NRXQ_10G 8 191 static int t4_nrxq10g = -1; 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 193 194 #define NTXQ_1G 4 195 static int t4_ntxq1g = -1; 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 197 198 #define NRXQ_1G 2 199 static int t4_nrxq1g = -1; 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 201 202 static int t4_rsrv_noflowq = 0; 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 204 205 #ifdef TCP_OFFLOAD 206 #define NOFLDTXQ_10G 8 207 static int t4_nofldtxq10g = -1; 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 209 210 #define NOFLDRXQ_10G 2 211 static int t4_nofldrxq10g = -1; 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 213 214 #define NOFLDTXQ_1G 2 215 static int t4_nofldtxq1g = -1; 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 217 218 #define NOFLDRXQ_1G 1 219 static int t4_nofldrxq1g = -1; 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 221 #endif 222 223 #ifdef DEV_NETMAP 224 #define NNMTXQ_10G 2 225 static int t4_nnmtxq10g = -1; 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 227 228 #define NNMRXQ_10G 2 229 static int t4_nnmrxq10g = -1; 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 231 232 #define NNMTXQ_1G 1 233 static int t4_nnmtxq1g = -1; 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 235 236 #define NNMRXQ_1G 1 237 static int t4_nnmrxq1g = -1; 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 239 #endif 240 241 /* 242 * Holdoff parameters for 10G and 1G ports. 243 */ 244 #define TMR_IDX_10G 1 245 static int t4_tmr_idx_10g = TMR_IDX_10G; 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 247 248 #define PKTC_IDX_10G (-1) 249 static int t4_pktc_idx_10g = PKTC_IDX_10G; 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 251 252 #define TMR_IDX_1G 1 253 static int t4_tmr_idx_1g = TMR_IDX_1G; 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 255 256 #define PKTC_IDX_1G (-1) 257 static int t4_pktc_idx_1g = PKTC_IDX_1G; 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 259 260 /* 261 * Size (# of entries) of each tx and rx queue. 262 */ 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 265 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 268 269 /* 270 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 271 */ 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 274 275 /* 276 * Configuration file. 277 */ 278 #define DEFAULT_CF "default" 279 #define FLASH_CF "flash" 280 #define UWIRE_CF "uwire" 281 #define FPGA_CF "fpga" 282 static char t4_cfg_file[32] = DEFAULT_CF; 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 284 285 /* 286 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 287 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 288 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 289 * mark or when signalled to do so, 0 to never emit PAUSE. 290 */ 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 293 294 /* 295 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 296 * encouraged respectively). 297 */ 298 static unsigned int t4_fw_install = 1; 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 300 301 /* 302 * ASIC features that will be used. Disable the ones you don't want so that the 303 * chip resources aren't wasted on features that will not be used. 304 */ 305 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 307 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 310 311 static int t4_toecaps_allowed = -1; 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 313 314 static int t4_rdmacaps_allowed = 0; 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 316 317 static int t4_iscsicaps_allowed = 0; 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 319 320 static int t4_fcoecaps_allowed = 0; 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 322 323 static int t5_write_combine = 0; 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 325 326 struct intrs_and_queues { 327 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 328 uint16_t nirq; /* Total # of vectors */ 329 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 330 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 331 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 332 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 333 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 334 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 335 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 336 #ifdef TCP_OFFLOAD 337 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 338 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 339 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 340 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 341 #endif 342 #ifdef DEV_NETMAP 343 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 344 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 345 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 346 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 347 #endif 348 }; 349 350 struct filter_entry { 351 uint32_t valid:1; /* filter allocated and valid */ 352 uint32_t locked:1; /* filter is administratively locked */ 353 uint32_t pending:1; /* filter action is pending firmware reply */ 354 uint32_t smtidx:8; /* Source MAC Table index for smac */ 355 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 356 357 struct t4_filter_specification fs; 358 }; 359 360 static int map_bars_0_and_4(struct adapter *); 361 static int map_bar_2(struct adapter *); 362 static void setup_memwin(struct adapter *); 363 static int validate_mem_range(struct adapter *, uint32_t, int); 364 static int fwmtype_to_hwmtype(int); 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 366 uint32_t *); 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 368 static uint32_t position_memwin(struct adapter *, int, uint32_t); 369 static int cfg_itype_and_nqueues(struct adapter *, int, int, 370 struct intrs_and_queues *); 371 static int prep_firmware(struct adapter *); 372 static int partition_resources(struct adapter *, const struct firmware *, 373 const char *); 374 static int get_params__pre_init(struct adapter *); 375 static int get_params__post_init(struct adapter *); 376 static int set_params__post_init(struct adapter *); 377 static void t4_set_desc(struct adapter *); 378 static void build_medialist(struct port_info *, struct ifmedia *); 379 static int cxgbe_init_synchronized(struct port_info *); 380 static int cxgbe_uninit_synchronized(struct port_info *); 381 static int setup_intr_handlers(struct adapter *); 382 static void quiesce_txq(struct adapter *, struct sge_txq *); 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *); 384 static void quiesce_iq(struct adapter *, struct sge_iq *); 385 static void quiesce_fl(struct adapter *, struct sge_fl *); 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 387 driver_intr_t *, void *, char *); 388 static int t4_free_irq(struct adapter *, struct irq *); 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 390 unsigned int); 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 393 static void cxgbe_tick(void *); 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 396 struct mbuf *); 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 398 static int fw_msg_not_handled(struct adapter *, const __be64 *); 399 static int t4_sysctls(struct adapter *); 400 static int cxgbe_sysctls(struct port_info *); 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 412 #ifdef SBUF_DRAIN 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS); 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 438 #endif 439 static uint32_t fconf_to_mode(uint32_t); 440 static uint32_t mode_to_fconf(uint32_t); 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *); 442 static int get_filter_mode(struct adapter *, uint32_t *); 443 static int set_filter_mode(struct adapter *, uint32_t); 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 445 static int get_filter(struct adapter *, struct t4_filter *); 446 static int set_filter(struct adapter *, struct t4_filter *); 447 static int del_filter(struct adapter *, struct t4_filter *); 448 static void clear_filter(struct filter_entry *); 449 static int set_filter_wr(struct adapter *, int); 450 static int del_filter_wr(struct adapter *, int); 451 static int get_sge_context(struct adapter *, struct t4_sge_context *); 452 static int load_fw(struct adapter *, struct t4_data *); 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 454 static int read_i2c(struct adapter *, struct t4_i2c_data *); 455 static int set_sched_class(struct adapter *, struct t4_sched_params *); 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 457 #ifdef TCP_OFFLOAD 458 static int toe_capability(struct port_info *, int); 459 #endif 460 static int mod_event(module_t, int, void *); 461 462 struct { 463 uint16_t device; 464 char *desc; 465 } t4_pciids[] = { 466 {0xa000, "Chelsio Terminator 4 FPGA"}, 467 {0x4400, "Chelsio T440-dbg"}, 468 {0x4401, "Chelsio T420-CR"}, 469 {0x4402, "Chelsio T422-CR"}, 470 {0x4403, "Chelsio T440-CR"}, 471 {0x4404, "Chelsio T420-BCH"}, 472 {0x4405, "Chelsio T440-BCH"}, 473 {0x4406, "Chelsio T440-CH"}, 474 {0x4407, "Chelsio T420-SO"}, 475 {0x4408, "Chelsio T420-CX"}, 476 {0x4409, "Chelsio T420-BT"}, 477 {0x440a, "Chelsio T404-BT"}, 478 {0x440e, "Chelsio T440-LP-CR"}, 479 }, t5_pciids[] = { 480 {0xb000, "Chelsio Terminator 5 FPGA"}, 481 {0x5400, "Chelsio T580-dbg"}, 482 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 483 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 484 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 485 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 486 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 487 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 488 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 489 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 490 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 491 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 492 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 493 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 494 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 495 #ifdef notyet 496 {0x5404, "Chelsio T520-BCH"}, 497 {0x5405, "Chelsio T540-BCH"}, 498 {0x5406, "Chelsio T540-CH"}, 499 {0x5408, "Chelsio T520-CX"}, 500 {0x540b, "Chelsio B520-SR"}, 501 {0x540c, "Chelsio B504-BT"}, 502 {0x540f, "Chelsio Amsterdam"}, 503 {0x5413, "Chelsio T580-CHR"}, 504 #endif 505 }; 506 507 #ifdef TCP_OFFLOAD 508 /* 509 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 510 * exactly the same for both rxq and ofld_rxq. 511 */ 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 514 #endif 515 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */ 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 519 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 521 522 static int 523 t4_probe(device_t dev) 524 { 525 int i; 526 uint16_t v = pci_get_vendor(dev); 527 uint16_t d = pci_get_device(dev); 528 uint8_t f = pci_get_function(dev); 529 530 if (v != PCI_VENDOR_ID_CHELSIO) 531 return (ENXIO); 532 533 /* Attach only to PF0 of the FPGA */ 534 if (d == 0xa000 && f != 0) 535 return (ENXIO); 536 537 for (i = 0; i < nitems(t4_pciids); i++) { 538 if (d == t4_pciids[i].device) { 539 device_set_desc(dev, t4_pciids[i].desc); 540 return (BUS_PROBE_DEFAULT); 541 } 542 } 543 544 return (ENXIO); 545 } 546 547 static int 548 t5_probe(device_t dev) 549 { 550 int i; 551 uint16_t v = pci_get_vendor(dev); 552 uint16_t d = pci_get_device(dev); 553 uint8_t f = pci_get_function(dev); 554 555 if (v != PCI_VENDOR_ID_CHELSIO) 556 return (ENXIO); 557 558 /* Attach only to PF0 of the FPGA */ 559 if (d == 0xb000 && f != 0) 560 return (ENXIO); 561 562 for (i = 0; i < nitems(t5_pciids); i++) { 563 if (d == t5_pciids[i].device) { 564 device_set_desc(dev, t5_pciids[i].desc); 565 return (BUS_PROBE_DEFAULT); 566 } 567 } 568 569 return (ENXIO); 570 } 571 572 static int 573 t4_attach(device_t dev) 574 { 575 struct adapter *sc; 576 int rc = 0, i, n10g, n1g, rqidx, tqidx; 577 struct intrs_and_queues iaq; 578 struct sge *s; 579 #ifdef TCP_OFFLOAD 580 int ofld_rqidx, ofld_tqidx; 581 #endif 582 #ifdef DEV_NETMAP 583 int nm_rqidx, nm_tqidx; 584 #endif 585 const char *pcie_ts; 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 590 pci_enable_busmaster(dev); 591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 592 uint32_t v; 593 594 pci_set_max_read_req(dev, 4096); 595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 598 599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 600 } 601 602 sc->traceq = -1; 603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 605 device_get_nameunit(dev)); 606 607 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 608 device_get_nameunit(dev)); 609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 610 sx_xlock(&t4_list_lock); 611 SLIST_INSERT_HEAD(&t4_list, sc, link); 612 sx_xunlock(&t4_list_lock); 613 614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 615 TAILQ_INIT(&sc->sfl); 616 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 617 618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 619 620 rc = map_bars_0_and_4(sc); 621 if (rc != 0) 622 goto done; /* error message displayed already */ 623 624 /* 625 * This is the real PF# to which we're attaching. Works from within PCI 626 * passthrough environments too, where pci_get_function() could return a 627 * different PF# depending on the passthrough configuration. We need to 628 * use the real PF# in all our communication with the firmware. 629 */ 630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 631 sc->mbox = sc->pf; 632 633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 634 sc->an_handler = an_not_handled; 635 for (i = 0; i < nitems(sc->cpl_handler); i++) 636 sc->cpl_handler[i] = cpl_not_handled; 637 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 638 sc->fw_msg_handler[i] = fw_msg_not_handled; 639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 642 t4_init_sge_cpl_handlers(sc); 643 644 /* Prepare the adapter for operation */ 645 rc = -t4_prep_adapter(sc); 646 if (rc != 0) { 647 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 648 goto done; 649 } 650 651 /* 652 * Do this really early, with the memory windows set up even before the 653 * character device. The userland tool's register i/o and mem read 654 * will work even in "recovery mode". 655 */ 656 setup_memwin(sc); 657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 659 device_get_nameunit(dev)); 660 if (sc->cdev == NULL) 661 device_printf(dev, "failed to create nexus char device.\n"); 662 else 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 671 #if defined(__i386__) 672 if ((cpu_feature & CPUID_CX8) == 0) { 673 device_printf(dev, "64 bit atomics not available.\n"); 674 rc = ENOTSUP; 675 goto done; 676 } 677 #endif 678 679 /* Prepare the firmware for operation */ 680 rc = prep_firmware(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = get_params__post_init(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = set_params__post_init(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 rc = map_bar_2(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 rc = t4_create_dma_tag(sc); 697 if (rc != 0) 698 goto done; /* error message displayed already */ 699 700 /* 701 * First pass over all the ports - allocate VIs and initialize some 702 * basic parameters like mac address, port type, etc. We also figure 703 * out whether a port is 10G or 1G and use that information when 704 * calculating how many interrupts to attempt to allocate. 705 */ 706 n10g = n1g = 0; 707 for_each_port(sc, i) { 708 struct port_info *pi; 709 710 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 711 sc->port[i] = pi; 712 713 /* These must be set before t4_port_init */ 714 pi->adapter = sc; 715 pi->port_id = i; 716 717 /* Allocate the vi and initialize parameters like mac addr */ 718 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 719 if (rc != 0) { 720 device_printf(dev, "unable to initialize port %d: %d\n", 721 i, rc); 722 free(pi, M_CXGBE); 723 sc->port[i] = NULL; 724 goto done; 725 } 726 727 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 728 pi->link_cfg.requested_fc |= t4_pause_settings; 729 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 730 pi->link_cfg.fc |= t4_pause_settings; 731 732 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 733 if (rc != 0) { 734 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 735 free(pi, M_CXGBE); 736 sc->port[i] = NULL; 737 goto done; 738 } 739 740 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 741 device_get_nameunit(dev), i); 742 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 743 sc->chan_map[pi->tx_chan] = i; 744 745 if (is_10G_port(pi) || is_40G_port(pi)) { 746 n10g++; 747 pi->tmr_idx = t4_tmr_idx_10g; 748 pi->pktc_idx = t4_pktc_idx_10g; 749 } else { 750 n1g++; 751 pi->tmr_idx = t4_tmr_idx_1g; 752 pi->pktc_idx = t4_pktc_idx_1g; 753 } 754 755 pi->xact_addr_filt = -1; 756 pi->linkdnrc = -1; 757 758 pi->qsize_rxq = t4_qsize_rxq; 759 pi->qsize_txq = t4_qsize_txq; 760 761 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 762 if (pi->dev == NULL) { 763 device_printf(dev, 764 "failed to add device for port %d.\n", i); 765 rc = ENXIO; 766 goto done; 767 } 768 device_set_softc(pi->dev, pi); 769 } 770 771 /* 772 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 773 */ 774 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 775 if (rc != 0) 776 goto done; /* error message displayed already */ 777 778 sc->intr_type = iaq.intr_type; 779 sc->intr_count = iaq.nirq; 780 781 s = &sc->sge; 782 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 783 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 784 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 785 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 786 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 787 #ifdef TCP_OFFLOAD 788 if (is_offload(sc)) { 789 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 790 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 791 s->neq += s->nofldtxq + s->nofldrxq; 792 s->niq += s->nofldrxq; 793 794 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 797 M_CXGBE, M_ZERO | M_WAITOK); 798 } 799 #endif 800 #ifdef DEV_NETMAP 801 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 802 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 803 s->neq += s->nnmtxq + s->nnmrxq; 804 s->niq += s->nnmrxq; 805 806 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 807 M_CXGBE, M_ZERO | M_WAITOK); 808 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 809 M_CXGBE, M_ZERO | M_WAITOK); 810 #endif 811 812 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 823 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 824 M_ZERO | M_WAITOK); 825 826 t4_init_l2t(sc, M_WAITOK); 827 828 /* 829 * Second pass over the ports. This time we know the number of rx and 830 * tx queues that each port should get. 831 */ 832 rqidx = tqidx = 0; 833 #ifdef TCP_OFFLOAD 834 ofld_rqidx = ofld_tqidx = 0; 835 #endif 836 #ifdef DEV_NETMAP 837 nm_rqidx = nm_tqidx = 0; 838 #endif 839 for_each_port(sc, i) { 840 struct port_info *pi = sc->port[i]; 841 842 if (pi == NULL) 843 continue; 844 845 pi->first_rxq = rqidx; 846 pi->first_txq = tqidx; 847 if (is_10G_port(pi) || is_40G_port(pi)) { 848 pi->flags |= iaq.intr_flags_10g; 849 pi->nrxq = iaq.nrxq10g; 850 pi->ntxq = iaq.ntxq10g; 851 } else { 852 pi->flags |= iaq.intr_flags_1g; 853 pi->nrxq = iaq.nrxq1g; 854 pi->ntxq = iaq.ntxq1g; 855 } 856 857 if (pi->ntxq > 1) 858 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 859 else 860 pi->rsrv_noflowq = 0; 861 862 rqidx += pi->nrxq; 863 tqidx += pi->ntxq; 864 #ifdef TCP_OFFLOAD 865 if (is_offload(sc)) { 866 pi->first_ofld_rxq = ofld_rqidx; 867 pi->first_ofld_txq = ofld_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nofldrxq = iaq.nofldrxq10g; 870 pi->nofldtxq = iaq.nofldtxq10g; 871 } else { 872 pi->nofldrxq = iaq.nofldrxq1g; 873 pi->nofldtxq = iaq.nofldtxq1g; 874 } 875 ofld_rqidx += pi->nofldrxq; 876 ofld_tqidx += pi->nofldtxq; 877 } 878 #endif 879 #ifdef DEV_NETMAP 880 pi->first_nm_rxq = nm_rqidx; 881 pi->first_nm_txq = nm_tqidx; 882 if (is_10G_port(pi) || is_40G_port(pi)) { 883 pi->nnmrxq = iaq.nnmrxq10g; 884 pi->nnmtxq = iaq.nnmtxq10g; 885 } else { 886 pi->nnmrxq = iaq.nnmrxq1g; 887 pi->nnmtxq = iaq.nnmtxq1g; 888 } 889 nm_rqidx += pi->nnmrxq; 890 nm_tqidx += pi->nnmtxq; 891 #endif 892 } 893 894 rc = setup_intr_handlers(sc); 895 if (rc != 0) { 896 device_printf(dev, 897 "failed to setup interrupt handlers: %d\n", rc); 898 goto done; 899 } 900 901 rc = bus_generic_attach(dev); 902 if (rc != 0) { 903 device_printf(dev, 904 "failed to attach all child ports: %d\n", rc); 905 goto done; 906 } 907 908 switch (sc->params.pci.speed) { 909 case 0x1: 910 pcie_ts = "2.5"; 911 break; 912 case 0x2: 913 pcie_ts = "5.0"; 914 break; 915 case 0x3: 916 pcie_ts = "8.0"; 917 break; 918 default: 919 pcie_ts = "??"; 920 break; 921 } 922 device_printf(dev, 923 "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n", 924 sc->params.pci.width, pcie_ts, sc->params.pci.speed, 925 sc->params.nports, sc->intr_count, 926 sc->intr_type == INTR_MSIX ? "MSI-X" : 927 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 928 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 929 930 t4_set_desc(sc); 931 932 done: 933 if (rc != 0 && sc->cdev) { 934 /* cdev was created and so cxgbetool works; recover that way. */ 935 device_printf(dev, 936 "error during attach, adapter is now in recovery mode.\n"); 937 rc = 0; 938 } 939 940 if (rc != 0) 941 t4_detach(dev); 942 else 943 t4_sysctls(sc); 944 945 return (rc); 946 } 947 948 /* 949 * Idempotent 950 */ 951 static int 952 t4_detach(device_t dev) 953 { 954 struct adapter *sc; 955 struct port_info *pi; 956 int i, rc; 957 958 sc = device_get_softc(dev); 959 960 if (sc->flags & FULL_INIT_DONE) 961 t4_intr_disable(sc); 962 963 if (sc->cdev) { 964 destroy_dev(sc->cdev); 965 sc->cdev = NULL; 966 } 967 968 rc = bus_generic_detach(dev); 969 if (rc) { 970 device_printf(dev, 971 "failed to detach child devices: %d\n", rc); 972 return (rc); 973 } 974 975 for (i = 0; i < sc->intr_count; i++) 976 t4_free_irq(sc, &sc->irq[i]); 977 978 for (i = 0; i < MAX_NPORTS; i++) { 979 pi = sc->port[i]; 980 if (pi) { 981 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 982 if (pi->dev) 983 device_delete_child(dev, pi->dev); 984 985 mtx_destroy(&pi->pi_lock); 986 free(pi, M_CXGBE); 987 } 988 } 989 990 if (sc->flags & FULL_INIT_DONE) 991 adapter_full_uninit(sc); 992 993 if (sc->flags & FW_OK) 994 t4_fw_bye(sc, sc->mbox); 995 996 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 997 pci_release_msi(dev); 998 999 if (sc->regs_res) 1000 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1001 sc->regs_res); 1002 1003 if (sc->udbs_res) 1004 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1005 sc->udbs_res); 1006 1007 if (sc->msix_res) 1008 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1009 sc->msix_res); 1010 1011 if (sc->l2t) 1012 t4_free_l2t(sc->l2t); 1013 1014 #ifdef TCP_OFFLOAD 1015 free(sc->sge.ofld_rxq, M_CXGBE); 1016 free(sc->sge.ofld_txq, M_CXGBE); 1017 #endif 1018 #ifdef DEV_NETMAP 1019 free(sc->sge.nm_rxq, M_CXGBE); 1020 free(sc->sge.nm_txq, M_CXGBE); 1021 #endif 1022 free(sc->irq, M_CXGBE); 1023 free(sc->sge.rxq, M_CXGBE); 1024 free(sc->sge.txq, M_CXGBE); 1025 free(sc->sge.ctrlq, M_CXGBE); 1026 free(sc->sge.iqmap, M_CXGBE); 1027 free(sc->sge.eqmap, M_CXGBE); 1028 free(sc->tids.ftid_tab, M_CXGBE); 1029 t4_destroy_dma_tag(sc); 1030 if (mtx_initialized(&sc->sc_lock)) { 1031 sx_xlock(&t4_list_lock); 1032 SLIST_REMOVE(&t4_list, sc, adapter, link); 1033 sx_xunlock(&t4_list_lock); 1034 mtx_destroy(&sc->sc_lock); 1035 } 1036 1037 if (mtx_initialized(&sc->tids.ftid_lock)) 1038 mtx_destroy(&sc->tids.ftid_lock); 1039 if (mtx_initialized(&sc->sfl_lock)) 1040 mtx_destroy(&sc->sfl_lock); 1041 if (mtx_initialized(&sc->ifp_lock)) 1042 mtx_destroy(&sc->ifp_lock); 1043 if (mtx_initialized(&sc->regwin_lock)) 1044 mtx_destroy(&sc->regwin_lock); 1045 1046 bzero(sc, sizeof(*sc)); 1047 1048 return (0); 1049 } 1050 1051 static int 1052 cxgbe_probe(device_t dev) 1053 { 1054 char buf[128]; 1055 struct port_info *pi = device_get_softc(dev); 1056 1057 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1058 device_set_desc_copy(dev, buf); 1059 1060 return (BUS_PROBE_DEFAULT); 1061 } 1062 1063 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1064 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1065 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1066 #define T4_CAP_ENABLE (T4_CAP) 1067 1068 static int 1069 cxgbe_attach(device_t dev) 1070 { 1071 struct port_info *pi = device_get_softc(dev); 1072 struct ifnet *ifp; 1073 char *s; 1074 int n, o; 1075 1076 /* Allocate an ifnet and set it up */ 1077 ifp = if_alloc(IFT_ETHER); 1078 if (ifp == NULL) { 1079 device_printf(dev, "Cannot allocate ifnet\n"); 1080 return (ENOMEM); 1081 } 1082 pi->ifp = ifp; 1083 ifp->if_softc = pi; 1084 1085 callout_init(&pi->tick, CALLOUT_MPSAFE); 1086 1087 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1088 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1089 1090 ifp->if_init = cxgbe_init; 1091 ifp->if_ioctl = cxgbe_ioctl; 1092 ifp->if_transmit = cxgbe_transmit; 1093 ifp->if_qflush = cxgbe_qflush; 1094 ifp->if_get_counter = cxgbe_get_counter; 1095 1096 ifp->if_capabilities = T4_CAP; 1097 #ifdef TCP_OFFLOAD 1098 if (is_offload(pi->adapter)) 1099 ifp->if_capabilities |= IFCAP_TOE; 1100 #endif 1101 ifp->if_capenable = T4_CAP_ENABLE; 1102 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1103 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1104 1105 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1106 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1107 ifp->if_hw_tsomaxsegsize = 65536; 1108 1109 /* Initialize ifmedia for this port */ 1110 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1111 cxgbe_media_status); 1112 build_medialist(pi, &pi->media); 1113 1114 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1115 EVENTHANDLER_PRI_ANY); 1116 1117 ether_ifattach(ifp, pi->hw_addr); 1118 1119 n = 128; 1120 s = malloc(n, M_CXGBE, M_WAITOK); 1121 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1122 MPASS(n > o); 1123 #ifdef TCP_OFFLOAD 1124 if (is_offload(pi->adapter)) { 1125 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1126 pi->nofldtxq, pi->nofldrxq); 1127 MPASS(n > o); 1128 } 1129 #endif 1130 #ifdef DEV_NETMAP 1131 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1132 pi->nnmrxq); 1133 MPASS(n > o); 1134 #endif 1135 device_printf(dev, "%s\n", s); 1136 free(s, M_CXGBE); 1137 1138 #ifdef DEV_NETMAP 1139 /* nm_media handled here to keep implementation private to this file */ 1140 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1141 cxgbe_media_status); 1142 build_medialist(pi, &pi->nm_media); 1143 create_netmap_ifnet(pi); /* logs errors it something fails */ 1144 #endif 1145 cxgbe_sysctls(pi); 1146 1147 return (0); 1148 } 1149 1150 static int 1151 cxgbe_detach(device_t dev) 1152 { 1153 struct port_info *pi = device_get_softc(dev); 1154 struct adapter *sc = pi->adapter; 1155 struct ifnet *ifp = pi->ifp; 1156 1157 /* Tell if_ioctl and if_init that the port is going away */ 1158 ADAPTER_LOCK(sc); 1159 SET_DOOMED(pi); 1160 wakeup(&sc->flags); 1161 while (IS_BUSY(sc)) 1162 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1163 SET_BUSY(sc); 1164 #ifdef INVARIANTS 1165 sc->last_op = "t4detach"; 1166 sc->last_op_thr = curthread; 1167 #endif 1168 ADAPTER_UNLOCK(sc); 1169 1170 if (pi->flags & HAS_TRACEQ) { 1171 sc->traceq = -1; /* cloner should not create ifnet */ 1172 t4_tracer_port_detach(sc); 1173 } 1174 1175 if (pi->vlan_c) 1176 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1177 1178 PORT_LOCK(pi); 1179 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1180 callout_stop(&pi->tick); 1181 PORT_UNLOCK(pi); 1182 callout_drain(&pi->tick); 1183 1184 /* Let detach proceed even if these fail. */ 1185 cxgbe_uninit_synchronized(pi); 1186 port_full_uninit(pi); 1187 1188 ifmedia_removeall(&pi->media); 1189 ether_ifdetach(pi->ifp); 1190 if_free(pi->ifp); 1191 1192 #ifdef DEV_NETMAP 1193 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1194 destroy_netmap_ifnet(pi); 1195 #endif 1196 1197 ADAPTER_LOCK(sc); 1198 CLR_BUSY(sc); 1199 wakeup(&sc->flags); 1200 ADAPTER_UNLOCK(sc); 1201 1202 return (0); 1203 } 1204 1205 static void 1206 cxgbe_init(void *arg) 1207 { 1208 struct port_info *pi = arg; 1209 struct adapter *sc = pi->adapter; 1210 1211 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1212 return; 1213 cxgbe_init_synchronized(pi); 1214 end_synchronized_op(sc, 0); 1215 } 1216 1217 static int 1218 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1219 { 1220 int rc = 0, mtu, flags, can_sleep; 1221 struct port_info *pi = ifp->if_softc; 1222 struct adapter *sc = pi->adapter; 1223 struct ifreq *ifr = (struct ifreq *)data; 1224 uint32_t mask; 1225 1226 switch (cmd) { 1227 case SIOCSIFMTU: 1228 mtu = ifr->ifr_mtu; 1229 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1230 return (EINVAL); 1231 1232 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1233 if (rc) 1234 return (rc); 1235 ifp->if_mtu = mtu; 1236 if (pi->flags & PORT_INIT_DONE) { 1237 t4_update_fl_bufsize(ifp); 1238 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1239 rc = update_mac_settings(ifp, XGMAC_MTU); 1240 } 1241 end_synchronized_op(sc, 0); 1242 break; 1243 1244 case SIOCSIFFLAGS: 1245 can_sleep = 0; 1246 redo_sifflags: 1247 rc = begin_synchronized_op(sc, pi, 1248 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1249 if (rc) 1250 return (rc); 1251 1252 if (ifp->if_flags & IFF_UP) { 1253 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1254 flags = pi->if_flags; 1255 if ((ifp->if_flags ^ flags) & 1256 (IFF_PROMISC | IFF_ALLMULTI)) { 1257 if (can_sleep == 1) { 1258 end_synchronized_op(sc, 0); 1259 can_sleep = 0; 1260 goto redo_sifflags; 1261 } 1262 rc = update_mac_settings(ifp, 1263 XGMAC_PROMISC | XGMAC_ALLMULTI); 1264 } 1265 } else { 1266 if (can_sleep == 0) { 1267 end_synchronized_op(sc, LOCK_HELD); 1268 can_sleep = 1; 1269 goto redo_sifflags; 1270 } 1271 rc = cxgbe_init_synchronized(pi); 1272 } 1273 pi->if_flags = ifp->if_flags; 1274 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1275 if (can_sleep == 0) { 1276 end_synchronized_op(sc, LOCK_HELD); 1277 can_sleep = 1; 1278 goto redo_sifflags; 1279 } 1280 rc = cxgbe_uninit_synchronized(pi); 1281 } 1282 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1283 break; 1284 1285 case SIOCADDMULTI: 1286 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1287 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1288 if (rc) 1289 return (rc); 1290 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1291 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1292 end_synchronized_op(sc, LOCK_HELD); 1293 break; 1294 1295 case SIOCSIFCAP: 1296 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1297 if (rc) 1298 return (rc); 1299 1300 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1301 if (mask & IFCAP_TXCSUM) { 1302 ifp->if_capenable ^= IFCAP_TXCSUM; 1303 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1304 1305 if (IFCAP_TSO4 & ifp->if_capenable && 1306 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1307 ifp->if_capenable &= ~IFCAP_TSO4; 1308 if_printf(ifp, 1309 "tso4 disabled due to -txcsum.\n"); 1310 } 1311 } 1312 if (mask & IFCAP_TXCSUM_IPV6) { 1313 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1314 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1315 1316 if (IFCAP_TSO6 & ifp->if_capenable && 1317 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1318 ifp->if_capenable &= ~IFCAP_TSO6; 1319 if_printf(ifp, 1320 "tso6 disabled due to -txcsum6.\n"); 1321 } 1322 } 1323 if (mask & IFCAP_RXCSUM) 1324 ifp->if_capenable ^= IFCAP_RXCSUM; 1325 if (mask & IFCAP_RXCSUM_IPV6) 1326 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1327 1328 /* 1329 * Note that we leave CSUM_TSO alone (it is always set). The 1330 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1331 * sending a TSO request our way, so it's sufficient to toggle 1332 * IFCAP_TSOx only. 1333 */ 1334 if (mask & IFCAP_TSO4) { 1335 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1336 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1337 if_printf(ifp, "enable txcsum first.\n"); 1338 rc = EAGAIN; 1339 goto fail; 1340 } 1341 ifp->if_capenable ^= IFCAP_TSO4; 1342 } 1343 if (mask & IFCAP_TSO6) { 1344 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1345 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1346 if_printf(ifp, "enable txcsum6 first.\n"); 1347 rc = EAGAIN; 1348 goto fail; 1349 } 1350 ifp->if_capenable ^= IFCAP_TSO6; 1351 } 1352 if (mask & IFCAP_LRO) { 1353 #if defined(INET) || defined(INET6) 1354 int i; 1355 struct sge_rxq *rxq; 1356 1357 ifp->if_capenable ^= IFCAP_LRO; 1358 for_each_rxq(pi, i, rxq) { 1359 if (ifp->if_capenable & IFCAP_LRO) 1360 rxq->iq.flags |= IQ_LRO_ENABLED; 1361 else 1362 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1363 } 1364 #endif 1365 } 1366 #ifdef TCP_OFFLOAD 1367 if (mask & IFCAP_TOE) { 1368 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1369 1370 rc = toe_capability(pi, enable); 1371 if (rc != 0) 1372 goto fail; 1373 1374 ifp->if_capenable ^= mask; 1375 } 1376 #endif 1377 if (mask & IFCAP_VLAN_HWTAGGING) { 1378 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1379 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1380 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1381 } 1382 if (mask & IFCAP_VLAN_MTU) { 1383 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1384 1385 /* Need to find out how to disable auto-mtu-inflation */ 1386 } 1387 if (mask & IFCAP_VLAN_HWTSO) 1388 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1389 if (mask & IFCAP_VLAN_HWCSUM) 1390 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1391 1392 #ifdef VLAN_CAPABILITIES 1393 VLAN_CAPABILITIES(ifp); 1394 #endif 1395 fail: 1396 end_synchronized_op(sc, 0); 1397 break; 1398 1399 case SIOCSIFMEDIA: 1400 case SIOCGIFMEDIA: 1401 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1402 break; 1403 1404 case SIOCGI2C: { 1405 struct ifi2creq i2c; 1406 1407 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1408 if (rc != 0) 1409 break; 1410 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1411 rc = EPERM; 1412 break; 1413 } 1414 if (i2c.len > sizeof(i2c.data)) { 1415 rc = EINVAL; 1416 break; 1417 } 1418 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1419 if (rc) 1420 return (rc); 1421 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1422 i2c.offset, i2c.len, &i2c.data[0]); 1423 end_synchronized_op(sc, 0); 1424 if (rc == 0) 1425 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1426 break; 1427 } 1428 1429 default: 1430 rc = ether_ioctl(ifp, cmd, data); 1431 } 1432 1433 return (rc); 1434 } 1435 1436 static int 1437 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1438 { 1439 struct port_info *pi = ifp->if_softc; 1440 struct adapter *sc = pi->adapter; 1441 struct sge_txq *txq; 1442 void *items[1]; 1443 int rc; 1444 1445 M_ASSERTPKTHDR(m); 1446 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1447 1448 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1449 m_freem(m); 1450 return (ENETDOWN); 1451 } 1452 1453 rc = parse_pkt(&m); 1454 if (__predict_false(rc != 0)) { 1455 MPASS(m == NULL); /* was freed already */ 1456 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1457 return (rc); 1458 } 1459 1460 /* Select a txq. */ 1461 txq = &sc->sge.txq[pi->first_txq]; 1462 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1463 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1464 pi->rsrv_noflowq); 1465 1466 items[0] = m; 1467 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1468 if (__predict_false(rc != 0)) 1469 m_freem(m); 1470 1471 return (rc); 1472 } 1473 1474 static void 1475 cxgbe_qflush(struct ifnet *ifp) 1476 { 1477 struct port_info *pi = ifp->if_softc; 1478 struct sge_txq *txq; 1479 int i; 1480 1481 /* queues do not exist if !PORT_INIT_DONE. */ 1482 if (pi->flags & PORT_INIT_DONE) { 1483 for_each_txq(pi, i, txq) { 1484 TXQ_LOCK(txq); 1485 txq->eq.flags &= ~EQ_ENABLED; 1486 TXQ_UNLOCK(txq); 1487 while (!mp_ring_is_idle(txq->r)) { 1488 mp_ring_check_drainage(txq->r, 0); 1489 pause("qflush", 1); 1490 } 1491 } 1492 } 1493 if_qflush(ifp); 1494 } 1495 1496 static uint64_t 1497 cxgbe_get_counter(struct ifnet *ifp, ift_counter c) 1498 { 1499 struct port_info *pi = ifp->if_softc; 1500 struct adapter *sc = pi->adapter; 1501 struct port_stats *s = &pi->stats; 1502 1503 cxgbe_refresh_stats(sc, pi); 1504 1505 switch (c) { 1506 case IFCOUNTER_IPACKETS: 1507 return (s->rx_frames - s->rx_pause); 1508 1509 case IFCOUNTER_IERRORS: 1510 return (s->rx_jabber + s->rx_runt + s->rx_too_long + 1511 s->rx_fcs_err + s->rx_len_err); 1512 1513 case IFCOUNTER_OPACKETS: 1514 return (s->tx_frames - s->tx_pause); 1515 1516 case IFCOUNTER_OERRORS: 1517 return (s->tx_error_frames); 1518 1519 case IFCOUNTER_IBYTES: 1520 return (s->rx_octets - s->rx_pause * 64); 1521 1522 case IFCOUNTER_OBYTES: 1523 return (s->tx_octets - s->tx_pause * 64); 1524 1525 case IFCOUNTER_IMCASTS: 1526 return (s->rx_mcast_frames - s->rx_pause); 1527 1528 case IFCOUNTER_OMCASTS: 1529 return (s->tx_mcast_frames - s->tx_pause); 1530 1531 case IFCOUNTER_IQDROPS: 1532 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 1533 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 1534 s->rx_trunc3 + pi->tnl_cong_drops); 1535 1536 case IFCOUNTER_OQDROPS: { 1537 uint64_t drops; 1538 1539 drops = s->tx_drop; 1540 if (pi->flags & PORT_INIT_DONE) { 1541 int i; 1542 struct sge_txq *txq; 1543 1544 for_each_txq(pi, i, txq) 1545 drops += counter_u64_fetch(txq->r->drops); 1546 } 1547 1548 return (drops); 1549 1550 } 1551 1552 default: 1553 return (if_get_counter_default(ifp, c)); 1554 } 1555 } 1556 1557 static int 1558 cxgbe_media_change(struct ifnet *ifp) 1559 { 1560 struct port_info *pi = ifp->if_softc; 1561 1562 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1563 1564 return (EOPNOTSUPP); 1565 } 1566 1567 static void 1568 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1569 { 1570 struct port_info *pi = ifp->if_softc; 1571 struct ifmedia *media = NULL; 1572 struct ifmedia_entry *cur; 1573 int speed = pi->link_cfg.speed; 1574 1575 if (ifp == pi->ifp) 1576 media = &pi->media; 1577 #ifdef DEV_NETMAP 1578 else if (ifp == pi->nm_ifp) 1579 media = &pi->nm_media; 1580 #endif 1581 MPASS(media != NULL); 1582 1583 cur = media->ifm_cur; 1584 1585 ifmr->ifm_status = IFM_AVALID; 1586 if (!pi->link_cfg.link_ok) 1587 return; 1588 1589 ifmr->ifm_status |= IFM_ACTIVE; 1590 1591 /* active and current will differ iff current media is autoselect. */ 1592 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1593 return; 1594 1595 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1596 if (speed == SPEED_10000) 1597 ifmr->ifm_active |= IFM_10G_T; 1598 else if (speed == SPEED_1000) 1599 ifmr->ifm_active |= IFM_1000_T; 1600 else if (speed == SPEED_100) 1601 ifmr->ifm_active |= IFM_100_TX; 1602 else if (speed == SPEED_10) 1603 ifmr->ifm_active |= IFM_10_T; 1604 else 1605 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1606 speed)); 1607 } 1608 1609 void 1610 t4_fatal_err(struct adapter *sc) 1611 { 1612 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1613 t4_intr_disable(sc); 1614 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1615 device_get_nameunit(sc->dev)); 1616 } 1617 1618 static int 1619 map_bars_0_and_4(struct adapter *sc) 1620 { 1621 sc->regs_rid = PCIR_BAR(0); 1622 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1623 &sc->regs_rid, RF_ACTIVE); 1624 if (sc->regs_res == NULL) { 1625 device_printf(sc->dev, "cannot map registers.\n"); 1626 return (ENXIO); 1627 } 1628 sc->bt = rman_get_bustag(sc->regs_res); 1629 sc->bh = rman_get_bushandle(sc->regs_res); 1630 sc->mmio_len = rman_get_size(sc->regs_res); 1631 setbit(&sc->doorbells, DOORBELL_KDB); 1632 1633 sc->msix_rid = PCIR_BAR(4); 1634 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1635 &sc->msix_rid, RF_ACTIVE); 1636 if (sc->msix_res == NULL) { 1637 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1638 return (ENXIO); 1639 } 1640 1641 return (0); 1642 } 1643 1644 static int 1645 map_bar_2(struct adapter *sc) 1646 { 1647 1648 /* 1649 * T4: only iWARP driver uses the userspace doorbells. There is no need 1650 * to map it if RDMA is disabled. 1651 */ 1652 if (is_t4(sc) && sc->rdmacaps == 0) 1653 return (0); 1654 1655 sc->udbs_rid = PCIR_BAR(2); 1656 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1657 &sc->udbs_rid, RF_ACTIVE); 1658 if (sc->udbs_res == NULL) { 1659 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1660 return (ENXIO); 1661 } 1662 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1663 1664 if (is_t5(sc)) { 1665 setbit(&sc->doorbells, DOORBELL_UDB); 1666 #if defined(__i386__) || defined(__amd64__) 1667 if (t5_write_combine) { 1668 int rc; 1669 1670 /* 1671 * Enable write combining on BAR2. This is the 1672 * userspace doorbell BAR and is split into 128B 1673 * (UDBS_SEG_SIZE) doorbell regions, each associated 1674 * with an egress queue. The first 64B has the doorbell 1675 * and the second 64B can be used to submit a tx work 1676 * request with an implicit doorbell. 1677 */ 1678 1679 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1680 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1681 if (rc == 0) { 1682 clrbit(&sc->doorbells, DOORBELL_UDB); 1683 setbit(&sc->doorbells, DOORBELL_WCWR); 1684 setbit(&sc->doorbells, DOORBELL_UDBWC); 1685 } else { 1686 device_printf(sc->dev, 1687 "couldn't enable write combining: %d\n", 1688 rc); 1689 } 1690 1691 t4_write_reg(sc, A_SGE_STAT_CFG, 1692 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1693 } 1694 #endif 1695 } 1696 1697 return (0); 1698 } 1699 1700 static const struct memwin t4_memwin[] = { 1701 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1702 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1703 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1704 }; 1705 1706 static const struct memwin t5_memwin[] = { 1707 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1708 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1709 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1710 }; 1711 1712 static void 1713 setup_memwin(struct adapter *sc) 1714 { 1715 const struct memwin *mw; 1716 int i, n; 1717 uint32_t bar0; 1718 1719 if (is_t4(sc)) { 1720 /* 1721 * Read low 32b of bar0 indirectly via the hardware backdoor 1722 * mechanism. Works from within PCI passthrough environments 1723 * too, where rman_get_start() can return a different value. We 1724 * need to program the T4 memory window decoders with the actual 1725 * addresses that will be coming across the PCIe link. 1726 */ 1727 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1728 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1729 1730 mw = &t4_memwin[0]; 1731 n = nitems(t4_memwin); 1732 } else { 1733 /* T5 uses the relative offset inside the PCIe BAR */ 1734 bar0 = 0; 1735 1736 mw = &t5_memwin[0]; 1737 n = nitems(t5_memwin); 1738 } 1739 1740 for (i = 0; i < n; i++, mw++) { 1741 t4_write_reg(sc, 1742 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1743 (mw->base + bar0) | V_BIR(0) | 1744 V_WINDOW(ilog2(mw->aperture) - 10)); 1745 } 1746 1747 /* flush */ 1748 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1749 } 1750 1751 /* 1752 * Verify that the memory range specified by the addr/len pair is valid and lies 1753 * entirely within a single region (EDCx or MCx). 1754 */ 1755 static int 1756 validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1757 { 1758 uint32_t em, addr_len, maddr, mlen; 1759 1760 /* Memory can only be accessed in naturally aligned 4 byte units */ 1761 if (addr & 3 || len & 3 || len == 0) 1762 return (EINVAL); 1763 1764 /* Enabled memories */ 1765 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1766 if (em & F_EDRAM0_ENABLE) { 1767 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1768 maddr = G_EDRAM0_BASE(addr_len) << 20; 1769 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1770 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1771 addr + len <= maddr + mlen) 1772 return (0); 1773 } 1774 if (em & F_EDRAM1_ENABLE) { 1775 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1776 maddr = G_EDRAM1_BASE(addr_len) << 20; 1777 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1778 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1779 addr + len <= maddr + mlen) 1780 return (0); 1781 } 1782 if (em & F_EXT_MEM_ENABLE) { 1783 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1784 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1785 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1786 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1787 addr + len <= maddr + mlen) 1788 return (0); 1789 } 1790 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1791 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1792 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1793 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1794 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1795 addr + len <= maddr + mlen) 1796 return (0); 1797 } 1798 1799 return (EFAULT); 1800 } 1801 1802 static int 1803 fwmtype_to_hwmtype(int mtype) 1804 { 1805 1806 switch (mtype) { 1807 case FW_MEMTYPE_EDC0: 1808 return (MEM_EDC0); 1809 case FW_MEMTYPE_EDC1: 1810 return (MEM_EDC1); 1811 case FW_MEMTYPE_EXTMEM: 1812 return (MEM_MC0); 1813 case FW_MEMTYPE_EXTMEM1: 1814 return (MEM_MC1); 1815 default: 1816 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1817 } 1818 } 1819 1820 /* 1821 * Verify that the memory range specified by the memtype/offset/len pair is 1822 * valid and lies entirely within the memtype specified. The global address of 1823 * the start of the range is returned in addr. 1824 */ 1825 static int 1826 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1827 uint32_t *addr) 1828 { 1829 uint32_t em, addr_len, maddr, mlen; 1830 1831 /* Memory can only be accessed in naturally aligned 4 byte units */ 1832 if (off & 3 || len & 3 || len == 0) 1833 return (EINVAL); 1834 1835 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1836 switch (fwmtype_to_hwmtype(mtype)) { 1837 case MEM_EDC0: 1838 if (!(em & F_EDRAM0_ENABLE)) 1839 return (EINVAL); 1840 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1841 maddr = G_EDRAM0_BASE(addr_len) << 20; 1842 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1843 break; 1844 case MEM_EDC1: 1845 if (!(em & F_EDRAM1_ENABLE)) 1846 return (EINVAL); 1847 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1848 maddr = G_EDRAM1_BASE(addr_len) << 20; 1849 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1850 break; 1851 case MEM_MC: 1852 if (!(em & F_EXT_MEM_ENABLE)) 1853 return (EINVAL); 1854 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1855 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1856 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1857 break; 1858 case MEM_MC1: 1859 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1860 return (EINVAL); 1861 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1862 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1863 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1864 break; 1865 default: 1866 return (EINVAL); 1867 } 1868 1869 if (mlen > 0 && off < mlen && off + len <= mlen) { 1870 *addr = maddr + off; /* global address */ 1871 return (0); 1872 } 1873 1874 return (EFAULT); 1875 } 1876 1877 static void 1878 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1879 { 1880 const struct memwin *mw; 1881 1882 if (is_t4(sc)) { 1883 KASSERT(win >= 0 && win < nitems(t4_memwin), 1884 ("%s: incorrect memwin# (%d)", __func__, win)); 1885 mw = &t4_memwin[win]; 1886 } else { 1887 KASSERT(win >= 0 && win < nitems(t5_memwin), 1888 ("%s: incorrect memwin# (%d)", __func__, win)); 1889 mw = &t5_memwin[win]; 1890 } 1891 1892 if (base != NULL) 1893 *base = mw->base; 1894 if (aperture != NULL) 1895 *aperture = mw->aperture; 1896 } 1897 1898 /* 1899 * Positions the memory window such that it can be used to access the specified 1900 * address in the chip's address space. The return value is the offset of addr 1901 * from the start of the window. 1902 */ 1903 static uint32_t 1904 position_memwin(struct adapter *sc, int n, uint32_t addr) 1905 { 1906 uint32_t start, pf; 1907 uint32_t reg; 1908 1909 KASSERT(n >= 0 && n <= 3, 1910 ("%s: invalid window %d.", __func__, n)); 1911 KASSERT((addr & 3) == 0, 1912 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1913 1914 if (is_t4(sc)) { 1915 pf = 0; 1916 start = addr & ~0xf; /* start must be 16B aligned */ 1917 } else { 1918 pf = V_PFNUM(sc->pf); 1919 start = addr & ~0x7f; /* start must be 128B aligned */ 1920 } 1921 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1922 1923 t4_write_reg(sc, reg, start | pf); 1924 t4_read_reg(sc, reg); 1925 1926 return (addr - start); 1927 } 1928 1929 static int 1930 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1931 struct intrs_and_queues *iaq) 1932 { 1933 int rc, itype, navail, nrxq10g, nrxq1g, n; 1934 int nofldrxq10g = 0, nofldrxq1g = 0; 1935 int nnmrxq10g = 0, nnmrxq1g = 0; 1936 1937 bzero(iaq, sizeof(*iaq)); 1938 1939 iaq->ntxq10g = t4_ntxq10g; 1940 iaq->ntxq1g = t4_ntxq1g; 1941 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1942 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1943 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1944 #ifdef TCP_OFFLOAD 1945 if (is_offload(sc)) { 1946 iaq->nofldtxq10g = t4_nofldtxq10g; 1947 iaq->nofldtxq1g = t4_nofldtxq1g; 1948 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1949 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1950 } 1951 #endif 1952 #ifdef DEV_NETMAP 1953 iaq->nnmtxq10g = t4_nnmtxq10g; 1954 iaq->nnmtxq1g = t4_nnmtxq1g; 1955 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1956 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1957 #endif 1958 1959 for (itype = INTR_MSIX; itype; itype >>= 1) { 1960 1961 if ((itype & t4_intr_types) == 0) 1962 continue; /* not allowed */ 1963 1964 if (itype == INTR_MSIX) 1965 navail = pci_msix_count(sc->dev); 1966 else if (itype == INTR_MSI) 1967 navail = pci_msi_count(sc->dev); 1968 else 1969 navail = 1; 1970 restart: 1971 if (navail == 0) 1972 continue; 1973 1974 iaq->intr_type = itype; 1975 iaq->intr_flags_10g = 0; 1976 iaq->intr_flags_1g = 0; 1977 1978 /* 1979 * Best option: an interrupt vector for errors, one for the 1980 * firmware event queue, and one for every rxq (NIC, TOE, and 1981 * netmap). 1982 */ 1983 iaq->nirq = T4_EXTRA_INTR; 1984 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1985 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1986 if (iaq->nirq <= navail && 1987 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1988 iaq->intr_flags_10g = INTR_ALL; 1989 iaq->intr_flags_1g = INTR_ALL; 1990 goto allocate; 1991 } 1992 1993 /* 1994 * Second best option: a vector for errors, one for the firmware 1995 * event queue, and vectors for either all the NIC rx queues or 1996 * all the TOE rx queues. The queues that don't get vectors 1997 * will forward their interrupts to those that do. 1998 * 1999 * Note: netmap rx queues cannot be created early and so they 2000 * can't be setup to receive forwarded interrupts for others. 2001 */ 2002 iaq->nirq = T4_EXTRA_INTR; 2003 if (nrxq10g >= nofldrxq10g) { 2004 iaq->intr_flags_10g = INTR_RXQ; 2005 iaq->nirq += n10g * nrxq10g; 2006 #ifdef DEV_NETMAP 2007 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 2008 #endif 2009 } else { 2010 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2011 iaq->nirq += n10g * nofldrxq10g; 2012 #ifdef DEV_NETMAP 2013 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 2014 #endif 2015 } 2016 if (nrxq1g >= nofldrxq1g) { 2017 iaq->intr_flags_1g = INTR_RXQ; 2018 iaq->nirq += n1g * nrxq1g; 2019 #ifdef DEV_NETMAP 2020 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 2021 #endif 2022 } else { 2023 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2024 iaq->nirq += n1g * nofldrxq1g; 2025 #ifdef DEV_NETMAP 2026 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 2027 #endif 2028 } 2029 if (iaq->nirq <= navail && 2030 (itype != INTR_MSI || powerof2(iaq->nirq))) 2031 goto allocate; 2032 2033 /* 2034 * Next best option: an interrupt vector for errors, one for the 2035 * firmware event queue, and at least one per port. At this 2036 * point we know we'll have to downsize nrxq and/or nofldrxq 2037 * and/or nnmrxq to fit what's available to us. 2038 */ 2039 iaq->nirq = T4_EXTRA_INTR; 2040 iaq->nirq += n10g + n1g; 2041 if (iaq->nirq <= navail) { 2042 int leftover = navail - iaq->nirq; 2043 2044 if (n10g > 0) { 2045 int target = max(nrxq10g, nofldrxq10g); 2046 2047 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2048 INTR_RXQ : INTR_OFLD_RXQ; 2049 2050 n = 1; 2051 while (n < target && leftover >= n10g) { 2052 leftover -= n10g; 2053 iaq->nirq += n10g; 2054 n++; 2055 } 2056 iaq->nrxq10g = min(n, nrxq10g); 2057 #ifdef TCP_OFFLOAD 2058 iaq->nofldrxq10g = min(n, nofldrxq10g); 2059 #endif 2060 #ifdef DEV_NETMAP 2061 iaq->nnmrxq10g = min(n, nnmrxq10g); 2062 #endif 2063 } 2064 2065 if (n1g > 0) { 2066 int target = max(nrxq1g, nofldrxq1g); 2067 2068 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2069 INTR_RXQ : INTR_OFLD_RXQ; 2070 2071 n = 1; 2072 while (n < target && leftover >= n1g) { 2073 leftover -= n1g; 2074 iaq->nirq += n1g; 2075 n++; 2076 } 2077 iaq->nrxq1g = min(n, nrxq1g); 2078 #ifdef TCP_OFFLOAD 2079 iaq->nofldrxq1g = min(n, nofldrxq1g); 2080 #endif 2081 #ifdef DEV_NETMAP 2082 iaq->nnmrxq1g = min(n, nnmrxq1g); 2083 #endif 2084 } 2085 2086 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2087 goto allocate; 2088 } 2089 2090 /* 2091 * Least desirable option: one interrupt vector for everything. 2092 */ 2093 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2094 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2095 #ifdef TCP_OFFLOAD 2096 if (is_offload(sc)) 2097 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2098 #endif 2099 #ifdef DEV_NETMAP 2100 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2101 #endif 2102 2103 allocate: 2104 navail = iaq->nirq; 2105 rc = 0; 2106 if (itype == INTR_MSIX) 2107 rc = pci_alloc_msix(sc->dev, &navail); 2108 else if (itype == INTR_MSI) 2109 rc = pci_alloc_msi(sc->dev, &navail); 2110 2111 if (rc == 0) { 2112 if (navail == iaq->nirq) 2113 return (0); 2114 2115 /* 2116 * Didn't get the number requested. Use whatever number 2117 * the kernel is willing to allocate (it's in navail). 2118 */ 2119 device_printf(sc->dev, "fewer vectors than requested, " 2120 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2121 itype, iaq->nirq, navail); 2122 pci_release_msi(sc->dev); 2123 goto restart; 2124 } 2125 2126 device_printf(sc->dev, 2127 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2128 itype, rc, iaq->nirq, navail); 2129 } 2130 2131 device_printf(sc->dev, 2132 "failed to find a usable interrupt type. " 2133 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2134 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2135 2136 return (ENXIO); 2137 } 2138 2139 #define FW_VERSION(chip) ( \ 2140 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2141 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2142 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2143 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2144 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2145 2146 struct fw_info { 2147 uint8_t chip; 2148 char *kld_name; 2149 char *fw_mod_name; 2150 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2151 } fw_info[] = { 2152 { 2153 .chip = CHELSIO_T4, 2154 .kld_name = "t4fw_cfg", 2155 .fw_mod_name = "t4fw", 2156 .fw_hdr = { 2157 .chip = FW_HDR_CHIP_T4, 2158 .fw_ver = htobe32_const(FW_VERSION(T4)), 2159 .intfver_nic = FW_INTFVER(T4, NIC), 2160 .intfver_vnic = FW_INTFVER(T4, VNIC), 2161 .intfver_ofld = FW_INTFVER(T4, OFLD), 2162 .intfver_ri = FW_INTFVER(T4, RI), 2163 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2164 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2165 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2166 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2167 }, 2168 }, { 2169 .chip = CHELSIO_T5, 2170 .kld_name = "t5fw_cfg", 2171 .fw_mod_name = "t5fw", 2172 .fw_hdr = { 2173 .chip = FW_HDR_CHIP_T5, 2174 .fw_ver = htobe32_const(FW_VERSION(T5)), 2175 .intfver_nic = FW_INTFVER(T5, NIC), 2176 .intfver_vnic = FW_INTFVER(T5, VNIC), 2177 .intfver_ofld = FW_INTFVER(T5, OFLD), 2178 .intfver_ri = FW_INTFVER(T5, RI), 2179 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2180 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2181 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2182 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2183 }, 2184 } 2185 }; 2186 2187 static struct fw_info * 2188 find_fw_info(int chip) 2189 { 2190 int i; 2191 2192 for (i = 0; i < nitems(fw_info); i++) { 2193 if (fw_info[i].chip == chip) 2194 return (&fw_info[i]); 2195 } 2196 return (NULL); 2197 } 2198 2199 /* 2200 * Is the given firmware API compatible with the one the driver was compiled 2201 * with? 2202 */ 2203 static int 2204 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2205 { 2206 2207 /* short circuit if it's the exact same firmware version */ 2208 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2209 return (1); 2210 2211 /* 2212 * XXX: Is this too conservative? Perhaps I should limit this to the 2213 * features that are supported in the driver. 2214 */ 2215 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2216 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2217 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2218 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2219 return (1); 2220 #undef SAME_INTF 2221 2222 return (0); 2223 } 2224 2225 /* 2226 * The firmware in the KLD is usable, but should it be installed? This routine 2227 * explains itself in detail if it indicates the KLD firmware should be 2228 * installed. 2229 */ 2230 static int 2231 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2232 { 2233 const char *reason; 2234 2235 if (!card_fw_usable) { 2236 reason = "incompatible or unusable"; 2237 goto install; 2238 } 2239 2240 if (k > c) { 2241 reason = "older than the version bundled with this driver"; 2242 goto install; 2243 } 2244 2245 if (t4_fw_install == 2 && k != c) { 2246 reason = "different than the version bundled with this driver"; 2247 goto install; 2248 } 2249 2250 return (0); 2251 2252 install: 2253 if (t4_fw_install == 0) { 2254 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2255 "but the driver is prohibited from installing a different " 2256 "firmware on the card.\n", 2257 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2258 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2259 2260 return (0); 2261 } 2262 2263 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2264 "installing firmware %u.%u.%u.%u on card.\n", 2265 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2266 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2267 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2268 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2269 2270 return (1); 2271 } 2272 /* 2273 * Establish contact with the firmware and determine if we are the master driver 2274 * or not, and whether we are responsible for chip initialization. 2275 */ 2276 static int 2277 prep_firmware(struct adapter *sc) 2278 { 2279 const struct firmware *fw = NULL, *default_cfg; 2280 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2281 enum dev_state state; 2282 struct fw_info *fw_info; 2283 struct fw_hdr *card_fw; /* fw on the card */ 2284 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2285 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2286 against */ 2287 2288 /* Contact firmware. */ 2289 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2290 if (rc < 0 || state == DEV_STATE_ERR) { 2291 rc = -rc; 2292 device_printf(sc->dev, 2293 "failed to connect to the firmware: %d, %d.\n", rc, state); 2294 return (rc); 2295 } 2296 pf = rc; 2297 if (pf == sc->mbox) 2298 sc->flags |= MASTER_PF; 2299 else if (state == DEV_STATE_UNINIT) { 2300 /* 2301 * We didn't get to be the master so we definitely won't be 2302 * configuring the chip. It's a bug if someone else hasn't 2303 * configured it already. 2304 */ 2305 device_printf(sc->dev, "couldn't be master(%d), " 2306 "device not already initialized either(%d).\n", rc, state); 2307 return (EDOOFUS); 2308 } 2309 2310 /* This is the firmware whose headers the driver was compiled against */ 2311 fw_info = find_fw_info(chip_id(sc)); 2312 if (fw_info == NULL) { 2313 device_printf(sc->dev, 2314 "unable to look up firmware information for chip %d.\n", 2315 chip_id(sc)); 2316 return (EINVAL); 2317 } 2318 drv_fw = &fw_info->fw_hdr; 2319 2320 /* 2321 * The firmware KLD contains many modules. The KLD name is also the 2322 * name of the module that contains the default config file. 2323 */ 2324 default_cfg = firmware_get(fw_info->kld_name); 2325 2326 /* Read the header of the firmware on the card */ 2327 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2328 rc = -t4_read_flash(sc, FLASH_FW_START, 2329 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2330 if (rc == 0) 2331 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2332 else { 2333 device_printf(sc->dev, 2334 "Unable to read card's firmware header: %d\n", rc); 2335 card_fw_usable = 0; 2336 } 2337 2338 /* This is the firmware in the KLD */ 2339 fw = firmware_get(fw_info->fw_mod_name); 2340 if (fw != NULL) { 2341 kld_fw = (const void *)fw->data; 2342 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2343 } else { 2344 kld_fw = NULL; 2345 kld_fw_usable = 0; 2346 } 2347 2348 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2349 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2350 /* 2351 * Common case: the firmware on the card is an exact match and 2352 * the KLD is an exact match too, or the KLD is 2353 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2354 * here -- use cxgbetool loadfw if you want to reinstall the 2355 * same firmware as the one on the card. 2356 */ 2357 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2358 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2359 be32toh(card_fw->fw_ver))) { 2360 2361 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2362 if (rc != 0) { 2363 device_printf(sc->dev, 2364 "failed to install firmware: %d\n", rc); 2365 goto done; 2366 } 2367 2368 /* Installed successfully, update the cached header too. */ 2369 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2370 card_fw_usable = 1; 2371 need_fw_reset = 0; /* already reset as part of load_fw */ 2372 } 2373 2374 if (!card_fw_usable) { 2375 uint32_t d, c, k; 2376 2377 d = ntohl(drv_fw->fw_ver); 2378 c = ntohl(card_fw->fw_ver); 2379 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2380 2381 device_printf(sc->dev, "Cannot find a usable firmware: " 2382 "fw_install %d, chip state %d, " 2383 "driver compiled with %d.%d.%d.%d, " 2384 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2385 t4_fw_install, state, 2386 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2387 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2388 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2389 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2390 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2391 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2392 rc = EINVAL; 2393 goto done; 2394 } 2395 2396 /* We're using whatever's on the card and it's known to be good. */ 2397 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2398 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2399 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2400 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2401 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2402 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2403 t4_get_tp_version(sc, &sc->params.tp_vers); 2404 2405 /* Reset device */ 2406 if (need_fw_reset && 2407 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2408 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2409 if (rc != ETIMEDOUT && rc != EIO) 2410 t4_fw_bye(sc, sc->mbox); 2411 goto done; 2412 } 2413 sc->flags |= FW_OK; 2414 2415 rc = get_params__pre_init(sc); 2416 if (rc != 0) 2417 goto done; /* error message displayed already */ 2418 2419 /* Partition adapter resources as specified in the config file. */ 2420 if (state == DEV_STATE_UNINIT) { 2421 2422 KASSERT(sc->flags & MASTER_PF, 2423 ("%s: trying to change chip settings when not master.", 2424 __func__)); 2425 2426 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2427 if (rc != 0) 2428 goto done; /* error message displayed already */ 2429 2430 t4_tweak_chip_settings(sc); 2431 2432 /* get basic stuff going */ 2433 rc = -t4_fw_initialize(sc, sc->mbox); 2434 if (rc != 0) { 2435 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2436 goto done; 2437 } 2438 } else { 2439 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2440 sc->cfcsum = 0; 2441 } 2442 2443 done: 2444 free(card_fw, M_CXGBE); 2445 if (fw != NULL) 2446 firmware_put(fw, FIRMWARE_UNLOAD); 2447 if (default_cfg != NULL) 2448 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2449 2450 return (rc); 2451 } 2452 2453 #define FW_PARAM_DEV(param) \ 2454 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2455 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2456 #define FW_PARAM_PFVF(param) \ 2457 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2458 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2459 2460 /* 2461 * Partition chip resources for use between various PFs, VFs, etc. 2462 */ 2463 static int 2464 partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2465 const char *name_prefix) 2466 { 2467 const struct firmware *cfg = NULL; 2468 int rc = 0; 2469 struct fw_caps_config_cmd caps; 2470 uint32_t mtype, moff, finicsum, cfcsum; 2471 2472 /* 2473 * Figure out what configuration file to use. Pick the default config 2474 * file for the card if the user hasn't specified one explicitly. 2475 */ 2476 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2477 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2478 /* Card specific overrides go here. */ 2479 if (pci_get_device(sc->dev) == 0x440a) 2480 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2481 if (is_fpga(sc)) 2482 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2483 } 2484 2485 /* 2486 * We need to load another module if the profile is anything except 2487 * "default" or "flash". 2488 */ 2489 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2490 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2491 char s[32]; 2492 2493 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2494 cfg = firmware_get(s); 2495 if (cfg == NULL) { 2496 if (default_cfg != NULL) { 2497 device_printf(sc->dev, 2498 "unable to load module \"%s\" for " 2499 "configuration profile \"%s\", will use " 2500 "the default config file instead.\n", 2501 s, sc->cfg_file); 2502 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2503 "%s", DEFAULT_CF); 2504 } else { 2505 device_printf(sc->dev, 2506 "unable to load module \"%s\" for " 2507 "configuration profile \"%s\", will use " 2508 "the config file on the card's flash " 2509 "instead.\n", s, sc->cfg_file); 2510 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2511 "%s", FLASH_CF); 2512 } 2513 } 2514 } 2515 2516 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2517 default_cfg == NULL) { 2518 device_printf(sc->dev, 2519 "default config file not available, will use the config " 2520 "file on the card's flash instead.\n"); 2521 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2522 } 2523 2524 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2525 u_int cflen, i, n; 2526 const uint32_t *cfdata; 2527 uint32_t param, val, addr, off, mw_base, mw_aperture; 2528 2529 KASSERT(cfg != NULL || default_cfg != NULL, 2530 ("%s: no config to upload", __func__)); 2531 2532 /* 2533 * Ask the firmware where it wants us to upload the config file. 2534 */ 2535 param = FW_PARAM_DEV(CF); 2536 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2537 if (rc != 0) { 2538 /* No support for config file? Shouldn't happen. */ 2539 device_printf(sc->dev, 2540 "failed to query config file location: %d.\n", rc); 2541 goto done; 2542 } 2543 mtype = G_FW_PARAMS_PARAM_Y(val); 2544 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2545 2546 /* 2547 * XXX: sheer laziness. We deliberately added 4 bytes of 2548 * useless stuffing/comments at the end of the config file so 2549 * it's ok to simply throw away the last remaining bytes when 2550 * the config file is not an exact multiple of 4. This also 2551 * helps with the validate_mt_off_len check. 2552 */ 2553 if (cfg != NULL) { 2554 cflen = cfg->datasize & ~3; 2555 cfdata = cfg->data; 2556 } else { 2557 cflen = default_cfg->datasize & ~3; 2558 cfdata = default_cfg->data; 2559 } 2560 2561 if (cflen > FLASH_CFG_MAX_SIZE) { 2562 device_printf(sc->dev, 2563 "config file too long (%d, max allowed is %d). " 2564 "Will try to use the config on the card, if any.\n", 2565 cflen, FLASH_CFG_MAX_SIZE); 2566 goto use_config_on_flash; 2567 } 2568 2569 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2570 if (rc != 0) { 2571 device_printf(sc->dev, 2572 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2573 "Will try to use the config on the card, if any.\n", 2574 __func__, mtype, moff, cflen, rc); 2575 goto use_config_on_flash; 2576 } 2577 2578 memwin_info(sc, 2, &mw_base, &mw_aperture); 2579 while (cflen) { 2580 off = position_memwin(sc, 2, addr); 2581 n = min(cflen, mw_aperture - off); 2582 for (i = 0; i < n; i += 4) 2583 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2584 cflen -= n; 2585 addr += n; 2586 } 2587 } else { 2588 use_config_on_flash: 2589 mtype = FW_MEMTYPE_FLASH; 2590 moff = t4_flash_cfg_addr(sc); 2591 } 2592 2593 bzero(&caps, sizeof(caps)); 2594 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2595 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2596 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2597 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2598 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2599 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2600 if (rc != 0) { 2601 device_printf(sc->dev, 2602 "failed to pre-process config file: %d " 2603 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2604 goto done; 2605 } 2606 2607 finicsum = be32toh(caps.finicsum); 2608 cfcsum = be32toh(caps.cfcsum); 2609 if (finicsum != cfcsum) { 2610 device_printf(sc->dev, 2611 "WARNING: config file checksum mismatch: %08x %08x\n", 2612 finicsum, cfcsum); 2613 } 2614 sc->cfcsum = cfcsum; 2615 2616 #define LIMIT_CAPS(x) do { \ 2617 caps.x &= htobe16(t4_##x##_allowed); \ 2618 } while (0) 2619 2620 /* 2621 * Let the firmware know what features will (not) be used so it can tune 2622 * things accordingly. 2623 */ 2624 LIMIT_CAPS(linkcaps); 2625 LIMIT_CAPS(niccaps); 2626 LIMIT_CAPS(toecaps); 2627 LIMIT_CAPS(rdmacaps); 2628 LIMIT_CAPS(iscsicaps); 2629 LIMIT_CAPS(fcoecaps); 2630 #undef LIMIT_CAPS 2631 2632 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2633 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2634 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2635 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2636 if (rc != 0) { 2637 device_printf(sc->dev, 2638 "failed to process config file: %d.\n", rc); 2639 } 2640 done: 2641 if (cfg != NULL) 2642 firmware_put(cfg, FIRMWARE_UNLOAD); 2643 return (rc); 2644 } 2645 2646 /* 2647 * Retrieve parameters that are needed (or nice to have) very early. 2648 */ 2649 static int 2650 get_params__pre_init(struct adapter *sc) 2651 { 2652 int rc; 2653 uint32_t param[2], val[2]; 2654 struct fw_devlog_cmd cmd; 2655 struct devlog_params *dlog = &sc->params.devlog; 2656 2657 param[0] = FW_PARAM_DEV(PORTVEC); 2658 param[1] = FW_PARAM_DEV(CCLK); 2659 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2660 if (rc != 0) { 2661 device_printf(sc->dev, 2662 "failed to query parameters (pre_init): %d.\n", rc); 2663 return (rc); 2664 } 2665 2666 sc->params.portvec = val[0]; 2667 sc->params.nports = bitcount32(val[0]); 2668 sc->params.vpd.cclk = val[1]; 2669 2670 /* Read device log parameters. */ 2671 bzero(&cmd, sizeof(cmd)); 2672 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2673 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2674 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2675 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2676 if (rc != 0) { 2677 device_printf(sc->dev, 2678 "failed to get devlog parameters: %d.\n", rc); 2679 bzero(dlog, sizeof (*dlog)); 2680 rc = 0; /* devlog isn't critical for device operation */ 2681 } else { 2682 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2683 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2684 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2685 dlog->size = be32toh(cmd.memsize_devlog); 2686 } 2687 2688 return (rc); 2689 } 2690 2691 /* 2692 * Retrieve various parameters that are of interest to the driver. The device 2693 * has been initialized by the firmware at this point. 2694 */ 2695 static int 2696 get_params__post_init(struct adapter *sc) 2697 { 2698 int rc; 2699 uint32_t param[7], val[7]; 2700 struct fw_caps_config_cmd caps; 2701 2702 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2703 param[1] = FW_PARAM_PFVF(EQ_START); 2704 param[2] = FW_PARAM_PFVF(FILTER_START); 2705 param[3] = FW_PARAM_PFVF(FILTER_END); 2706 param[4] = FW_PARAM_PFVF(L2T_START); 2707 param[5] = FW_PARAM_PFVF(L2T_END); 2708 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2709 if (rc != 0) { 2710 device_printf(sc->dev, 2711 "failed to query parameters (post_init): %d.\n", rc); 2712 return (rc); 2713 } 2714 2715 sc->sge.iq_start = val[0]; 2716 sc->sge.eq_start = val[1]; 2717 sc->tids.ftid_base = val[2]; 2718 sc->tids.nftids = val[3] - val[2] + 1; 2719 sc->params.ftid_min = val[2]; 2720 sc->params.ftid_max = val[3]; 2721 sc->vres.l2t.start = val[4]; 2722 sc->vres.l2t.size = val[5] - val[4] + 1; 2723 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2724 ("%s: L2 table size (%u) larger than expected (%u)", 2725 __func__, sc->vres.l2t.size, L2T_SIZE)); 2726 2727 /* get capabilites */ 2728 bzero(&caps, sizeof(caps)); 2729 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2730 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2731 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2732 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2733 if (rc != 0) { 2734 device_printf(sc->dev, 2735 "failed to get card capabilities: %d.\n", rc); 2736 return (rc); 2737 } 2738 2739 #define READ_CAPS(x) do { \ 2740 sc->x = htobe16(caps.x); \ 2741 } while (0) 2742 READ_CAPS(linkcaps); 2743 READ_CAPS(niccaps); 2744 READ_CAPS(toecaps); 2745 READ_CAPS(rdmacaps); 2746 READ_CAPS(iscsicaps); 2747 READ_CAPS(fcoecaps); 2748 2749 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2750 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2751 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2752 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2753 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2754 if (rc != 0) { 2755 device_printf(sc->dev, 2756 "failed to query NIC parameters: %d.\n", rc); 2757 return (rc); 2758 } 2759 sc->tids.etid_base = val[0]; 2760 sc->params.etid_min = val[0]; 2761 sc->tids.netids = val[1] - val[0] + 1; 2762 sc->params.netids = sc->tids.netids; 2763 sc->params.eo_wr_cred = val[2]; 2764 sc->params.ethoffload = 1; 2765 } 2766 2767 if (sc->toecaps) { 2768 /* query offload-related parameters */ 2769 param[0] = FW_PARAM_DEV(NTID); 2770 param[1] = FW_PARAM_PFVF(SERVER_START); 2771 param[2] = FW_PARAM_PFVF(SERVER_END); 2772 param[3] = FW_PARAM_PFVF(TDDP_START); 2773 param[4] = FW_PARAM_PFVF(TDDP_END); 2774 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2775 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2776 if (rc != 0) { 2777 device_printf(sc->dev, 2778 "failed to query TOE parameters: %d.\n", rc); 2779 return (rc); 2780 } 2781 sc->tids.ntids = val[0]; 2782 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2783 sc->tids.stid_base = val[1]; 2784 sc->tids.nstids = val[2] - val[1] + 1; 2785 sc->vres.ddp.start = val[3]; 2786 sc->vres.ddp.size = val[4] - val[3] + 1; 2787 sc->params.ofldq_wr_cred = val[5]; 2788 sc->params.offload = 1; 2789 } 2790 if (sc->rdmacaps) { 2791 param[0] = FW_PARAM_PFVF(STAG_START); 2792 param[1] = FW_PARAM_PFVF(STAG_END); 2793 param[2] = FW_PARAM_PFVF(RQ_START); 2794 param[3] = FW_PARAM_PFVF(RQ_END); 2795 param[4] = FW_PARAM_PFVF(PBL_START); 2796 param[5] = FW_PARAM_PFVF(PBL_END); 2797 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2798 if (rc != 0) { 2799 device_printf(sc->dev, 2800 "failed to query RDMA parameters(1): %d.\n", rc); 2801 return (rc); 2802 } 2803 sc->vres.stag.start = val[0]; 2804 sc->vres.stag.size = val[1] - val[0] + 1; 2805 sc->vres.rq.start = val[2]; 2806 sc->vres.rq.size = val[3] - val[2] + 1; 2807 sc->vres.pbl.start = val[4]; 2808 sc->vres.pbl.size = val[5] - val[4] + 1; 2809 2810 param[0] = FW_PARAM_PFVF(SQRQ_START); 2811 param[1] = FW_PARAM_PFVF(SQRQ_END); 2812 param[2] = FW_PARAM_PFVF(CQ_START); 2813 param[3] = FW_PARAM_PFVF(CQ_END); 2814 param[4] = FW_PARAM_PFVF(OCQ_START); 2815 param[5] = FW_PARAM_PFVF(OCQ_END); 2816 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2817 if (rc != 0) { 2818 device_printf(sc->dev, 2819 "failed to query RDMA parameters(2): %d.\n", rc); 2820 return (rc); 2821 } 2822 sc->vres.qp.start = val[0]; 2823 sc->vres.qp.size = val[1] - val[0] + 1; 2824 sc->vres.cq.start = val[2]; 2825 sc->vres.cq.size = val[3] - val[2] + 1; 2826 sc->vres.ocq.start = val[4]; 2827 sc->vres.ocq.size = val[5] - val[4] + 1; 2828 } 2829 if (sc->iscsicaps) { 2830 param[0] = FW_PARAM_PFVF(ISCSI_START); 2831 param[1] = FW_PARAM_PFVF(ISCSI_END); 2832 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2833 if (rc != 0) { 2834 device_printf(sc->dev, 2835 "failed to query iSCSI parameters: %d.\n", rc); 2836 return (rc); 2837 } 2838 sc->vres.iscsi.start = val[0]; 2839 sc->vres.iscsi.size = val[1] - val[0] + 1; 2840 } 2841 2842 /* 2843 * We've got the params we wanted to query via the firmware. Now grab 2844 * some others directly from the chip. 2845 */ 2846 rc = t4_read_chip_settings(sc); 2847 2848 return (rc); 2849 } 2850 2851 static int 2852 set_params__post_init(struct adapter *sc) 2853 { 2854 uint32_t param, val; 2855 2856 /* ask for encapsulated CPLs */ 2857 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2858 val = 1; 2859 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2860 2861 return (0); 2862 } 2863 2864 #undef FW_PARAM_PFVF 2865 #undef FW_PARAM_DEV 2866 2867 static void 2868 t4_set_desc(struct adapter *sc) 2869 { 2870 char buf[128]; 2871 struct adapter_params *p = &sc->params; 2872 2873 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2874 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2875 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2876 2877 device_set_desc_copy(sc->dev, buf); 2878 } 2879 2880 static void 2881 build_medialist(struct port_info *pi, struct ifmedia *media) 2882 { 2883 int m; 2884 2885 PORT_LOCK(pi); 2886 2887 ifmedia_removeall(media); 2888 2889 m = IFM_ETHER | IFM_FDX; 2890 2891 switch(pi->port_type) { 2892 case FW_PORT_TYPE_BT_XFI: 2893 case FW_PORT_TYPE_BT_XAUI: 2894 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 2895 /* fall through */ 2896 2897 case FW_PORT_TYPE_BT_SGMII: 2898 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 2899 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 2900 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 2901 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2902 break; 2903 2904 case FW_PORT_TYPE_CX4: 2905 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 2906 ifmedia_set(media, m | IFM_10G_CX4); 2907 break; 2908 2909 case FW_PORT_TYPE_QSFP_10G: 2910 case FW_PORT_TYPE_SFP: 2911 case FW_PORT_TYPE_FIBER_XFI: 2912 case FW_PORT_TYPE_FIBER_XAUI: 2913 switch (pi->mod_type) { 2914 2915 case FW_PORT_MOD_TYPE_LR: 2916 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 2917 ifmedia_set(media, m | IFM_10G_LR); 2918 break; 2919 2920 case FW_PORT_MOD_TYPE_SR: 2921 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 2922 ifmedia_set(media, m | IFM_10G_SR); 2923 break; 2924 2925 case FW_PORT_MOD_TYPE_LRM: 2926 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 2927 ifmedia_set(media, m | IFM_10G_LRM); 2928 break; 2929 2930 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2931 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2932 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 2933 ifmedia_set(media, m | IFM_10G_TWINAX); 2934 break; 2935 2936 case FW_PORT_MOD_TYPE_NONE: 2937 m &= ~IFM_FDX; 2938 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2939 ifmedia_set(media, m | IFM_NONE); 2940 break; 2941 2942 case FW_PORT_MOD_TYPE_NA: 2943 case FW_PORT_MOD_TYPE_ER: 2944 default: 2945 device_printf(pi->dev, 2946 "unknown port_type (%d), mod_type (%d)\n", 2947 pi->port_type, pi->mod_type); 2948 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2949 ifmedia_set(media, m | IFM_UNKNOWN); 2950 break; 2951 } 2952 break; 2953 2954 case FW_PORT_TYPE_QSFP: 2955 switch (pi->mod_type) { 2956 2957 case FW_PORT_MOD_TYPE_LR: 2958 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 2959 ifmedia_set(media, m | IFM_40G_LR4); 2960 break; 2961 2962 case FW_PORT_MOD_TYPE_SR: 2963 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 2964 ifmedia_set(media, m | IFM_40G_SR4); 2965 break; 2966 2967 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2968 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2969 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 2970 ifmedia_set(media, m | IFM_40G_CR4); 2971 break; 2972 2973 case FW_PORT_MOD_TYPE_NONE: 2974 m &= ~IFM_FDX; 2975 ifmedia_add(media, m | IFM_NONE, 0, NULL); 2976 ifmedia_set(media, m | IFM_NONE); 2977 break; 2978 2979 default: 2980 device_printf(pi->dev, 2981 "unknown port_type (%d), mod_type (%d)\n", 2982 pi->port_type, pi->mod_type); 2983 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2984 ifmedia_set(media, m | IFM_UNKNOWN); 2985 break; 2986 } 2987 break; 2988 2989 default: 2990 device_printf(pi->dev, 2991 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2992 pi->mod_type); 2993 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 2994 ifmedia_set(media, m | IFM_UNKNOWN); 2995 break; 2996 } 2997 2998 PORT_UNLOCK(pi); 2999 } 3000 3001 #define FW_MAC_EXACT_CHUNK 7 3002 3003 /* 3004 * Program the port's XGMAC based on parameters in ifnet. The caller also 3005 * indicates which parameters should be programmed (the rest are left alone). 3006 */ 3007 int 3008 update_mac_settings(struct ifnet *ifp, int flags) 3009 { 3010 int rc = 0; 3011 struct port_info *pi = ifp->if_softc; 3012 struct adapter *sc = pi->adapter; 3013 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3014 uint16_t viid = 0xffff; 3015 int16_t *xact_addr_filt = NULL; 3016 3017 ASSERT_SYNCHRONIZED_OP(sc); 3018 KASSERT(flags, ("%s: not told what to update.", __func__)); 3019 3020 if (ifp == pi->ifp) { 3021 viid = pi->viid; 3022 xact_addr_filt = &pi->xact_addr_filt; 3023 } 3024 #ifdef DEV_NETMAP 3025 else if (ifp == pi->nm_ifp) { 3026 viid = pi->nm_viid; 3027 xact_addr_filt = &pi->nm_xact_addr_filt; 3028 } 3029 #endif 3030 if (flags & XGMAC_MTU) 3031 mtu = ifp->if_mtu; 3032 3033 if (flags & XGMAC_PROMISC) 3034 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3035 3036 if (flags & XGMAC_ALLMULTI) 3037 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3038 3039 if (flags & XGMAC_VLANEX) 3040 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3041 3042 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3043 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3044 1, vlanex, false); 3045 if (rc) { 3046 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3047 rc); 3048 return (rc); 3049 } 3050 } 3051 3052 if (flags & XGMAC_UCADDR) { 3053 uint8_t ucaddr[ETHER_ADDR_LEN]; 3054 3055 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3056 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3057 true, true); 3058 if (rc < 0) { 3059 rc = -rc; 3060 if_printf(ifp, "change_mac failed: %d\n", rc); 3061 return (rc); 3062 } else { 3063 *xact_addr_filt = rc; 3064 rc = 0; 3065 } 3066 } 3067 3068 if (flags & XGMAC_MCADDRS) { 3069 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3070 int del = 1; 3071 uint64_t hash = 0; 3072 struct ifmultiaddr *ifma; 3073 int i = 0, j; 3074 3075 if_maddr_rlock(ifp); 3076 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3077 if (ifma->ifma_addr->sa_family != AF_LINK) 3078 continue; 3079 mcaddr[i] = 3080 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3081 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3082 i++; 3083 3084 if (i == FW_MAC_EXACT_CHUNK) { 3085 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3086 i, mcaddr, NULL, &hash, 0); 3087 if (rc < 0) { 3088 rc = -rc; 3089 for (j = 0; j < i; j++) { 3090 if_printf(ifp, 3091 "failed to add mc address" 3092 " %02x:%02x:%02x:" 3093 "%02x:%02x:%02x rc=%d\n", 3094 mcaddr[j][0], mcaddr[j][1], 3095 mcaddr[j][2], mcaddr[j][3], 3096 mcaddr[j][4], mcaddr[j][5], 3097 rc); 3098 } 3099 goto mcfail; 3100 } 3101 del = 0; 3102 i = 0; 3103 } 3104 } 3105 if (i > 0) { 3106 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3107 mcaddr, NULL, &hash, 0); 3108 if (rc < 0) { 3109 rc = -rc; 3110 for (j = 0; j < i; j++) { 3111 if_printf(ifp, 3112 "failed to add mc address" 3113 " %02x:%02x:%02x:" 3114 "%02x:%02x:%02x rc=%d\n", 3115 mcaddr[j][0], mcaddr[j][1], 3116 mcaddr[j][2], mcaddr[j][3], 3117 mcaddr[j][4], mcaddr[j][5], 3118 rc); 3119 } 3120 goto mcfail; 3121 } 3122 } 3123 3124 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3125 if (rc != 0) 3126 if_printf(ifp, "failed to set mc address hash: %d", rc); 3127 mcfail: 3128 if_maddr_runlock(ifp); 3129 } 3130 3131 return (rc); 3132 } 3133 3134 /* 3135 * {begin|end}_synchronized_op must be called from the same thread. 3136 */ 3137 int 3138 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3139 char *wmesg) 3140 { 3141 int rc, pri; 3142 3143 #ifdef WITNESS 3144 /* the caller thinks it's ok to sleep, but is it really? */ 3145 if (flags & SLEEP_OK) 3146 pause("t4slptst", 1); 3147 #endif 3148 3149 if (INTR_OK) 3150 pri = PCATCH; 3151 else 3152 pri = 0; 3153 3154 ADAPTER_LOCK(sc); 3155 for (;;) { 3156 3157 if (pi && IS_DOOMED(pi)) { 3158 rc = ENXIO; 3159 goto done; 3160 } 3161 3162 if (!IS_BUSY(sc)) { 3163 rc = 0; 3164 break; 3165 } 3166 3167 if (!(flags & SLEEP_OK)) { 3168 rc = EBUSY; 3169 goto done; 3170 } 3171 3172 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3173 rc = EINTR; 3174 goto done; 3175 } 3176 } 3177 3178 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3179 SET_BUSY(sc); 3180 #ifdef INVARIANTS 3181 sc->last_op = wmesg; 3182 sc->last_op_thr = curthread; 3183 #endif 3184 3185 done: 3186 if (!(flags & HOLD_LOCK) || rc) 3187 ADAPTER_UNLOCK(sc); 3188 3189 return (rc); 3190 } 3191 3192 /* 3193 * {begin|end}_synchronized_op must be called from the same thread. 3194 */ 3195 void 3196 end_synchronized_op(struct adapter *sc, int flags) 3197 { 3198 3199 if (flags & LOCK_HELD) 3200 ADAPTER_LOCK_ASSERT_OWNED(sc); 3201 else 3202 ADAPTER_LOCK(sc); 3203 3204 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3205 CLR_BUSY(sc); 3206 wakeup(&sc->flags); 3207 ADAPTER_UNLOCK(sc); 3208 } 3209 3210 static int 3211 cxgbe_init_synchronized(struct port_info *pi) 3212 { 3213 struct adapter *sc = pi->adapter; 3214 struct ifnet *ifp = pi->ifp; 3215 int rc = 0, i; 3216 struct sge_txq *txq; 3217 3218 ASSERT_SYNCHRONIZED_OP(sc); 3219 3220 if (isset(&sc->open_device_map, pi->port_id)) { 3221 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3222 ("mismatch between open_device_map and if_drv_flags")); 3223 return (0); /* already running */ 3224 } 3225 3226 if (!(sc->flags & FULL_INIT_DONE) && 3227 ((rc = adapter_full_init(sc)) != 0)) 3228 return (rc); /* error message displayed already */ 3229 3230 if (!(pi->flags & PORT_INIT_DONE) && 3231 ((rc = port_full_init(pi)) != 0)) 3232 return (rc); /* error message displayed already */ 3233 3234 rc = update_mac_settings(ifp, XGMAC_ALL); 3235 if (rc) 3236 goto done; /* error message displayed already */ 3237 3238 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3239 if (rc != 0) { 3240 if_printf(ifp, "enable_vi failed: %d\n", rc); 3241 goto done; 3242 } 3243 3244 /* 3245 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3246 * if this changes. 3247 */ 3248 3249 for_each_txq(pi, i, txq) { 3250 TXQ_LOCK(txq); 3251 txq->eq.flags |= EQ_ENABLED; 3252 TXQ_UNLOCK(txq); 3253 } 3254 3255 /* 3256 * The first iq of the first port to come up is used for tracing. 3257 */ 3258 if (sc->traceq < 0) { 3259 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3260 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3261 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3262 V_QUEUENUMBER(sc->traceq)); 3263 pi->flags |= HAS_TRACEQ; 3264 } 3265 3266 /* all ok */ 3267 setbit(&sc->open_device_map, pi->port_id); 3268 PORT_LOCK(pi); 3269 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3270 PORT_UNLOCK(pi); 3271 3272 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3273 done: 3274 if (rc != 0) 3275 cxgbe_uninit_synchronized(pi); 3276 3277 return (rc); 3278 } 3279 3280 /* 3281 * Idempotent. 3282 */ 3283 static int 3284 cxgbe_uninit_synchronized(struct port_info *pi) 3285 { 3286 struct adapter *sc = pi->adapter; 3287 struct ifnet *ifp = pi->ifp; 3288 int rc, i; 3289 struct sge_txq *txq; 3290 3291 ASSERT_SYNCHRONIZED_OP(sc); 3292 3293 if (!(pi->flags & PORT_INIT_DONE)) { 3294 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3295 ("uninited port is running")); 3296 return (0); 3297 } 3298 3299 /* 3300 * Disable the VI so that all its data in either direction is discarded 3301 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3302 * tick) intact as the TP can deliver negative advice or data that it's 3303 * holding in its RAM (for an offloaded connection) even after the VI is 3304 * disabled. 3305 */ 3306 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3307 if (rc) { 3308 if_printf(ifp, "disable_vi failed: %d\n", rc); 3309 return (rc); 3310 } 3311 3312 for_each_txq(pi, i, txq) { 3313 TXQ_LOCK(txq); 3314 txq->eq.flags &= ~EQ_ENABLED; 3315 TXQ_UNLOCK(txq); 3316 } 3317 3318 clrbit(&sc->open_device_map, pi->port_id); 3319 PORT_LOCK(pi); 3320 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3321 PORT_UNLOCK(pi); 3322 3323 pi->link_cfg.link_ok = 0; 3324 pi->link_cfg.speed = 0; 3325 pi->linkdnrc = -1; 3326 t4_os_link_changed(sc, pi->port_id, 0, -1); 3327 3328 return (0); 3329 } 3330 3331 /* 3332 * It is ok for this function to fail midway and return right away. t4_detach 3333 * will walk the entire sc->irq list and clean up whatever is valid. 3334 */ 3335 static int 3336 setup_intr_handlers(struct adapter *sc) 3337 { 3338 int rc, rid, p, q; 3339 char s[8]; 3340 struct irq *irq; 3341 struct port_info *pi; 3342 struct sge_rxq *rxq; 3343 #ifdef TCP_OFFLOAD 3344 struct sge_ofld_rxq *ofld_rxq; 3345 #endif 3346 #ifdef DEV_NETMAP 3347 struct sge_nm_rxq *nm_rxq; 3348 #endif 3349 3350 /* 3351 * Setup interrupts. 3352 */ 3353 irq = &sc->irq[0]; 3354 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3355 if (sc->intr_count == 1) 3356 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3357 3358 /* Multiple interrupts. */ 3359 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3360 ("%s: too few intr.", __func__)); 3361 3362 /* The first one is always error intr */ 3363 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3364 if (rc != 0) 3365 return (rc); 3366 irq++; 3367 rid++; 3368 3369 /* The second one is always the firmware event queue */ 3370 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3371 if (rc != 0) 3372 return (rc); 3373 irq++; 3374 rid++; 3375 3376 for_each_port(sc, p) { 3377 pi = sc->port[p]; 3378 3379 if (pi->flags & INTR_RXQ) { 3380 for_each_rxq(pi, q, rxq) { 3381 snprintf(s, sizeof(s), "%d.%d", p, q); 3382 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3383 s); 3384 if (rc != 0) 3385 return (rc); 3386 irq++; 3387 rid++; 3388 } 3389 } 3390 #ifdef TCP_OFFLOAD 3391 if (pi->flags & INTR_OFLD_RXQ) { 3392 for_each_ofld_rxq(pi, q, ofld_rxq) { 3393 snprintf(s, sizeof(s), "%d,%d", p, q); 3394 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3395 ofld_rxq, s); 3396 if (rc != 0) 3397 return (rc); 3398 irq++; 3399 rid++; 3400 } 3401 } 3402 #endif 3403 #ifdef DEV_NETMAP 3404 if (pi->flags & INTR_NM_RXQ) { 3405 for_each_nm_rxq(pi, q, nm_rxq) { 3406 snprintf(s, sizeof(s), "%d-%d", p, q); 3407 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3408 nm_rxq, s); 3409 if (rc != 0) 3410 return (rc); 3411 irq++; 3412 rid++; 3413 } 3414 } 3415 #endif 3416 } 3417 MPASS(irq == &sc->irq[sc->intr_count]); 3418 3419 return (0); 3420 } 3421 3422 int 3423 adapter_full_init(struct adapter *sc) 3424 { 3425 int rc, i; 3426 3427 ASSERT_SYNCHRONIZED_OP(sc); 3428 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3429 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3430 ("%s: FULL_INIT_DONE already", __func__)); 3431 3432 /* 3433 * queues that belong to the adapter (not any particular port). 3434 */ 3435 rc = t4_setup_adapter_queues(sc); 3436 if (rc != 0) 3437 goto done; 3438 3439 for (i = 0; i < nitems(sc->tq); i++) { 3440 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3441 taskqueue_thread_enqueue, &sc->tq[i]); 3442 if (sc->tq[i] == NULL) { 3443 device_printf(sc->dev, 3444 "failed to allocate task queue %d\n", i); 3445 rc = ENOMEM; 3446 goto done; 3447 } 3448 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3449 device_get_nameunit(sc->dev), i); 3450 } 3451 3452 t4_intr_enable(sc); 3453 sc->flags |= FULL_INIT_DONE; 3454 done: 3455 if (rc != 0) 3456 adapter_full_uninit(sc); 3457 3458 return (rc); 3459 } 3460 3461 int 3462 adapter_full_uninit(struct adapter *sc) 3463 { 3464 int i; 3465 3466 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3467 3468 t4_teardown_adapter_queues(sc); 3469 3470 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3471 taskqueue_free(sc->tq[i]); 3472 sc->tq[i] = NULL; 3473 } 3474 3475 sc->flags &= ~FULL_INIT_DONE; 3476 3477 return (0); 3478 } 3479 3480 int 3481 port_full_init(struct port_info *pi) 3482 { 3483 struct adapter *sc = pi->adapter; 3484 struct ifnet *ifp = pi->ifp; 3485 uint16_t *rss; 3486 struct sge_rxq *rxq; 3487 int rc, i, j; 3488 3489 ASSERT_SYNCHRONIZED_OP(sc); 3490 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3491 ("%s: PORT_INIT_DONE already", __func__)); 3492 3493 sysctl_ctx_init(&pi->ctx); 3494 pi->flags |= PORT_SYSCTL_CTX; 3495 3496 /* 3497 * Allocate tx/rx/fl queues for this port. 3498 */ 3499 rc = t4_setup_port_queues(pi); 3500 if (rc != 0) 3501 goto done; /* error message displayed already */ 3502 3503 /* 3504 * Setup RSS for this port. Save a copy of the RSS table for later use. 3505 */ 3506 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3507 for (i = 0; i < pi->rss_size;) { 3508 for_each_rxq(pi, j, rxq) { 3509 rss[i++] = rxq->iq.abs_id; 3510 if (i == pi->rss_size) 3511 break; 3512 } 3513 } 3514 3515 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3516 pi->rss_size); 3517 if (rc != 0) { 3518 if_printf(ifp, "rss_config failed: %d\n", rc); 3519 goto done; 3520 } 3521 3522 pi->rss = rss; 3523 pi->flags |= PORT_INIT_DONE; 3524 done: 3525 if (rc != 0) 3526 port_full_uninit(pi); 3527 3528 return (rc); 3529 } 3530 3531 /* 3532 * Idempotent. 3533 */ 3534 int 3535 port_full_uninit(struct port_info *pi) 3536 { 3537 struct adapter *sc = pi->adapter; 3538 int i; 3539 struct sge_rxq *rxq; 3540 struct sge_txq *txq; 3541 #ifdef TCP_OFFLOAD 3542 struct sge_ofld_rxq *ofld_rxq; 3543 struct sge_wrq *ofld_txq; 3544 #endif 3545 3546 if (pi->flags & PORT_INIT_DONE) { 3547 3548 /* Need to quiesce queues. */ 3549 3550 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3551 3552 for_each_txq(pi, i, txq) { 3553 quiesce_txq(sc, txq); 3554 } 3555 3556 #ifdef TCP_OFFLOAD 3557 for_each_ofld_txq(pi, i, ofld_txq) { 3558 quiesce_wrq(sc, ofld_txq); 3559 } 3560 #endif 3561 3562 for_each_rxq(pi, i, rxq) { 3563 quiesce_iq(sc, &rxq->iq); 3564 quiesce_fl(sc, &rxq->fl); 3565 } 3566 3567 #ifdef TCP_OFFLOAD 3568 for_each_ofld_rxq(pi, i, ofld_rxq) { 3569 quiesce_iq(sc, &ofld_rxq->iq); 3570 quiesce_fl(sc, &ofld_rxq->fl); 3571 } 3572 #endif 3573 free(pi->rss, M_CXGBE); 3574 } 3575 3576 t4_teardown_port_queues(pi); 3577 pi->flags &= ~PORT_INIT_DONE; 3578 3579 return (0); 3580 } 3581 3582 static void 3583 quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3584 { 3585 struct sge_eq *eq = &txq->eq; 3586 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3587 3588 (void) sc; /* unused */ 3589 3590 #ifdef INVARIANTS 3591 TXQ_LOCK(txq); 3592 MPASS((eq->flags & EQ_ENABLED) == 0); 3593 TXQ_UNLOCK(txq); 3594 #endif 3595 3596 /* Wait for the mp_ring to empty. */ 3597 while (!mp_ring_is_idle(txq->r)) { 3598 mp_ring_check_drainage(txq->r, 0); 3599 pause("rquiesce", 1); 3600 } 3601 3602 /* Then wait for the hardware to finish. */ 3603 while (spg->cidx != htobe16(eq->pidx)) 3604 pause("equiesce", 1); 3605 3606 /* Finally, wait for the driver to reclaim all descriptors. */ 3607 while (eq->cidx != eq->pidx) 3608 pause("dquiesce", 1); 3609 } 3610 3611 static void 3612 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3613 { 3614 3615 /* XXXTX */ 3616 } 3617 3618 static void 3619 quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3620 { 3621 (void) sc; /* unused */ 3622 3623 /* Synchronize with the interrupt handler */ 3624 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3625 pause("iqfree", 1); 3626 } 3627 3628 static void 3629 quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3630 { 3631 mtx_lock(&sc->sfl_lock); 3632 FL_LOCK(fl); 3633 fl->flags |= FL_DOOMED; 3634 FL_UNLOCK(fl); 3635 mtx_unlock(&sc->sfl_lock); 3636 3637 callout_drain(&sc->sfl_callout); 3638 KASSERT((fl->flags & FL_STARVING) == 0, 3639 ("%s: still starving", __func__)); 3640 } 3641 3642 static int 3643 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3644 driver_intr_t *handler, void *arg, char *name) 3645 { 3646 int rc; 3647 3648 irq->rid = rid; 3649 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3650 RF_SHAREABLE | RF_ACTIVE); 3651 if (irq->res == NULL) { 3652 device_printf(sc->dev, 3653 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3654 return (ENOMEM); 3655 } 3656 3657 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3658 NULL, handler, arg, &irq->tag); 3659 if (rc != 0) { 3660 device_printf(sc->dev, 3661 "failed to setup interrupt for rid %d, name %s: %d\n", 3662 rid, name, rc); 3663 } else if (name) 3664 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3665 3666 return (rc); 3667 } 3668 3669 static int 3670 t4_free_irq(struct adapter *sc, struct irq *irq) 3671 { 3672 if (irq->tag) 3673 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3674 if (irq->res) 3675 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3676 3677 bzero(irq, sizeof(*irq)); 3678 3679 return (0); 3680 } 3681 3682 static void 3683 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3684 unsigned int end) 3685 { 3686 uint32_t *p = (uint32_t *)(buf + start); 3687 3688 for ( ; start <= end; start += sizeof(uint32_t)) 3689 *p++ = t4_read_reg(sc, start); 3690 } 3691 3692 static void 3693 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3694 { 3695 int i, n; 3696 const unsigned int *reg_ranges; 3697 static const unsigned int t4_reg_ranges[] = { 3698 0x1008, 0x1108, 3699 0x1180, 0x11b4, 3700 0x11fc, 0x123c, 3701 0x1300, 0x173c, 3702 0x1800, 0x18fc, 3703 0x3000, 0x30d8, 3704 0x30e0, 0x5924, 3705 0x5960, 0x59d4, 3706 0x5a00, 0x5af8, 3707 0x6000, 0x6098, 3708 0x6100, 0x6150, 3709 0x6200, 0x6208, 3710 0x6240, 0x6248, 3711 0x6280, 0x6338, 3712 0x6370, 0x638c, 3713 0x6400, 0x643c, 3714 0x6500, 0x6524, 3715 0x6a00, 0x6a38, 3716 0x6a60, 0x6a78, 3717 0x6b00, 0x6b84, 3718 0x6bf0, 0x6c84, 3719 0x6cf0, 0x6d84, 3720 0x6df0, 0x6e84, 3721 0x6ef0, 0x6f84, 3722 0x6ff0, 0x7084, 3723 0x70f0, 0x7184, 3724 0x71f0, 0x7284, 3725 0x72f0, 0x7384, 3726 0x73f0, 0x7450, 3727 0x7500, 0x7530, 3728 0x7600, 0x761c, 3729 0x7680, 0x76cc, 3730 0x7700, 0x7798, 3731 0x77c0, 0x77fc, 3732 0x7900, 0x79fc, 3733 0x7b00, 0x7c38, 3734 0x7d00, 0x7efc, 3735 0x8dc0, 0x8e1c, 3736 0x8e30, 0x8e78, 3737 0x8ea0, 0x8f6c, 3738 0x8fc0, 0x9074, 3739 0x90fc, 0x90fc, 3740 0x9400, 0x9458, 3741 0x9600, 0x96bc, 3742 0x9800, 0x9808, 3743 0x9820, 0x983c, 3744 0x9850, 0x9864, 3745 0x9c00, 0x9c6c, 3746 0x9c80, 0x9cec, 3747 0x9d00, 0x9d6c, 3748 0x9d80, 0x9dec, 3749 0x9e00, 0x9e6c, 3750 0x9e80, 0x9eec, 3751 0x9f00, 0x9f6c, 3752 0x9f80, 0x9fec, 3753 0xd004, 0xd03c, 3754 0xdfc0, 0xdfe0, 3755 0xe000, 0xea7c, 3756 0xf000, 0x11110, 3757 0x11118, 0x11190, 3758 0x19040, 0x1906c, 3759 0x19078, 0x19080, 3760 0x1908c, 0x19124, 3761 0x19150, 0x191b0, 3762 0x191d0, 0x191e8, 3763 0x19238, 0x1924c, 3764 0x193f8, 0x19474, 3765 0x19490, 0x194f8, 3766 0x19800, 0x19f30, 3767 0x1a000, 0x1a06c, 3768 0x1a0b0, 0x1a120, 3769 0x1a128, 0x1a138, 3770 0x1a190, 0x1a1c4, 3771 0x1a1fc, 0x1a1fc, 3772 0x1e040, 0x1e04c, 3773 0x1e284, 0x1e28c, 3774 0x1e2c0, 0x1e2c0, 3775 0x1e2e0, 0x1e2e0, 3776 0x1e300, 0x1e384, 3777 0x1e3c0, 0x1e3c8, 3778 0x1e440, 0x1e44c, 3779 0x1e684, 0x1e68c, 3780 0x1e6c0, 0x1e6c0, 3781 0x1e6e0, 0x1e6e0, 3782 0x1e700, 0x1e784, 3783 0x1e7c0, 0x1e7c8, 3784 0x1e840, 0x1e84c, 3785 0x1ea84, 0x1ea8c, 3786 0x1eac0, 0x1eac0, 3787 0x1eae0, 0x1eae0, 3788 0x1eb00, 0x1eb84, 3789 0x1ebc0, 0x1ebc8, 3790 0x1ec40, 0x1ec4c, 3791 0x1ee84, 0x1ee8c, 3792 0x1eec0, 0x1eec0, 3793 0x1eee0, 0x1eee0, 3794 0x1ef00, 0x1ef84, 3795 0x1efc0, 0x1efc8, 3796 0x1f040, 0x1f04c, 3797 0x1f284, 0x1f28c, 3798 0x1f2c0, 0x1f2c0, 3799 0x1f2e0, 0x1f2e0, 3800 0x1f300, 0x1f384, 3801 0x1f3c0, 0x1f3c8, 3802 0x1f440, 0x1f44c, 3803 0x1f684, 0x1f68c, 3804 0x1f6c0, 0x1f6c0, 3805 0x1f6e0, 0x1f6e0, 3806 0x1f700, 0x1f784, 3807 0x1f7c0, 0x1f7c8, 3808 0x1f840, 0x1f84c, 3809 0x1fa84, 0x1fa8c, 3810 0x1fac0, 0x1fac0, 3811 0x1fae0, 0x1fae0, 3812 0x1fb00, 0x1fb84, 3813 0x1fbc0, 0x1fbc8, 3814 0x1fc40, 0x1fc4c, 3815 0x1fe84, 0x1fe8c, 3816 0x1fec0, 0x1fec0, 3817 0x1fee0, 0x1fee0, 3818 0x1ff00, 0x1ff84, 3819 0x1ffc0, 0x1ffc8, 3820 0x20000, 0x2002c, 3821 0x20100, 0x2013c, 3822 0x20190, 0x201c8, 3823 0x20200, 0x20318, 3824 0x20400, 0x20528, 3825 0x20540, 0x20614, 3826 0x21000, 0x21040, 3827 0x2104c, 0x21060, 3828 0x210c0, 0x210ec, 3829 0x21200, 0x21268, 3830 0x21270, 0x21284, 3831 0x212fc, 0x21388, 3832 0x21400, 0x21404, 3833 0x21500, 0x21518, 3834 0x2152c, 0x2153c, 3835 0x21550, 0x21554, 3836 0x21600, 0x21600, 3837 0x21608, 0x21628, 3838 0x21630, 0x2163c, 3839 0x21700, 0x2171c, 3840 0x21780, 0x2178c, 3841 0x21800, 0x21c38, 3842 0x21c80, 0x21d7c, 3843 0x21e00, 0x21e04, 3844 0x22000, 0x2202c, 3845 0x22100, 0x2213c, 3846 0x22190, 0x221c8, 3847 0x22200, 0x22318, 3848 0x22400, 0x22528, 3849 0x22540, 0x22614, 3850 0x23000, 0x23040, 3851 0x2304c, 0x23060, 3852 0x230c0, 0x230ec, 3853 0x23200, 0x23268, 3854 0x23270, 0x23284, 3855 0x232fc, 0x23388, 3856 0x23400, 0x23404, 3857 0x23500, 0x23518, 3858 0x2352c, 0x2353c, 3859 0x23550, 0x23554, 3860 0x23600, 0x23600, 3861 0x23608, 0x23628, 3862 0x23630, 0x2363c, 3863 0x23700, 0x2371c, 3864 0x23780, 0x2378c, 3865 0x23800, 0x23c38, 3866 0x23c80, 0x23d7c, 3867 0x23e00, 0x23e04, 3868 0x24000, 0x2402c, 3869 0x24100, 0x2413c, 3870 0x24190, 0x241c8, 3871 0x24200, 0x24318, 3872 0x24400, 0x24528, 3873 0x24540, 0x24614, 3874 0x25000, 0x25040, 3875 0x2504c, 0x25060, 3876 0x250c0, 0x250ec, 3877 0x25200, 0x25268, 3878 0x25270, 0x25284, 3879 0x252fc, 0x25388, 3880 0x25400, 0x25404, 3881 0x25500, 0x25518, 3882 0x2552c, 0x2553c, 3883 0x25550, 0x25554, 3884 0x25600, 0x25600, 3885 0x25608, 0x25628, 3886 0x25630, 0x2563c, 3887 0x25700, 0x2571c, 3888 0x25780, 0x2578c, 3889 0x25800, 0x25c38, 3890 0x25c80, 0x25d7c, 3891 0x25e00, 0x25e04, 3892 0x26000, 0x2602c, 3893 0x26100, 0x2613c, 3894 0x26190, 0x261c8, 3895 0x26200, 0x26318, 3896 0x26400, 0x26528, 3897 0x26540, 0x26614, 3898 0x27000, 0x27040, 3899 0x2704c, 0x27060, 3900 0x270c0, 0x270ec, 3901 0x27200, 0x27268, 3902 0x27270, 0x27284, 3903 0x272fc, 0x27388, 3904 0x27400, 0x27404, 3905 0x27500, 0x27518, 3906 0x2752c, 0x2753c, 3907 0x27550, 0x27554, 3908 0x27600, 0x27600, 3909 0x27608, 0x27628, 3910 0x27630, 0x2763c, 3911 0x27700, 0x2771c, 3912 0x27780, 0x2778c, 3913 0x27800, 0x27c38, 3914 0x27c80, 0x27d7c, 3915 0x27e00, 0x27e04 3916 }; 3917 static const unsigned int t5_reg_ranges[] = { 3918 0x1008, 0x1148, 3919 0x1180, 0x11b4, 3920 0x11fc, 0x123c, 3921 0x1280, 0x173c, 3922 0x1800, 0x18fc, 3923 0x3000, 0x3028, 3924 0x3060, 0x30d8, 3925 0x30e0, 0x30fc, 3926 0x3140, 0x357c, 3927 0x35a8, 0x35cc, 3928 0x35ec, 0x35ec, 3929 0x3600, 0x5624, 3930 0x56cc, 0x575c, 3931 0x580c, 0x5814, 3932 0x5890, 0x58bc, 3933 0x5940, 0x59dc, 3934 0x59fc, 0x5a18, 3935 0x5a60, 0x5a9c, 3936 0x5b94, 0x5bfc, 3937 0x6000, 0x6040, 3938 0x6058, 0x614c, 3939 0x7700, 0x7798, 3940 0x77c0, 0x78fc, 3941 0x7b00, 0x7c54, 3942 0x7d00, 0x7efc, 3943 0x8dc0, 0x8de0, 3944 0x8df8, 0x8e84, 3945 0x8ea0, 0x8f84, 3946 0x8fc0, 0x90f8, 3947 0x9400, 0x9470, 3948 0x9600, 0x96f4, 3949 0x9800, 0x9808, 3950 0x9820, 0x983c, 3951 0x9850, 0x9864, 3952 0x9c00, 0x9c6c, 3953 0x9c80, 0x9cec, 3954 0x9d00, 0x9d6c, 3955 0x9d80, 0x9dec, 3956 0x9e00, 0x9e6c, 3957 0x9e80, 0x9eec, 3958 0x9f00, 0x9f6c, 3959 0x9f80, 0xa020, 3960 0xd004, 0xd03c, 3961 0xdfc0, 0xdfe0, 3962 0xe000, 0x11088, 3963 0x1109c, 0x11110, 3964 0x11118, 0x1117c, 3965 0x11190, 0x11204, 3966 0x19040, 0x1906c, 3967 0x19078, 0x19080, 3968 0x1908c, 0x19124, 3969 0x19150, 0x191b0, 3970 0x191d0, 0x191e8, 3971 0x19238, 0x19290, 3972 0x193f8, 0x19474, 3973 0x19490, 0x194cc, 3974 0x194f0, 0x194f8, 3975 0x19c00, 0x19c60, 3976 0x19c94, 0x19e10, 3977 0x19e50, 0x19f34, 3978 0x19f40, 0x19f50, 3979 0x19f90, 0x19fe4, 3980 0x1a000, 0x1a06c, 3981 0x1a0b0, 0x1a120, 3982 0x1a128, 0x1a138, 3983 0x1a190, 0x1a1c4, 3984 0x1a1fc, 0x1a1fc, 3985 0x1e008, 0x1e00c, 3986 0x1e040, 0x1e04c, 3987 0x1e284, 0x1e290, 3988 0x1e2c0, 0x1e2c0, 3989 0x1e2e0, 0x1e2e0, 3990 0x1e300, 0x1e384, 3991 0x1e3c0, 0x1e3c8, 3992 0x1e408, 0x1e40c, 3993 0x1e440, 0x1e44c, 3994 0x1e684, 0x1e690, 3995 0x1e6c0, 0x1e6c0, 3996 0x1e6e0, 0x1e6e0, 3997 0x1e700, 0x1e784, 3998 0x1e7c0, 0x1e7c8, 3999 0x1e808, 0x1e80c, 4000 0x1e840, 0x1e84c, 4001 0x1ea84, 0x1ea90, 4002 0x1eac0, 0x1eac0, 4003 0x1eae0, 0x1eae0, 4004 0x1eb00, 0x1eb84, 4005 0x1ebc0, 0x1ebc8, 4006 0x1ec08, 0x1ec0c, 4007 0x1ec40, 0x1ec4c, 4008 0x1ee84, 0x1ee90, 4009 0x1eec0, 0x1eec0, 4010 0x1eee0, 0x1eee0, 4011 0x1ef00, 0x1ef84, 4012 0x1efc0, 0x1efc8, 4013 0x1f008, 0x1f00c, 4014 0x1f040, 0x1f04c, 4015 0x1f284, 0x1f290, 4016 0x1f2c0, 0x1f2c0, 4017 0x1f2e0, 0x1f2e0, 4018 0x1f300, 0x1f384, 4019 0x1f3c0, 0x1f3c8, 4020 0x1f408, 0x1f40c, 4021 0x1f440, 0x1f44c, 4022 0x1f684, 0x1f690, 4023 0x1f6c0, 0x1f6c0, 4024 0x1f6e0, 0x1f6e0, 4025 0x1f700, 0x1f784, 4026 0x1f7c0, 0x1f7c8, 4027 0x1f808, 0x1f80c, 4028 0x1f840, 0x1f84c, 4029 0x1fa84, 0x1fa90, 4030 0x1fac0, 0x1fac0, 4031 0x1fae0, 0x1fae0, 4032 0x1fb00, 0x1fb84, 4033 0x1fbc0, 0x1fbc8, 4034 0x1fc08, 0x1fc0c, 4035 0x1fc40, 0x1fc4c, 4036 0x1fe84, 0x1fe90, 4037 0x1fec0, 0x1fec0, 4038 0x1fee0, 0x1fee0, 4039 0x1ff00, 0x1ff84, 4040 0x1ffc0, 0x1ffc8, 4041 0x30000, 0x30030, 4042 0x30100, 0x30144, 4043 0x30190, 0x301d0, 4044 0x30200, 0x30318, 4045 0x30400, 0x3052c, 4046 0x30540, 0x3061c, 4047 0x30800, 0x30834, 4048 0x308c0, 0x30908, 4049 0x30910, 0x309ac, 4050 0x30a00, 0x30a2c, 4051 0x30a44, 0x30a50, 4052 0x30a74, 0x30c24, 4053 0x30d00, 0x30d00, 4054 0x30d08, 0x30d14, 4055 0x30d1c, 0x30d20, 4056 0x30d3c, 0x30d50, 4057 0x31200, 0x3120c, 4058 0x31220, 0x31220, 4059 0x31240, 0x31240, 4060 0x31600, 0x3160c, 4061 0x31a00, 0x31a1c, 4062 0x31e00, 0x31e20, 4063 0x31e38, 0x31e3c, 4064 0x31e80, 0x31e80, 4065 0x31e88, 0x31ea8, 4066 0x31eb0, 0x31eb4, 4067 0x31ec8, 0x31ed4, 4068 0x31fb8, 0x32004, 4069 0x32200, 0x32200, 4070 0x32208, 0x32240, 4071 0x32248, 0x32280, 4072 0x32288, 0x322c0, 4073 0x322c8, 0x322fc, 4074 0x32600, 0x32630, 4075 0x32a00, 0x32abc, 4076 0x32b00, 0x32b70, 4077 0x33000, 0x33048, 4078 0x33060, 0x3309c, 4079 0x330f0, 0x33148, 4080 0x33160, 0x3319c, 4081 0x331f0, 0x332e4, 4082 0x332f8, 0x333e4, 4083 0x333f8, 0x33448, 4084 0x33460, 0x3349c, 4085 0x334f0, 0x33548, 4086 0x33560, 0x3359c, 4087 0x335f0, 0x336e4, 4088 0x336f8, 0x337e4, 4089 0x337f8, 0x337fc, 4090 0x33814, 0x33814, 4091 0x3382c, 0x3382c, 4092 0x33880, 0x3388c, 4093 0x338e8, 0x338ec, 4094 0x33900, 0x33948, 4095 0x33960, 0x3399c, 4096 0x339f0, 0x33ae4, 4097 0x33af8, 0x33b10, 4098 0x33b28, 0x33b28, 4099 0x33b3c, 0x33b50, 4100 0x33bf0, 0x33c10, 4101 0x33c28, 0x33c28, 4102 0x33c3c, 0x33c50, 4103 0x33cf0, 0x33cfc, 4104 0x34000, 0x34030, 4105 0x34100, 0x34144, 4106 0x34190, 0x341d0, 4107 0x34200, 0x34318, 4108 0x34400, 0x3452c, 4109 0x34540, 0x3461c, 4110 0x34800, 0x34834, 4111 0x348c0, 0x34908, 4112 0x34910, 0x349ac, 4113 0x34a00, 0x34a2c, 4114 0x34a44, 0x34a50, 4115 0x34a74, 0x34c24, 4116 0x34d00, 0x34d00, 4117 0x34d08, 0x34d14, 4118 0x34d1c, 0x34d20, 4119 0x34d3c, 0x34d50, 4120 0x35200, 0x3520c, 4121 0x35220, 0x35220, 4122 0x35240, 0x35240, 4123 0x35600, 0x3560c, 4124 0x35a00, 0x35a1c, 4125 0x35e00, 0x35e20, 4126 0x35e38, 0x35e3c, 4127 0x35e80, 0x35e80, 4128 0x35e88, 0x35ea8, 4129 0x35eb0, 0x35eb4, 4130 0x35ec8, 0x35ed4, 4131 0x35fb8, 0x36004, 4132 0x36200, 0x36200, 4133 0x36208, 0x36240, 4134 0x36248, 0x36280, 4135 0x36288, 0x362c0, 4136 0x362c8, 0x362fc, 4137 0x36600, 0x36630, 4138 0x36a00, 0x36abc, 4139 0x36b00, 0x36b70, 4140 0x37000, 0x37048, 4141 0x37060, 0x3709c, 4142 0x370f0, 0x37148, 4143 0x37160, 0x3719c, 4144 0x371f0, 0x372e4, 4145 0x372f8, 0x373e4, 4146 0x373f8, 0x37448, 4147 0x37460, 0x3749c, 4148 0x374f0, 0x37548, 4149 0x37560, 0x3759c, 4150 0x375f0, 0x376e4, 4151 0x376f8, 0x377e4, 4152 0x377f8, 0x377fc, 4153 0x37814, 0x37814, 4154 0x3782c, 0x3782c, 4155 0x37880, 0x3788c, 4156 0x378e8, 0x378ec, 4157 0x37900, 0x37948, 4158 0x37960, 0x3799c, 4159 0x379f0, 0x37ae4, 4160 0x37af8, 0x37b10, 4161 0x37b28, 0x37b28, 4162 0x37b3c, 0x37b50, 4163 0x37bf0, 0x37c10, 4164 0x37c28, 0x37c28, 4165 0x37c3c, 0x37c50, 4166 0x37cf0, 0x37cfc, 4167 0x38000, 0x38030, 4168 0x38100, 0x38144, 4169 0x38190, 0x381d0, 4170 0x38200, 0x38318, 4171 0x38400, 0x3852c, 4172 0x38540, 0x3861c, 4173 0x38800, 0x38834, 4174 0x388c0, 0x38908, 4175 0x38910, 0x389ac, 4176 0x38a00, 0x38a2c, 4177 0x38a44, 0x38a50, 4178 0x38a74, 0x38c24, 4179 0x38d00, 0x38d00, 4180 0x38d08, 0x38d14, 4181 0x38d1c, 0x38d20, 4182 0x38d3c, 0x38d50, 4183 0x39200, 0x3920c, 4184 0x39220, 0x39220, 4185 0x39240, 0x39240, 4186 0x39600, 0x3960c, 4187 0x39a00, 0x39a1c, 4188 0x39e00, 0x39e20, 4189 0x39e38, 0x39e3c, 4190 0x39e80, 0x39e80, 4191 0x39e88, 0x39ea8, 4192 0x39eb0, 0x39eb4, 4193 0x39ec8, 0x39ed4, 4194 0x39fb8, 0x3a004, 4195 0x3a200, 0x3a200, 4196 0x3a208, 0x3a240, 4197 0x3a248, 0x3a280, 4198 0x3a288, 0x3a2c0, 4199 0x3a2c8, 0x3a2fc, 4200 0x3a600, 0x3a630, 4201 0x3aa00, 0x3aabc, 4202 0x3ab00, 0x3ab70, 4203 0x3b000, 0x3b048, 4204 0x3b060, 0x3b09c, 4205 0x3b0f0, 0x3b148, 4206 0x3b160, 0x3b19c, 4207 0x3b1f0, 0x3b2e4, 4208 0x3b2f8, 0x3b3e4, 4209 0x3b3f8, 0x3b448, 4210 0x3b460, 0x3b49c, 4211 0x3b4f0, 0x3b548, 4212 0x3b560, 0x3b59c, 4213 0x3b5f0, 0x3b6e4, 4214 0x3b6f8, 0x3b7e4, 4215 0x3b7f8, 0x3b7fc, 4216 0x3b814, 0x3b814, 4217 0x3b82c, 0x3b82c, 4218 0x3b880, 0x3b88c, 4219 0x3b8e8, 0x3b8ec, 4220 0x3b900, 0x3b948, 4221 0x3b960, 0x3b99c, 4222 0x3b9f0, 0x3bae4, 4223 0x3baf8, 0x3bb10, 4224 0x3bb28, 0x3bb28, 4225 0x3bb3c, 0x3bb50, 4226 0x3bbf0, 0x3bc10, 4227 0x3bc28, 0x3bc28, 4228 0x3bc3c, 0x3bc50, 4229 0x3bcf0, 0x3bcfc, 4230 0x3c000, 0x3c030, 4231 0x3c100, 0x3c144, 4232 0x3c190, 0x3c1d0, 4233 0x3c200, 0x3c318, 4234 0x3c400, 0x3c52c, 4235 0x3c540, 0x3c61c, 4236 0x3c800, 0x3c834, 4237 0x3c8c0, 0x3c908, 4238 0x3c910, 0x3c9ac, 4239 0x3ca00, 0x3ca2c, 4240 0x3ca44, 0x3ca50, 4241 0x3ca74, 0x3cc24, 4242 0x3cd00, 0x3cd00, 4243 0x3cd08, 0x3cd14, 4244 0x3cd1c, 0x3cd20, 4245 0x3cd3c, 0x3cd50, 4246 0x3d200, 0x3d20c, 4247 0x3d220, 0x3d220, 4248 0x3d240, 0x3d240, 4249 0x3d600, 0x3d60c, 4250 0x3da00, 0x3da1c, 4251 0x3de00, 0x3de20, 4252 0x3de38, 0x3de3c, 4253 0x3de80, 0x3de80, 4254 0x3de88, 0x3dea8, 4255 0x3deb0, 0x3deb4, 4256 0x3dec8, 0x3ded4, 4257 0x3dfb8, 0x3e004, 4258 0x3e200, 0x3e200, 4259 0x3e208, 0x3e240, 4260 0x3e248, 0x3e280, 4261 0x3e288, 0x3e2c0, 4262 0x3e2c8, 0x3e2fc, 4263 0x3e600, 0x3e630, 4264 0x3ea00, 0x3eabc, 4265 0x3eb00, 0x3eb70, 4266 0x3f000, 0x3f048, 4267 0x3f060, 0x3f09c, 4268 0x3f0f0, 0x3f148, 4269 0x3f160, 0x3f19c, 4270 0x3f1f0, 0x3f2e4, 4271 0x3f2f8, 0x3f3e4, 4272 0x3f3f8, 0x3f448, 4273 0x3f460, 0x3f49c, 4274 0x3f4f0, 0x3f548, 4275 0x3f560, 0x3f59c, 4276 0x3f5f0, 0x3f6e4, 4277 0x3f6f8, 0x3f7e4, 4278 0x3f7f8, 0x3f7fc, 4279 0x3f814, 0x3f814, 4280 0x3f82c, 0x3f82c, 4281 0x3f880, 0x3f88c, 4282 0x3f8e8, 0x3f8ec, 4283 0x3f900, 0x3f948, 4284 0x3f960, 0x3f99c, 4285 0x3f9f0, 0x3fae4, 4286 0x3faf8, 0x3fb10, 4287 0x3fb28, 0x3fb28, 4288 0x3fb3c, 0x3fb50, 4289 0x3fbf0, 0x3fc10, 4290 0x3fc28, 0x3fc28, 4291 0x3fc3c, 0x3fc50, 4292 0x3fcf0, 0x3fcfc, 4293 0x40000, 0x4000c, 4294 0x40040, 0x40068, 4295 0x4007c, 0x40144, 4296 0x40180, 0x4018c, 4297 0x40200, 0x40298, 4298 0x402ac, 0x4033c, 4299 0x403f8, 0x403fc, 4300 0x41304, 0x413c4, 4301 0x41400, 0x4141c, 4302 0x41480, 0x414d0, 4303 0x44000, 0x44078, 4304 0x440c0, 0x44278, 4305 0x442c0, 0x44478, 4306 0x444c0, 0x44678, 4307 0x446c0, 0x44878, 4308 0x448c0, 0x449fc, 4309 0x45000, 0x45068, 4310 0x45080, 0x45084, 4311 0x450a0, 0x450b0, 4312 0x45200, 0x45268, 4313 0x45280, 0x45284, 4314 0x452a0, 0x452b0, 4315 0x460c0, 0x460e4, 4316 0x47000, 0x4708c, 4317 0x47200, 0x47250, 4318 0x47400, 0x47420, 4319 0x47600, 0x47618, 4320 0x47800, 0x47814, 4321 0x48000, 0x4800c, 4322 0x48040, 0x48068, 4323 0x4807c, 0x48144, 4324 0x48180, 0x4818c, 4325 0x48200, 0x48298, 4326 0x482ac, 0x4833c, 4327 0x483f8, 0x483fc, 4328 0x49304, 0x493c4, 4329 0x49400, 0x4941c, 4330 0x49480, 0x494d0, 4331 0x4c000, 0x4c078, 4332 0x4c0c0, 0x4c278, 4333 0x4c2c0, 0x4c478, 4334 0x4c4c0, 0x4c678, 4335 0x4c6c0, 0x4c878, 4336 0x4c8c0, 0x4c9fc, 4337 0x4d000, 0x4d068, 4338 0x4d080, 0x4d084, 4339 0x4d0a0, 0x4d0b0, 4340 0x4d200, 0x4d268, 4341 0x4d280, 0x4d284, 4342 0x4d2a0, 0x4d2b0, 4343 0x4e0c0, 0x4e0e4, 4344 0x4f000, 0x4f08c, 4345 0x4f200, 0x4f250, 4346 0x4f400, 0x4f420, 4347 0x4f600, 0x4f618, 4348 0x4f800, 0x4f814, 4349 0x50000, 0x500cc, 4350 0x50400, 0x50400, 4351 0x50800, 0x508cc, 4352 0x50c00, 0x50c00, 4353 0x51000, 0x5101c, 4354 0x51300, 0x51308, 4355 }; 4356 4357 if (is_t4(sc)) { 4358 reg_ranges = &t4_reg_ranges[0]; 4359 n = nitems(t4_reg_ranges); 4360 } else { 4361 reg_ranges = &t5_reg_ranges[0]; 4362 n = nitems(t5_reg_ranges); 4363 } 4364 4365 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4366 for (i = 0; i < n; i += 2) 4367 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4368 } 4369 4370 static void 4371 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4372 { 4373 int i; 4374 u_int v, tnl_cong_drops; 4375 struct timeval tv; 4376 const struct timeval interval = {0, 250000}; /* 250ms */ 4377 4378 getmicrotime(&tv); 4379 timevalsub(&tv, &interval); 4380 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4381 return; 4382 4383 tnl_cong_drops = 0; 4384 t4_get_port_stats(sc, pi->tx_chan, &pi->stats); 4385 for (i = 0; i < NCHAN; i++) { 4386 if (pi->rx_chan_map & (1 << i)) { 4387 mtx_lock(&sc->regwin_lock); 4388 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4389 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4390 mtx_unlock(&sc->regwin_lock); 4391 tnl_cong_drops += v; 4392 } 4393 } 4394 pi->tnl_cong_drops = tnl_cong_drops; 4395 getmicrotime(&pi->last_refreshed); 4396 } 4397 4398 static void 4399 cxgbe_tick(void *arg) 4400 { 4401 struct port_info *pi = arg; 4402 struct adapter *sc = pi->adapter; 4403 struct ifnet *ifp = pi->ifp; 4404 4405 PORT_LOCK(pi); 4406 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4407 PORT_UNLOCK(pi); 4408 return; /* without scheduling another callout */ 4409 } 4410 4411 cxgbe_refresh_stats(sc, pi); 4412 4413 callout_schedule(&pi->tick, hz); 4414 PORT_UNLOCK(pi); 4415 } 4416 4417 static void 4418 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4419 { 4420 struct ifnet *vlan; 4421 4422 if (arg != ifp || ifp->if_type != IFT_ETHER) 4423 return; 4424 4425 vlan = VLAN_DEVAT(ifp, vid); 4426 VLAN_SETCOOKIE(vlan, ifp); 4427 } 4428 4429 static int 4430 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4431 { 4432 4433 #ifdef INVARIANTS 4434 panic("%s: opcode 0x%02x on iq %p with payload %p", 4435 __func__, rss->opcode, iq, m); 4436 #else 4437 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4438 __func__, rss->opcode, iq, m); 4439 m_freem(m); 4440 #endif 4441 return (EDOOFUS); 4442 } 4443 4444 int 4445 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4446 { 4447 uintptr_t *loc, new; 4448 4449 if (opcode >= nitems(sc->cpl_handler)) 4450 return (EINVAL); 4451 4452 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4453 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4454 atomic_store_rel_ptr(loc, new); 4455 4456 return (0); 4457 } 4458 4459 static int 4460 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4461 { 4462 4463 #ifdef INVARIANTS 4464 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4465 #else 4466 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4467 __func__, iq, ctrl); 4468 #endif 4469 return (EDOOFUS); 4470 } 4471 4472 int 4473 t4_register_an_handler(struct adapter *sc, an_handler_t h) 4474 { 4475 uintptr_t *loc, new; 4476 4477 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4478 loc = (uintptr_t *) &sc->an_handler; 4479 atomic_store_rel_ptr(loc, new); 4480 4481 return (0); 4482 } 4483 4484 static int 4485 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4486 { 4487 const struct cpl_fw6_msg *cpl = 4488 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4489 4490 #ifdef INVARIANTS 4491 panic("%s: fw_msg type %d", __func__, cpl->type); 4492 #else 4493 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4494 #endif 4495 return (EDOOFUS); 4496 } 4497 4498 int 4499 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4500 { 4501 uintptr_t *loc, new; 4502 4503 if (type >= nitems(sc->fw_msg_handler)) 4504 return (EINVAL); 4505 4506 /* 4507 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4508 * handler dispatch table. Reject any attempt to install a handler for 4509 * this subtype. 4510 */ 4511 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4512 return (EINVAL); 4513 4514 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4515 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4516 atomic_store_rel_ptr(loc, new); 4517 4518 return (0); 4519 } 4520 4521 static int 4522 t4_sysctls(struct adapter *sc) 4523 { 4524 struct sysctl_ctx_list *ctx; 4525 struct sysctl_oid *oid; 4526 struct sysctl_oid_list *children, *c0; 4527 static char *caps[] = { 4528 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4529 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4530 "\6HASHFILTER\7ETHOFLD", 4531 "\20\1TOE", /* caps[2] toecaps */ 4532 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4533 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4534 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4535 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4536 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4537 "\4PO_INITIAOR\5PO_TARGET" 4538 }; 4539 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4540 4541 ctx = device_get_sysctl_ctx(sc->dev); 4542 4543 /* 4544 * dev.t4nex.X. 4545 */ 4546 oid = device_get_sysctl_tree(sc->dev); 4547 c0 = children = SYSCTL_CHILDREN(oid); 4548 4549 sc->sc_do_rxcopy = 1; 4550 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4551 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4552 4553 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4554 sc->params.nports, "# of ports"); 4555 4556 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4557 NULL, chip_rev(sc), "chip hardware revision"); 4558 4559 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4560 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4561 4562 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4563 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4564 4565 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4566 sc->cfcsum, "config file checksum"); 4567 4568 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4569 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4570 sysctl_bitfield, "A", "available doorbells"); 4571 4572 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4573 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4574 sysctl_bitfield, "A", "available link capabilities"); 4575 4576 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4577 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4578 sysctl_bitfield, "A", "available NIC capabilities"); 4579 4580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4581 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4582 sysctl_bitfield, "A", "available TCP offload capabilities"); 4583 4584 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4585 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4586 sysctl_bitfield, "A", "available RDMA capabilities"); 4587 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4589 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4590 sysctl_bitfield, "A", "available iSCSI capabilities"); 4591 4592 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4593 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4594 sysctl_bitfield, "A", "available FCoE capabilities"); 4595 4596 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4597 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4598 4599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4600 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4601 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4602 "interrupt holdoff timer values (us)"); 4603 4604 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4605 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4606 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4607 "interrupt holdoff packet counter values"); 4608 4609 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4610 NULL, sc->tids.nftids, "number of filters"); 4611 4612 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4613 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4614 "chip temperature (in Celsius)"); 4615 4616 t4_sge_sysctls(sc, ctx, children); 4617 4618 sc->lro_timeout = 100; 4619 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4620 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4621 4622 #ifdef SBUF_DRAIN 4623 /* 4624 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4625 */ 4626 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4627 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4628 "logs and miscellaneous information"); 4629 children = SYSCTL_CHILDREN(oid); 4630 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4633 sysctl_cctrl, "A", "congestion control"); 4634 4635 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4636 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4637 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4638 4639 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4640 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4641 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4642 4643 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4644 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4645 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4646 4647 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4648 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4649 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4650 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4653 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4654 4655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4656 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4657 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4658 4659 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4660 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4661 sysctl_cim_la, "A", "CIM logic analyzer"); 4662 4663 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4664 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4665 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4666 4667 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4668 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4669 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4670 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4673 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4674 4675 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4676 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4677 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4678 4679 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4680 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4681 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4682 4683 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4684 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4685 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4686 4687 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4688 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4689 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4690 4691 if (is_t5(sc)) { 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4694 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4698 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4699 } 4700 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4704 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4710 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4711 sysctl_cpl_stats, "A", "CPL statistics"); 4712 4713 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4714 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4715 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4716 4717 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4718 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4719 sysctl_devlog, "A", "firmware's device log"); 4720 4721 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4722 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4723 sysctl_fcoe_stats, "A", "FCoE statistics"); 4724 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4726 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4727 sysctl_hw_sched, "A", "hardware scheduler "); 4728 4729 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4730 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4731 sysctl_l2t, "A", "hardware L2 table"); 4732 4733 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4734 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4735 sysctl_lb_stats, "A", "loopback statistics"); 4736 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4738 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4739 sysctl_meminfo, "A", "memory regions"); 4740 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4742 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4743 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4744 4745 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4746 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4747 sysctl_path_mtus, "A", "path MTUs"); 4748 4749 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4750 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4751 sysctl_pm_stats, "A", "PM statistics"); 4752 4753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4754 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4755 sysctl_rdma_stats, "A", "RDMA statistics"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4758 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4759 sysctl_tcp_stats, "A", "TCP statistics"); 4760 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_tids, "A", "TID information"); 4764 4765 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4766 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4767 sysctl_tp_err_stats, "A", "TP error statistics"); 4768 4769 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4770 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4771 sysctl_tp_la, "A", "TP logic analyzer"); 4772 4773 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4774 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4775 sysctl_tx_rate, "A", "Tx rate"); 4776 4777 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4778 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4779 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4780 4781 if (is_t5(sc)) { 4782 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4783 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4784 sysctl_wcwr_stats, "A", "write combined work requests"); 4785 } 4786 #endif 4787 4788 #ifdef TCP_OFFLOAD 4789 if (is_offload(sc)) { 4790 /* 4791 * dev.t4nex.X.toe. 4792 */ 4793 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4794 NULL, "TOE parameters"); 4795 children = SYSCTL_CHILDREN(oid); 4796 4797 sc->tt.sndbuf = 256 * 1024; 4798 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4799 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4800 4801 sc->tt.ddp = 0; 4802 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4803 &sc->tt.ddp, 0, "DDP allowed"); 4804 4805 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4807 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4808 4809 sc->tt.ddp_thres = 4810 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4811 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4812 &sc->tt.ddp_thres, 0, "DDP threshold"); 4813 4814 sc->tt.rx_coalesce = 1; 4815 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4816 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4817 4818 sc->tt.tx_align = 1; 4819 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4820 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4821 } 4822 #endif 4823 4824 4825 return (0); 4826 } 4827 4828 static int 4829 cxgbe_sysctls(struct port_info *pi) 4830 { 4831 struct sysctl_ctx_list *ctx; 4832 struct sysctl_oid *oid; 4833 struct sysctl_oid_list *children; 4834 struct adapter *sc = pi->adapter; 4835 4836 ctx = device_get_sysctl_ctx(pi->dev); 4837 4838 /* 4839 * dev.cxgbe.X. 4840 */ 4841 oid = device_get_sysctl_tree(pi->dev); 4842 children = SYSCTL_CHILDREN(oid); 4843 4844 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4845 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4846 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4847 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4848 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4849 "PHY temperature (in Celsius)"); 4850 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4851 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4852 "PHY firmware version"); 4853 } 4854 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4855 &pi->nrxq, 0, "# of rx queues"); 4856 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4857 &pi->ntxq, 0, "# of tx queues"); 4858 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4859 &pi->first_rxq, 0, "index of first rx queue"); 4860 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4861 &pi->first_txq, 0, "index of first tx queue"); 4862 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4863 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4864 "Reserve queue 0 for non-flowid packets"); 4865 4866 #ifdef TCP_OFFLOAD 4867 if (is_offload(sc)) { 4868 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4869 &pi->nofldrxq, 0, 4870 "# of rx queues for offloaded TCP connections"); 4871 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4872 &pi->nofldtxq, 0, 4873 "# of tx queues for offloaded TCP connections"); 4874 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4875 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4876 "index of first TOE rx queue"); 4877 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4878 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4879 "index of first TOE tx queue"); 4880 } 4881 #endif 4882 #ifdef DEV_NETMAP 4883 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4884 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4885 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4886 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4887 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4888 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4889 "index of first netmap rx queue"); 4890 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4891 CTLFLAG_RD, &pi->first_nm_txq, 0, 4892 "index of first netmap tx queue"); 4893 #endif 4894 4895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4896 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4897 "holdoff timer index"); 4898 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4899 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4900 "holdoff packet counter index"); 4901 4902 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4903 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4904 "rx queue size"); 4905 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4906 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4907 "tx queue size"); 4908 4909 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4910 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4911 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4912 4913 /* 4914 * dev.cxgbe.X.stats. 4915 */ 4916 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4917 NULL, "port statistics"); 4918 children = SYSCTL_CHILDREN(oid); 4919 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4920 &pi->tx_parse_error, 0, 4921 "# of tx packets with invalid length or # of segments"); 4922 4923 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4924 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4925 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4926 sysctl_handle_t4_reg64, "QU", desc) 4927 4928 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4929 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4930 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4931 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4932 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4941 "# of tx frames in this range", 4942 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4943 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4944 "# of tx frames in this range", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4947 "# of tx frames in this range", 4948 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4949 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4950 "# of tx frames in this range", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4953 "# of tx frames in this range", 4954 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4955 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4956 "# of tx frames in this range", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4959 "# of tx frames in this range", 4960 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4961 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4971 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4972 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4973 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4978 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4979 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4981 4982 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4983 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4984 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4985 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4986 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4987 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4988 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4989 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4990 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4991 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4992 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4995 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4996 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4997 "# of frames received with bad FCS", 4998 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4999 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5000 "# of frames received with length error", 5001 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5002 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5003 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5004 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5005 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5006 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5007 "# of rx frames in this range", 5008 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5009 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5010 "# of rx frames in this range", 5011 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5012 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5013 "# of rx frames in this range", 5014 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5015 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5016 "# of rx frames in this range", 5017 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5018 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5019 "# of rx frames in this range", 5020 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5021 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5022 "# of rx frames in this range", 5023 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5024 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5025 "# of rx frames in this range", 5026 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5027 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5028 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5029 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5030 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5031 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5032 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5033 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5034 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5035 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5036 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5037 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5038 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5039 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5040 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5041 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5042 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5043 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5044 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5045 5046 #undef SYSCTL_ADD_T4_REG64 5047 5048 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5049 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5050 &pi->stats.name, desc) 5051 5052 /* We get these from port_stats and they may be stale by upto 1s */ 5053 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5054 "# drops due to buffer-group 0 overflows"); 5055 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5056 "# drops due to buffer-group 1 overflows"); 5057 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5058 "# drops due to buffer-group 2 overflows"); 5059 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5060 "# drops due to buffer-group 3 overflows"); 5061 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5062 "# of buffer-group 0 truncated packets"); 5063 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5064 "# of buffer-group 1 truncated packets"); 5065 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5066 "# of buffer-group 2 truncated packets"); 5067 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5068 "# of buffer-group 3 truncated packets"); 5069 5070 #undef SYSCTL_ADD_T4_PORTSTAT 5071 5072 return (0); 5073 } 5074 5075 static int 5076 sysctl_int_array(SYSCTL_HANDLER_ARGS) 5077 { 5078 int rc, *i, space = 0; 5079 struct sbuf sb; 5080 5081 sbuf_new_for_sysctl(&sb, NULL, 64, req); 5082 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5083 if (space) 5084 sbuf_printf(&sb, " "); 5085 sbuf_printf(&sb, "%d", *i); 5086 space = 1; 5087 } 5088 rc = sbuf_finish(&sb); 5089 sbuf_delete(&sb); 5090 return (rc); 5091 } 5092 5093 static int 5094 sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5095 { 5096 int rc; 5097 struct sbuf *sb; 5098 5099 rc = sysctl_wire_old_buffer(req, 0); 5100 if (rc != 0) 5101 return(rc); 5102 5103 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5104 if (sb == NULL) 5105 return (ENOMEM); 5106 5107 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5108 rc = sbuf_finish(sb); 5109 sbuf_delete(sb); 5110 5111 return (rc); 5112 } 5113 5114 static int 5115 sysctl_btphy(SYSCTL_HANDLER_ARGS) 5116 { 5117 struct port_info *pi = arg1; 5118 int op = arg2; 5119 struct adapter *sc = pi->adapter; 5120 u_int v; 5121 int rc; 5122 5123 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5124 if (rc) 5125 return (rc); 5126 /* XXX: magic numbers */ 5127 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5128 &v); 5129 end_synchronized_op(sc, 0); 5130 if (rc) 5131 return (rc); 5132 if (op == 0) 5133 v /= 256; 5134 5135 rc = sysctl_handle_int(oidp, &v, 0, req); 5136 return (rc); 5137 } 5138 5139 static int 5140 sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5141 { 5142 struct port_info *pi = arg1; 5143 int rc, val; 5144 5145 val = pi->rsrv_noflowq; 5146 rc = sysctl_handle_int(oidp, &val, 0, req); 5147 if (rc != 0 || req->newptr == NULL) 5148 return (rc); 5149 5150 if ((val >= 1) && (pi->ntxq > 1)) 5151 pi->rsrv_noflowq = 1; 5152 else 5153 pi->rsrv_noflowq = 0; 5154 5155 return (rc); 5156 } 5157 5158 static int 5159 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5160 { 5161 struct port_info *pi = arg1; 5162 struct adapter *sc = pi->adapter; 5163 int idx, rc, i; 5164 struct sge_rxq *rxq; 5165 #ifdef TCP_OFFLOAD 5166 struct sge_ofld_rxq *ofld_rxq; 5167 #endif 5168 uint8_t v; 5169 5170 idx = pi->tmr_idx; 5171 5172 rc = sysctl_handle_int(oidp, &idx, 0, req); 5173 if (rc != 0 || req->newptr == NULL) 5174 return (rc); 5175 5176 if (idx < 0 || idx >= SGE_NTIMERS) 5177 return (EINVAL); 5178 5179 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5180 "t4tmr"); 5181 if (rc) 5182 return (rc); 5183 5184 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5185 for_each_rxq(pi, i, rxq) { 5186 #ifdef atomic_store_rel_8 5187 atomic_store_rel_8(&rxq->iq.intr_params, v); 5188 #else 5189 rxq->iq.intr_params = v; 5190 #endif 5191 } 5192 #ifdef TCP_OFFLOAD 5193 for_each_ofld_rxq(pi, i, ofld_rxq) { 5194 #ifdef atomic_store_rel_8 5195 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5196 #else 5197 ofld_rxq->iq.intr_params = v; 5198 #endif 5199 } 5200 #endif 5201 pi->tmr_idx = idx; 5202 5203 end_synchronized_op(sc, LOCK_HELD); 5204 return (0); 5205 } 5206 5207 static int 5208 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5209 { 5210 struct port_info *pi = arg1; 5211 struct adapter *sc = pi->adapter; 5212 int idx, rc; 5213 5214 idx = pi->pktc_idx; 5215 5216 rc = sysctl_handle_int(oidp, &idx, 0, req); 5217 if (rc != 0 || req->newptr == NULL) 5218 return (rc); 5219 5220 if (idx < -1 || idx >= SGE_NCOUNTERS) 5221 return (EINVAL); 5222 5223 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5224 "t4pktc"); 5225 if (rc) 5226 return (rc); 5227 5228 if (pi->flags & PORT_INIT_DONE) 5229 rc = EBUSY; /* cannot be changed once the queues are created */ 5230 else 5231 pi->pktc_idx = idx; 5232 5233 end_synchronized_op(sc, LOCK_HELD); 5234 return (rc); 5235 } 5236 5237 static int 5238 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5239 { 5240 struct port_info *pi = arg1; 5241 struct adapter *sc = pi->adapter; 5242 int qsize, rc; 5243 5244 qsize = pi->qsize_rxq; 5245 5246 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5247 if (rc != 0 || req->newptr == NULL) 5248 return (rc); 5249 5250 if (qsize < 128 || (qsize & 7)) 5251 return (EINVAL); 5252 5253 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5254 "t4rxqs"); 5255 if (rc) 5256 return (rc); 5257 5258 if (pi->flags & PORT_INIT_DONE) 5259 rc = EBUSY; /* cannot be changed once the queues are created */ 5260 else 5261 pi->qsize_rxq = qsize; 5262 5263 end_synchronized_op(sc, LOCK_HELD); 5264 return (rc); 5265 } 5266 5267 static int 5268 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5269 { 5270 struct port_info *pi = arg1; 5271 struct adapter *sc = pi->adapter; 5272 int qsize, rc; 5273 5274 qsize = pi->qsize_txq; 5275 5276 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5277 if (rc != 0 || req->newptr == NULL) 5278 return (rc); 5279 5280 if (qsize < 128 || qsize > 65536) 5281 return (EINVAL); 5282 5283 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5284 "t4txqs"); 5285 if (rc) 5286 return (rc); 5287 5288 if (pi->flags & PORT_INIT_DONE) 5289 rc = EBUSY; /* cannot be changed once the queues are created */ 5290 else 5291 pi->qsize_txq = qsize; 5292 5293 end_synchronized_op(sc, LOCK_HELD); 5294 return (rc); 5295 } 5296 5297 static int 5298 sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5299 { 5300 struct port_info *pi = arg1; 5301 struct adapter *sc = pi->adapter; 5302 struct link_config *lc = &pi->link_cfg; 5303 int rc; 5304 5305 if (req->newptr == NULL) { 5306 struct sbuf *sb; 5307 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5308 5309 rc = sysctl_wire_old_buffer(req, 0); 5310 if (rc != 0) 5311 return(rc); 5312 5313 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5314 if (sb == NULL) 5315 return (ENOMEM); 5316 5317 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5318 rc = sbuf_finish(sb); 5319 sbuf_delete(sb); 5320 } else { 5321 char s[2]; 5322 int n; 5323 5324 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5325 s[1] = 0; 5326 5327 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5328 if (rc != 0) 5329 return(rc); 5330 5331 if (s[1] != 0) 5332 return (EINVAL); 5333 if (s[0] < '0' || s[0] > '9') 5334 return (EINVAL); /* not a number */ 5335 n = s[0] - '0'; 5336 if (n & ~(PAUSE_TX | PAUSE_RX)) 5337 return (EINVAL); /* some other bit is set too */ 5338 5339 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5340 if (rc) 5341 return (rc); 5342 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5343 int link_ok = lc->link_ok; 5344 5345 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5346 lc->requested_fc |= n; 5347 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5348 lc->link_ok = link_ok; /* restore */ 5349 } 5350 end_synchronized_op(sc, 0); 5351 } 5352 5353 return (rc); 5354 } 5355 5356 static int 5357 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5358 { 5359 struct adapter *sc = arg1; 5360 int reg = arg2; 5361 uint64_t val; 5362 5363 val = t4_read_reg64(sc, reg); 5364 5365 return (sysctl_handle_64(oidp, &val, 0, req)); 5366 } 5367 5368 static int 5369 sysctl_temperature(SYSCTL_HANDLER_ARGS) 5370 { 5371 struct adapter *sc = arg1; 5372 int rc, t; 5373 uint32_t param, val; 5374 5375 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5376 if (rc) 5377 return (rc); 5378 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5379 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5380 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5381 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5382 end_synchronized_op(sc, 0); 5383 if (rc) 5384 return (rc); 5385 5386 /* unknown is returned as 0 but we display -1 in that case */ 5387 t = val == 0 ? -1 : val; 5388 5389 rc = sysctl_handle_int(oidp, &t, 0, req); 5390 return (rc); 5391 } 5392 5393 #ifdef SBUF_DRAIN 5394 static int 5395 sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5396 { 5397 struct adapter *sc = arg1; 5398 struct sbuf *sb; 5399 int rc, i; 5400 uint16_t incr[NMTUS][NCCTRL_WIN]; 5401 static const char *dec_fac[] = { 5402 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5403 "0.9375" 5404 }; 5405 5406 rc = sysctl_wire_old_buffer(req, 0); 5407 if (rc != 0) 5408 return (rc); 5409 5410 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5411 if (sb == NULL) 5412 return (ENOMEM); 5413 5414 t4_read_cong_tbl(sc, incr); 5415 5416 for (i = 0; i < NCCTRL_WIN; ++i) { 5417 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5418 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5419 incr[5][i], incr[6][i], incr[7][i]); 5420 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5421 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5422 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5423 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5424 } 5425 5426 rc = sbuf_finish(sb); 5427 sbuf_delete(sb); 5428 5429 return (rc); 5430 } 5431 5432 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5433 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5434 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5435 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5436 }; 5437 5438 static int 5439 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5440 { 5441 struct adapter *sc = arg1; 5442 struct sbuf *sb; 5443 int rc, i, n, qid = arg2; 5444 uint32_t *buf, *p; 5445 char *qtype; 5446 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5447 5448 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5449 ("%s: bad qid %d\n", __func__, qid)); 5450 5451 if (qid < CIM_NUM_IBQ) { 5452 /* inbound queue */ 5453 qtype = "IBQ"; 5454 n = 4 * CIM_IBQ_SIZE; 5455 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5456 rc = t4_read_cim_ibq(sc, qid, buf, n); 5457 } else { 5458 /* outbound queue */ 5459 qtype = "OBQ"; 5460 qid -= CIM_NUM_IBQ; 5461 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5462 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5463 rc = t4_read_cim_obq(sc, qid, buf, n); 5464 } 5465 5466 if (rc < 0) { 5467 rc = -rc; 5468 goto done; 5469 } 5470 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5471 5472 rc = sysctl_wire_old_buffer(req, 0); 5473 if (rc != 0) 5474 goto done; 5475 5476 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5477 if (sb == NULL) { 5478 rc = ENOMEM; 5479 goto done; 5480 } 5481 5482 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5483 for (i = 0, p = buf; i < n; i += 16, p += 4) 5484 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5485 p[2], p[3]); 5486 5487 rc = sbuf_finish(sb); 5488 sbuf_delete(sb); 5489 done: 5490 free(buf, M_CXGBE); 5491 return (rc); 5492 } 5493 5494 static int 5495 sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5496 { 5497 struct adapter *sc = arg1; 5498 u_int cfg; 5499 struct sbuf *sb; 5500 uint32_t *buf, *p; 5501 int rc; 5502 5503 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5504 if (rc != 0) 5505 return (rc); 5506 5507 rc = sysctl_wire_old_buffer(req, 0); 5508 if (rc != 0) 5509 return (rc); 5510 5511 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5512 if (sb == NULL) 5513 return (ENOMEM); 5514 5515 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5516 M_ZERO | M_WAITOK); 5517 5518 rc = -t4_cim_read_la(sc, buf, NULL); 5519 if (rc != 0) 5520 goto done; 5521 5522 sbuf_printf(sb, "Status Data PC%s", 5523 cfg & F_UPDBGLACAPTPCONLY ? "" : 5524 " LS0Stat LS0Addr LS0Data"); 5525 5526 KASSERT((sc->params.cim_la_size & 7) == 0, 5527 ("%s: p will walk off the end of buf", __func__)); 5528 5529 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5530 if (cfg & F_UPDBGLACAPTPCONLY) { 5531 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5532 p[6], p[7]); 5533 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5534 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5535 p[4] & 0xff, p[5] >> 8); 5536 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5537 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5538 p[1] & 0xf, p[2] >> 4); 5539 } else { 5540 sbuf_printf(sb, 5541 "\n %02x %x%07x %x%07x %08x %08x " 5542 "%08x%08x%08x%08x", 5543 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5544 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5545 p[6], p[7]); 5546 } 5547 } 5548 5549 rc = sbuf_finish(sb); 5550 sbuf_delete(sb); 5551 done: 5552 free(buf, M_CXGBE); 5553 return (rc); 5554 } 5555 5556 static int 5557 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5558 { 5559 struct adapter *sc = arg1; 5560 u_int i; 5561 struct sbuf *sb; 5562 uint32_t *buf, *p; 5563 int rc; 5564 5565 rc = sysctl_wire_old_buffer(req, 0); 5566 if (rc != 0) 5567 return (rc); 5568 5569 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5570 if (sb == NULL) 5571 return (ENOMEM); 5572 5573 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5574 M_ZERO | M_WAITOK); 5575 5576 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5577 p = buf; 5578 5579 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5580 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5581 p[1], p[0]); 5582 } 5583 5584 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5585 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5586 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5587 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5588 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5589 (p[1] >> 2) | ((p[2] & 3) << 30), 5590 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5591 p[0] & 1); 5592 } 5593 5594 rc = sbuf_finish(sb); 5595 sbuf_delete(sb); 5596 free(buf, M_CXGBE); 5597 return (rc); 5598 } 5599 5600 static int 5601 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5602 { 5603 struct adapter *sc = arg1; 5604 u_int i; 5605 struct sbuf *sb; 5606 uint32_t *buf, *p; 5607 int rc; 5608 5609 rc = sysctl_wire_old_buffer(req, 0); 5610 if (rc != 0) 5611 return (rc); 5612 5613 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5614 if (sb == NULL) 5615 return (ENOMEM); 5616 5617 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5618 M_ZERO | M_WAITOK); 5619 5620 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5621 p = buf; 5622 5623 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5624 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5625 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5626 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5627 p[4], p[3], p[2], p[1], p[0]); 5628 } 5629 5630 sbuf_printf(sb, "\n\nCntl ID Data"); 5631 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5632 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5633 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5634 } 5635 5636 rc = sbuf_finish(sb); 5637 sbuf_delete(sb); 5638 free(buf, M_CXGBE); 5639 return (rc); 5640 } 5641 5642 static int 5643 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5644 { 5645 struct adapter *sc = arg1; 5646 struct sbuf *sb; 5647 int rc, i; 5648 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5649 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5650 uint16_t thres[CIM_NUM_IBQ]; 5651 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5652 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5653 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5654 5655 if (is_t4(sc)) { 5656 cim_num_obq = CIM_NUM_OBQ; 5657 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5658 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5659 } else { 5660 cim_num_obq = CIM_NUM_OBQ_T5; 5661 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5662 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5663 } 5664 nq = CIM_NUM_IBQ + cim_num_obq; 5665 5666 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5667 if (rc == 0) 5668 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5669 if (rc != 0) 5670 return (rc); 5671 5672 t4_read_cimq_cfg(sc, base, size, thres); 5673 5674 rc = sysctl_wire_old_buffer(req, 0); 5675 if (rc != 0) 5676 return (rc); 5677 5678 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5679 if (sb == NULL) 5680 return (ENOMEM); 5681 5682 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5683 5684 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5685 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5686 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5687 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5688 G_QUEREMFLITS(p[2]) * 16); 5689 for ( ; i < nq; i++, p += 4, wr += 2) 5690 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5691 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5692 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5693 G_QUEREMFLITS(p[2]) * 16); 5694 5695 rc = sbuf_finish(sb); 5696 sbuf_delete(sb); 5697 5698 return (rc); 5699 } 5700 5701 static int 5702 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5703 { 5704 struct adapter *sc = arg1; 5705 struct sbuf *sb; 5706 int rc; 5707 struct tp_cpl_stats stats; 5708 5709 rc = sysctl_wire_old_buffer(req, 0); 5710 if (rc != 0) 5711 return (rc); 5712 5713 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5714 if (sb == NULL) 5715 return (ENOMEM); 5716 5717 t4_tp_get_cpl_stats(sc, &stats); 5718 5719 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5720 "channel 3\n"); 5721 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5722 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5723 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5724 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5725 5726 rc = sbuf_finish(sb); 5727 sbuf_delete(sb); 5728 5729 return (rc); 5730 } 5731 5732 static int 5733 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5734 { 5735 struct adapter *sc = arg1; 5736 struct sbuf *sb; 5737 int rc; 5738 struct tp_usm_stats stats; 5739 5740 rc = sysctl_wire_old_buffer(req, 0); 5741 if (rc != 0) 5742 return(rc); 5743 5744 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5745 if (sb == NULL) 5746 return (ENOMEM); 5747 5748 t4_get_usm_stats(sc, &stats); 5749 5750 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5751 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5752 sbuf_printf(sb, "Drops: %u", stats.drops); 5753 5754 rc = sbuf_finish(sb); 5755 sbuf_delete(sb); 5756 5757 return (rc); 5758 } 5759 5760 const char *devlog_level_strings[] = { 5761 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5762 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5763 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5764 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5765 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5766 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5767 }; 5768 5769 const char *devlog_facility_strings[] = { 5770 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5771 [FW_DEVLOG_FACILITY_CF] = "CF", 5772 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5773 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5774 [FW_DEVLOG_FACILITY_RES] = "RES", 5775 [FW_DEVLOG_FACILITY_HW] = "HW", 5776 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5777 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5778 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5779 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5780 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5781 [FW_DEVLOG_FACILITY_VI] = "VI", 5782 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5783 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5784 [FW_DEVLOG_FACILITY_TM] = "TM", 5785 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5786 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5787 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5788 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5789 [FW_DEVLOG_FACILITY_RI] = "RI", 5790 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5791 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5792 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5793 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5794 }; 5795 5796 static int 5797 sysctl_devlog(SYSCTL_HANDLER_ARGS) 5798 { 5799 struct adapter *sc = arg1; 5800 struct devlog_params *dparams = &sc->params.devlog; 5801 struct fw_devlog_e *buf, *e; 5802 int i, j, rc, nentries, first = 0, m; 5803 struct sbuf *sb; 5804 uint64_t ftstamp = UINT64_MAX; 5805 5806 if (dparams->start == 0) { 5807 dparams->memtype = FW_MEMTYPE_EDC0; 5808 dparams->start = 0x84000; 5809 dparams->size = 32768; 5810 } 5811 5812 nentries = dparams->size / sizeof(struct fw_devlog_e); 5813 5814 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5815 if (buf == NULL) 5816 return (ENOMEM); 5817 5818 m = fwmtype_to_hwmtype(dparams->memtype); 5819 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5820 if (rc != 0) 5821 goto done; 5822 5823 for (i = 0; i < nentries; i++) { 5824 e = &buf[i]; 5825 5826 if (e->timestamp == 0) 5827 break; /* end */ 5828 5829 e->timestamp = be64toh(e->timestamp); 5830 e->seqno = be32toh(e->seqno); 5831 for (j = 0; j < 8; j++) 5832 e->params[j] = be32toh(e->params[j]); 5833 5834 if (e->timestamp < ftstamp) { 5835 ftstamp = e->timestamp; 5836 first = i; 5837 } 5838 } 5839 5840 if (buf[first].timestamp == 0) 5841 goto done; /* nothing in the log */ 5842 5843 rc = sysctl_wire_old_buffer(req, 0); 5844 if (rc != 0) 5845 goto done; 5846 5847 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5848 if (sb == NULL) { 5849 rc = ENOMEM; 5850 goto done; 5851 } 5852 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5853 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5854 5855 i = first; 5856 do { 5857 e = &buf[i]; 5858 if (e->timestamp == 0) 5859 break; /* end */ 5860 5861 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5862 e->seqno, e->timestamp, 5863 (e->level < nitems(devlog_level_strings) ? 5864 devlog_level_strings[e->level] : "UNKNOWN"), 5865 (e->facility < nitems(devlog_facility_strings) ? 5866 devlog_facility_strings[e->facility] : "UNKNOWN")); 5867 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5868 e->params[2], e->params[3], e->params[4], 5869 e->params[5], e->params[6], e->params[7]); 5870 5871 if (++i == nentries) 5872 i = 0; 5873 } while (i != first); 5874 5875 rc = sbuf_finish(sb); 5876 sbuf_delete(sb); 5877 done: 5878 free(buf, M_CXGBE); 5879 return (rc); 5880 } 5881 5882 static int 5883 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5884 { 5885 struct adapter *sc = arg1; 5886 struct sbuf *sb; 5887 int rc; 5888 struct tp_fcoe_stats stats[4]; 5889 5890 rc = sysctl_wire_old_buffer(req, 0); 5891 if (rc != 0) 5892 return (rc); 5893 5894 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5895 if (sb == NULL) 5896 return (ENOMEM); 5897 5898 t4_get_fcoe_stats(sc, 0, &stats[0]); 5899 t4_get_fcoe_stats(sc, 1, &stats[1]); 5900 t4_get_fcoe_stats(sc, 2, &stats[2]); 5901 t4_get_fcoe_stats(sc, 3, &stats[3]); 5902 5903 sbuf_printf(sb, " channel 0 channel 1 " 5904 "channel 2 channel 3\n"); 5905 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5906 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5907 stats[3].octetsDDP); 5908 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5909 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5910 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5911 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5912 stats[3].framesDrop); 5913 5914 rc = sbuf_finish(sb); 5915 sbuf_delete(sb); 5916 5917 return (rc); 5918 } 5919 5920 static int 5921 sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5922 { 5923 struct adapter *sc = arg1; 5924 struct sbuf *sb; 5925 int rc, i; 5926 unsigned int map, kbps, ipg, mode; 5927 unsigned int pace_tab[NTX_SCHED]; 5928 5929 rc = sysctl_wire_old_buffer(req, 0); 5930 if (rc != 0) 5931 return (rc); 5932 5933 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5934 if (sb == NULL) 5935 return (ENOMEM); 5936 5937 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5938 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5939 t4_read_pace_tbl(sc, pace_tab); 5940 5941 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5942 "Class IPG (0.1 ns) Flow IPG (us)"); 5943 5944 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5945 t4_get_tx_sched(sc, i, &kbps, &ipg); 5946 sbuf_printf(sb, "\n %u %-5s %u ", i, 5947 (mode & (1 << i)) ? "flow" : "class", map & 3); 5948 if (kbps) 5949 sbuf_printf(sb, "%9u ", kbps); 5950 else 5951 sbuf_printf(sb, " disabled "); 5952 5953 if (ipg) 5954 sbuf_printf(sb, "%13u ", ipg); 5955 else 5956 sbuf_printf(sb, " disabled "); 5957 5958 if (pace_tab[i]) 5959 sbuf_printf(sb, "%10u", pace_tab[i]); 5960 else 5961 sbuf_printf(sb, " disabled"); 5962 } 5963 5964 rc = sbuf_finish(sb); 5965 sbuf_delete(sb); 5966 5967 return (rc); 5968 } 5969 5970 static int 5971 sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5972 { 5973 struct adapter *sc = arg1; 5974 struct sbuf *sb; 5975 int rc, i, j; 5976 uint64_t *p0, *p1; 5977 struct lb_port_stats s[2]; 5978 static const char *stat_name[] = { 5979 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5980 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5981 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5982 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5983 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5984 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5985 "BG2FramesTrunc:", "BG3FramesTrunc:" 5986 }; 5987 5988 rc = sysctl_wire_old_buffer(req, 0); 5989 if (rc != 0) 5990 return (rc); 5991 5992 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5993 if (sb == NULL) 5994 return (ENOMEM); 5995 5996 memset(s, 0, sizeof(s)); 5997 5998 for (i = 0; i < 4; i += 2) { 5999 t4_get_lb_stats(sc, i, &s[0]); 6000 t4_get_lb_stats(sc, i + 1, &s[1]); 6001 6002 p0 = &s[0].octets; 6003 p1 = &s[1].octets; 6004 sbuf_printf(sb, "%s Loopback %u" 6005 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6006 6007 for (j = 0; j < nitems(stat_name); j++) 6008 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6009 *p0++, *p1++); 6010 } 6011 6012 rc = sbuf_finish(sb); 6013 sbuf_delete(sb); 6014 6015 return (rc); 6016 } 6017 6018 static int 6019 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6020 { 6021 int rc = 0; 6022 struct port_info *pi = arg1; 6023 struct sbuf *sb; 6024 static const char *linkdnreasons[] = { 6025 "non-specific", "remote fault", "autoneg failed", "reserved3", 6026 "PHY overheated", "unknown", "rx los", "reserved7" 6027 }; 6028 6029 rc = sysctl_wire_old_buffer(req, 0); 6030 if (rc != 0) 6031 return(rc); 6032 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6033 if (sb == NULL) 6034 return (ENOMEM); 6035 6036 if (pi->linkdnrc < 0) 6037 sbuf_printf(sb, "n/a"); 6038 else if (pi->linkdnrc < nitems(linkdnreasons)) 6039 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6040 else 6041 sbuf_printf(sb, "%d", pi->linkdnrc); 6042 6043 rc = sbuf_finish(sb); 6044 sbuf_delete(sb); 6045 6046 return (rc); 6047 } 6048 6049 struct mem_desc { 6050 unsigned int base; 6051 unsigned int limit; 6052 unsigned int idx; 6053 }; 6054 6055 static int 6056 mem_desc_cmp(const void *a, const void *b) 6057 { 6058 return ((const struct mem_desc *)a)->base - 6059 ((const struct mem_desc *)b)->base; 6060 } 6061 6062 static void 6063 mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6064 unsigned int to) 6065 { 6066 unsigned int size; 6067 6068 size = to - from + 1; 6069 if (size == 0) 6070 return; 6071 6072 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6073 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6074 } 6075 6076 static int 6077 sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6078 { 6079 struct adapter *sc = arg1; 6080 struct sbuf *sb; 6081 int rc, i, n; 6082 uint32_t lo, hi, used, alloc; 6083 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6084 static const char *region[] = { 6085 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6086 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6087 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6088 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6089 "RQUDP region:", "PBL region:", "TXPBL region:", 6090 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6091 "On-chip queues:" 6092 }; 6093 struct mem_desc avail[4]; 6094 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6095 struct mem_desc *md = mem; 6096 6097 rc = sysctl_wire_old_buffer(req, 0); 6098 if (rc != 0) 6099 return (rc); 6100 6101 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6102 if (sb == NULL) 6103 return (ENOMEM); 6104 6105 for (i = 0; i < nitems(mem); i++) { 6106 mem[i].limit = 0; 6107 mem[i].idx = i; 6108 } 6109 6110 /* Find and sort the populated memory ranges */ 6111 i = 0; 6112 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6113 if (lo & F_EDRAM0_ENABLE) { 6114 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6115 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6116 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6117 avail[i].idx = 0; 6118 i++; 6119 } 6120 if (lo & F_EDRAM1_ENABLE) { 6121 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6122 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6123 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6124 avail[i].idx = 1; 6125 i++; 6126 } 6127 if (lo & F_EXT_MEM_ENABLE) { 6128 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6129 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6130 avail[i].limit = avail[i].base + 6131 (G_EXT_MEM_SIZE(hi) << 20); 6132 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6133 i++; 6134 } 6135 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6136 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6137 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6138 avail[i].limit = avail[i].base + 6139 (G_EXT_MEM1_SIZE(hi) << 20); 6140 avail[i].idx = 4; 6141 i++; 6142 } 6143 if (!i) /* no memory available */ 6144 return 0; 6145 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6146 6147 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6148 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6149 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6150 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6151 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6152 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6153 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6154 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6155 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6156 6157 /* the next few have explicit upper bounds */ 6158 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6159 md->limit = md->base - 1 + 6160 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6161 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6162 md++; 6163 6164 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6165 md->limit = md->base - 1 + 6166 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6167 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6168 md++; 6169 6170 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6171 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6172 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6173 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6174 } else { 6175 md->base = 0; 6176 md->idx = nitems(region); /* hide it */ 6177 } 6178 md++; 6179 6180 #define ulp_region(reg) \ 6181 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6182 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6183 6184 ulp_region(RX_ISCSI); 6185 ulp_region(RX_TDDP); 6186 ulp_region(TX_TPT); 6187 ulp_region(RX_STAG); 6188 ulp_region(RX_RQ); 6189 ulp_region(RX_RQUDP); 6190 ulp_region(RX_PBL); 6191 ulp_region(TX_PBL); 6192 #undef ulp_region 6193 6194 md->base = 0; 6195 md->idx = nitems(region); 6196 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6197 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6198 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6199 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6200 } 6201 md++; 6202 6203 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6204 md->limit = md->base + sc->tids.ntids - 1; 6205 md++; 6206 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6207 md->limit = md->base + sc->tids.ntids - 1; 6208 md++; 6209 6210 md->base = sc->vres.ocq.start; 6211 if (sc->vres.ocq.size) 6212 md->limit = md->base + sc->vres.ocq.size - 1; 6213 else 6214 md->idx = nitems(region); /* hide it */ 6215 md++; 6216 6217 /* add any address-space holes, there can be up to 3 */ 6218 for (n = 0; n < i - 1; n++) 6219 if (avail[n].limit < avail[n + 1].base) 6220 (md++)->base = avail[n].limit; 6221 if (avail[n].limit) 6222 (md++)->base = avail[n].limit; 6223 6224 n = md - mem; 6225 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6226 6227 for (lo = 0; lo < i; lo++) 6228 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6229 avail[lo].limit - 1); 6230 6231 sbuf_printf(sb, "\n"); 6232 for (i = 0; i < n; i++) { 6233 if (mem[i].idx >= nitems(region)) 6234 continue; /* skip holes */ 6235 if (!mem[i].limit) 6236 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6237 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6238 mem[i].limit); 6239 } 6240 6241 sbuf_printf(sb, "\n"); 6242 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6243 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6244 mem_region_show(sb, "uP RAM:", lo, hi); 6245 6246 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6247 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6248 mem_region_show(sb, "uP Extmem2:", lo, hi); 6249 6250 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6251 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6252 G_PMRXMAXPAGE(lo), 6253 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6254 (lo & F_PMRXNUMCHN) ? 2 : 1); 6255 6256 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6257 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6258 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6259 G_PMTXMAXPAGE(lo), 6260 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6261 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6262 sbuf_printf(sb, "%u p-structs\n", 6263 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6264 6265 for (i = 0; i < 4; i++) { 6266 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6267 if (is_t4(sc)) { 6268 used = G_USED(lo); 6269 alloc = G_ALLOC(lo); 6270 } else { 6271 used = G_T5_USED(lo); 6272 alloc = G_T5_ALLOC(lo); 6273 } 6274 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6275 i, used, alloc); 6276 } 6277 for (i = 0; i < 4; i++) { 6278 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6279 if (is_t4(sc)) { 6280 used = G_USED(lo); 6281 alloc = G_ALLOC(lo); 6282 } else { 6283 used = G_T5_USED(lo); 6284 alloc = G_T5_ALLOC(lo); 6285 } 6286 sbuf_printf(sb, 6287 "\nLoopback %d using %u pages out of %u allocated", 6288 i, used, alloc); 6289 } 6290 6291 rc = sbuf_finish(sb); 6292 sbuf_delete(sb); 6293 6294 return (rc); 6295 } 6296 6297 static inline void 6298 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6299 { 6300 *mask = x | y; 6301 y = htobe64(y); 6302 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6303 } 6304 6305 static int 6306 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6307 { 6308 struct adapter *sc = arg1; 6309 struct sbuf *sb; 6310 int rc, i, n; 6311 6312 rc = sysctl_wire_old_buffer(req, 0); 6313 if (rc != 0) 6314 return (rc); 6315 6316 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6317 if (sb == NULL) 6318 return (ENOMEM); 6319 6320 sbuf_printf(sb, 6321 "Idx Ethernet address Mask Vld Ports PF" 6322 " VF Replication P0 P1 P2 P3 ML"); 6323 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6324 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6325 for (i = 0; i < n; i++) { 6326 uint64_t tcamx, tcamy, mask; 6327 uint32_t cls_lo, cls_hi; 6328 uint8_t addr[ETHER_ADDR_LEN]; 6329 6330 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6331 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6332 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6333 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6334 6335 if (tcamx & tcamy) 6336 continue; 6337 6338 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6339 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6340 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6341 addr[3], addr[4], addr[5], (uintmax_t)mask, 6342 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6343 G_PORTMAP(cls_hi), G_PF(cls_lo), 6344 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6345 6346 if (cls_lo & F_REPLICATE) { 6347 struct fw_ldst_cmd ldst_cmd; 6348 6349 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6350 ldst_cmd.op_to_addrspace = 6351 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6352 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6353 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6354 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6355 ldst_cmd.u.mps.fid_ctl = 6356 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6357 V_FW_LDST_CMD_CTL(i)); 6358 6359 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6360 "t4mps"); 6361 if (rc) 6362 break; 6363 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6364 sizeof(ldst_cmd), &ldst_cmd); 6365 end_synchronized_op(sc, 0); 6366 6367 if (rc != 0) { 6368 sbuf_printf(sb, 6369 " ------------ error %3u ------------", rc); 6370 rc = 0; 6371 } else { 6372 sbuf_printf(sb, " %08x %08x %08x %08x", 6373 be32toh(ldst_cmd.u.mps.rplc127_96), 6374 be32toh(ldst_cmd.u.mps.rplc95_64), 6375 be32toh(ldst_cmd.u.mps.rplc63_32), 6376 be32toh(ldst_cmd.u.mps.rplc31_0)); 6377 } 6378 } else 6379 sbuf_printf(sb, "%36s", ""); 6380 6381 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6382 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6383 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6384 } 6385 6386 if (rc) 6387 (void) sbuf_finish(sb); 6388 else 6389 rc = sbuf_finish(sb); 6390 sbuf_delete(sb); 6391 6392 return (rc); 6393 } 6394 6395 static int 6396 sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6397 { 6398 struct adapter *sc = arg1; 6399 struct sbuf *sb; 6400 int rc; 6401 uint16_t mtus[NMTUS]; 6402 6403 rc = sysctl_wire_old_buffer(req, 0); 6404 if (rc != 0) 6405 return (rc); 6406 6407 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6408 if (sb == NULL) 6409 return (ENOMEM); 6410 6411 t4_read_mtu_tbl(sc, mtus, NULL); 6412 6413 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6414 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6415 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6416 mtus[14], mtus[15]); 6417 6418 rc = sbuf_finish(sb); 6419 sbuf_delete(sb); 6420 6421 return (rc); 6422 } 6423 6424 static int 6425 sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6426 { 6427 struct adapter *sc = arg1; 6428 struct sbuf *sb; 6429 int rc, i; 6430 uint32_t cnt[PM_NSTATS]; 6431 uint64_t cyc[PM_NSTATS]; 6432 static const char *rx_stats[] = { 6433 "Read:", "Write bypass:", "Write mem:", "Flush:" 6434 }; 6435 static const char *tx_stats[] = { 6436 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6437 }; 6438 6439 rc = sysctl_wire_old_buffer(req, 0); 6440 if (rc != 0) 6441 return (rc); 6442 6443 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6444 if (sb == NULL) 6445 return (ENOMEM); 6446 6447 t4_pmtx_get_stats(sc, cnt, cyc); 6448 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6449 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6450 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6451 cyc[i]); 6452 6453 t4_pmrx_get_stats(sc, cnt, cyc); 6454 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6455 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6456 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6457 cyc[i]); 6458 6459 rc = sbuf_finish(sb); 6460 sbuf_delete(sb); 6461 6462 return (rc); 6463 } 6464 6465 static int 6466 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6467 { 6468 struct adapter *sc = arg1; 6469 struct sbuf *sb; 6470 int rc; 6471 struct tp_rdma_stats stats; 6472 6473 rc = sysctl_wire_old_buffer(req, 0); 6474 if (rc != 0) 6475 return (rc); 6476 6477 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6478 if (sb == NULL) 6479 return (ENOMEM); 6480 6481 t4_tp_get_rdma_stats(sc, &stats); 6482 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6483 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6484 6485 rc = sbuf_finish(sb); 6486 sbuf_delete(sb); 6487 6488 return (rc); 6489 } 6490 6491 static int 6492 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6493 { 6494 struct adapter *sc = arg1; 6495 struct sbuf *sb; 6496 int rc; 6497 struct tp_tcp_stats v4, v6; 6498 6499 rc = sysctl_wire_old_buffer(req, 0); 6500 if (rc != 0) 6501 return (rc); 6502 6503 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6504 if (sb == NULL) 6505 return (ENOMEM); 6506 6507 t4_tp_get_tcp_stats(sc, &v4, &v6); 6508 sbuf_printf(sb, 6509 " IP IPv6\n"); 6510 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6511 v4.tcpOutRsts, v6.tcpOutRsts); 6512 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6513 v4.tcpInSegs, v6.tcpInSegs); 6514 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6515 v4.tcpOutSegs, v6.tcpOutSegs); 6516 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6517 v4.tcpRetransSegs, v6.tcpRetransSegs); 6518 6519 rc = sbuf_finish(sb); 6520 sbuf_delete(sb); 6521 6522 return (rc); 6523 } 6524 6525 static int 6526 sysctl_tids(SYSCTL_HANDLER_ARGS) 6527 { 6528 struct adapter *sc = arg1; 6529 struct sbuf *sb; 6530 int rc; 6531 struct tid_info *t = &sc->tids; 6532 6533 rc = sysctl_wire_old_buffer(req, 0); 6534 if (rc != 0) 6535 return (rc); 6536 6537 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6538 if (sb == NULL) 6539 return (ENOMEM); 6540 6541 if (t->natids) { 6542 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6543 t->atids_in_use); 6544 } 6545 6546 if (t->ntids) { 6547 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6548 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6549 6550 if (b) { 6551 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6552 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6553 t->ntids - 1); 6554 } else { 6555 sbuf_printf(sb, "TID range: %u-%u", 6556 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6557 t->ntids - 1); 6558 } 6559 } else 6560 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6561 sbuf_printf(sb, ", in use: %u\n", 6562 atomic_load_acq_int(&t->tids_in_use)); 6563 } 6564 6565 if (t->nstids) { 6566 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6567 t->stid_base + t->nstids - 1, t->stids_in_use); 6568 } 6569 6570 if (t->nftids) { 6571 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6572 t->ftid_base + t->nftids - 1); 6573 } 6574 6575 if (t->netids) { 6576 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6577 t->etid_base + t->netids - 1); 6578 } 6579 6580 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6581 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6582 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6583 6584 rc = sbuf_finish(sb); 6585 sbuf_delete(sb); 6586 6587 return (rc); 6588 } 6589 6590 static int 6591 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6592 { 6593 struct adapter *sc = arg1; 6594 struct sbuf *sb; 6595 int rc; 6596 struct tp_err_stats stats; 6597 6598 rc = sysctl_wire_old_buffer(req, 0); 6599 if (rc != 0) 6600 return (rc); 6601 6602 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6603 if (sb == NULL) 6604 return (ENOMEM); 6605 6606 t4_tp_get_err_stats(sc, &stats); 6607 6608 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6609 "channel 3\n"); 6610 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6611 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6612 stats.macInErrs[3]); 6613 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6614 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6615 stats.hdrInErrs[3]); 6616 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6617 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6618 stats.tcpInErrs[3]); 6619 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6620 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6621 stats.tcp6InErrs[3]); 6622 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6623 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6624 stats.tnlCongDrops[3]); 6625 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6626 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6627 stats.tnlTxDrops[3]); 6628 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6629 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6630 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6631 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6632 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6633 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6634 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6635 stats.ofldNoNeigh, stats.ofldCongDefer); 6636 6637 rc = sbuf_finish(sb); 6638 sbuf_delete(sb); 6639 6640 return (rc); 6641 } 6642 6643 struct field_desc { 6644 const char *name; 6645 u_int start; 6646 u_int width; 6647 }; 6648 6649 static void 6650 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6651 { 6652 char buf[32]; 6653 int line_size = 0; 6654 6655 while (f->name) { 6656 uint64_t mask = (1ULL << f->width) - 1; 6657 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6658 ((uintmax_t)v >> f->start) & mask); 6659 6660 if (line_size + len >= 79) { 6661 line_size = 8; 6662 sbuf_printf(sb, "\n "); 6663 } 6664 sbuf_printf(sb, "%s ", buf); 6665 line_size += len + 1; 6666 f++; 6667 } 6668 sbuf_printf(sb, "\n"); 6669 } 6670 6671 static struct field_desc tp_la0[] = { 6672 { "RcfOpCodeOut", 60, 4 }, 6673 { "State", 56, 4 }, 6674 { "WcfState", 52, 4 }, 6675 { "RcfOpcSrcOut", 50, 2 }, 6676 { "CRxError", 49, 1 }, 6677 { "ERxError", 48, 1 }, 6678 { "SanityFailed", 47, 1 }, 6679 { "SpuriousMsg", 46, 1 }, 6680 { "FlushInputMsg", 45, 1 }, 6681 { "FlushInputCpl", 44, 1 }, 6682 { "RssUpBit", 43, 1 }, 6683 { "RssFilterHit", 42, 1 }, 6684 { "Tid", 32, 10 }, 6685 { "InitTcb", 31, 1 }, 6686 { "LineNumber", 24, 7 }, 6687 { "Emsg", 23, 1 }, 6688 { "EdataOut", 22, 1 }, 6689 { "Cmsg", 21, 1 }, 6690 { "CdataOut", 20, 1 }, 6691 { "EreadPdu", 19, 1 }, 6692 { "CreadPdu", 18, 1 }, 6693 { "TunnelPkt", 17, 1 }, 6694 { "RcfPeerFin", 16, 1 }, 6695 { "RcfReasonOut", 12, 4 }, 6696 { "TxCchannel", 10, 2 }, 6697 { "RcfTxChannel", 8, 2 }, 6698 { "RxEchannel", 6, 2 }, 6699 { "RcfRxChannel", 5, 1 }, 6700 { "RcfDataOutSrdy", 4, 1 }, 6701 { "RxDvld", 3, 1 }, 6702 { "RxOoDvld", 2, 1 }, 6703 { "RxCongestion", 1, 1 }, 6704 { "TxCongestion", 0, 1 }, 6705 { NULL } 6706 }; 6707 6708 static struct field_desc tp_la1[] = { 6709 { "CplCmdIn", 56, 8 }, 6710 { "CplCmdOut", 48, 8 }, 6711 { "ESynOut", 47, 1 }, 6712 { "EAckOut", 46, 1 }, 6713 { "EFinOut", 45, 1 }, 6714 { "ERstOut", 44, 1 }, 6715 { "SynIn", 43, 1 }, 6716 { "AckIn", 42, 1 }, 6717 { "FinIn", 41, 1 }, 6718 { "RstIn", 40, 1 }, 6719 { "DataIn", 39, 1 }, 6720 { "DataInVld", 38, 1 }, 6721 { "PadIn", 37, 1 }, 6722 { "RxBufEmpty", 36, 1 }, 6723 { "RxDdp", 35, 1 }, 6724 { "RxFbCongestion", 34, 1 }, 6725 { "TxFbCongestion", 33, 1 }, 6726 { "TxPktSumSrdy", 32, 1 }, 6727 { "RcfUlpType", 28, 4 }, 6728 { "Eread", 27, 1 }, 6729 { "Ebypass", 26, 1 }, 6730 { "Esave", 25, 1 }, 6731 { "Static0", 24, 1 }, 6732 { "Cread", 23, 1 }, 6733 { "Cbypass", 22, 1 }, 6734 { "Csave", 21, 1 }, 6735 { "CPktOut", 20, 1 }, 6736 { "RxPagePoolFull", 18, 2 }, 6737 { "RxLpbkPkt", 17, 1 }, 6738 { "TxLpbkPkt", 16, 1 }, 6739 { "RxVfValid", 15, 1 }, 6740 { "SynLearned", 14, 1 }, 6741 { "SetDelEntry", 13, 1 }, 6742 { "SetInvEntry", 12, 1 }, 6743 { "CpcmdDvld", 11, 1 }, 6744 { "CpcmdSave", 10, 1 }, 6745 { "RxPstructsFull", 8, 2 }, 6746 { "EpcmdDvld", 7, 1 }, 6747 { "EpcmdFlush", 6, 1 }, 6748 { "EpcmdTrimPrefix", 5, 1 }, 6749 { "EpcmdTrimPostfix", 4, 1 }, 6750 { "ERssIp4Pkt", 3, 1 }, 6751 { "ERssIp6Pkt", 2, 1 }, 6752 { "ERssTcpUdpPkt", 1, 1 }, 6753 { "ERssFceFipPkt", 0, 1 }, 6754 { NULL } 6755 }; 6756 6757 static struct field_desc tp_la2[] = { 6758 { "CplCmdIn", 56, 8 }, 6759 { "MpsVfVld", 55, 1 }, 6760 { "MpsPf", 52, 3 }, 6761 { "MpsVf", 44, 8 }, 6762 { "SynIn", 43, 1 }, 6763 { "AckIn", 42, 1 }, 6764 { "FinIn", 41, 1 }, 6765 { "RstIn", 40, 1 }, 6766 { "DataIn", 39, 1 }, 6767 { "DataInVld", 38, 1 }, 6768 { "PadIn", 37, 1 }, 6769 { "RxBufEmpty", 36, 1 }, 6770 { "RxDdp", 35, 1 }, 6771 { "RxFbCongestion", 34, 1 }, 6772 { "TxFbCongestion", 33, 1 }, 6773 { "TxPktSumSrdy", 32, 1 }, 6774 { "RcfUlpType", 28, 4 }, 6775 { "Eread", 27, 1 }, 6776 { "Ebypass", 26, 1 }, 6777 { "Esave", 25, 1 }, 6778 { "Static0", 24, 1 }, 6779 { "Cread", 23, 1 }, 6780 { "Cbypass", 22, 1 }, 6781 { "Csave", 21, 1 }, 6782 { "CPktOut", 20, 1 }, 6783 { "RxPagePoolFull", 18, 2 }, 6784 { "RxLpbkPkt", 17, 1 }, 6785 { "TxLpbkPkt", 16, 1 }, 6786 { "RxVfValid", 15, 1 }, 6787 { "SynLearned", 14, 1 }, 6788 { "SetDelEntry", 13, 1 }, 6789 { "SetInvEntry", 12, 1 }, 6790 { "CpcmdDvld", 11, 1 }, 6791 { "CpcmdSave", 10, 1 }, 6792 { "RxPstructsFull", 8, 2 }, 6793 { "EpcmdDvld", 7, 1 }, 6794 { "EpcmdFlush", 6, 1 }, 6795 { "EpcmdTrimPrefix", 5, 1 }, 6796 { "EpcmdTrimPostfix", 4, 1 }, 6797 { "ERssIp4Pkt", 3, 1 }, 6798 { "ERssIp6Pkt", 2, 1 }, 6799 { "ERssTcpUdpPkt", 1, 1 }, 6800 { "ERssFceFipPkt", 0, 1 }, 6801 { NULL } 6802 }; 6803 6804 static void 6805 tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6806 { 6807 6808 field_desc_show(sb, *p, tp_la0); 6809 } 6810 6811 static void 6812 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6813 { 6814 6815 if (idx) 6816 sbuf_printf(sb, "\n"); 6817 field_desc_show(sb, p[0], tp_la0); 6818 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6819 field_desc_show(sb, p[1], tp_la0); 6820 } 6821 6822 static void 6823 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6824 { 6825 6826 if (idx) 6827 sbuf_printf(sb, "\n"); 6828 field_desc_show(sb, p[0], tp_la0); 6829 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6830 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6831 } 6832 6833 static int 6834 sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6835 { 6836 struct adapter *sc = arg1; 6837 struct sbuf *sb; 6838 uint64_t *buf, *p; 6839 int rc; 6840 u_int i, inc; 6841 void (*show_func)(struct sbuf *, uint64_t *, int); 6842 6843 rc = sysctl_wire_old_buffer(req, 0); 6844 if (rc != 0) 6845 return (rc); 6846 6847 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6848 if (sb == NULL) 6849 return (ENOMEM); 6850 6851 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6852 6853 t4_tp_read_la(sc, buf, NULL); 6854 p = buf; 6855 6856 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6857 case 2: 6858 inc = 2; 6859 show_func = tp_la_show2; 6860 break; 6861 case 3: 6862 inc = 2; 6863 show_func = tp_la_show3; 6864 break; 6865 default: 6866 inc = 1; 6867 show_func = tp_la_show; 6868 } 6869 6870 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6871 (*show_func)(sb, p, i); 6872 6873 rc = sbuf_finish(sb); 6874 sbuf_delete(sb); 6875 free(buf, M_CXGBE); 6876 return (rc); 6877 } 6878 6879 static int 6880 sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6881 { 6882 struct adapter *sc = arg1; 6883 struct sbuf *sb; 6884 int rc; 6885 u64 nrate[NCHAN], orate[NCHAN]; 6886 6887 rc = sysctl_wire_old_buffer(req, 0); 6888 if (rc != 0) 6889 return (rc); 6890 6891 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6892 if (sb == NULL) 6893 return (ENOMEM); 6894 6895 t4_get_chan_txrate(sc, nrate, orate); 6896 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6897 "channel 3\n"); 6898 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6899 nrate[0], nrate[1], nrate[2], nrate[3]); 6900 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6901 orate[0], orate[1], orate[2], orate[3]); 6902 6903 rc = sbuf_finish(sb); 6904 sbuf_delete(sb); 6905 6906 return (rc); 6907 } 6908 6909 static int 6910 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6911 { 6912 struct adapter *sc = arg1; 6913 struct sbuf *sb; 6914 uint32_t *buf, *p; 6915 int rc, i; 6916 6917 rc = sysctl_wire_old_buffer(req, 0); 6918 if (rc != 0) 6919 return (rc); 6920 6921 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6922 if (sb == NULL) 6923 return (ENOMEM); 6924 6925 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6926 M_ZERO | M_WAITOK); 6927 6928 t4_ulprx_read_la(sc, buf); 6929 p = buf; 6930 6931 sbuf_printf(sb, " Pcmd Type Message" 6932 " Data"); 6933 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6934 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6935 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6936 } 6937 6938 rc = sbuf_finish(sb); 6939 sbuf_delete(sb); 6940 free(buf, M_CXGBE); 6941 return (rc); 6942 } 6943 6944 static int 6945 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6946 { 6947 struct adapter *sc = arg1; 6948 struct sbuf *sb; 6949 int rc, v; 6950 6951 rc = sysctl_wire_old_buffer(req, 0); 6952 if (rc != 0) 6953 return (rc); 6954 6955 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6956 if (sb == NULL) 6957 return (ENOMEM); 6958 6959 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6960 if (G_STATSOURCE_T5(v) == 7) { 6961 if (G_STATMODE(v) == 0) { 6962 sbuf_printf(sb, "total %d, incomplete %d", 6963 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6964 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6965 } else if (G_STATMODE(v) == 1) { 6966 sbuf_printf(sb, "total %d, data overflow %d", 6967 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6968 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6969 } 6970 } 6971 rc = sbuf_finish(sb); 6972 sbuf_delete(sb); 6973 6974 return (rc); 6975 } 6976 #endif 6977 6978 static uint32_t 6979 fconf_to_mode(uint32_t fconf) 6980 { 6981 uint32_t mode; 6982 6983 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6984 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6985 6986 if (fconf & F_FRAGMENTATION) 6987 mode |= T4_FILTER_IP_FRAGMENT; 6988 6989 if (fconf & F_MPSHITTYPE) 6990 mode |= T4_FILTER_MPS_HIT_TYPE; 6991 6992 if (fconf & F_MACMATCH) 6993 mode |= T4_FILTER_MAC_IDX; 6994 6995 if (fconf & F_ETHERTYPE) 6996 mode |= T4_FILTER_ETH_TYPE; 6997 6998 if (fconf & F_PROTOCOL) 6999 mode |= T4_FILTER_IP_PROTO; 7000 7001 if (fconf & F_TOS) 7002 mode |= T4_FILTER_IP_TOS; 7003 7004 if (fconf & F_VLAN) 7005 mode |= T4_FILTER_VLAN; 7006 7007 if (fconf & F_VNIC_ID) 7008 mode |= T4_FILTER_VNIC; 7009 7010 if (fconf & F_PORT) 7011 mode |= T4_FILTER_PORT; 7012 7013 if (fconf & F_FCOE) 7014 mode |= T4_FILTER_FCoE; 7015 7016 return (mode); 7017 } 7018 7019 static uint32_t 7020 mode_to_fconf(uint32_t mode) 7021 { 7022 uint32_t fconf = 0; 7023 7024 if (mode & T4_FILTER_IP_FRAGMENT) 7025 fconf |= F_FRAGMENTATION; 7026 7027 if (mode & T4_FILTER_MPS_HIT_TYPE) 7028 fconf |= F_MPSHITTYPE; 7029 7030 if (mode & T4_FILTER_MAC_IDX) 7031 fconf |= F_MACMATCH; 7032 7033 if (mode & T4_FILTER_ETH_TYPE) 7034 fconf |= F_ETHERTYPE; 7035 7036 if (mode & T4_FILTER_IP_PROTO) 7037 fconf |= F_PROTOCOL; 7038 7039 if (mode & T4_FILTER_IP_TOS) 7040 fconf |= F_TOS; 7041 7042 if (mode & T4_FILTER_VLAN) 7043 fconf |= F_VLAN; 7044 7045 if (mode & T4_FILTER_VNIC) 7046 fconf |= F_VNIC_ID; 7047 7048 if (mode & T4_FILTER_PORT) 7049 fconf |= F_PORT; 7050 7051 if (mode & T4_FILTER_FCoE) 7052 fconf |= F_FCOE; 7053 7054 return (fconf); 7055 } 7056 7057 static uint32_t 7058 fspec_to_fconf(struct t4_filter_specification *fs) 7059 { 7060 uint32_t fconf = 0; 7061 7062 if (fs->val.frag || fs->mask.frag) 7063 fconf |= F_FRAGMENTATION; 7064 7065 if (fs->val.matchtype || fs->mask.matchtype) 7066 fconf |= F_MPSHITTYPE; 7067 7068 if (fs->val.macidx || fs->mask.macidx) 7069 fconf |= F_MACMATCH; 7070 7071 if (fs->val.ethtype || fs->mask.ethtype) 7072 fconf |= F_ETHERTYPE; 7073 7074 if (fs->val.proto || fs->mask.proto) 7075 fconf |= F_PROTOCOL; 7076 7077 if (fs->val.tos || fs->mask.tos) 7078 fconf |= F_TOS; 7079 7080 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7081 fconf |= F_VLAN; 7082 7083 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7084 fconf |= F_VNIC_ID; 7085 7086 if (fs->val.iport || fs->mask.iport) 7087 fconf |= F_PORT; 7088 7089 if (fs->val.fcoe || fs->mask.fcoe) 7090 fconf |= F_FCOE; 7091 7092 return (fconf); 7093 } 7094 7095 static int 7096 get_filter_mode(struct adapter *sc, uint32_t *mode) 7097 { 7098 int rc; 7099 uint32_t fconf; 7100 7101 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7102 "t4getfm"); 7103 if (rc) 7104 return (rc); 7105 7106 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7107 A_TP_VLAN_PRI_MAP); 7108 7109 if (sc->params.tp.vlan_pri_map != fconf) { 7110 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7111 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7112 fconf); 7113 } 7114 7115 *mode = fconf_to_mode(fconf); 7116 7117 end_synchronized_op(sc, LOCK_HELD); 7118 return (0); 7119 } 7120 7121 static int 7122 set_filter_mode(struct adapter *sc, uint32_t mode) 7123 { 7124 uint32_t fconf; 7125 int rc; 7126 7127 fconf = mode_to_fconf(mode); 7128 7129 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7130 "t4setfm"); 7131 if (rc) 7132 return (rc); 7133 7134 if (sc->tids.ftids_in_use > 0) { 7135 rc = EBUSY; 7136 goto done; 7137 } 7138 7139 #ifdef TCP_OFFLOAD 7140 if (uld_active(sc, ULD_TOM)) { 7141 rc = EBUSY; 7142 goto done; 7143 } 7144 #endif 7145 7146 rc = -t4_set_filter_mode(sc, fconf); 7147 done: 7148 end_synchronized_op(sc, LOCK_HELD); 7149 return (rc); 7150 } 7151 7152 static inline uint64_t 7153 get_filter_hits(struct adapter *sc, uint32_t fid) 7154 { 7155 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7156 uint64_t hits; 7157 7158 memwin_info(sc, 0, &mw_base, NULL); 7159 off = position_memwin(sc, 0, 7160 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7161 if (is_t4(sc)) { 7162 hits = t4_read_reg64(sc, mw_base + off + 16); 7163 hits = be64toh(hits); 7164 } else { 7165 hits = t4_read_reg(sc, mw_base + off + 24); 7166 hits = be32toh(hits); 7167 } 7168 7169 return (hits); 7170 } 7171 7172 static int 7173 get_filter(struct adapter *sc, struct t4_filter *t) 7174 { 7175 int i, rc, nfilters = sc->tids.nftids; 7176 struct filter_entry *f; 7177 7178 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7179 "t4getf"); 7180 if (rc) 7181 return (rc); 7182 7183 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7184 t->idx >= nfilters) { 7185 t->idx = 0xffffffff; 7186 goto done; 7187 } 7188 7189 f = &sc->tids.ftid_tab[t->idx]; 7190 for (i = t->idx; i < nfilters; i++, f++) { 7191 if (f->valid) { 7192 t->idx = i; 7193 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7194 t->smtidx = f->smtidx; 7195 if (f->fs.hitcnts) 7196 t->hits = get_filter_hits(sc, t->idx); 7197 else 7198 t->hits = UINT64_MAX; 7199 t->fs = f->fs; 7200 7201 goto done; 7202 } 7203 } 7204 7205 t->idx = 0xffffffff; 7206 done: 7207 end_synchronized_op(sc, LOCK_HELD); 7208 return (0); 7209 } 7210 7211 static int 7212 set_filter(struct adapter *sc, struct t4_filter *t) 7213 { 7214 unsigned int nfilters, nports; 7215 struct filter_entry *f; 7216 int i, rc; 7217 7218 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7219 if (rc) 7220 return (rc); 7221 7222 nfilters = sc->tids.nftids; 7223 nports = sc->params.nports; 7224 7225 if (nfilters == 0) { 7226 rc = ENOTSUP; 7227 goto done; 7228 } 7229 7230 if (!(sc->flags & FULL_INIT_DONE)) { 7231 rc = EAGAIN; 7232 goto done; 7233 } 7234 7235 if (t->idx >= nfilters) { 7236 rc = EINVAL; 7237 goto done; 7238 } 7239 7240 /* Validate against the global filter mode */ 7241 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7242 sc->params.tp.vlan_pri_map) { 7243 rc = E2BIG; 7244 goto done; 7245 } 7246 7247 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7248 rc = EINVAL; 7249 goto done; 7250 } 7251 7252 if (t->fs.val.iport >= nports) { 7253 rc = EINVAL; 7254 goto done; 7255 } 7256 7257 /* Can't specify an iq if not steering to it */ 7258 if (!t->fs.dirsteer && t->fs.iq) { 7259 rc = EINVAL; 7260 goto done; 7261 } 7262 7263 /* IPv6 filter idx must be 4 aligned */ 7264 if (t->fs.type == 1 && 7265 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7266 rc = EINVAL; 7267 goto done; 7268 } 7269 7270 if (sc->tids.ftid_tab == NULL) { 7271 KASSERT(sc->tids.ftids_in_use == 0, 7272 ("%s: no memory allocated but filters_in_use > 0", 7273 __func__)); 7274 7275 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7276 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7277 if (sc->tids.ftid_tab == NULL) { 7278 rc = ENOMEM; 7279 goto done; 7280 } 7281 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7282 } 7283 7284 for (i = 0; i < 4; i++) { 7285 f = &sc->tids.ftid_tab[t->idx + i]; 7286 7287 if (f->pending || f->valid) { 7288 rc = EBUSY; 7289 goto done; 7290 } 7291 if (f->locked) { 7292 rc = EPERM; 7293 goto done; 7294 } 7295 7296 if (t->fs.type == 0) 7297 break; 7298 } 7299 7300 f = &sc->tids.ftid_tab[t->idx]; 7301 f->fs = t->fs; 7302 7303 rc = set_filter_wr(sc, t->idx); 7304 done: 7305 end_synchronized_op(sc, 0); 7306 7307 if (rc == 0) { 7308 mtx_lock(&sc->tids.ftid_lock); 7309 for (;;) { 7310 if (f->pending == 0) { 7311 rc = f->valid ? 0 : EIO; 7312 break; 7313 } 7314 7315 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7316 PCATCH, "t4setfw", 0)) { 7317 rc = EINPROGRESS; 7318 break; 7319 } 7320 } 7321 mtx_unlock(&sc->tids.ftid_lock); 7322 } 7323 return (rc); 7324 } 7325 7326 static int 7327 del_filter(struct adapter *sc, struct t4_filter *t) 7328 { 7329 unsigned int nfilters; 7330 struct filter_entry *f; 7331 int rc; 7332 7333 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7334 if (rc) 7335 return (rc); 7336 7337 nfilters = sc->tids.nftids; 7338 7339 if (nfilters == 0) { 7340 rc = ENOTSUP; 7341 goto done; 7342 } 7343 7344 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7345 t->idx >= nfilters) { 7346 rc = EINVAL; 7347 goto done; 7348 } 7349 7350 if (!(sc->flags & FULL_INIT_DONE)) { 7351 rc = EAGAIN; 7352 goto done; 7353 } 7354 7355 f = &sc->tids.ftid_tab[t->idx]; 7356 7357 if (f->pending) { 7358 rc = EBUSY; 7359 goto done; 7360 } 7361 if (f->locked) { 7362 rc = EPERM; 7363 goto done; 7364 } 7365 7366 if (f->valid) { 7367 t->fs = f->fs; /* extra info for the caller */ 7368 rc = del_filter_wr(sc, t->idx); 7369 } 7370 7371 done: 7372 end_synchronized_op(sc, 0); 7373 7374 if (rc == 0) { 7375 mtx_lock(&sc->tids.ftid_lock); 7376 for (;;) { 7377 if (f->pending == 0) { 7378 rc = f->valid ? EIO : 0; 7379 break; 7380 } 7381 7382 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7383 PCATCH, "t4delfw", 0)) { 7384 rc = EINPROGRESS; 7385 break; 7386 } 7387 } 7388 mtx_unlock(&sc->tids.ftid_lock); 7389 } 7390 7391 return (rc); 7392 } 7393 7394 static void 7395 clear_filter(struct filter_entry *f) 7396 { 7397 if (f->l2t) 7398 t4_l2t_release(f->l2t); 7399 7400 bzero(f, sizeof (*f)); 7401 } 7402 7403 static int 7404 set_filter_wr(struct adapter *sc, int fidx) 7405 { 7406 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7407 struct fw_filter_wr *fwr; 7408 unsigned int ftid; 7409 struct wrq_cookie cookie; 7410 7411 ASSERT_SYNCHRONIZED_OP(sc); 7412 7413 if (f->fs.newdmac || f->fs.newvlan) { 7414 /* This filter needs an L2T entry; allocate one. */ 7415 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7416 if (f->l2t == NULL) 7417 return (EAGAIN); 7418 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7419 f->fs.dmac)) { 7420 t4_l2t_release(f->l2t); 7421 f->l2t = NULL; 7422 return (ENOMEM); 7423 } 7424 } 7425 7426 ftid = sc->tids.ftid_base + fidx; 7427 7428 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7429 if (fwr == NULL) 7430 return (ENOMEM); 7431 bzero(fwr, sizeof(*fwr)); 7432 7433 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7434 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7435 fwr->tid_to_iq = 7436 htobe32(V_FW_FILTER_WR_TID(ftid) | 7437 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7438 V_FW_FILTER_WR_NOREPLY(0) | 7439 V_FW_FILTER_WR_IQ(f->fs.iq)); 7440 fwr->del_filter_to_l2tix = 7441 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7442 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7443 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7444 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7445 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7446 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7447 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7448 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7449 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7450 f->fs.newvlan == VLAN_REWRITE) | 7451 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7452 f->fs.newvlan == VLAN_REWRITE) | 7453 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7454 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7455 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7456 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7457 fwr->ethtype = htobe16(f->fs.val.ethtype); 7458 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7459 fwr->frag_to_ovlan_vldm = 7460 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7461 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7462 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7463 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7464 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7465 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7466 fwr->smac_sel = 0; 7467 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7468 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7469 fwr->maci_to_matchtypem = 7470 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7471 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7472 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7473 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7474 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7475 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7476 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7477 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7478 fwr->ptcl = f->fs.val.proto; 7479 fwr->ptclm = f->fs.mask.proto; 7480 fwr->ttyp = f->fs.val.tos; 7481 fwr->ttypm = f->fs.mask.tos; 7482 fwr->ivlan = htobe16(f->fs.val.vlan); 7483 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7484 fwr->ovlan = htobe16(f->fs.val.vnic); 7485 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7486 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7487 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7488 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7489 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7490 fwr->lp = htobe16(f->fs.val.dport); 7491 fwr->lpm = htobe16(f->fs.mask.dport); 7492 fwr->fp = htobe16(f->fs.val.sport); 7493 fwr->fpm = htobe16(f->fs.mask.sport); 7494 if (f->fs.newsmac) 7495 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7496 7497 f->pending = 1; 7498 sc->tids.ftids_in_use++; 7499 7500 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7501 return (0); 7502 } 7503 7504 static int 7505 del_filter_wr(struct adapter *sc, int fidx) 7506 { 7507 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7508 struct fw_filter_wr *fwr; 7509 unsigned int ftid; 7510 struct wrq_cookie cookie; 7511 7512 ftid = sc->tids.ftid_base + fidx; 7513 7514 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7515 if (fwr == NULL) 7516 return (ENOMEM); 7517 bzero(fwr, sizeof (*fwr)); 7518 7519 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7520 7521 f->pending = 1; 7522 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7523 return (0); 7524 } 7525 7526 int 7527 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7528 { 7529 struct adapter *sc = iq->adapter; 7530 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7531 unsigned int idx = GET_TID(rpl); 7532 unsigned int rc; 7533 struct filter_entry *f; 7534 7535 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7536 rss->opcode)); 7537 7538 if (is_ftid(sc, idx)) { 7539 7540 idx -= sc->tids.ftid_base; 7541 f = &sc->tids.ftid_tab[idx]; 7542 rc = G_COOKIE(rpl->cookie); 7543 7544 mtx_lock(&sc->tids.ftid_lock); 7545 if (rc == FW_FILTER_WR_FLT_ADDED) { 7546 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7547 __func__, idx)); 7548 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7549 f->pending = 0; /* asynchronous setup completed */ 7550 f->valid = 1; 7551 } else { 7552 if (rc != FW_FILTER_WR_FLT_DELETED) { 7553 /* Add or delete failed, display an error */ 7554 log(LOG_ERR, 7555 "filter %u setup failed with error %u\n", 7556 idx, rc); 7557 } 7558 7559 clear_filter(f); 7560 sc->tids.ftids_in_use--; 7561 } 7562 wakeup(&sc->tids.ftid_tab); 7563 mtx_unlock(&sc->tids.ftid_lock); 7564 } 7565 7566 return (0); 7567 } 7568 7569 static int 7570 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7571 { 7572 int rc; 7573 7574 if (cntxt->cid > M_CTXTQID) 7575 return (EINVAL); 7576 7577 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7578 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7579 return (EINVAL); 7580 7581 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7582 if (rc) 7583 return (rc); 7584 7585 if (sc->flags & FW_OK) { 7586 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7587 &cntxt->data[0]); 7588 if (rc == 0) 7589 goto done; 7590 } 7591 7592 /* 7593 * Read via firmware failed or wasn't even attempted. Read directly via 7594 * the backdoor. 7595 */ 7596 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7597 done: 7598 end_synchronized_op(sc, 0); 7599 return (rc); 7600 } 7601 7602 static int 7603 load_fw(struct adapter *sc, struct t4_data *fw) 7604 { 7605 int rc; 7606 uint8_t *fw_data; 7607 7608 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7609 if (rc) 7610 return (rc); 7611 7612 if (sc->flags & FULL_INIT_DONE) { 7613 rc = EBUSY; 7614 goto done; 7615 } 7616 7617 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7618 if (fw_data == NULL) { 7619 rc = ENOMEM; 7620 goto done; 7621 } 7622 7623 rc = copyin(fw->data, fw_data, fw->len); 7624 if (rc == 0) 7625 rc = -t4_load_fw(sc, fw_data, fw->len); 7626 7627 free(fw_data, M_CXGBE); 7628 done: 7629 end_synchronized_op(sc, 0); 7630 return (rc); 7631 } 7632 7633 static int 7634 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7635 { 7636 uint32_t addr, off, remaining, i, n; 7637 uint32_t *buf, *b; 7638 uint32_t mw_base, mw_aperture; 7639 int rc; 7640 uint8_t *dst; 7641 7642 rc = validate_mem_range(sc, mr->addr, mr->len); 7643 if (rc != 0) 7644 return (rc); 7645 7646 memwin_info(sc, win, &mw_base, &mw_aperture); 7647 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7648 addr = mr->addr; 7649 remaining = mr->len; 7650 dst = (void *)mr->data; 7651 7652 while (remaining) { 7653 off = position_memwin(sc, win, addr); 7654 7655 /* number of bytes that we'll copy in the inner loop */ 7656 n = min(remaining, mw_aperture - off); 7657 for (i = 0; i < n; i += 4) 7658 *b++ = t4_read_reg(sc, mw_base + off + i); 7659 7660 rc = copyout(buf, dst, n); 7661 if (rc != 0) 7662 break; 7663 7664 b = buf; 7665 dst += n; 7666 remaining -= n; 7667 addr += n; 7668 } 7669 7670 free(buf, M_CXGBE); 7671 return (rc); 7672 } 7673 7674 static int 7675 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7676 { 7677 int rc; 7678 7679 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7680 return (EINVAL); 7681 7682 if (i2cd->len > sizeof(i2cd->data)) 7683 return (EFBIG); 7684 7685 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7686 if (rc) 7687 return (rc); 7688 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7689 i2cd->offset, i2cd->len, &i2cd->data[0]); 7690 end_synchronized_op(sc, 0); 7691 7692 return (rc); 7693 } 7694 7695 static int 7696 in_range(int val, int lo, int hi) 7697 { 7698 7699 return (val < 0 || (val <= hi && val >= lo)); 7700 } 7701 7702 static int 7703 set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7704 { 7705 int fw_subcmd, fw_type, rc; 7706 7707 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7708 if (rc) 7709 return (rc); 7710 7711 if (!(sc->flags & FULL_INIT_DONE)) { 7712 rc = EAGAIN; 7713 goto done; 7714 } 7715 7716 /* 7717 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7718 * sub-command and type are in common locations.) 7719 */ 7720 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7721 fw_subcmd = FW_SCHED_SC_CONFIG; 7722 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7723 fw_subcmd = FW_SCHED_SC_PARAMS; 7724 else { 7725 rc = EINVAL; 7726 goto done; 7727 } 7728 if (p->type == SCHED_CLASS_TYPE_PACKET) 7729 fw_type = FW_SCHED_TYPE_PKTSCHED; 7730 else { 7731 rc = EINVAL; 7732 goto done; 7733 } 7734 7735 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7736 /* Vet our parameters ..*/ 7737 if (p->u.config.minmax < 0) { 7738 rc = EINVAL; 7739 goto done; 7740 } 7741 7742 /* And pass the request to the firmware ...*/ 7743 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7744 goto done; 7745 } 7746 7747 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7748 int fw_level; 7749 int fw_mode; 7750 int fw_rateunit; 7751 int fw_ratemode; 7752 7753 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7754 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7755 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7756 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7757 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7758 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7759 else { 7760 rc = EINVAL; 7761 goto done; 7762 } 7763 7764 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7765 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7766 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7767 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7768 else { 7769 rc = EINVAL; 7770 goto done; 7771 } 7772 7773 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7774 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7775 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7776 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7777 else { 7778 rc = EINVAL; 7779 goto done; 7780 } 7781 7782 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7783 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7784 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7785 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7786 else { 7787 rc = EINVAL; 7788 goto done; 7789 } 7790 7791 /* Vet our parameters ... */ 7792 if (!in_range(p->u.params.channel, 0, 3) || 7793 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7794 !in_range(p->u.params.minrate, 0, 10000000) || 7795 !in_range(p->u.params.maxrate, 0, 10000000) || 7796 !in_range(p->u.params.weight, 0, 100)) { 7797 rc = ERANGE; 7798 goto done; 7799 } 7800 7801 /* 7802 * Translate any unset parameters into the firmware's 7803 * nomenclature and/or fail the call if the parameters 7804 * are required ... 7805 */ 7806 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7807 p->u.params.channel < 0 || p->u.params.cl < 0) { 7808 rc = EINVAL; 7809 goto done; 7810 } 7811 if (p->u.params.minrate < 0) 7812 p->u.params.minrate = 0; 7813 if (p->u.params.maxrate < 0) { 7814 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7815 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7816 rc = EINVAL; 7817 goto done; 7818 } else 7819 p->u.params.maxrate = 0; 7820 } 7821 if (p->u.params.weight < 0) { 7822 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7823 rc = EINVAL; 7824 goto done; 7825 } else 7826 p->u.params.weight = 0; 7827 } 7828 if (p->u.params.pktsize < 0) { 7829 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7830 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7831 rc = EINVAL; 7832 goto done; 7833 } else 7834 p->u.params.pktsize = 0; 7835 } 7836 7837 /* See what the firmware thinks of the request ... */ 7838 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7839 fw_rateunit, fw_ratemode, p->u.params.channel, 7840 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7841 p->u.params.weight, p->u.params.pktsize, 1); 7842 goto done; 7843 } 7844 7845 rc = EINVAL; 7846 done: 7847 end_synchronized_op(sc, 0); 7848 return (rc); 7849 } 7850 7851 static int 7852 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7853 { 7854 struct port_info *pi = NULL; 7855 struct sge_txq *txq; 7856 uint32_t fw_mnem, fw_queue, fw_class; 7857 int i, rc; 7858 7859 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7860 if (rc) 7861 return (rc); 7862 7863 if (!(sc->flags & FULL_INIT_DONE)) { 7864 rc = EAGAIN; 7865 goto done; 7866 } 7867 7868 if (p->port >= sc->params.nports) { 7869 rc = EINVAL; 7870 goto done; 7871 } 7872 7873 pi = sc->port[p->port]; 7874 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7875 rc = EINVAL; 7876 goto done; 7877 } 7878 7879 /* 7880 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7881 * Scheduling Class in this case). 7882 */ 7883 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7884 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7885 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7886 7887 /* 7888 * If op.queue is non-negative, then we're only changing the scheduling 7889 * on a single specified TX queue. 7890 */ 7891 if (p->queue >= 0) { 7892 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7893 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7894 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7895 &fw_class); 7896 goto done; 7897 } 7898 7899 /* 7900 * Change the scheduling on all the TX queues for the 7901 * interface. 7902 */ 7903 for_each_txq(pi, i, txq) { 7904 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7905 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7906 &fw_class); 7907 if (rc) 7908 goto done; 7909 } 7910 7911 rc = 0; 7912 done: 7913 end_synchronized_op(sc, 0); 7914 return (rc); 7915 } 7916 7917 int 7918 t4_os_find_pci_capability(struct adapter *sc, int cap) 7919 { 7920 int i; 7921 7922 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7923 } 7924 7925 int 7926 t4_os_pci_save_state(struct adapter *sc) 7927 { 7928 device_t dev; 7929 struct pci_devinfo *dinfo; 7930 7931 dev = sc->dev; 7932 dinfo = device_get_ivars(dev); 7933 7934 pci_cfg_save(dev, dinfo, 0); 7935 return (0); 7936 } 7937 7938 int 7939 t4_os_pci_restore_state(struct adapter *sc) 7940 { 7941 device_t dev; 7942 struct pci_devinfo *dinfo; 7943 7944 dev = sc->dev; 7945 dinfo = device_get_ivars(dev); 7946 7947 pci_cfg_restore(dev, dinfo); 7948 return (0); 7949 } 7950 7951 void 7952 t4_os_portmod_changed(const struct adapter *sc, int idx) 7953 { 7954 struct port_info *pi = sc->port[idx]; 7955 static const char *mod_str[] = { 7956 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7957 }; 7958 7959 build_medialist(pi, &pi->media); 7960 #ifdef DEV_NETMAP 7961 build_medialist(pi, &pi->nm_media); 7962 #endif 7963 7964 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7965 if_printf(pi->ifp, "transceiver unplugged.\n"); 7966 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7967 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7968 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7969 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7970 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7971 if_printf(pi->ifp, "%s transceiver inserted.\n", 7972 mod_str[pi->mod_type]); 7973 } else { 7974 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7975 pi->mod_type); 7976 } 7977 } 7978 7979 void 7980 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7981 { 7982 struct port_info *pi = sc->port[idx]; 7983 struct ifnet *ifp = pi->ifp; 7984 7985 if (link_stat) { 7986 pi->linkdnrc = -1; 7987 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7988 if_link_state_change(ifp, LINK_STATE_UP); 7989 } else { 7990 if (reason >= 0) 7991 pi->linkdnrc = reason; 7992 if_link_state_change(ifp, LINK_STATE_DOWN); 7993 } 7994 } 7995 7996 void 7997 t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7998 { 7999 struct adapter *sc; 8000 8001 sx_slock(&t4_list_lock); 8002 SLIST_FOREACH(sc, &t4_list, link) { 8003 /* 8004 * func should not make any assumptions about what state sc is 8005 * in - the only guarantee is that sc->sc_lock is a valid lock. 8006 */ 8007 func(sc, arg); 8008 } 8009 sx_sunlock(&t4_list_lock); 8010 } 8011 8012 static int 8013 t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8014 { 8015 return (0); 8016 } 8017 8018 static int 8019 t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8020 { 8021 return (0); 8022 } 8023 8024 static int 8025 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8026 struct thread *td) 8027 { 8028 int rc; 8029 struct adapter *sc = dev->si_drv1; 8030 8031 rc = priv_check(td, PRIV_DRIVER); 8032 if (rc != 0) 8033 return (rc); 8034 8035 switch (cmd) { 8036 case CHELSIO_T4_GETREG: { 8037 struct t4_reg *edata = (struct t4_reg *)data; 8038 8039 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8040 return (EFAULT); 8041 8042 if (edata->size == 4) 8043 edata->val = t4_read_reg(sc, edata->addr); 8044 else if (edata->size == 8) 8045 edata->val = t4_read_reg64(sc, edata->addr); 8046 else 8047 return (EINVAL); 8048 8049 break; 8050 } 8051 case CHELSIO_T4_SETREG: { 8052 struct t4_reg *edata = (struct t4_reg *)data; 8053 8054 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8055 return (EFAULT); 8056 8057 if (edata->size == 4) { 8058 if (edata->val & 0xffffffff00000000) 8059 return (EINVAL); 8060 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8061 } else if (edata->size == 8) 8062 t4_write_reg64(sc, edata->addr, edata->val); 8063 else 8064 return (EINVAL); 8065 break; 8066 } 8067 case CHELSIO_T4_REGDUMP: { 8068 struct t4_regdump *regs = (struct t4_regdump *)data; 8069 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8070 uint8_t *buf; 8071 8072 if (regs->len < reglen) { 8073 regs->len = reglen; /* hint to the caller */ 8074 return (ENOBUFS); 8075 } 8076 8077 regs->len = reglen; 8078 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8079 t4_get_regs(sc, regs, buf); 8080 rc = copyout(buf, regs->data, reglen); 8081 free(buf, M_CXGBE); 8082 break; 8083 } 8084 case CHELSIO_T4_GET_FILTER_MODE: 8085 rc = get_filter_mode(sc, (uint32_t *)data); 8086 break; 8087 case CHELSIO_T4_SET_FILTER_MODE: 8088 rc = set_filter_mode(sc, *(uint32_t *)data); 8089 break; 8090 case CHELSIO_T4_GET_FILTER: 8091 rc = get_filter(sc, (struct t4_filter *)data); 8092 break; 8093 case CHELSIO_T4_SET_FILTER: 8094 rc = set_filter(sc, (struct t4_filter *)data); 8095 break; 8096 case CHELSIO_T4_DEL_FILTER: 8097 rc = del_filter(sc, (struct t4_filter *)data); 8098 break; 8099 case CHELSIO_T4_GET_SGE_CONTEXT: 8100 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8101 break; 8102 case CHELSIO_T4_LOAD_FW: 8103 rc = load_fw(sc, (struct t4_data *)data); 8104 break; 8105 case CHELSIO_T4_GET_MEM: 8106 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8107 break; 8108 case CHELSIO_T4_GET_I2C: 8109 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8110 break; 8111 case CHELSIO_T4_CLEAR_STATS: { 8112 int i; 8113 u_int port_id = *(uint32_t *)data; 8114 struct port_info *pi; 8115 8116 if (port_id >= sc->params.nports) 8117 return (EINVAL); 8118 pi = sc->port[port_id]; 8119 8120 /* MAC stats */ 8121 t4_clr_port_stats(sc, pi->tx_chan); 8122 pi->tx_parse_error = 0; 8123 8124 if (pi->flags & PORT_INIT_DONE) { 8125 struct sge_rxq *rxq; 8126 struct sge_txq *txq; 8127 struct sge_wrq *wrq; 8128 8129 for_each_rxq(pi, i, rxq) { 8130 #if defined(INET) || defined(INET6) 8131 rxq->lro.lro_queued = 0; 8132 rxq->lro.lro_flushed = 0; 8133 #endif 8134 rxq->rxcsum = 0; 8135 rxq->vlan_extraction = 0; 8136 } 8137 8138 for_each_txq(pi, i, txq) { 8139 txq->txcsum = 0; 8140 txq->tso_wrs = 0; 8141 txq->vlan_insertion = 0; 8142 txq->imm_wrs = 0; 8143 txq->sgl_wrs = 0; 8144 txq->txpkt_wrs = 0; 8145 txq->txpkts0_wrs = 0; 8146 txq->txpkts1_wrs = 0; 8147 txq->txpkts0_pkts = 0; 8148 txq->txpkts1_pkts = 0; 8149 mp_ring_reset_stats(txq->r); 8150 } 8151 8152 #ifdef TCP_OFFLOAD 8153 /* nothing to clear for each ofld_rxq */ 8154 8155 for_each_ofld_txq(pi, i, wrq) { 8156 wrq->tx_wrs_direct = 0; 8157 wrq->tx_wrs_copied = 0; 8158 } 8159 #endif 8160 wrq = &sc->sge.ctrlq[pi->port_id]; 8161 wrq->tx_wrs_direct = 0; 8162 wrq->tx_wrs_copied = 0; 8163 } 8164 break; 8165 } 8166 case CHELSIO_T4_SCHED_CLASS: 8167 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8168 break; 8169 case CHELSIO_T4_SCHED_QUEUE: 8170 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8171 break; 8172 case CHELSIO_T4_GET_TRACER: 8173 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8174 break; 8175 case CHELSIO_T4_SET_TRACER: 8176 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8177 break; 8178 default: 8179 rc = EINVAL; 8180 } 8181 8182 return (rc); 8183 } 8184 8185 #ifdef TCP_OFFLOAD 8186 void 8187 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8188 const unsigned int *pgsz_order) 8189 { 8190 struct port_info *pi = ifp->if_softc; 8191 struct adapter *sc = pi->adapter; 8192 8193 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8194 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8195 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8196 V_HPZ3(pgsz_order[3])); 8197 } 8198 8199 static int 8200 toe_capability(struct port_info *pi, int enable) 8201 { 8202 int rc; 8203 struct adapter *sc = pi->adapter; 8204 8205 ASSERT_SYNCHRONIZED_OP(sc); 8206 8207 if (!is_offload(sc)) 8208 return (ENODEV); 8209 8210 if (enable) { 8211 /* 8212 * We need the port's queues around so that we're able to send 8213 * and receive CPLs to/from the TOE even if the ifnet for this 8214 * port has never been UP'd administratively. 8215 */ 8216 if (!(pi->flags & PORT_INIT_DONE)) { 8217 rc = cxgbe_init_synchronized(pi); 8218 if (rc) 8219 return (rc); 8220 } 8221 8222 if (isset(&sc->offload_map, pi->port_id)) 8223 return (0); 8224 8225 if (!uld_active(sc, ULD_TOM)) { 8226 rc = t4_activate_uld(sc, ULD_TOM); 8227 if (rc == EAGAIN) { 8228 log(LOG_WARNING, 8229 "You must kldload t4_tom.ko before trying " 8230 "to enable TOE on a cxgbe interface.\n"); 8231 } 8232 if (rc != 0) 8233 return (rc); 8234 KASSERT(sc->tom_softc != NULL, 8235 ("%s: TOM activated but softc NULL", __func__)); 8236 KASSERT(uld_active(sc, ULD_TOM), 8237 ("%s: TOM activated but flag not set", __func__)); 8238 } 8239 8240 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8241 if (!uld_active(sc, ULD_IWARP)) 8242 (void) t4_activate_uld(sc, ULD_IWARP); 8243 if (!uld_active(sc, ULD_ISCSI)) 8244 (void) t4_activate_uld(sc, ULD_ISCSI); 8245 8246 setbit(&sc->offload_map, pi->port_id); 8247 } else { 8248 if (!isset(&sc->offload_map, pi->port_id)) 8249 return (0); 8250 8251 KASSERT(uld_active(sc, ULD_TOM), 8252 ("%s: TOM never initialized?", __func__)); 8253 clrbit(&sc->offload_map, pi->port_id); 8254 } 8255 8256 return (0); 8257 } 8258 8259 /* 8260 * Add an upper layer driver to the global list. 8261 */ 8262 int 8263 t4_register_uld(struct uld_info *ui) 8264 { 8265 int rc = 0; 8266 struct uld_info *u; 8267 8268 sx_xlock(&t4_uld_list_lock); 8269 SLIST_FOREACH(u, &t4_uld_list, link) { 8270 if (u->uld_id == ui->uld_id) { 8271 rc = EEXIST; 8272 goto done; 8273 } 8274 } 8275 8276 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8277 ui->refcount = 0; 8278 done: 8279 sx_xunlock(&t4_uld_list_lock); 8280 return (rc); 8281 } 8282 8283 int 8284 t4_unregister_uld(struct uld_info *ui) 8285 { 8286 int rc = EINVAL; 8287 struct uld_info *u; 8288 8289 sx_xlock(&t4_uld_list_lock); 8290 8291 SLIST_FOREACH(u, &t4_uld_list, link) { 8292 if (u == ui) { 8293 if (ui->refcount > 0) { 8294 rc = EBUSY; 8295 goto done; 8296 } 8297 8298 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8299 rc = 0; 8300 goto done; 8301 } 8302 } 8303 done: 8304 sx_xunlock(&t4_uld_list_lock); 8305 return (rc); 8306 } 8307 8308 int 8309 t4_activate_uld(struct adapter *sc, int id) 8310 { 8311 int rc; 8312 struct uld_info *ui; 8313 8314 ASSERT_SYNCHRONIZED_OP(sc); 8315 8316 if (id < 0 || id > ULD_MAX) 8317 return (EINVAL); 8318 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8319 8320 sx_slock(&t4_uld_list_lock); 8321 8322 SLIST_FOREACH(ui, &t4_uld_list, link) { 8323 if (ui->uld_id == id) { 8324 if (!(sc->flags & FULL_INIT_DONE)) { 8325 rc = adapter_full_init(sc); 8326 if (rc != 0) 8327 break; 8328 } 8329 8330 rc = ui->activate(sc); 8331 if (rc == 0) { 8332 setbit(&sc->active_ulds, id); 8333 ui->refcount++; 8334 } 8335 break; 8336 } 8337 } 8338 8339 sx_sunlock(&t4_uld_list_lock); 8340 8341 return (rc); 8342 } 8343 8344 int 8345 t4_deactivate_uld(struct adapter *sc, int id) 8346 { 8347 int rc; 8348 struct uld_info *ui; 8349 8350 ASSERT_SYNCHRONIZED_OP(sc); 8351 8352 if (id < 0 || id > ULD_MAX) 8353 return (EINVAL); 8354 rc = ENXIO; 8355 8356 sx_slock(&t4_uld_list_lock); 8357 8358 SLIST_FOREACH(ui, &t4_uld_list, link) { 8359 if (ui->uld_id == id) { 8360 rc = ui->deactivate(sc); 8361 if (rc == 0) { 8362 clrbit(&sc->active_ulds, id); 8363 ui->refcount--; 8364 } 8365 break; 8366 } 8367 } 8368 8369 sx_sunlock(&t4_uld_list_lock); 8370 8371 return (rc); 8372 } 8373 8374 int 8375 uld_active(struct adapter *sc, int uld_id) 8376 { 8377 8378 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8379 8380 return (isset(&sc->active_ulds, uld_id)); 8381 } 8382 #endif 8383 8384 /* 8385 * Come up with reasonable defaults for some of the tunables, provided they're 8386 * not set by the user (in which case we'll use the values as is). 8387 */ 8388 static void 8389 tweak_tunables(void) 8390 { 8391 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8392 8393 if (t4_ntxq10g < 1) 8394 t4_ntxq10g = min(nc, NTXQ_10G); 8395 8396 if (t4_ntxq1g < 1) 8397 t4_ntxq1g = min(nc, NTXQ_1G); 8398 8399 if (t4_nrxq10g < 1) 8400 t4_nrxq10g = min(nc, NRXQ_10G); 8401 8402 if (t4_nrxq1g < 1) 8403 t4_nrxq1g = min(nc, NRXQ_1G); 8404 8405 #ifdef TCP_OFFLOAD 8406 if (t4_nofldtxq10g < 1) 8407 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8408 8409 if (t4_nofldtxq1g < 1) 8410 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8411 8412 if (t4_nofldrxq10g < 1) 8413 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8414 8415 if (t4_nofldrxq1g < 1) 8416 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8417 8418 if (t4_toecaps_allowed == -1) 8419 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8420 #else 8421 if (t4_toecaps_allowed == -1) 8422 t4_toecaps_allowed = 0; 8423 #endif 8424 8425 #ifdef DEV_NETMAP 8426 if (t4_nnmtxq10g < 1) 8427 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8428 8429 if (t4_nnmtxq1g < 1) 8430 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8431 8432 if (t4_nnmrxq10g < 1) 8433 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8434 8435 if (t4_nnmrxq1g < 1) 8436 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8437 #endif 8438 8439 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8440 t4_tmr_idx_10g = TMR_IDX_10G; 8441 8442 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8443 t4_pktc_idx_10g = PKTC_IDX_10G; 8444 8445 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8446 t4_tmr_idx_1g = TMR_IDX_1G; 8447 8448 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8449 t4_pktc_idx_1g = PKTC_IDX_1G; 8450 8451 if (t4_qsize_txq < 128) 8452 t4_qsize_txq = 128; 8453 8454 if (t4_qsize_rxq < 128) 8455 t4_qsize_rxq = 128; 8456 while (t4_qsize_rxq & 7) 8457 t4_qsize_rxq++; 8458 8459 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8460 } 8461 8462 static struct sx mlu; /* mod load unload */ 8463 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8464 8465 static int 8466 mod_event(module_t mod, int cmd, void *arg) 8467 { 8468 int rc = 0; 8469 static int loaded = 0; 8470 8471 switch (cmd) { 8472 case MOD_LOAD: 8473 sx_xlock(&mlu); 8474 if (loaded++ == 0) { 8475 t4_sge_modload(); 8476 sx_init(&t4_list_lock, "T4/T5 adapters"); 8477 SLIST_INIT(&t4_list); 8478 #ifdef TCP_OFFLOAD 8479 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8480 SLIST_INIT(&t4_uld_list); 8481 #endif 8482 t4_tracer_modload(); 8483 tweak_tunables(); 8484 } 8485 sx_xunlock(&mlu); 8486 break; 8487 8488 case MOD_UNLOAD: 8489 sx_xlock(&mlu); 8490 if (--loaded == 0) { 8491 int tries; 8492 8493 sx_slock(&t4_list_lock); 8494 if (!SLIST_EMPTY(&t4_list)) { 8495 rc = EBUSY; 8496 sx_sunlock(&t4_list_lock); 8497 goto done_unload; 8498 } 8499 #ifdef TCP_OFFLOAD 8500 sx_slock(&t4_uld_list_lock); 8501 if (!SLIST_EMPTY(&t4_uld_list)) { 8502 rc = EBUSY; 8503 sx_sunlock(&t4_uld_list_lock); 8504 sx_sunlock(&t4_list_lock); 8505 goto done_unload; 8506 } 8507 #endif 8508 tries = 0; 8509 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8510 uprintf("%ju clusters with custom free routine " 8511 "still is use.\n", t4_sge_extfree_refs()); 8512 pause("t4unload", 2 * hz); 8513 } 8514 #ifdef TCP_OFFLOAD 8515 sx_sunlock(&t4_uld_list_lock); 8516 #endif 8517 sx_sunlock(&t4_list_lock); 8518 8519 if (t4_sge_extfree_refs() == 0) { 8520 t4_tracer_modunload(); 8521 #ifdef TCP_OFFLOAD 8522 sx_destroy(&t4_uld_list_lock); 8523 #endif 8524 sx_destroy(&t4_list_lock); 8525 t4_sge_modunload(); 8526 loaded = 0; 8527 } else { 8528 rc = EBUSY; 8529 loaded++; /* undo earlier decrement */ 8530 } 8531 } 8532 done_unload: 8533 sx_xunlock(&mlu); 8534 break; 8535 } 8536 8537 return (rc); 8538 } 8539 8540 static devclass_t t4_devclass, t5_devclass; 8541 static devclass_t cxgbe_devclass, cxl_devclass; 8542 8543 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8544 MODULE_VERSION(t4nex, 1); 8545 MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8546 8547 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8548 MODULE_VERSION(t5nex, 1); 8549 MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8550 8551 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8552 MODULE_VERSION(cxgbe, 1); 8553 8554 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8555 MODULE_VERSION(cxl, 1); 8556